From 46203ea7612a1a2766ee80ecd79c16ed0d5f0ee3 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 20:38:29 -0500 Subject: [PATCH 01/36] test(format): keep the 2026-09-20 B-tree v2 fuzz crash as a regression An 82-byte fuzz_btree_v2 crash input from 2026-09-20 was left untracked in fuzz/artifacts. Replayed today it runs cleanly: the depth cap and record budget added to B-tree v2 traversal that day fixed it. It is now in the committed fuzz corpus, and a robustness test replays the fuzz target's exact code path on it so a regression fails CI rather than waiting for someone to run the fuzzer. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../fuzz_btree_v2/regression-crash-f98c19dc | Bin 0 -> 82 bytes .../clawhdf5-format/tests/robustness_tests.rs | 46 ++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 crates/clawhdf5-format/fuzz/corpus/fuzz_btree_v2/regression-crash-f98c19dc diff --git a/crates/clawhdf5-format/fuzz/corpus/fuzz_btree_v2/regression-crash-f98c19dc b/crates/clawhdf5-format/fuzz/corpus/fuzz_btree_v2/regression-crash-f98c19dc new file mode 100644 index 0000000000000000000000000000000000000000..afb7da7203eb9efd8600a7c9abefcce35057c8b8 GIT binary patch literal 82 zcmZSICOd_A`Z().unwrap(); + let header = BTreeV2Header { + tree_type: fields[0], + node_size: u32::from_le_bytes([fields[1], fields[2], fields[3], fields[4]]), + record_size: u16::from_le_bytes([fields[5], fields[6]]), + depth: u16::from_le_bytes([fields[7], fields[8]]), + root_node_address: u64::from(u32::from_le_bytes([ + fields[9], fields[10], fields[11], fields[12], + ])), + num_records_in_root: u16::from_le_bytes([fields[13], fields[14]]), + total_records: u64::from(u32::from_le_bytes([ + fields[15], fields[16], fields[17], fields[18], + ])), + }; + let offset_size = if fields[19] & 1 == 0 { 4 } else { 8 }; + let _ = collect_btree_v2_records(file, &header, offset_size, 8); +} From 9ea44d473d0a4bd8047fc081afed314561562605 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:01:57 -0500 Subject: [PATCH 02/36] fix(format): read v1 chunk B-tree key offsets as 8 bytes A type-1 (raw data chunk) B-tree key holds the chunk size, the filter mask and one offset per dimension, and those offsets are always 8 bytes: they are dataset coordinates, not file addresses. The reader used the superblock's size-of-offsets for them, so in a file with 4-byte offsets every key was misparsed. Unfiltered chunked datasets read as zeros (with stray bytes where a misread address landed on data) and filtered ones failed with "deflate: truncated stream". Only the sibling and child addresses follow size-of-offsets now. The unit-test B-tree builder wrote keys the same wrong way, which is why its tests passed; it now matches the format. Regression: h5py_four_byte_offsets_chunked_reads (h5py, set_sizes(4, 4) and (4, 8); 1-D and 2-D, unfiltered and gzip) and the unit test collect_chunks_with_four_byte_addresses. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 54 ++++++++- .../clawhdf5/tests/h5py_chunked_read_tests.rs | 105 ++++++++++++++++++ 2 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 crates/clawhdf5/tests/h5py_chunked_read_tests.rs diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 8dade54..b0f5141 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -223,6 +223,10 @@ pub fn collect_chunk_info( collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0) } +/// Width of each chunk offset in a v1 chunk B-tree key, independent of the +/// file's size-of-offsets. +const CHUNK_KEY_OFFSET_SIZE: u8 = 8; + /// Maximum recursion depth for chunk B-tree traversal (malformed/cyclic data /// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`. const MAX_CHUNK_BTREE_DEPTH: usize = 64; @@ -260,8 +264,14 @@ fn collect_chunk_info_inner( let mut pos = offset + 8 + os * 2; // skip left/right sibling - // Key size: chunk_size(4) + filter_mask(4) + ndims * offset_size - let key_size = 4 + 4 + ndims * os; + // Key: chunk_size(4) + filter_mask(4) + one offset per dimension. The + // offsets are always 8 bytes each — they are dataset coordinates, not file + // addresses, so they do not follow the superblock's size-of-offsets (only + // the sibling and child addresses do). + let key_size = ndims + .checked_mul(CHUNK_KEY_OFFSET_SIZE as usize) + .and_then(|n| n.checked_add(8)) + .ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?; if node_level == 0 { // Leaf node: keys and children interleaved @@ -287,8 +297,8 @@ fn collect_chunk_info_inner( let mut offsets = Vec::with_capacity(ndims); let mut kp = pos + 8; for _ in 0..ndims { - offsets.push(read_offset(file_data, kp, offset_size)?); - kp += os; + offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?); + kp += CHUNK_KEY_OFFSET_SIZE as usize; } pos += key_size; @@ -1592,7 +1602,8 @@ mod tests { } else { 0 }; - write_offset(&mut buf, off, offset_size); + // Key offsets are always 8 bytes (they are coordinates). + write_offset(&mut buf, off, 8); } // Child: address write_offset(&mut buf, chunk.address, offset_size); @@ -1602,7 +1613,7 @@ mod tests { buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask for _ in 0..ndims { - write_offset(&mut buf, u64::MAX, offset_size); + write_offset(&mut buf, u64::MAX, 8); } buf @@ -1680,6 +1691,37 @@ mod tests { assert_eq!(result[2].address, 0x300); } + #[test] + fn collect_chunks_with_four_byte_addresses() { + // Sibling and child addresses are 4 bytes; the key offsets stay 8. + let ndims = 3; + let os: u8 = 4; + let chunks = vec![ + ChunkInfo { + chunk_size: 80, + filter_mask: 2, + offsets: vec![0, 5, 0], + address: 0x1000, + }, + ChunkInfo { + chunk_size: 96, + filter_mask: 0, + offsets: vec![8, 10, 0], + address: 0x2000, + }, + ]; + let btree = build_chunk_btree_leaf(&chunks, ndims, os); + assert_eq!(btree.len(), 8 + 2 * 4 + 2 * (8 + 3 * 8 + 4) + (8 + 3 * 8)); + let result = collect_chunk_info(&btree, 0, ndims, os, os).unwrap(); + assert_eq!(result.len(), 2); + for (got, want) in result.iter().zip(&chunks) { + assert_eq!(got.offsets, want.offsets); + assert_eq!(got.address, want.address); + assert_eq!(got.chunk_size, want.chunk_size); + assert_eq!(got.filter_mask, want.filter_mask); + } + } + #[test] fn collect_empty_btree() { let ndims = 2; diff --git a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs new file mode 100644 index 0000000..d46cff5 --- /dev/null +++ b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs @@ -0,0 +1,105 @@ +//! Chunked-read regressions against files written by h5py / libhdf5. +//! +//! Each test builds its input with h5py (or uses a small committed fixture +//! when h5py cannot produce the feature) and compares clawhdf5's read with the +//! known contents. Tests are skipped if python3 with h5py is not available, +//! unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; + +/// The Python interpreter to drive interop checks with (see +/// `h5py_interop_tests.rs`). +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python3"); + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + panic!("Python script failed:\nSTDOUT: {stdout}\nSTDERR: {stderr}"); + } +} + +// --------------------------------------------------------------------------- +// Files with 4-byte addresses (superblock size-of-offsets = 4) +// --------------------------------------------------------------------------- + +/// The chunk B-tree (v1, type 1) stores each chunk offset in its keys as a +/// fixed 8-byte value whatever the file's size-of-offsets. Reading them with +/// the offset width misparsed every key in a 4-byte-offset file: unfiltered +/// datasets came back as zeros and filtered ones failed to inflate. +#[test] +fn h5py_four_byte_offsets_chunked_reads() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sizes_4.h5"); + let p = path.display().to_string(); + run_python(&format!( + r#" +import h5py, numpy as np +for name, lengths in (("{p}", 4), ("{p}.l8", 8)): + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE) + fcpl.set_sizes(4, lengths) + fid = h5py.h5f.create(name.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl) + with h5py.File(fid) as f: + f.create_dataset("plain", data=np.arange(100.0), chunks=(10,)) + f.create_dataset("gzip", data=np.arange(100.0), chunks=(10,), compression="gzip") + f.create_dataset("grid", data=np.arange(35 * 13, dtype=" = (0..100).map(f64::from).collect(); + let grid: Vec = (0..35 * 13).collect(); + for name in [p.clone(), format!("{p}.l8")] { + let file = File::open(&name).unwrap(); + for ds in ["plain", "gzip"] { + assert_eq!( + file.dataset(ds).unwrap().read_f64().unwrap(), + expect, + "{name}:{ds}" + ); + } + for ds in ["grid", "grid_gzip"] { + assert_eq!( + file.dataset(ds).unwrap().read_i32().unwrap(), + grid, + "{name}:{ds}" + ); + } + } +} From aef8e766ae9ea9206613bd4fe8dbad7bffc998e5 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:03:49 -0500 Subject: [PATCH 03/36] fix(format): write and read the registered HDF5 LZ4 filter format Filter 32004 chunks were framed as a 4-byte little-endian size plus one LZ4 block. That is not the registered HDF5 LZ4 format (H5Zlz4.c: 8-byte big-endian total size, 4-byte big-endian block size, then per block a 4-byte big-endian compressed length and the block, stored raw when the length equals the block size), so libhdf5 + hdf5plugin could not read our LZ4 datasets and we could not read theirs (h5ex_d_lz4.h5: "lz4: 0 is not a valid match offset"). Write the registered format (cd_values[0] is honoured as the block size, default 1 GiB like the plugin) and read it, multi-block and raw blocks included. Chunks in the old framing stay readable: an HDF5 chunk is under 4 GiB, so a registered chunk always starts with four zero bytes and is at least 12 bytes long, while an old one starts with four zero bytes only when empty (5 bytes). Tests: lz4_reads_registered_hdf5_format (chunk of the HDF Group's h5ex_d_lz4.h5, block size 3), lz4_writes_registered_hdf5_format, lz4_reads_legacy_clawhdf5_format, and hdf5plugin_reads_our_lz4 (ignored interop test; failed before with "filter returned failure during read"). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 223 ++++++++++++++++-- .../tests/fixtures/filters/README.md | 10 + .../tests/fixtures/filters/h5ex_d_lz4.h5 | Bin 0 -> 23984 bytes .../tests/writer_h5py_tests.rs | 45 ++++ 4 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/README.md create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index c10cea9..d12302b 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -69,7 +69,7 @@ pub fn compress_chunk( let level = filter.client_data.first().copied().unwrap_or(6); deflate_compress(&result, level)? } - FILTER_LZ4 => lz4_compress(&result)?, + FILTER_LZ4 => lz4_compress(&result, &filter.client_data)?, FILTER_ZSTD => { let level = filter.client_data.first().copied().unwrap_or(3); zstd_compress(&result, level)? @@ -813,12 +813,32 @@ fn deflate_compress(_data: &[u8], _level: u32) -> Result, FormatError> { Err(FormatError::UnsupportedFilter(FILTER_DEFLATE)) } -/// Decompress LZ4 data. Format: 4 bytes LE original size + LZ4 block data. +/// Default LZ4 block size of the registered HDF5 LZ4 filter (`H5Zlz4.c`, +/// `DEFAULT_BLOCK_SIZE`): 1 GiB, so an HDF5 chunk is normally one block. +#[cfg(feature = "lz4")] +const LZ4_DEFAULT_BLOCK_SIZE: usize = 1 << 30; + +/// Decompress an LZ4 (filter 32004) chunk. /// -/// The 4-byte "original size" header is part of the attacker-controlled -/// compressed payload itself, so it is bounded against `expected_bytes` (the -/// pipeline's declared chunk size) before being used to size the output -/// allocation — otherwise a crafted 4-byte value can request up to ~4 GiB. +/// Two framings are read: +/// +/// * The registered HDF5 LZ4 filter format (`H5Zlz4.c`, what libhdf5 + +/// hdf5plugin write, and what clawhdf5 writes after 2.7.0): an 8-byte +/// big-endian total decompressed size, a 4-byte big-endian block size, then +/// per block a 4-byte big-endian compressed length followed by the block. A +/// block whose compressed length equals its decompressed length is stored +/// raw. +/// * The legacy clawhdf5 framing (up to 2.7.0): a 4-byte little-endian size +/// followed by one raw LZ4 block. libhdf5 cannot read it. +/// +/// They are told apart unambiguously: an HDF5 chunk is smaller than 4 GiB, so +/// the registered format's big-endian `u64` size always starts with four zero +/// bytes and the whole chunk is at least 12 bytes; a legacy chunk starts with +/// four zero bytes only when it is empty, and is then 5 bytes long. +/// +/// Every size read from the payload is bounded against `expected_bytes` (the +/// pipeline's declared chunk size) before it sizes an allocation, so a crafted +/// header cannot request gigabytes. #[cfg(feature = "lz4")] fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result, FormatError> { if data.len() < 4 { @@ -826,38 +846,112 @@ fn lz4_decompress(data: &[u8], expected_bytes: usize) -> Result, FormatE "lz4: data too short".into(), )); } + let check_size = |size: usize| -> Result<(), FormatError> { + if expected_bytes != 0 && size > expected_bytes { + return Err(FormatError::DecompressionError( + "lz4: declared size exceeds chunk size".into(), + )); + } + if size > MAX_DECOMPRESS_SIZE { + return Err(FormatError::DecompressionError( + "lz4: declared size exceeds limit".into(), + )); + } + Ok(()) + }; + if data.len() >= 12 && data[..4] == [0, 0, 0, 0] { + return lz4_decompress_hdf5(data, check_size); + } + // Legacy clawhdf5 framing: 4-byte LE size + one LZ4 block. let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; - if expected_bytes != 0 && orig_size > expected_bytes { - return Err(FormatError::DecompressionError( - "lz4: declared size exceeds chunk size".into(), - )); - } - if orig_size > MAX_DECOMPRESS_SIZE { - return Err(FormatError::DecompressionError( - "lz4: declared size exceeds limit".into(), - )); - } + check_size(orig_size)?; lz4_flex::block::decompress(&data[4..], orig_size) .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}"))) } +/// Decode the registered HDF5 LZ4 framing (see [`lz4_decompress`]). +#[cfg(feature = "lz4")] +fn lz4_decompress_hdf5( + data: &[u8], + check_size: impl Fn(usize) -> Result<(), FormatError>, +) -> Result, FormatError> { + let err = |m: &str| FormatError::DecompressionError(format!("lz4: {m}")); + let be32 = |b: &[u8]| u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize; + // The first four bytes are zero (checked by the caller), so the size is + // the low 32 bits of the big-endian u64. + let orig_size = be32(&data[4..8]); + check_size(orig_size)?; + let block_size = be32(&data[8..12]).min(orig_size); + if block_size == 0 && orig_size != 0 { + return Err(err("zero block size")); + } + let mut out = vec![0u8; orig_size]; + let mut pos = 12usize; + let mut done = 0usize; + while done < orig_size { + let this_block = block_size.min(orig_size - done); + let comp_len = be32( + data.get(pos..pos + 4) + .ok_or_else(|| err("truncated block header"))?, + ); + pos += 4; + let block = data + .get(pos..pos.saturating_add(comp_len)) + .ok_or_else(|| err("truncated block"))?; + let dst = &mut out[done..done + this_block]; + if comp_len == this_block { + dst.copy_from_slice(block); + } else { + let n = lz4_flex::block::decompress_into(block, dst) + .map_err(|e| FormatError::DecompressionError(format!("lz4: {e}")))?; + if n != this_block { + return Err(err("block decompressed to the wrong size")); + } + } + pos += comp_len; + done += this_block; + } + Ok(out) +} + #[cfg(not(feature = "lz4"))] fn lz4_decompress(_data: &[u8], _expected_bytes: usize) -> Result, FormatError> { Err(FormatError::UnsupportedFilter(FILTER_LZ4)) } -/// Compress data with LZ4 block format. Format: 4 bytes LE original size + LZ4 block data. +/// Compress data in the registered HDF5 LZ4 filter format (see +/// [`lz4_decompress`]), so libhdf5 with the LZ4 plugin (e.g. hdf5plugin) can +/// read it. `cd[0]`, when present and non-zero, is the block size in bytes, +/// as in `H5Zlz4.c`; otherwise the 1 GiB default applies. #[cfg(feature = "lz4")] -fn lz4_compress(data: &[u8]) -> Result, FormatError> { - let compressed = lz4_flex::block::compress(data); - let mut result = Vec::with_capacity(4 + compressed.len()); - result.extend_from_slice(&(data.len() as u32).to_le_bytes()); - result.extend_from_slice(&compressed); +fn lz4_compress(data: &[u8], cd: &[u32]) -> Result, FormatError> { + let block_size = match cd.first() { + Some(&b) if b != 0 => b as usize, + _ => LZ4_DEFAULT_BLOCK_SIZE, + } + .min(data.len()); + let mut result = Vec::with_capacity(16 + data.len() / 2); + result.extend_from_slice(&(data.len() as u64).to_be_bytes()); + result.extend_from_slice(&(block_size as u32).to_be_bytes()); + if block_size == 0 { + return Ok(result); + } + for block in data.chunks(block_size) { + let compressed = lz4_flex::block::compress(block); + if compressed.len() >= block.len() { + // Incompressible: stored raw, marked by length == block length. + result.extend_from_slice(&(block.len() as u32).to_be_bytes()); + result.extend_from_slice(block); + } else { + result.extend_from_slice(&(compressed.len() as u32).to_be_bytes()); + result.extend_from_slice(&compressed); + } + } Ok(result) } #[cfg(not(feature = "lz4"))] -fn lz4_compress(_data: &[u8]) -> Result, FormatError> { +fn lz4_compress(_data: &[u8], _cd: &[u32]) -> Result, FormatError> { Err(FormatError::UnsupportedFilter(FILTER_LZ4)) } @@ -1480,11 +1574,92 @@ mod tests { // --- LZ4 tests --- + /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example, + /// written by libhdf5's registered LZ4 plugin with a 3-byte block size + /// (so it has many blocks, some stored raw). Byte range from h5py's + /// `get_chunk_info`; values are `i*j - j` (i32 LE), as h5py reads them. + #[test] + #[cfg(feature = "lz4")] + fn lz4_reads_registered_hdf5_format() { + let file: &[u8] = include_bytes!("../tests/fixtures/filters/h5ex_d_lz4.h5"); + let chunk = &file[4016..4016 + 312]; + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_LZ4, + name: None, + flags: 1, + client_data: vec![3], + }], + }; + let out = decompress_chunk(chunk, &pipeline, 4 * 8 * 4, 4).unwrap(); + let expected: Vec = (0..4i32) + .flat_map(|i| (0..8i32).map(move |j| i * j - j)) + .flat_map(i32::to_le_bytes) + .collect(); + assert_eq!(out, expected); + } + + #[test] + #[cfg(feature = "lz4")] + fn lz4_writes_registered_hdf5_format() { + // Compressible data, one block: 8-byte BE size, 4-byte BE block size, + // 4-byte BE compressed length, block. + let data = vec![7u8; 1000]; + let c = lz4_compress(&data, &[]).unwrap(); + assert_eq!(&c[0..8], &1000u64.to_be_bytes()); + assert_eq!(&c[8..12], &1000u32.to_be_bytes()); + let len = u32::from_be_bytes(c[12..16].try_into().unwrap()) as usize; + assert_eq!(c.len(), 16 + len); + assert!(len < 1000); + assert_eq!(lz4_decompress(&c, 1000).unwrap(), data); + + // Several blocks, incompressible ones stored raw (length == block). + let data: Vec = (0..10u8).collect(); + let c = lz4_compress(&data, &[3]).unwrap(); + assert_eq!(&c[8..12], &3u32.to_be_bytes()); + assert_eq!(&c[12..16], &3u32.to_be_bytes()); + assert_eq!(&c[16..19], &[0, 1, 2]); + assert_eq!(c.len(), 12 + 3 * (4 + 3) + (4 + 1)); + assert_eq!(lz4_decompress(&c, 10).unwrap(), data); + + let c = lz4_compress(&[], &[]).unwrap(); + assert_eq!(lz4_decompress(&c, 0).unwrap(), Vec::::new()); + } + + /// Chunks written by clawhdf5 up to 2.7.0 (4-byte LE size + one LZ4 + /// block) must stay readable. + #[test] + #[cfg(feature = "lz4")] + fn lz4_reads_legacy_clawhdf5_format() { + for data in [vec![], vec![5u8; 300], (0..=255u8).collect::>()] { + let mut legacy = (data.len() as u32).to_le_bytes().to_vec(); + legacy.extend_from_slice(&lz4_flex::block::compress(&data)); + assert_eq!(lz4_decompress(&legacy, data.len()).unwrap(), data); + } + } + + #[test] + #[cfg(feature = "lz4")] + fn lz4_registered_format_rejects_hostile_sizes() { + // Declared total larger than the chunk. + let mut c = 1000u64.to_be_bytes().to_vec(); + c.extend_from_slice(&1000u32.to_be_bytes()); + c.extend_from_slice(&[0u8; 8]); + assert!(lz4_decompress(&c, 64).is_err()); + // Truncated block. + let mut c = 16u64.to_be_bytes().to_vec(); + c.extend_from_slice(&16u32.to_be_bytes()); + c.extend_from_slice(&16u32.to_be_bytes()); + c.extend_from_slice(&[1u8; 4]); + assert!(lz4_decompress(&c, 16).is_err()); + } + #[test] #[cfg(feature = "lz4")] fn lz4_compress_decompress_roundtrip() { let data: Vec = (0..256).map(|i| (i % 256) as u8).collect(); - let compressed = lz4_compress(&data).unwrap(); + let compressed = lz4_compress(&data, &[]).unwrap(); let decompressed = lz4_decompress(&compressed, data.len()).unwrap(); assert_eq!(decompressed, data); } diff --git a/crates/clawhdf5-format/tests/fixtures/filters/README.md b/crates/clawhdf5-format/tests/fixtures/filters/README.md new file mode 100644 index 0000000..d2095a0 --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/filters/README.md @@ -0,0 +1,10 @@ +# Filter conformance fixtures + +Files written by libhdf5 (and its registered filter plugins), used by the +filter regression tests in `src/filters.rs` to compare our decoders against +the values h5py/libhdf5 read from the same bytes. Chunk byte ranges quoted in +the tests come from h5py's `DatasetID.get_chunk_info`. + +| File | Origin | Licence | +|------|--------|---------| +| `h5ex_d_lz4.h5` | HDF Group `HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5` (hdf5 repository) | HDF5 licence (BSD-3-Clause style) | diff --git a/crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 b/crates/clawhdf5-format/tests/fixtures/filters/h5ex_d_lz4.h5 new file mode 100644 index 0000000000000000000000000000000000000000..d2790150378c8e9b603a64dc6fcaf43c4f9dea4b GIT binary patch literal 23984 zcmeI4f0$NPwZ}IcqzMs6BSTYsGea^3R6-Q4jx^9zNRyJ1kxl$^EYi7%LWVx0p_12X z@|ICqO-fH>W|LW$yyl(6OIT!+mkJYYEJ89;GE6S{c+$_nfoW+3#x~ZEBoxz_7!HCH&RZr6H-FZ~f&Z{dK)3d%W$uMQ;Ylu996l zIKLT`@V}Hs$$yaYm*ZxiHGO)rNYC$9vNT~xeply?zuW&;Gte~sjAjgYjz;Zg=|-Jr zJ{gT?pTM_%J~lb{U)n+GWjufL3(nB7Isf2%gqrd;$d=oMo$rXOw-Yy?!Z~8UR7d~d zA}88EwX!`(9UpQ9hC3+P<@v_i;>#!27G1b_>B1#bYL_frSZiInwC&XKd}-Ubi#u>Cy`?zUY#1txG?+*pl(Dkgd}d zSqv_g@0We(9xg9-^I*P34^o=a(K9!Pq;&SIw>RR7Mj-xbo-jB6{`xbRN*$aajZjvHHw+xW`r zxCy1W^s4H($)&i->adSj%S9Zr zJi0orJb$Z>td1+sUw5rKt~`HxUNbPRJRg~+G~ubUa!Z5dM? zSIh3c*LqEqp9kJAx%=XpUZ3BV*Ch6RYFO^SAts1LMl$RQFQX zr0Tfx{OvfoI9p#&^88IcePEn! z!`T^L+Zw9l%JaAOjOw`Z{Pj+&jw{dKsK)BJT6XunHrrDG{`~y&ag^t8)r{)6^89tb zxjL>qe|wq+#+Cb@sonqHS{+xOzt*=^$Cc-=b7pm1`S@GTtd6T?ci(IM+pFWs^Vif| z9ao;e_IFgrmFI8WSp(w^&&|Ji-F0?#TzUS+zOy>6Jbx|cRL7O)Z}qv=akcF3d+nK1 z9ao;ey7Q{z%JbLo?&`Sm{4MXRuycGZ8$E6KvxA%|wc9&X-m1O}t0U2R>7^I5Ve17s zpBMXUV0*>>Q_N6*GBo}VS}0|`gZVM7$Zd0??sAr&KZ-O|hkaRKa3PZVl8*7lcsygVK%NhdckT8MA;lhz|2d8SFL2tX1I)8xyT$$-*fU~(+6VLI<9IoaXa8XG z#!Qj+XDcO)%h>_&S&L&dCms!)3-+AhQ;)5p$x{b(XK&BC!mun73UT z|I-@(Q_yPO{wTIn><@uGDfat)FmD{I4d9?Kj{I0tBtEtni{sCa9!26j7{eomkH2Q` z(K;%dbTnINq#~D~MZP9YfYQ))$K{a3vaMp?c4+)hX#Cru)x7;qY@66`1N)8Gpk%8s=UXe~T9|RKvn*^UZdCxP3Qp$5q6=RuG zF>k#Z|F1Rv$Dq}`Ju0?U?2*8JCHC+>n711GYy-d}laGVvi?v=W7SEgfh*Knwpr{y0((g8 zXZv8@xQ5yQ@aW~^z%_&xAD_&L>&V&hV!WoziANo0&0;J&%sD%-bHwJr8pBfUy1dL5 z>3pz?dF#>mf2#362(9MrCt{n#ejM13#2(lO^Tun-27pI19|x~9Yn^01IOFw}AEk=K z>&_S+g(jrKCvI{gL(7&4Ic4){J8g^H6@#b`;W7KfX`aI7n$?)z}m!?$gQ%TRN8e^3XYH9VUE>-)iCT+3WQUn+)(c|8HWht!LWgf-`=zz!F4 zgSyOh!{kJb0Vnj%@|Ifvy71b-5ROQU3xe1e9|QUTHaZ)&aQ}pp_lJskqsF+eQ&F3@ zvI?%<$Q$pse!tkaKciIEUbL9dcU;{tCZNxgBB7 zc)p#HL(&XJrt~%l=u3qk&#&RAn=3JZ&TfBD7xm*m_r0Z6~XRYgDD$X~K%{V?Ih~k{*N92NQ5EvdcJ+F_0 zU!R(@C9nnd9+9ot4Crf{*itd{*>kQC^Ko1e*wtd!!J6QPz&nO3C1G`0R zEv)6dBe1)~?uMjUc++aR_nu!qF9h&>kAHnAsRE9Q-# zt@Yr;;MS3Nu3iP7IXMr;@L79bdH!1KV>yIlb+!gRWBxhb&Q^AIWy5hJ*}A~)5!(oB z>?eWw7=J0YHL%CwGr{(3(jVADBNe-{Nq=E0rRU+KFX zF>n6)|1+ag!ny8wKMS9=cpjMZOM&@0j&lNn?!fMc&zwH~_!;+zZO$PHpS2$kbAcAe zYR+c^+bi}`4$1o=8sp0~#u2cldj*VGO>B(Vaez}<*r;sSs^Qg~W3x#U*g_)}le6I~bf_Oa^}%9ZFC85i zYaxq7EdW7W_bD^yFtL$2gsstp{e3wxu&H7*U@PX$`-)@Xn27Vfo(Z3|W@nS$%@!J| zn3qlB+F^nQIO$78lUOsXIp<`*CV^;@ugBaK7ig`o7p+ro|8^y5pc+Rb2+hHr_ZLPSk@87kyYrQ|b&ztkdf%*FOF!aW{17knT*(0_k zuwM9#?Z}2}fFEy<7~+xh@od<760hd`b2bCk#r4takiZykIsJZm zRA9#`C-0dR^VW-erhOut^b}iY*gknS>?6j3@p*rSDepnBK5+Gw>$*AX#YP34uWw_; zCIpth6D$&AEa&S3^R;h^bd7SEu@BQx$tU-+HqUFRr$H#EKn6C><0$U+=Eo{ZS zajtEQM}k}4F+qF0v=hbRIyFt~En*Ep*CfWZ%yP~NtVPWG+7{UI68p!%xUN|TKceH| z-r?i_v~*t-`)XhvVxJRR6WASM>%{I0Y?Ih8U@PX$*XeVmJwIp{O1oI>lE8c%SBbR; zwn{qo+2(C^V4Y%XrRxfe{kOU462o(V&&_?({a9>sV2_Gz7kfIeUa=>|b_e#NSRLlY z`gt%+o}=P$TH8a0%|3gR+U=i3+C5y_Pbe?zUiwn;1+g_^xF`EK?iBl;Sa)C>#d^eW zzjB?AV~5zT!1hQ-YrSKL#xYFecm=F6)J8tW*NKf2J2|ikVyBAn`lc@xv&80z%?rB4 zVwb`4z1c8tY*|P4#~bUEq$wI32&DfS6QW~bL7{+!hk*y0@0R~JF=IXlE|hBYVFb7yynwThuGb9S{D z`T!^N&b}swS^&=VfwhTwZO8i}ef_l*`T$+5CGMAwbwkCxb&3D0v>S2?wtgnIRSavc z_w^aE7sR>)+a!kT6M}7l?Gk%ltPYIVXaBQwkJzJNcyGq*kmuOa_w~|oO}EI=Xq(^- zFk-!kHz(=@FdWx&dQC7&y0|8CEwo6ke--mq$NctvuC$|(SI2z4n6IaNei7rkZh7A+ zT`lOm_L(f!7?`g^bH#9O`SJXj&~?)TJB!D)eM{+ZPP^_yVyqc#QdpnUmkK`~YYt;k z=W849@y6o4o%eJbAMfK8^VXtcen9=XEcAzKrD<;v`%GZBNcRn~_P`i#y47N90^?d~ z>>ja=fo)dKPO)zUcDL9M#Wn}_xY$!-e+{f#>_IVHJ3ilBW34ZH#D;?9IVwid+V}jh z74x=EdDlz(V95Kh*fz1azV8*=4>|3#>J7S`V&0!2VENcZJ+18(NH;dHiDF(CU|+#? z?7tmoymV6ody91Eh|Leo*VEI)W(Ibibgg1MXXs1CO0gSZv(Mh7cKc_eO?~5dHJEvL zjpw-H1X^4B#NLb$v9rY9Bi0buOfg^I7X;QOwnDm9fqhDhG1iwW0=r)9Gt%)KrY{xW z61!K7&x2x}Vt0t$6LS7YY>U|Tz;=rb#w_u@*)VUMYn!l3q{X`9bMsMYIUdu0DX`nb zz9U^{V7H5{6T3gK9qUG4{B0?SXaVkkkolx~^>4>c*=% zd$P&*Mf&>pE^zHa7x$644`fcSmp0`PMxtJUz-x_{*vd}t%Pl!%z&_H39KyYN91pdQ z8T=KfSJ2{~NyWVF634M1LS60a%@Ml4v#u~UB`~i;-h~)H7~}1HlddKW#f7Fy9Y(lX9Y-_kQ|w^krf=2b{4F)tMg^`#iC(z`DhH#CikUCFb=Do&$XRUWfQ| zeXJ+`C>@{EZEkSS@q2Wa82f81)-T)#u(A8PKP>3D-W%gHZ^gXzD(};>e zUo)@{&p8S{C!2sZfa&n(eXWMt~(*HY0{l7-Gaa_6}wi9 zbtZkO@Z)j)x4uLjpV{qrtE9uTqmRMYHm=b*m(kRY5d34UU~eywHr6oLFuq8>p56*U z&QaVc_5OcX2Iq@*qi=&^Sr-SUff%KZoEIkmG{+P%6qKxo+5^% z&V22!iSc}su0gtHv6i4~6PqA*Mqp=&%@mUt(U*#Du^zGBpxY(3 zPV9leekJw?F|2pq&jW^7!^SWg*7$fB`fQDsZm`(Mz{W^7S?sNW%@w-{wqoAQYk&Q} z(;mZCcAk<=nh9&ie{VKyEy1ffugONOY|u9_JVQY@H5;~?@oLTm*>JCkLo??MVs7}H z#YQS{4MBH$HUoZd${NxbYb{I0&zJEydKMg#&4AyxP7~u^nnhk@Z5ON*!#%HR-d2l$ z9o9tmX2X{6BW)AI9**brJs8$1=IjjYF2q~p#%vPTKKlA=v>{rDMj?`GQz}^Rm~O3f z-v^V<>%}L;c)p4GKGGQFjP(-Eb!1_!X31h-o)P1okg?H-R9}1_iS<^+`8I)ZMS7Et z$!85|FBIcfxTSr^zQv1a957?|(*_SIqh@fW8mI~jqkz0xR+r$xqneHZyS~O5%rbp0)3XA@kJU5Yk7}> z$vOJ!kTb=4!!dU&XPwyLK{tW=A~lNP8u6UIr|dY-(V>H^FCgW4O*eCu;~BG@rwn!(doXTa?$I>sj~axZ<;7tYs|kzlFRH ziTw`NSbJc8Os^;I3C!2;ZPM)tjO(@~>lFJ=U|xs#y1zTHm)r5t34eCSGpw%}-RkEr zlyhg$aj(O4UWbeex<=(Z50*L1hI!kgy!(UOKVIY1q~ZixJ2%A6MTpq@#jX&m1vXL4 z_Zj8})+Xj_^y7V**p7Dy zCtBaj&}YvJ(ruTHpP_UIOE*@!(~w2F*<$a9t(Z3)5yyO0HvI43aGKa(F+H0!m#u-n z)4pEp%VN=t62o(Xr5^AdX^Yr$v5p+V7V8fP*h=Y8v)(#UPV51~xgap?3&VL+HvA2+ z>xhgUCN`0g^cFZrpli)$!0)*5S%cyFtD3iU@EiYWHvEmOAM*vVky>l{T(7lun$}F; zN9tylEWkO0oV&8&e;4a(e@(i{%Gr!a<@CK7ulpWn79>f<-fa55cRLPF>88`#e@g{x zonIDMk91GM8sljv-Dv5&FJ}eaVljS4R&lI?UwEa&FX7k>X`IlW#wD(Lu~iZOoIllx~h4-;YD*VDl)FYDeMSA3Axeorp;d4!1h ze#ir24T1R{OPkoLz&fcf(mFAm51w;}81>c{{vL_GR9r81t917S-Ot6I5_=h#>0#Ih z8yNNGTLUNKQ^7qVV_T%d^Mf<5iN+vLj<+}c^VUmWk)D%w1hhKl>uBx!1z7ucc3@tI zTqS0xPse)51QW!X0&9WK*mALsz}AY@#PEFVeewN}4~VS{>!kNKz=YZ-gLKiCj^F>@4ckxqd%)(yk> zRBVyX>xnur>1xvX9z=6stzurk@G_Uq>yVSsw*0Z4SSa0K<4BAhNF^94Bf&EJCPtvsq#yY~1vF0!x?{ADxMXz*wq~rZf zYd`COpyNK0bia}AuhNag@uWMRwn#Hz?Rea`5%c5itNpIhGz+Ow<6xdxkBs~OMF>hXL{TbZ)!*xOHyZ;`kkvAHh|6;zc!|%LI z*PG?c@;qKG*(m8Iq9q;oh>Wp5G9AxFG43rH+mYq@IZNy)THCjTHQn5h^Fv}ciG4e; zUC3nueh+H8SoisNnZDn3ZP0mNSpQVa8*41f>))?!4_e7j~SgRQ8 z4!W%xhu3H$gKh%#MVbj~y4Jv0@0hbk%>2UA#oc_Jcb7DM)r8_~oS+K@G5OQ`S-kkhi%HHE1;yJyx z@tWwIp!2XX}_31?@-GAMUBxF}ZI^Un;x~c~-iXpj#mw>k-r46`1c& zaPPxd9hla)qqV;IKFU{u&i6xpt>bb0>dT?hO@g(Ytk3DQ^;CU%P`aHV=P>D5|Cnw@ z=nHEIOLjG4D(0v4Q|u?wHH2f%74v%HQ-OJ1ut~ZdfejgI z4eNy0FCPoc>yRHxw=J+i(v60-eold*ufOiOSGui1w^#l2I%HDN%~sAOuod&>H4~3x z$HaQ=$2>(^<}h|~VAqR%1=d(Cu&H8m#FhruA$Ggi#=y3fbe9Hpqu4j4+Z5RE#fI|1 iRR6sv-doT7cz0Cjo|LXmx-k$)*GOyse=V>!Z~p~iOFr-b literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 9707d04..44c42f6 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -940,3 +940,48 @@ fn provenance_verify_written_file() { .unwrap(); assert_eq!(result, clawhdf5_format::provenance::VerifyResult::Ok); } + +// ---- hdf5plugin interop: registered third-party compression filters ---- + +/// Write `data` (f64, 1-D, chunked) with `configure` applied, then read it +/// back with h5py + hdf5plugin (libhdf5's registered filter plugins) and +/// return the values it decodes. +#[cfg(any(feature = "lz4", feature = "zstd"))] +fn hdf5plugin_roundtrip( + tag: &str, + data: &[f64], + configure: impl FnOnce(&mut clawhdf5_format::type_builders::DatasetBuilder), +) -> Vec { + let mut fw = FileWriter::new(); + let ds = fw.create_dataset("data"); + ds.with_f64_data(data) + .with_shape(&[data.len() as u64]) + .with_chunks(&[250]); + configure(ds); + let bytes = fw.finish().unwrap(); + let path = std::env::temp_dir().join(format!("clawhdf5_hdf5plugin_{tag}.h5")); + std::fs::write(&path, &bytes).unwrap(); + let script = format!( + "import h5py,hdf5plugin,json; f=h5py.File('{}','r'); print(json.dumps(f['data'][:].tolist()))", + path.display() + ); + let stdout = h5py_read(&path, &script); + serde_json::from_str(&stdout).unwrap() +} + +/// libhdf5's LZ4 plugin must decode what we write (it could not while we +/// wrote a private 4-byte-LE-size framing). +#[cfg(feature = "lz4")] +#[test] +#[ignore = "requires Python h5py + hdf5plugin"] +fn hdf5plugin_reads_our_lz4() { + let data: Vec = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect(); + let got = hdf5plugin_roundtrip("lz4", &data, |ds| { + ds.with_lz4(); + }); + assert_eq!(got, data); + let got = hdf5plugin_roundtrip("lz4_noshuffle", &data, |ds| { + ds.with_lz4().without_shuffle(); + }); + assert_eq!(got, data); +} From b36998ef01a78a914f72670c7a1ea7f8a56b1499 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:04:07 -0500 Subject: [PATCH 04/36] fix(format): refuse object header messages over 64 KiB A v2 object header message has a 2-byte size field. The writer truncated larger sizes to 16 bits, so an attribute over ~64 KiB (or a compact dataset of 65532-65535 bytes, whose layout message adds 4 bytes) produced a file libhdf5 rejects ("message of unshareable class flagged as shareable", "bad flag combination"). ObjectHeaderWriter::serialize now returns a Result and fails on any message over MAX_MESSAGE_SIZE; FileWriter::finish propagates it. Compact storage falls back to contiguous above 65531 bytes, the real limit. Dense storage for large attributes remains future work. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 45 +++--- .../src/object_header_writer.rs | 72 +++++++-- crates/clawhdf5-format/src/type_builders.rs | 2 +- .../tests/writer_meta_tests.rs | 153 ++++++++++++++++++ 4 files changed, 239 insertions(+), 33 deletions(-) create mode 100644 crates/clawhdf5-format/tests/writer_meta_tests.rs diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index c262cd4..d5260c2 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -33,6 +33,11 @@ pub(crate) const OFFSET_SIZE: u8 = 8; pub(crate) const LENGTH_SIZE: u8 = 8; const SUPERBLOCK_SIZE: usize = 48; +/// Largest raw data a compact dataset can hold: the layout message (version, +/// class, 2-byte size, data) must fit an object header message, whose size +/// field is 2 bytes. Bigger "compact" requests fall back to contiguous storage. +const MAX_COMPACT_DATA_SIZE: usize = crate::object_header_writer::MAX_MESSAGE_SIZE - 4; + /// Threshold for switching from compact (inline) to dense attribute storage. const DENSE_ATTR_THRESHOLD: usize = 8; @@ -51,7 +56,7 @@ pub(crate) fn build_chunked_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_time: FillTime, -) -> Vec { +) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); @@ -78,7 +83,7 @@ pub(crate) fn build_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_time: FillTime, -) -> Vec { +) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); @@ -113,7 +118,7 @@ pub(crate) fn build_compact_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_time: FillTime, -) -> Vec { +) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); @@ -140,7 +145,7 @@ pub(crate) fn build_group_oh( dense_link_info: Option<&[u8]>, attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, -) -> Vec { +) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); if let Some(li) = dense_link_info { // Dense link storage: a LinkInfo pointing at the fractal heap + name @@ -903,7 +908,7 @@ pub(crate) fn build_vds_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_time: FillTime, -) -> Vec { +) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); @@ -1130,7 +1135,9 @@ impl FileWriter { let is_compact: Vec = all_ds .iter() .enumerate() - .map(|(i, d)| !is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= 65535) + .map(|(i, d)| { + !is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= MAX_COMPACT_DATA_SIZE + }) .collect(); let root_dense = root_attrs.len() > DENSE_ATTR_THRESHOLD; let group_dense: Vec = groups @@ -1169,9 +1176,9 @@ impl FileWriter { } let attr_blob = group_dense[gi].then(|| build_dense_attrs(&g.attrs, 0)); let dl = group_links_dense[gi].then_some(dummy_link_info.as_slice()); - build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).len() + build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).map(|oh| oh.len()) }) - .collect(); + .collect::>()?; let root_dummy_links: Vec = { let mut links = Vec::new(); @@ -1186,7 +1193,7 @@ impl FileWriter { let root_oh_size = { let attr_blob = root_dense.then(|| build_dense_attrs(&root_attrs, 0)); let dl = root_links_dense.then_some(dummy_link_info.as_slice()); - build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref()).len() + build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref())?.len() }; struct DataBlob { @@ -1215,7 +1222,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), d.fill_time, - ); + )?; // Global heap blob size is address-independent; compute it now // so pass 2 can place it correctly. let vds_mappings = d.virtual_sources.as_deref().unwrap_or(&[]); @@ -1259,7 +1266,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), d.fill_time, - ); + )?; dummy_blobs.push(DataBlob { data: result.data_bytes, oh_bytes: oh, @@ -1278,7 +1285,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), d.fill_time, - ); + )?; dummy_blobs.push(DataBlob { data: vec![], oh_bytes: oh, @@ -1298,7 +1305,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), d.fill_time, - ); + )?; dummy_blobs.push(DataBlob { data: d.raw.clone(), oh_bytes: oh, @@ -1407,7 +1414,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), d.fill_time, - ); + )?; ds_blobs2.push(DataBlob { data: gcol_bytes.clone(), oh_bytes: oh, @@ -1434,7 +1441,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), d.fill_time, - ); + )?; ds_blobs2.push(DataBlob { data: result.data_bytes, oh_bytes: oh, @@ -1449,7 +1456,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), d.fill_time, - ); + )?; ds_blobs2.push(DataBlob { data: vec![], oh_bytes: oh, @@ -1474,7 +1481,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), d.fill_time, - ); + )?; let mut data = vec![0u8; padding]; data.extend_from_slice(&d.raw); cursor2 += d.raw.len(); @@ -1530,7 +1537,7 @@ impl FileWriter { root_dl, &root_attrs, root_dense_blob.as_ref(), - )); + )?); if let Some(ref b) = root_link_blob { buf.extend_from_slice(&b.blob); } @@ -1555,7 +1562,7 @@ impl FileWriter { dl, &g.attrs, group_dense_blobs[gi].as_ref(), - )); + )?); if let Some(ref b) = link_blob { buf.extend_from_slice(&b.blob); } diff --git a/crates/clawhdf5-format/src/object_header_writer.rs b/crates/clawhdf5-format/src/object_header_writer.rs index 9142452..8d52ad4 100644 --- a/crates/clawhdf5-format/src/object_header_writer.rs +++ b/crates/clawhdf5-format/src/object_header_writer.rs @@ -1,11 +1,17 @@ //! Object header writer for v2 format. #[cfg(not(feature = "std"))] -use alloc::vec::Vec; +use alloc::{format, vec::Vec}; use crate::checksum::jenkins_lookup3; +use crate::error::FormatError; use crate::message_type::MessageType; +/// Largest message payload a v2 object header can describe: the per-message +/// size field is 2 bytes. A bigger message cannot be encoded at all — writing +/// its size truncated to 16 bits produced files libhdf5 refuses. +pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize; + /// Writer for v2 object headers with proper checksums. pub struct ObjectHeaderWriter { messages: Vec<(MessageType, Vec, u8)>, // (type, data, msg_flags) @@ -30,7 +36,22 @@ impl ObjectHeaderWriter { } /// Serialize the complete v2 object header (OHDR + messages + checksum). - pub fn serialize(&self) -> Vec { + /// + /// Fails with [`FormatError::SerializationError`] when a message is larger + /// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would + /// need dense attribute storage), rather than writing a corrupt header. + pub fn serialize(&self) -> Result, FormatError> { + if let Some((msg_type, data, _)) = self + .messages + .iter() + .find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE) + { + return Err(FormatError::SerializationError(format!( + "{msg_type:?} message is {} bytes; an object header message holds at most \ + {MAX_MESSAGE_SIZE} bytes", + data.len() + ))); + } // Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data let msg_bytes_total: usize = self .messages @@ -80,7 +101,7 @@ impl ObjectHeaderWriter { let checksum = jenkins_lookup3(&buf); buf.extend_from_slice(&checksum.to_le_bytes()); - buf + Ok(buf) } } @@ -125,15 +146,22 @@ impl BatchObjectHeaderWriter { /// Compute the serialized size of each header without actually serializing. /// Returns sizes in the same order as headers were added. - pub fn compute_sizes(&self) -> Vec { - self.headers.iter().map(|h| h.serialize().len()).collect() + pub fn compute_sizes(&self) -> Result, FormatError> { + self.headers + .iter() + .map(|h| h.serialize().map(|b| b.len())) + .collect() } /// Serialize all headers into a single contiguous buffer. /// Returns `(combined_bytes, offsets)` where `offsets[i]` is the byte /// offset of header `i` within the combined buffer. - pub fn serialize_all(&self) -> (Vec, Vec) { - let serialized: Vec> = self.headers.iter().map(|h| h.serialize()).collect(); + pub fn serialize_all(&self) -> Result<(Vec, Vec), FormatError> { + let serialized: Vec> = self + .headers + .iter() + .map(|h| h.serialize()) + .collect::>()?; let total: usize = serialized.iter().map(|s| s.len()).sum(); let mut buf = Vec::with_capacity(total); let mut offsets = Vec::with_capacity(serialized.len()); @@ -141,7 +169,7 @@ impl BatchObjectHeaderWriter { offsets.push(buf.len()); buf.extend_from_slice(s); } - (buf, offsets) + Ok((buf, offsets)) } } @@ -159,7 +187,7 @@ mod tests { #[test] fn empty_header_roundtrip() { let writer = ObjectHeaderWriter::new(); - let bytes = writer.serialize(); + let bytes = writer.serialize().unwrap(); let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap(); assert_eq!(hdr.version, 2); assert_eq!(hdr.messages.len(), 0); @@ -170,7 +198,7 @@ mod tests { let mut writer = ObjectHeaderWriter::new(); writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]); writer.add_message(MessageType::Datatype, vec![5, 6]); - let bytes = writer.serialize(); + let bytes = writer.serialize().unwrap(); let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap(); assert_eq!(hdr.messages.len(), 2); assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace); @@ -184,12 +212,30 @@ mod tests { let mut writer = ObjectHeaderWriter::new(); // Add a message with >255 bytes of payload writer.add_message(MessageType::Datatype, vec![0xAA; 300]); - let bytes = writer.serialize(); + let bytes = writer.serialize().unwrap(); let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap(); assert_eq!(hdr.messages.len(), 1); assert_eq!(hdr.messages[0].data.len(), 300); } + #[test] + fn oversized_message_is_an_error_not_a_truncated_size() { + // 65535 bytes is the largest encodable payload. + let mut writer = ObjectHeaderWriter::new(); + writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE]); + let bytes = writer.serialize().unwrap(); + let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap(); + assert_eq!(hdr.messages[0].data.len(), MAX_MESSAGE_SIZE); + + // One byte more used to be written with its size wrapped to 0. + let mut writer = ObjectHeaderWriter::new(); + writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE + 1]); + assert!(matches!( + writer.serialize(), + Err(FormatError::SerializationError(_)) + )); + } + #[test] fn batch_writer_serialize_all() { let mut batch = BatchObjectHeaderWriter::new(); @@ -204,7 +250,7 @@ mod tests { batch.add(w2); assert_eq!(batch.len(), 2); - let (buf, offsets) = batch.serialize_all(); + let (buf, offsets) = batch.serialize_all().unwrap(); assert_eq!(offsets.len(), 2); assert_eq!(offsets[0], 0); @@ -222,7 +268,7 @@ mod tests { fn batch_writer_empty() { let batch = BatchObjectHeaderWriter::new(); assert!(batch.is_empty()); - let (buf, offsets) = batch.serialize_all(); + let (buf, offsets) = batch.serialize_all().unwrap(); assert!(buf.is_empty()); assert!(offsets.is_empty()); } diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index e698899..9ee2520 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -718,7 +718,7 @@ impl DatasetBuilder { /// Use compact (inline) storage for this dataset. /// /// The raw data is stored directly in the dataset's object header rather - /// than as a separate data blob. Only effective when raw data <= 65536 bytes + /// than as a separate data blob. Only effective when raw data <= 65531 bytes /// and the dataset is not chunked. pub fn compact(&mut self) -> &mut Self { self.compact = true; diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs new file mode 100644 index 0000000..1175ab4 --- /dev/null +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -0,0 +1,153 @@ +//! Regression tests for writer metadata bugs that produced files libhdf5 +//! refuses (or reads differently from us), plus the reader-side counterparts. +//! +//! The plain tests check the bytes we write with our own parser. The +//! `#[ignore]`d ones are the interop half: they open what we write in h5py +//! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it. + +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::file_writer::{AttrValue, FileWriter}; +use clawhdf5_format::group_v2::resolve_path_any; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::signature; +use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::type_builders::make_u8_type; + +// ---- helpers ---- + +fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) { + let sig = signature::find_signature(bytes).unwrap(); + let sb = Superblock::parse(bytes, sig).unwrap(); + let addr = if path == "/" { + sb.root_group_address + } else { + resolve_path_any(bytes, &sb, path).unwrap() + }; + let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap(); + (sb, oh) +} + +fn layout_of(bytes: &[u8], path: &str) -> DataLayout { + let (sb, oh) = header_at(bytes, path); + let msg = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + .unwrap(); + DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap() +} + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5")); + std::fs::write(&path, bytes).unwrap(); + path +} + +/// Run `script` (with `path` bound to the file) under h5py; return stdout. +fn h5py(path: &std::path::Path, script: &str) -> String { + let full = format!( + "import h5py, numpy as np, json\npath = {:?}\n{script}", + path.display().to_string() + ); + let o = std::process::Command::new(python()) + .args(["-c", &full]) + .output() + .expect("python interpreter"); + assert!( + o.status.success(), + "h5py failed: {}", + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8(o.stdout).unwrap().trim().to_string() +} + +/// `h5dump` must read the whole file without error. +fn h5dump_ok(path: &std::path::Path) { + let o = std::process::Command::new("h5dump") + .arg(path) + .output() + .expect("h5dump"); + assert!( + o.status.success(), + "h5dump failed: {}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); +} + +fn u8_ramp(n: usize) -> Vec { + (0..n).map(|i| (i % 251) as u8).collect() +} + +// ---- 1. object header message size limit ---- + +#[test] +fn attribute_too_big_for_a_header_message_is_an_error() { + // Measured: a 70000-byte attribute was written with its message size + // wrapped to 16 bits, and libhdf5 refused the whole root group. + let mut fw = FileWriter::new(); + fw.set_root_attr( + "a", + AttrValue::Raw { + datatype: make_u8_type(), + shape: vec![70_000], + data: u8_ramp(70_000), + }, + ); + assert!(fw.finish().is_err()); + + // 65500 bytes still fits and still works. + let mut fw = FileWriter::new(); + fw.set_root_attr( + "a", + AttrValue::Raw { + datatype: make_u8_type(), + shape: vec![65_500], + data: u8_ramp(65_500), + }, + ); + let bytes = fw.finish().unwrap(); + let (sb, oh) = header_at(&bytes, "/"); + let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap(); + assert_eq!(attrs[0].raw_data, u8_ramp(65_500)); +} + +#[test] +fn compact_layout_falls_back_to_contiguous_past_the_message_limit() { + // Layout message = 4 bytes + data; data may be at most 65531 bytes. + for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] { + let mut fw = FileWriter::new(); + fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact(); + let bytes = fw.finish().unwrap(); + match layout_of(&bytes, "d") { + DataLayout::Compact { data } => { + assert!(compact, "{n} bytes must not be compact"); + assert_eq!(data, u8_ramp(n)); + } + DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"), + other => panic!("unexpected layout {other:?}"), + } + } +} + +#[test] +#[ignore = "requires Python h5py module and h5dump"] +fn h5py_reads_compact_datasets_at_the_limit() { + for n in [65_531usize, 65_534] { + let mut fw = FileWriter::new(); + fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact(); + let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap()); + let out = h5py( + &path, + "f = h5py.File(path, 'r'); v = f['d'][()]\n\ + print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)", + ); + assert_eq!(out, format!("True {n}")); + h5dump_ok(&path); + } +} From bba156041665355611904dcc64efbbd33520195a Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:04:19 -0500 Subject: [PATCH 05/36] fix(format): lay Fixed/Extensible Array chunk indexes out by max dims Both indexes place each chunk at a linear index computed from the dataset's maximum dimensions (libhdf5's max_down_chunks), and the Extensible Array first swizzles its unlimited dimension to the slowest position. We linearised by the current dimensions, so any dataset whose shape was smaller than its maxshape, or whose unlimited dimension was not the first, read back scrambled without an error: h5py libver="latest" files with maxshape (10, None) or (20, 10), and the libhdf5 test files h5fc_ext*.h5 and test_ld.h5. The linearisation now lives in chunk_grid (shared with the writers), and slots beyond the current extent are ignored as the library does. read_fixed_array_chunks / read_extensible_array_chunks take the dataspace's max dimensions. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunk_grid.rs | 201 ++++++++++++++ crates/clawhdf5-format/src/chunked_read.rs | 2 + .../clawhdf5-format/src/extensible_array.rs | 159 ++++------- crates/clawhdf5-format/src/fixed_array.rs | 101 ++----- crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5/tests/chunk_index_interop.rs | 256 ++++++++++++++++++ 6 files changed, 546 insertions(+), 174 deletions(-) create mode 100644 crates/clawhdf5-format/src/chunk_grid.rs create mode 100644 crates/clawhdf5/tests/chunk_index_interop.rs diff --git a/crates/clawhdf5-format/src/chunk_grid.rs b/crates/clawhdf5-format/src/chunk_grid.rs new file mode 100644 index 0000000..c5a06fd --- /dev/null +++ b/crates/clawhdf5-format/src/chunk_grid.rs @@ -0,0 +1,201 @@ +//! Chunk-index linearisation shared by the Fixed Array and Extensible Array +//! chunk indexes (reader and writer). +//! +//! Both indexes store one element per chunk at a *linear* index, and the +//! library derives that index from the chunk's scaled coordinates +//! (`offset / chunk_dim`) using the dataset's **maximum** dimensions, not its +//! current ones (`H5D__farray_idx_get_addr` / `H5D__earray_idx_get_addr`, +//! via `layout->max_down_chunks`). A dataset whose current shape is smaller +//! than its maxshape therefore has gaps in the index, and laying it out by the +//! current shape puts every chunk after the first row in the wrong place. +//! +//! The Extensible Array adds one more step: its one unlimited dimension has no +//! finite chunk count, so the library *swizzles* the coordinates to make that +//! dimension the slowest-varying one (`H5VM_swizzle_coords`, which moves +//! `coords[unlim_dim]` to the front and shifts the dimensions before it right +//! by one) before linearising with `swizzled_max_down_chunks`. When the +//! unlimited dimension is already dimension 0 no swizzle happens. + +#[cfg(not(feature = "std"))] +extern crate alloc; + +#[cfg(not(feature = "std"))] +use alloc::{vec, vec::Vec}; + +use crate::error::FormatError; + +/// How a chunk index maps linear element indexes to chunk coordinates. +#[derive(Debug, Clone)] +pub(crate) struct ChunkGrid { + /// Spatial chunk dimensions, in dataset order. + chunk_dims: Vec, + /// Chunks per dimension covering the *current* extent, in dataset order. + cur_chunks: Vec, + /// Dataset dimension stored at each linearisation position (slowest + /// first). The identity except for a swizzled Extensible Array. + order: Vec, + /// Linear stride of each linearisation position. + down: Vec, +} + +impl ChunkGrid { + /// Grid for a Fixed Array index: row-major over the chunk counts of the + /// maximum dimensions (`max_dims`, falling back to the current dimensions + /// when the dataspace records none). + pub(crate) fn fixed_array( + cur_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dims: &[u64], + ) -> Result { + Self::build(cur_dims, max_dims, chunk_dims, None) + } + + /// Grid for an Extensible Array index: like the Fixed Array, but the + /// unlimited dimension (the one whose maximum is `H5S_UNLIMITED`) is moved + /// to the slowest-varying position first. + pub(crate) fn extensible_array( + cur_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dims: &[u64], + ) -> Result { + let unlim = max_dims.and_then(|m| m.iter().position(|&d| d == u64::MAX)); + Self::build(cur_dims, max_dims, chunk_dims, unlim) + } + + fn build( + cur_dims: &[u64], + max_dims: Option<&[u64]>, + chunk_dims: &[u64], + unlim: Option, + ) -> Result { + let rank = chunk_dims.len(); + if cur_dims.len() != rank || max_dims.is_some_and(|m| m.len() != rank) { + return Err(FormatError::ChunkedReadError( + "chunk index rank does not match the dataspace".into(), + )); + } + if chunk_dims.contains(&0) { + return Err(FormatError::ChunkedReadError( + "chunk dimension is zero".into(), + )); + } + let cur_chunks: Vec = cur_dims + .iter() + .zip(chunk_dims) + .map(|(&d, &c)| d.div_ceil(c)) + .collect(); + // Chunk counts of the maximum extent. An unlimited dimension has no + // finite count; it only ever sits in the slowest position, where its + // count never enters a stride. A (corrupt) maximum smaller than the + // current extent is widened so no allocated chunk becomes unreachable. + let max_chunks: Vec = (0..rank) + .map(|d| { + let max = max_dims.map_or(cur_dims[d], |m| m[d]); + if max == u64::MAX { + u64::MAX + } else { + max.div_ceil(chunk_dims[d]).max(cur_chunks[d]) + } + }) + .collect(); + + let mut order: Vec = (0..rank).collect(); + if let Some(u) = unlim { + order.remove(u); + order.insert(0, u); + } + let mut down = vec![1u64; rank]; + for p in (0..rank.saturating_sub(1)).rev() { + let next = max_chunks[order[p + 1]]; + if next == u64::MAX { + // Only reachable with more than one unlimited dimension, which + // neither index type can describe. + return Err(FormatError::ChunkedReadError( + "array chunk index with more than one unlimited dimension".into(), + )); + } + down[p] = down[p + 1].checked_mul(next).ok_or_else(|| { + FormatError::Overflow("chunk index linear stride overflows u64".into()) + })?; + } + Ok(Self { + chunk_dims: chunk_dims.to_vec(), + cur_chunks, + order, + down, + }) + } + + /// Dataset-space offsets of the chunk stored at linear `index`, or `None` + /// when that chunk lies outside the current extent (the index still has a + /// slot for it; the library ignores such chunks on read). + pub(crate) fn offsets(&self, index: u64) -> Option> { + let rank = self.chunk_dims.len(); + let mut offsets = vec![0u64; rank]; + let mut rem = index; + for p in 0..rank { + let d = self.order[p]; + let scaled = rem / self.down[p]; + rem %= self.down[p]; + if scaled >= self.cur_chunks[d] { + return None; + } + offsets[d] = scaled * self.chunk_dims[d]; + } + Some(offsets) + } + + /// Linear index of the chunk with scaled coordinates `scaled` + /// (`offset / chunk_dim` per dimension, in dataset order). + #[allow(dead_code)] // used by the writer + pub(crate) fn linear_index(&self, scaled: &[u64]) -> u64 { + self.order + .iter() + .zip(&self.down) + .map(|(&d, &stride)| scaled[d] * stride) + .sum() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_array_uses_max_dims() { + // shape (4, 6), chunks (2, 3), maxshape (20, 10): 10 x 4 chunk grid. + let g = ChunkGrid::fixed_array(&[4, 6], Some(&[20, 10]), &[2, 3]).unwrap(); + assert_eq!(g.offsets(0), Some(vec![0, 0])); + assert_eq!(g.offsets(1), Some(vec![0, 3])); + assert_eq!(g.offsets(2), None); // column chunk 2 is beyond the extent + assert_eq!(g.offsets(4), Some(vec![2, 0])); + assert_eq!(g.offsets(5), Some(vec![2, 3])); + assert_eq!(g.offsets(8), None); // row chunk 2 is beyond the extent + assert_eq!(g.linear_index(&[1, 1]), 5); + } + + #[test] + fn extensible_array_swizzles_unlimited_dim() { + // maxshape (10, None): dim 1 is unlimited and becomes slowest. + let g = ChunkGrid::extensible_array(&[4, 6], Some(&[10, u64::MAX]), &[2, 3]).unwrap(); + // max chunks of dim 0 = 5, so index = c1 * 5 + c0. + assert_eq!(g.linear_index(&[1, 0]), 1); + assert_eq!(g.linear_index(&[0, 1]), 5); + assert_eq!(g.offsets(5), Some(vec![0, 3])); + assert_eq!(g.offsets(6), Some(vec![2, 3])); + assert_eq!(g.offsets(2), None); + } + + #[test] + fn extensible_array_unlimited_first_is_row_major() { + let g = ChunkGrid::extensible_array(&[4, 6], Some(&[u64::MAX, 30]), &[2, 3]).unwrap(); + // max chunks of dim 1 = 10. + assert_eq!(g.linear_index(&[1, 1]), 11); + assert_eq!(g.offsets(11), Some(vec![2, 3])); + } + + #[test] + fn rejects_two_unlimited_dims_after_the_first() { + assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 8dade54..f07ae27 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -593,6 +593,7 @@ pub fn list_chunks( file_data, &header, &dataspace.dimensions, + dataspace.max_dimensions.as_deref(), spatial_chunk_dims, elem_size as u32, offset_size, @@ -608,6 +609,7 @@ pub fn list_chunks( file_data, &header, &dataspace.dimensions, + dataspace.max_dimensions.as_deref(), spatial_chunk_dims, elem_size as u32, offset_size, diff --git a/crates/clawhdf5-format/src/extensible_array.rs b/crates/clawhdf5-format/src/extensible_array.rs index 4416a76..ba1c822 100644 --- a/crates/clawhdf5-format/src/extensible_array.rs +++ b/crates/clawhdf5-format/src/extensible_array.rs @@ -9,6 +9,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; @@ -203,8 +204,7 @@ fn read_element( offset_size: u8, chunk_byte_size: u64, linear_index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], + grid: &ChunkGrid, ) -> Result<(Option, usize), FormatError> { let os = offset_size as usize; @@ -220,7 +220,10 @@ fn read_element( return Ok((None, os)); } let address = read_offset(data, pos, offset_size)?; - let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions); + // A slot beyond the current extent is ignored, as the library does. + let Some(offsets) = grid.offsets(linear_index as u64) else { + return Ok((None, os)); + }; Ok(( Some(ChunkInfo { chunk_size: chunk_byte_size as u32, @@ -261,7 +264,9 @@ fn read_element( data[fm_off + 2], data[fm_off + 3], ]); - let offsets = index_to_chunk_offsets(linear_index, num_chunks_per_dim, chunk_dimensions); + let Some(offsets) = grid.offsets(linear_index as u64) else { + return Ok((None, elem_total)); + }; Ok(( Some(ChunkInfo { chunk_size: chunk_size as u32, @@ -274,27 +279,6 @@ fn read_element( } } -/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space. -fn index_to_chunk_offsets( - index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], -) -> Vec { - let rank = num_chunks_per_dim.len(); - let mut offsets = vec![0u64; rank]; - let mut remaining = index as u64; - for d in (0..rank).rev() { - let nchunks = num_chunks_per_dim[d]; - if nchunks == 0 { - continue; - } - let chunk_idx = remaining % nchunks; - remaining /= nchunks; - offsets[d] = chunk_idx * chunk_dimensions[d] as u64; - } - offsets -} - /// Collect elements from a data block at the given offset. #[allow(clippy::too_many_arguments)] /// Layout of super block `u`, per the HDF5 spec: the number of data blocks it @@ -339,8 +323,7 @@ fn read_data_block_elements( offset_size: u8, chunk_byte_size: u64, start_index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], + grid: &ChunkGrid, page_init: &[u8], first_page: usize, ) -> Result, FormatError> { @@ -376,8 +359,7 @@ fn read_data_block_elements( offset_size, chunk_byte_size, first_index + i, - num_chunks_per_dim, - chunk_dimensions, + grid, )?; if let Some(ci) = info { chunks.push(ci); @@ -449,25 +431,19 @@ pub fn read_extensible_array_chunks( file_data: &[u8], header: &ExtensibleArrayHeader, dataset_dims: &[u64], + max_dims: Option<&[u64]>, chunk_dimensions: &[u32], element_size: u32, offset_size: u8, _length_size: u8, ) -> Result, FormatError> { - let rank = chunk_dimensions.len(); let os = offset_size as usize; - let mut num_chunks_per_dim = Vec::with_capacity(rank); - for d in 0..rank { - let ch_dim = chunk_dimensions[d] as u64; - if ch_dim == 0 { - return Err(FormatError::ChunkedReadError( - "chunk dimension is zero".into(), - )); - } - let ds_dim = dataset_dims[d]; - num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim)); - } + // Linear indexes follow the maximum dimensions, with the unlimited + // dimension swizzled to the slowest position (see `chunk_grid`). + let dims_u64: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); + let grid = ChunkGrid::extensible_array(dataset_dims, max_dims, &dims_u64)?; + let grid = &grid; let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; @@ -557,8 +533,7 @@ pub fn read_extensible_array_chunks( offset_size, chunk_byte_size, i, - &num_chunks_per_dim, - chunk_dimensions, + grid, )?; if let Some(ci) = info { chunks.push(ci); @@ -594,8 +569,7 @@ pub fn read_extensible_array_chunks( offset_size, chunk_byte_size, global_index, - &num_chunks_per_dim, - chunk_dimensions, + grid, &[], 0, )?); @@ -625,8 +599,7 @@ pub fn read_extensible_array_chunks( offset_size, chunk_byte_size, global_index, - &num_chunks_per_dim, - chunk_dimensions, + grid, )?); } global_index = @@ -653,8 +626,7 @@ fn read_super_block( offset_size: u8, chunk_byte_size: u64, start_index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], + grid: &ChunkGrid, ) -> Result, FormatError> { let os = offset_size as usize; let sb_header_size = 4 + 1 + 1 + os + arr_off_size(header); @@ -710,8 +682,7 @@ fn read_super_block( offset_size, chunk_byte_size, global_idx, - num_chunks_per_dim, - chunk_dimensions, + grid, bitmap, i * npages, )?); @@ -735,35 +706,18 @@ mod tests { } #[test] fn index_to_offsets_1d() { - let num_chunks = vec![5u64]; - let chunk_dims = vec![20u32]; - assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![20] - ); - assert_eq!( - index_to_chunk_offsets(4, &num_chunks, &chunk_dims), - vec![80] - ); + let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0]); + assert_eq!(g.offsets(1).unwrap(), vec![20]); + assert_eq!(g.offsets(4).unwrap(), vec![80]); } #[test] fn index_to_offsets_2d() { - let num_chunks = vec![3u64, 2]; - let chunk_dims = vec![4u32, 3]; - assert_eq!( - index_to_chunk_offsets(0, &num_chunks, &chunk_dims), - vec![0, 0] - ); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![0, 3] - ); - assert_eq!( - index_to_chunk_offsets(2, &num_chunks, &chunk_dims), - vec![4, 0] - ); + let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0, 0]); + assert_eq!(g.offsets(1).unwrap(), vec![0, 3]); + assert_eq!(g.offsets(2).unwrap(), vec![4, 0]); } #[test] @@ -830,7 +784,7 @@ mod tests { index_block_address: (usize::MAX - 4) as u64, }; let buf = vec![0u8; 64]; - let r = read_extensible_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_extensible_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -913,9 +867,17 @@ mod tests { let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap(); let ds_dims = vec![40u64]; // 2 chunks × 20 elements let chunk_dims = vec![20u32]; - let chunks = - read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls) - .unwrap(); + let chunks = read_extensible_array_chunks( + &file_data, + &header, + &ds_dims, + None, + &chunk_dims, + 8, + os, + ls, + ) + .unwrap(); assert_eq!(chunks.len(), 2); assert_eq!(chunks[0].address, base_addr); @@ -1023,9 +985,17 @@ mod tests { let header = ExtensibleArrayHeader::parse(&file_data, aehd_offset, os, ls).unwrap(); let ds_dims = vec![40u64]; let chunk_dims = vec![10u32]; - let chunks = - read_extensible_array_chunks(&file_data, &header, &ds_dims, &chunk_dims, 8, os, ls) - .unwrap(); + let chunks = read_extensible_array_chunks( + &file_data, + &header, + &ds_dims, + None, + &chunk_dims, + 8, + os, + ls, + ) + .unwrap(); assert_eq!(chunks.len(), 4); for (i, c) in chunks.iter().enumerate() { @@ -1047,10 +1017,8 @@ mod tests { #[test] fn read_element_unallocated() { let data = vec![0xFFu8; 16]; - let num_chunks = vec![5u64]; - let chunk_dims = vec![10u32]; - let (info, consumed) = - read_element(&data, 0, 0, 8, 8, 80, 0, &num_chunks, &chunk_dims).unwrap(); + let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); + let (info, consumed) = read_element(&data, 0, 0, 8, 8, 80, 0, &grid).unwrap(); assert!(info.is_none()); assert_eq!(consumed, 8); } @@ -1069,20 +1037,9 @@ mod tests { // Filter mask data[12..16].copy_from_slice(&0u32.to_le_bytes()); - let num_chunks = vec![5u64]; - let chunk_dims = vec![10u32]; - let (info, consumed) = read_element( - &data, - 0, - 1, - elem_size as u8, - os, - 80, - 2, - &num_chunks, - &chunk_dims, - ) - .unwrap(); + let grid = ChunkGrid::fixed_array(&[50], None, &[10]).unwrap(); + let (info, consumed) = + read_element(&data, 0, 1, elem_size as u8, os, 80, 2, &grid).unwrap(); let ci = info.unwrap(); assert_eq!(ci.address, 0x2000); assert_eq!(ci.chunk_size, 120); diff --git a/crates/clawhdf5-format/src/fixed_array.rs b/crates/clawhdf5-format/src/fixed_array.rs index 4c79f03..8546224 100644 --- a/crates/clawhdf5-format/src/fixed_array.rs +++ b/crates/clawhdf5-format/src/fixed_array.rs @@ -6,6 +6,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] use alloc::{format, vec, vec::Vec}; +use crate::chunk_grid::ChunkGrid; use crate::chunked_read::ChunkInfo; use crate::error::FormatError; @@ -151,13 +152,13 @@ pub fn read_fixed_array_chunks( file_data: &[u8], header: &FixedArrayHeader, dataset_dims: &[u64], + max_dims: Option<&[u64]>, chunk_dimensions: &[u32], element_size: u32, offset_size: u8, _length_size: u8, ) -> Result, FormatError> { let db_offset = header.data_block_address as usize; - let rank = chunk_dimensions.len(); // Parse data block header: FADB(4) + version(1) + client_id(1) + header_address(offset_size) let db_header_size = 4 + 1 + 1 + offset_size as usize; @@ -198,19 +199,10 @@ pub fn read_fixed_array_chunks( )) }; - // Compute chunk offsets based on index. - // Chunks are stored in row-major order within the dataset space. - let mut num_chunks_per_dim = Vec::with_capacity(rank); - for d_idx in 0..rank { - let ch_dim = chunk_dimensions[d_idx] as u64; - if ch_dim == 0 { - return Err(FormatError::ChunkedReadError( - "chunk dimension is zero".into(), - )); - } - let ds_dim = dataset_dims[d_idx]; - num_chunks_per_dim.push(ds_dim.div_ceil(ch_dim)); - } + // The index is laid out over the chunk grid of the *maximum* dimensions + // (row-major), so a dataset smaller than its maxshape has gaps. + let dims_u64: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); + let grid = ChunkGrid::fixed_array(dataset_dims, max_dims, &dims_u64)?; let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; @@ -226,7 +218,11 @@ pub fn read_fixed_array_chunks( header.element_size, chunk_byte_size, )? { - let offsets = index_to_chunk_offsets(i, &num_chunks_per_dim, chunk_dimensions); + // A slot beyond the current extent is ignored, as the + // library does. + let Some(offsets) = grid.offsets(i as u64) else { + return Ok(()); + }; chunks.push(ChunkInfo { chunk_size, filter_mask, @@ -367,27 +363,6 @@ fn parse_fa_element( } } -/// Convert a linear chunk index to N-dimensional chunk offsets in dataset space. -fn index_to_chunk_offsets( - index: usize, - num_chunks_per_dim: &[u64], - chunk_dimensions: &[u32], -) -> Vec { - let rank = num_chunks_per_dim.len(); - let mut offsets = vec![0u64; rank]; - let mut remaining = index as u64; - for d in (0..rank).rev() { - let nchunks = num_chunks_per_dim[d]; - if nchunks == 0 { - continue; - } - let chunk_idx = remaining % nchunks; - remaining /= nchunks; - offsets[d] = chunk_idx * chunk_dimensions[d] as u64; - } - offsets -} - /// Read a variable-length little-endian unsigned integer. fn read_variable_length(data: &[u8], size: usize) -> Result { if size > 8 || data.len() < size { @@ -416,44 +391,21 @@ mod tests { #[test] fn index_to_offsets_1d() { - let num_chunks = vec![5u64]; - let chunk_dims = vec![20u32]; - assert_eq!(index_to_chunk_offsets(0, &num_chunks, &chunk_dims), vec![0]); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![20] - ); - assert_eq!( - index_to_chunk_offsets(4, &num_chunks, &chunk_dims), - vec![80] - ); + let g = ChunkGrid::fixed_array(&[100], None, &[20]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0]); + assert_eq!(g.offsets(1).unwrap(), vec![20]); + assert_eq!(g.offsets(4).unwrap(), vec![80]); } #[test] fn index_to_offsets_2d() { // 10x6 dataset with 4x3 chunks => ceil(10/4)=3, ceil(6/3)=2 => 6 chunks - let num_chunks = vec![3u64, 2]; - let chunk_dims = vec![4u32, 3]; - assert_eq!( - index_to_chunk_offsets(0, &num_chunks, &chunk_dims), - vec![0, 0] - ); - assert_eq!( - index_to_chunk_offsets(1, &num_chunks, &chunk_dims), - vec![0, 3] - ); - assert_eq!( - index_to_chunk_offsets(2, &num_chunks, &chunk_dims), - vec![4, 0] - ); - assert_eq!( - index_to_chunk_offsets(3, &num_chunks, &chunk_dims), - vec![4, 3] - ); - assert_eq!( - index_to_chunk_offsets(5, &num_chunks, &chunk_dims), - vec![8, 3] - ); + let g = ChunkGrid::fixed_array(&[10, 6], None, &[4, 3]).unwrap(); + assert_eq!(g.offsets(0).unwrap(), vec![0, 0]); + assert_eq!(g.offsets(1).unwrap(), vec![0, 3]); + assert_eq!(g.offsets(2).unwrap(), vec![4, 0]); + assert_eq!(g.offsets(3).unwrap(), vec![4, 3]); + assert_eq!(g.offsets(5).unwrap(), vec![8, 3]); } #[test] @@ -517,7 +469,7 @@ mod tests { let read = |f: &[u8], fahd: usize| -> Result, FormatError> { let h = FixedArrayHeader::parse(f, fahd, 8, 8)?; - read_fixed_array_chunks(f, &h, &[60], &[20], 8, 8, 8) + read_fixed_array_chunks(f, &h, &[60], None, &[20], 8, 8, 8) }; let (clean, fahd) = build(); @@ -562,7 +514,7 @@ mod tests { let db = 0x100usize; buf[db..db + 4].copy_from_slice(b"FADB"); let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap(); - let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -579,7 +531,7 @@ mod tests { stamp_checksum(&mut buf, fahd, fahd + 24); buf[0x80..0x84].copy_from_slice(b"FADB"); let header = FixedArrayHeader::parse(&buf, fahd, 8, 8).unwrap(); - let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -602,7 +554,7 @@ mod tests { data_block_address: (usize::MAX - 4) as u64, }; let buf = vec![0u8; 64]; - let r = read_fixed_array_chunks(&buf, &header, &[100], &[20], 8, 8, 8); + let r = read_fixed_array_chunks(&buf, &header, &[100], None, &[20], 8, 8, 8); assert!(r.is_err()); } @@ -664,6 +616,7 @@ mod tests { &file_data, &header, &ds_dims, + None, &chunk_dims, 8, offset_size, @@ -740,6 +693,7 @@ mod tests { &file_data, &header, &ds_dims, + None, &chunk_dims, 8, offset_size, @@ -840,6 +794,7 @@ mod tests { &file_data, &header, &ds_dims, + None, &chunk_dims, 8, offset_size, diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 54bd326..4a6c2a1 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -54,6 +54,7 @@ pub mod btree_v1; pub mod btree_v2; pub mod checksum; pub mod chunk_cache; +mod chunk_grid; pub mod chunk_index; pub mod chunked_read; pub mod chunked_write; diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs new file mode 100644 index 0000000..9c030a3 --- /dev/null +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -0,0 +1,256 @@ +//! Fixed Array / Extensible Array chunk-index interop with libhdf5 (via h5py). +//! +//! Both indexes place each chunk at a linear index computed from the +//! dataset's *maximum* dimensions, and the Extensible Array additionally +//! moves its unlimited dimension to the slowest-varying position. Getting +//! either wrong reads (or writes) every chunk after the first row in the +//! wrong place, silently, so these tests compare every value. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Row-major `arange` of `shape`, cropped to `crop` (the current extent). +fn arange_cropped(full: &[usize], crop: &[usize]) -> Vec { + let n: usize = crop.iter().product(); + let mut out = Vec::with_capacity(n); + for flat in 0..n { + let mut rem = flat; + let mut src = 0usize; + let mut stride = 1usize; + let mut coords = vec![0usize; crop.len()]; + for d in (0..crop.len()).rev() { + coords[d] = rem % crop[d]; + rem /= crop[d]; + } + for d in (0..full.len()).rev() { + src += coords[d] * stride; + stride *= full[d]; + } + out.push(src as i32); + } + out +} + +/// One `i4` dataset, filled with `arange` over `full` and then resized to +/// `shape` (equal to `full` unless the case shrinks it). +struct Case { + name: &'static str, + full: Vec, + shape: Vec, + chunks: Vec, + maxshape: &'static str, + extra: &'static str, + index: &'static str, +} + +fn py_tuple(v: &[usize]) -> String { + let parts: Vec = v.iter().map(|x| x.to_string()).collect(); + format!("({},)", parts.join(",")) +} + +/// Have h5py (`libver="latest"`, so Fixed/Extensible Array indexes) write +/// every case to one file, then read each back and compare every value. +fn check_h5py_written(cases: &[Case]) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("h5py_chunk_index.h5"); + let path_str = path.display().to_string(); + + let mut script = + format!("import h5py, numpy as np\nf = h5py.File(r'{path_str}', 'w', libver='latest')\n"); + for c in cases { + script += &format!( + "d = f.create_dataset('{name}', data=np.arange({n}, dtype='i4').reshape({full}), \ + chunks={chunks}, maxshape={maxshape}{extra})\n\ + d.resize({shape})\n", + name = c.name, + n = c.full.iter().product::(), + full = py_tuple(&c.full), + chunks = py_tuple(&c.chunks), + maxshape = c.maxshape, + extra = c.extra, + shape = py_tuple(&c.shape), + ); + } + script += "f.close()\n"; + run_python(&script); + + let file = File::open(&path).unwrap(); + for c in cases { + let ds = file.dataset(c.name).unwrap(); + let shape: Vec = ds.shape().unwrap().iter().map(|&d| d as usize).collect(); + assert_eq!(shape, c.shape, "{}: shape", c.name); + let got = ds.read_i32().unwrap(); + let want = arange_cropped(&c.full, &c.shape); + let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count(); + assert_eq!( + got, + want, + "{}: {bad} of {} values differ (index {})", + c.name, + want.len(), + c.index + ); + } +} + +/// h5py-written Extensible Array whose unlimited dimension is not the first, +/// with the current shape smaller than the finite maximum: the library +/// swizzles the unlimited dimension to the slowest position and strides the +/// rest by their maximum chunk counts. +#[test] +fn h5py_extensible_array_partial_extent_reads_correctly() { + skip_if_no_python!(); + check_h5py_written(&[ + // The `ea_fa_partial.h5` repro from the conformance sweep. + Case { + name: "ea_10_none", + full: vec![4, 6], + shape: vec![4, 6], + chunks: vec![2, 3], + maxshape: "(10, None)", + extra: "", + index: "EA, unlimited dim 1", + }, + Case { + name: "ea_none_10", + full: vec![4, 6], + shape: vec![4, 6], + chunks: vec![2, 3], + maxshape: "(None, 10)", + extra: "", + index: "EA, unlimited dim 0", + }, + Case { + name: "ea_3d_mid", + full: vec![3, 4, 5], + shape: vec![3, 4, 5], + chunks: vec![2, 3, 2], + maxshape: "(5, None, 7)", + extra: "", + index: "EA, unlimited dim 1 of 3", + }, + Case { + name: "ea_3d_last_gzip", + full: vec![3, 4, 5], + shape: vec![3, 4, 5], + chunks: vec![2, 3, 2], + maxshape: "(5, 9, None)", + extra: ", compression='gzip'", + index: "EA, unlimited dim 2 of 3, filtered", + }, + // Many chunks: crosses data blocks, super blocks and paging. + Case { + name: "ea_many", + full: vec![3, 1500], + shape: vec![3, 1500], + chunks: vec![1, 1], + maxshape: "(4, None)", + extra: "", + index: "EA, 4500 slots", + }, + // Shrunk after writing: chunks beyond the extent must be ignored. + Case { + name: "ea_shrunk", + full: vec![8, 9], + shape: vec![3, 4], + chunks: vec![2, 3], + maxshape: "(10, None)", + extra: "", + index: "EA, shrunk", + }, + ]); +} + +/// h5py-written Fixed Array with the current shape smaller than a finite +/// maxshape: the index has one slot per chunk of the *maximum* extent. +#[test] +fn h5py_fixed_array_partial_extent_reads_correctly() { + skip_if_no_python!(); + check_h5py_written(&[ + Case { + name: "fa_20_10", + full: vec![4, 6], + shape: vec![4, 6], + chunks: vec![2, 3], + maxshape: "(20, 10)", + extra: "", + index: "FA", + }, + Case { + name: "fa_3d_gzip", + full: vec![3, 4, 5], + shape: vec![3, 4, 5], + chunks: vec![2, 3, 2], + maxshape: "(6, 8, 10)", + extra: ", compression='gzip'", + index: "FA, filtered", + }, + // Paged (> 1024 slots) with most of them beyond the extent. + Case { + name: "fa_paged", + full: vec![30, 50], + shape: vec![30, 50], + chunks: vec![1, 1], + maxshape: "(40, 60)", + extra: "", + index: "FA, 2400 slots, paged", + }, + Case { + name: "fa_shrunk", + full: vec![8, 9], + shape: vec![5, 2], + chunks: vec![2, 3], + maxshape: "(20, 10)", + extra: "", + index: "FA, shrunk", + }, + ]); +} From 183d96ee265008b686416fb6ef0f1f78381098de Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:04:29 -0500 Subject: [PATCH 06/36] fix(format): record the content size in zstd frames Filter 32015 chunks were written with the streaming encoder (zstd::encode_all), whose frames carry no content size. The registered HDF5 Zstandard filter (H5Zzstd.c, libhdf5 + hdf5plugin) sizes its output from ZSTD_getFrameContentSize and fails on such frames, so h5py could not read our zstd datasets ("filter returned failure during read"). Compress with the one-shot API, which records the size. Tests: zstd_frames_record_content_size (content size was None before), hdf5plugin_reads_our_zstd (ignored interop test; failed before). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 25 +++++++++++++++++-- .../tests/writer_h5py_tests.rs | 13 ++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index d12302b..a28d4be 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -988,10 +988,14 @@ fn zstd_decompress(_data: &[u8], _expected_bytes: usize) -> Result, Form Err(FormatError::UnsupportedFilter(FILTER_ZSTD)) } -/// Compress data with zstd. +/// Compress data with zstd as one frame whose header records the content +/// size. The registered HDF5 Zstandard filter (`H5Zzstd.c`, used by +/// libhdf5 + hdf5plugin) sizes its output buffer from +/// `ZSTD_getFrameContentSize` and fails on a frame without it, which is what +/// the streaming encoder (`zstd::encode_all`) produced. #[cfg(feature = "zstd")] fn zstd_compress(data: &[u8], level: u32) -> Result, FormatError> { - zstd::encode_all(data, level as i32) + zstd::bulk::compress(data, level as i32) .map_err(|e| FormatError::CompressionError(format!("zstd: {e}"))) } @@ -1710,6 +1714,23 @@ mod tests { // --- Zstd tests --- + /// libhdf5's zstd plugin needs the frame content size to size its + /// output; frames without it fail to decode there. + #[test] + #[cfg(feature = "zstd")] + fn zstd_frames_record_content_size() { + for n in [0usize, 1, 200, 100_000] { + let data: Vec = (0..n).map(|i| (i % 7) as u8).collect(); + let c = zstd_compress(&data, 3).unwrap(); + assert_eq!( + zstd::zstd_safe::get_frame_content_size(&c).unwrap(), + Some(n as u64), + "{n} bytes" + ); + assert_eq!(zstd_decompress(&c, n).unwrap(), data); + } + } + #[test] #[cfg(feature = "zstd")] fn zstd_compress_decompress_roundtrip() { diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 44c42f6..cfbaaa8 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -985,3 +985,16 @@ fn hdf5plugin_reads_our_lz4() { }); assert_eq!(got, data); } + +/// libhdf5's Zstandard plugin must decode what we write (it could not while +/// our frames lacked the content size). +#[cfg(feature = "zstd")] +#[test] +#[ignore = "requires Python h5py + hdf5plugin"] +fn hdf5plugin_reads_our_zstd() { + let data: Vec = (0..1000).map(|i| (i % 37) as f64 * 0.5).collect(); + let got = hdf5plugin_roundtrip("zstd", &data, |ds| { + ds.with_zstd(3); + }); + assert_eq!(got, data); +} From 585e14d5e213bac5c4a271ece5136d6a0371f14f Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:05:07 -0500 Subject: [PATCH 07/36] fix(format): honour each bit of a chunk's filter mask A chunk's filter mask has one bit per pipeline filter; bit i set means filter i was not applied to that chunk (an optional filter that declined, or a direct chunk write). Every read path treated any nonzero mask as "no filters applied" and returned the stored bytes, so a chunk that skipped only gzip in a shuffle+gzip pipeline came back still shuffled (h5py write_direct_chunk with filter_mask=0b10: 8 of 32 values wrong). decompress_chunk_masked undoes the filters the mask leaves set and skips the rest; an unsupported filter is no longer an error when the chunk skipped it. The full, cached, sweep, indexed, parallel and selection (partial_read) paths all use it, and a chunk is copied straight from the file only when every filter was skipped. decompress_chunk is the mask-0 case. Regression: h5py_partial_filter_mask_skips_only_masked_filters (1-D shuffle+gzip with masks 0, 0b10 and 0b11; 2-D with 0b01; full and hyperslab reads) and filter_mask_skips_only_the_masked_filters. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 55 +++++--- crates/clawhdf5-format/src/filters.rs | 124 +++++++++++++++--- crates/clawhdf5-format/src/parallel_read.rs | 38 +++--- crates/clawhdf5-format/src/partial_read.rs | 16 ++- .../clawhdf5/tests/h5py_chunked_read_tests.rs | 70 ++++++++++ 5 files changed, 244 insertions(+), 59 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index b0f5141..2dab147 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -15,7 +15,7 @@ use crate::datatype::Datatype; use crate::error::FormatError; use crate::extensible_array::{ExtensibleArrayHeader, read_extensible_array_chunks}; use crate::filter_pipeline::FilterPipeline; -use crate::filters::decompress_chunk; +use crate::filters::{all_filters_skipped, decompress_chunk_masked}; use crate::fixed_array::{FixedArrayHeader, read_fixed_array_chunks}; #[cfg(feature = "std")] use std::sync::Arc; @@ -65,11 +65,13 @@ fn decompress_all_chunks( let raw_chunk = &file_data[c_addr..c_addr + size]; let decompressed = if let Some(pl) = pipeline { - if chunk_info.filter_mask == 0 { - decompress_chunk(raw_chunk, pl, chunk_total_bytes, element_size)? - } else { - raw_chunk.to_vec() - } + decompress_chunk_masked( + raw_chunk, + pl, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + )? } else { raw_chunk.to_vec() }; @@ -886,10 +888,11 @@ pub fn read_chunked_data_cached( }; // Chunks stored as-is (no pipeline, or the filter mask says this chunk - // skipped it) are copied straight from the file bytes: they are already in - // memory, so routing them through a Vec and then an aligned cache buffer - // was two extra copies of the whole dataset for nothing. - let stored_raw = |c: &ChunkInfo| pipeline.is_none() || c.filter_mask != 0; + // skipped every filter) are copied straight from the file bytes: they are + // already in memory, so routing them through a Vec and then an aligned + // cache buffer was two extra copies of the whole dataset for nothing. + let stored_raw = + |c: &ChunkInfo| pipeline.is_none_or(|pl| all_filters_skipped(pl, c.filter_mask)); let mut misses: Vec<&ChunkInfo> = Vec::new(); for chunk_info in &chunks { if stored_raw(chunk_info) { @@ -911,7 +914,13 @@ pub fn read_chunked_data_cached( let cache_them = total_bytes <= cache.max_bytes(); if let Some(pl) = pipeline { let decode = |c: &&ChunkInfo| -> Result, FormatError> { - decompress_chunk(raw_bytes(c)?, pl, chunk_total_bytes, elem_size as u32) + decompress_chunk_masked( + raw_bytes(c)?, + pl, + chunk_total_bytes, + elem_size as u32, + c.filter_mask, + ) }; for batch in misses.chunks(DECODE_BATCH) { #[cfg(feature = "parallel")] @@ -1194,11 +1203,13 @@ pub fn read_chunked_data_sweep( ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; let dec = if let Some(pl) = pipeline { - if chunk_info.filter_mask == 0 { - decompress_chunk(raw_chunk, pl, chunk_total_bytes, elem_size as u32)? - } else { - raw_chunk.to_vec() - } + decompress_chunk_masked( + raw_chunk, + pl, + chunk_total_bytes, + elem_size as u32, + chunk_info.filter_mask, + )? } else { raw_chunk.to_vec() }; @@ -1335,11 +1346,13 @@ pub fn read_chunked_data_indexed( ensure_len(file_data, c_addr, size)?; let raw_chunk = &file_data[c_addr..c_addr + size]; let decompressed = if let Some(pl) = pipeline { - if *filter_mask == 0 { - decompress_chunk(raw_chunk, pl, chunk_total_bytes, elem_size as u32)? - } else { - raw_chunk.to_vec() - } + decompress_chunk_masked( + raw_chunk, + pl, + chunk_total_bytes, + elem_size as u32, + *filter_mask, + )? } else { raw_chunk.to_vec() }; diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index c10cea9..31c024a 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -19,33 +19,64 @@ pub(crate) const MAX_DECOMPRESS_SIZE: usize = 256 * 1024 * 1024; /// Apply a filter pipeline to decompress a chunk. /// Filters are applied in REVERSE order for decompression. +/// +/// Equivalent to [`decompress_chunk_masked`] with a filter mask of 0 (every +/// filter was applied when the chunk was written). pub fn decompress_chunk( compressed: &[u8], pipeline: &FilterPipeline, chunk_size: usize, element_size: u32, ) -> Result, FormatError> { - let mut data = compressed.to_vec(); + decompress_chunk_masked(compressed, pipeline, chunk_size, element_size, 0) +} - for filter in pipeline.filters.iter().rev() { +/// Whether bit `index` of a chunk's filter mask says filter `index` was +/// skipped when the chunk was written. +fn filter_skipped(filter_mask: u32, index: usize) -> bool { + index < 32 && filter_mask & (1u32 << index) != 0 +} + +/// Whether `filter_mask` says none of `pipeline`'s filters were applied, so +/// the stored bytes are the chunk itself. +pub fn all_filters_skipped(pipeline: &FilterPipeline, filter_mask: u32) -> bool { + (0..pipeline.filters.len()).all(|i| filter_skipped(filter_mask, i)) +} + +/// Decompress a chunk whose filter mask is `filter_mask`: bit *i* set means +/// filter *i* of the pipeline was not applied when the chunk was written (an +/// optional filter that declined, or a direct chunk write), so only that +/// filter is skipped here; the others are still undone, in reverse order. +/// +/// `chunk_size` is the chunk's decoded size (0 if unknown); it caps each +/// stage's output. +pub fn decompress_chunk_masked( + compressed: &[u8], + pipeline: &FilterPipeline, + chunk_size: usize, + element_size: u32, + filter_mask: u32, +) -> Result, FormatError> { + let mut data = compressed.to_vec(); + for (i, filter) in pipeline.filters.iter().enumerate().rev() { + if filter_skipped(filter_mask, i) { + continue; + } + let bound = chunk_size; data = match filter.filter_id { FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?, - // `chunk_size` is the expected decompressed size (shuffle/fletcher32 - // are size-preserving, so it bounds these too); pass it so these - // decoders can't be forced into unbounded allocation by a hostile - // or corrupted compressed payload. - FILTER_DEFLATE => deflate_decompress(&data, chunk_size)?, - FILTER_LZ4 => lz4_decompress(&data, chunk_size)?, - FILTER_ZSTD => zstd_decompress(&data, chunk_size)?, + // `bound` caps the decoded size so these decoders can't be forced + // into unbounded allocation by a hostile or corrupted payload. + FILTER_DEFLATE => deflate_decompress(&data, bound)?, + FILTER_LZ4 => lz4_decompress(&data, bound)?, + FILTER_ZSTD => zstd_decompress(&data, bound)?, FILTER_FLETCHER32 => fletcher32_verify(&data)?, - FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?, - // `chunk_size` is the expected decompressed size; pass it so these - // decoders can reject an element count that would over-allocate. - FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?, - FILTER_NBIT => nbit_decompress(&data, &filter.client_data, chunk_size)?, - FILTER_SZIP => { - crate::filters_szip::szip_decompress(&data, &filter.client_data, chunk_size)? - } + FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, bound)?, + // These decoders also reject an element count that would + // over-allocate past `bound`. + FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, bound)?, + FILTER_NBIT => nbit_decompress(&data, &filter.client_data, bound)?, + FILTER_SZIP => crate::filters_szip::szip_decompress(&data, &filter.client_data, bound)?, other => return Err(FormatError::UnsupportedFilter(other)), }; } @@ -1413,6 +1444,65 @@ mod tests { assert_eq!(decompressed, data); } + fn filter(filter_id: u16) -> FilterDescription { + FilterDescription { + filter_id, + name: None, + flags: 0, + client_data: vec![], + } + } + + #[test] + #[cfg(feature = "deflate")] + fn filter_mask_skips_only_the_masked_filters() { + let pipeline = FilterPipeline { + version: 2, + filters: vec![filter(FILTER_SHUFFLE), filter(FILTER_DEFLATE)], + }; + let only_shuffle = FilterPipeline { + version: 2, + filters: vec![filter(FILTER_SHUFFLE)], + }; + let only_deflate = FilterPipeline { + version: 2, + filters: vec![filter(FILTER_DEFLATE)], + }; + let data: Vec = (0..200).map(|i| (i * 7 % 256) as u8).collect(); + let n = data.len(); + + let shuffled = compress_chunk(&data, &only_shuffle, 8).unwrap(); + assert_ne!(shuffled, data); + let deflated = compress_chunk(&data, &only_deflate, 8).unwrap(); + + // Bit 1: deflate skipped, shuffle still undone. + assert_eq!( + decompress_chunk_masked(&shuffled, &pipeline, n, 8, 0b10).unwrap(), + data + ); + // Bit 0: shuffle skipped, deflate still undone. + assert_eq!( + decompress_chunk_masked(&deflated, &pipeline, n, 8, 0b01).unwrap(), + data + ); + // Both bits (and bits past the pipeline): stored as-is. + assert_eq!( + decompress_chunk_masked(&data, &pipeline, n, 8, u32::MAX).unwrap(), + data + ); + assert!(all_filters_skipped(&pipeline, 0b11)); + assert!(!all_filters_skipped(&pipeline, 0b10)); + // An unsupported filter is fine when the chunk skipped it. + let unknown = FilterPipeline { + version: 2, + filters: vec![filter(32000), filter(FILTER_DEFLATE)], + }; + assert_eq!( + decompress_chunk_masked(&deflated, &unknown, n, 8, 0b01).unwrap(), + data + ); + } + #[test] #[cfg(feature = "deflate")] fn pipeline_compress_decompress_roundtrip() { diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 14f5613..0bb2785 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -10,7 +10,7 @@ use crate::chunked_read::ChunkInfo; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::filters::decompress_chunk; +use crate::filters::decompress_chunk_masked; use crate::lane_partition::{self, LaneStats, PartitionStats}; /// Threshold: only use parallel decompression when chunk count exceeds this. @@ -84,11 +84,13 @@ pub fn decompress_chunks_lane_partitioned( } let raw_chunk = &file_data[c_addr..c_addr + size]; - let decompressed = if chunk_info.filter_mask == 0 { - decompress_chunk(raw_chunk, pipeline, chunk_total_bytes, element_size)? - } else { - raw_chunk.to_vec() - }; + let decompressed = decompress_chunk_masked( + raw_chunk, + pipeline, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + )?; stats.chunks_processed += 1; stats.compressed_bytes += size as u64; @@ -158,11 +160,13 @@ pub fn decompress_chunks_parallel( } let raw_chunk = &file_data[c_addr..c_addr + size]; - let decompressed = if chunk_info.filter_mask == 0 { - decompress_chunk(raw_chunk, pipeline, chunk_total_bytes, element_size)? - } else { - raw_chunk.to_vec() - }; + let decompressed = decompress_chunk_masked( + raw_chunk, + pipeline, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + )?; Ok(DecompressedChunk { index, @@ -200,11 +204,13 @@ pub fn decompress_chunks_sequential( let raw_chunk = &file_data[c_addr..c_addr + size]; let decompressed = if let Some(pl) = pipeline { - if chunk_info.filter_mask == 0 { - decompress_chunk(raw_chunk, pl, chunk_total_bytes, element_size)? - } else { - raw_chunk.to_vec() - } + decompress_chunk_masked( + raw_chunk, + pl, + chunk_total_bytes, + element_size, + chunk_info.filter_mask, + )? } else { raw_chunk.to_vec() }; diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 7cdd2de..5cc3aaa 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -22,7 +22,7 @@ use crate::data_read::extract_selection_from_buffer; use crate::dataspace::Dataspace; use crate::error::FormatError; use crate::filter_pipeline::FilterPipeline; -use crate::filters::decompress_chunk; +use crate::filters::{all_filters_skipped, decompress_chunk_masked}; use crate::selection::Selection; /// The smallest axis-aligned box containing every selected element, as @@ -325,12 +325,18 @@ pub fn read_selection( expected: at.saturating_add(chunk.chunk_size as usize), available: file_data.len(), })?; - // Mirrors the full-read path: a non-zero filter mask means the - // chunk was stored unfiltered. + // Mirrors the full-read path: filter-mask bit i set means + // filter i was not applied to this chunk. let decoded; let data: &[u8] = match pipeline { - Some(pl) if chunk.filter_mask == 0 => { - decoded = decompress_chunk(raw, pl, chunk_bytes, elem_size as u32)?; + Some(pl) if !all_filters_skipped(pl, chunk.filter_mask) => { + decoded = decompress_chunk_masked( + raw, + pl, + chunk_bytes, + elem_size as u32, + chunk.filter_mask, + )?; &decoded } _ => raw, diff --git a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs index d46cff5..58fe290 100644 --- a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs +++ b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs @@ -8,6 +8,7 @@ use std::process::Command; use clawhdf5::File; +use clawhdf5_format::selection::Selection; /// The Python interpreter to drive interop checks with (see /// `h5py_interop_tests.rs`). @@ -103,3 +104,72 @@ for name, lengths in (("{p}", 4), ("{p}.l8", 8)): } } } + +// --------------------------------------------------------------------------- +// Per-chunk filter masks +// --------------------------------------------------------------------------- + +/// A chunk's filter mask has one bit per pipeline filter: bit i set means +/// filter i was not applied to that chunk. Any nonzero mask used to skip the +/// whole pipeline, so a chunk that skipped only gzip was handed back still +/// shuffled. +#[test] +fn h5py_partial_filter_mask_skips_only_masked_filters() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("mask.h5"); + let p = path.display().to_string(); + run_python(&format!( + r#" +import h5py, numpy as np, zlib +def shuffle(b, es): + a = np.frombuffer(b, dtype=np.uint8).reshape(-1, es) + return a.T.copy().tobytes() +with h5py.File("{p}", "w") as f: + # shuffle (0) + gzip (1). Even chunks: both applied. Odd chunks: mask + # 0b10, gzip skipped, shuffle applied. Chunk 3: mask 0b11, raw. + ds = f.create_dataset("shuf_gzip", shape=(32,), chunks=(8,), dtype=" = (1000..1032).collect(); + assert_eq!(file.dataset("shuf_gzip").unwrap().read_i32().unwrap(), want); + let grid: Vec = (0..64).map(f64::from).collect(); + assert_eq!(file.dataset("grid").unwrap().read_f64().unwrap(), grid); + // The selection path decodes chunks on its own. + let part = file + .dataset("shuf_gzip") + .unwrap() + .read_selection(&Selection::Hyperslab { + start: vec![6], + stride: vec![1], + count: vec![1], + block: vec![20], + }) + .unwrap(); + let want: Vec = (1006..1026i32).flat_map(i32::to_le_bytes).collect(); + assert_eq!(part, want); +} From 06dda26d856d1af3b915196c48f3cbf7333fa1bf Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:06:07 -0500 Subject: [PATCH 08/36] fix(format): stop writing pcodec under Granular BitRound's filter ID Pcodec chunks were written as filter 32023, which the HDF Group registry assigns to Granular BitRound (GBR). Pcodec has no registered ID (checked 2026-09-25 against hdf5_plugins/docs/RegisteredFilterPlugins.md, which ends at 32033 with no pcodec entry). GBR's decode is a pass-through, so libhdf5 with that plugin loaded would have returned the compressed bytes as the dataset's values. Write pcodec as 480, from the registry's testing/private range (256-511), named "pcodec (clawhdf5 private)", and document it as non-interoperable: only clawhdf5 with the `pcodec` feature reads it. Chunks under 32023 are still read as pcodec when the filter is named exactly "pcodec" (what clawhdf5 <= 2.7.0 wrote); any other 32023 is UnsupportedFilter. Test: pcodec_uses_private_id_and_reads_legacy_32023. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- crates/clawhdf5-format/src/chunked_write.rs | 9 +-- crates/clawhdf5-format/src/filter_pipeline.rs | 19 +++++- crates/clawhdf5-format/src/filters.rs | 61 ++++++++++++++++++- crates/clawhdf5-format/src/type_builders.rs | 7 ++- 5 files changed, 88 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d21462c..5d4c707 100644 --- a/README.md +++ b/README.md @@ -696,7 +696,7 @@ stores keep their setting. Opt out with `float16 = false` or | `fast-checksum` | no | crc32fast-accelerated checksums | | `lz4` | no | LZ4 block compression filter (id 32004) | | `zstd` | no | Zstandard compression filter (id 32015) | -| `pcodec` | no | Pcodec lossless numerical codec (id 32023, via `pco` crate) | +| `pcodec` | no | Pcodec lossless numerical codec (via `pco` crate). Private, unregistered filter id 480: **only clawhdf5 can read these datasets** (h5py/libhdf5 cannot). Files from clawhdf5 <= 2.7.0 used id 32023, which is registered to Granular BitRound; they still read. | | `system-zlib` | no | System zlib backend for deflate (C) | | `blake3_hash` | no | BLAKE3 content hashing for provenance | | `szip` | no | SZIP filter (id 4) via libaec (C, through the internal `libaec-sys` crate) | diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 6816546..aa8bb8f 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -11,8 +11,8 @@ use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line}; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_SHUFFLE, FILTER_ZSTD, - FilterDescription, FilterPipeline, + FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_PCODEC, FILTER_PCODEC_NAME, + FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; use crate::filters::compress_chunk; /// Round a file offset up to the next cache-line boundary. @@ -44,7 +44,8 @@ pub struct ChunkOptions { pub lz4: bool, /// Zstandard compression level (1-22), None = no zstd. Filter ID 32015. pub zstd_level: Option, - /// Pcodec lossless numerical compression. Filter ID 32023. + /// Pcodec lossless numerical compression. Private, unregistered filter + /// ID [`FILTER_PCODEC`] (480): only clawhdf5 can read it. pub pcodec: bool, } @@ -115,7 +116,7 @@ impl ChunkOptions { if self.pcodec { filters.push(FilterDescription { filter_id: FILTER_PCODEC, - name: Some("pcodec".into()), + name: Some(FILTER_PCODEC_NAME.into()), flags: 0, client_data: vec![element_size], }); diff --git a/crates/clawhdf5-format/src/filter_pipeline.rs b/crates/clawhdf5-format/src/filter_pipeline.rs index f64219d..74bc727 100644 --- a/crates/clawhdf5-format/src/filter_pipeline.rs +++ b/crates/clawhdf5-format/src/filter_pipeline.rs @@ -19,8 +19,23 @@ pub const FILTER_SCALEOFFSET: u16 = 6; pub const FILTER_LZ4: u16 = 32004; /// Zstandard compression. pub const FILTER_ZSTD: u16 = 32015; -/// Pcodec lossless numerical codec (clawhdf5 internal; not yet HDF5-registered). -pub const FILTER_PCODEC: u16 = 32023; +/// Pcodec lossless numerical codec — a **private, unregistered** clawhdf5 +/// filter. Pcodec has no ID in the HDF Group's filter registry (checked +/// 2026-09-25, `hdf5_plugins/docs/RegisteredFilterPlugins.md`), so it uses an +/// ID from the registry's testing/private range (256–511). No libhdf5 plugin +/// decodes it: h5py/libhdf5 report the filter as unavailable. Only clawhdf5 +/// (with the `pcodec` feature) reads these datasets. +pub const FILTER_PCODEC: u16 = 480; +/// Filter name written with [`FILTER_PCODEC`]. +pub const FILTER_PCODEC_NAME: &str = "pcodec (clawhdf5 private)"; +/// The ID clawhdf5 up to 2.7.0 wrote pcodec under. It is registered to +/// Granular BitRound (GBR), whose decode is a pass-through, so libhdf5 with +/// that plugin would have returned the compressed bytes as data. Read as +/// pcodec only when the filter is named exactly [`FILTER_PCODEC_LEGACY_NAME`], +/// the name those versions wrote; never written. +pub const FILTER_PCODEC_LEGACY: u16 = 32023; +/// The filter name clawhdf5 up to 2.7.0 wrote with [`FILTER_PCODEC_LEGACY`]. +pub const FILTER_PCODEC_LEGACY_NAME: &str = "pcodec"; /// Description of a single filter in a pipeline. #[derive(Debug, Clone, PartialEq)] diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index a28d4be..c0bf701 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -8,8 +8,9 @@ use alloc::{boxed::Box, vec, vec::Vec}; use crate::error::FormatError; use crate::filter_pipeline::{ - FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, FILTER_SCALEOFFSET, - FILTER_SHUFFLE, FILTER_SZIP, FILTER_ZSTD, FilterPipeline, + FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_LZ4, FILTER_NBIT, FILTER_PCODEC, + FILTER_PCODEC_LEGACY, FILTER_PCODEC_LEGACY_NAME, FILTER_SCALEOFFSET, FILTER_SHUFFLE, + FILTER_SZIP, FILTER_ZSTD, FilterPipeline, }; /// Absolute ceiling on a single decompressed chunk's output size, used only @@ -39,6 +40,11 @@ pub fn decompress_chunk( FILTER_ZSTD => zstd_decompress(&data, chunk_size)?, FILTER_FLETCHER32 => fletcher32_verify(&data)?, FILTER_PCODEC => pcodec_decompress(&data, element_size as usize, chunk_size)?, + // Pcodec chunks written by clawhdf5 <= 2.7.0 under the ID registered + // to Granular BitRound; recognised by the name those versions wrote. + FILTER_PCODEC_LEGACY if filter.name.as_deref() == Some(FILTER_PCODEC_LEGACY_NAME) => { + pcodec_decompress(&data, element_size as usize, chunk_size)? + } // `chunk_size` is the expected decompressed size; pass it so these // decoders can reject an element count that would over-allocate. FILTER_SCALEOFFSET => scaleoffset_decompress(&data, &filter.client_data, chunk_size)?, @@ -2192,6 +2198,57 @@ mod tests { assert!(pcodec_decompress(&compressed, 4, 16).is_err()); } + /// Pcodec is written under the private ID 480, not 32023 (registered to + /// Granular BitRound, whose pass-through decode would hand libhdf5 users + /// the compressed bytes as data). Chunks under 32023 are read as pcodec + /// only with the name clawhdf5 <= 2.7.0 wrote. + #[test] + #[cfg(feature = "pcodec")] + fn pcodec_uses_private_id_and_reads_legacy_32023() { + use crate::chunked_write::ChunkOptions; + let opts = ChunkOptions { + pcodec: true, + ..Default::default() + }; + let pl = opts.build_pipeline(8).unwrap(); + let f = pl.filters.iter().find(|f| f.filter_id == 480).unwrap(); + assert_eq!( + f.name.as_deref(), + Some(crate::filter_pipeline::FILTER_PCODEC_NAME) + ); + assert!(pl.filters.iter().all(|f| f.filter_id != 32023)); + + let data: Vec = (0..100).map(|i| i as f64 * 0.25).collect(); + let raw: Vec = data.iter().flat_map(|x| x.to_le_bytes()).collect(); + let compressed = pcodec_compress(&raw, 8).unwrap(); + let pipeline = |id: u16, name: Option<&str>| FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: id, + name: name.map(Into::into), + flags: 0, + client_data: vec![8], + }], + }; + let legacy = pipeline(32023, Some("pcodec")); + assert_eq!( + decompress_chunk(&compressed, &legacy, raw.len(), 8).unwrap(), + raw + ); + let current = pipeline(480, None); + assert_eq!( + decompress_chunk(&compressed, ¤t, raw.len(), 8).unwrap(), + raw + ); + // A real Granular BitRound filter is not pcodec. + for name in [None, Some("Granular BitRound")] { + assert!(matches!( + decompress_chunk(&compressed, &pipeline(32023, name), raw.len(), 8), + Err(FormatError::UnsupportedFilter(32023)) + )); + } + } + #[test] #[cfg(feature = "lz4")] fn decompress_chunk_rejects_hostile_lz4_size_via_public_entrypoint() { diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index e698899..7b45d70 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -671,7 +671,12 @@ impl DatasetBuilder { self } - /// Enable Pcodec lossless numerical compression (clawhdf5 filter ID 32023). + /// Enable Pcodec lossless numerical compression (private clawhdf5 filter + /// ID 480). + /// + /// **Not interoperable:** pcodec has no registered HDF5 filter ID and no + /// libhdf5 plugin, so h5py and other HDF5 readers cannot read the + /// dataset — only clawhdf5 built with the `pcodec` feature can. /// /// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64 /// columns at 1–5 GiB/s decompression speed (arXiv:2502.06112). Requires From e162c013fdd92c80e95e092ba42e7b9e5c2c2628 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:06:22 -0500 Subject: [PATCH 09/36] fix(format): bound each filter stage by what the stages before it produce Every decode stage was capped at the chunk's decoded size. That holds only when every filter ahead of the codec preserves size; Fletcher32 does not (it appends a 4-byte checksum), so a pipeline with Fletcher32 before deflate (NetCDF-4's fletcher32 -> shuffle -> deflate ordering, h5repack's "all filters") failed with "deflate: output exceeds size limit" on every chunk. decompress_chunk_masked now computes each stage's bound by running the chunk size forward through the filters that precede it in write order (and that the chunk's mask did not skip): shuffle keeps the size, Fletcher32 adds 4, any codec adds at most n/8 + 64. The cap is still a small constant factor of the chunk, so a decompression bomb is rejected as before (tested). Shuffle also had to learn libhdf5's handling of a length that is not a whole number of elements (chunk + checksum): shuffle the whole elements and leave the trailing bytes in place, in both directions. It used to refuse such data. Regression: h5py_fletcher32_before_deflate_reads (fletcher->shuffle-> gzip, fletcher->gzip, shuffle->fletcher->gzip, and a 2-D i32 grid), fletcher32_ahead_of_deflate_stays_bounded and shuffle_leaves_a_partial_trailing_element_in_place. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 111 +++++++++++++++--- .../clawhdf5/tests/h5py_chunked_read_tests.rs | 64 ++++++++++ 2 files changed, 160 insertions(+), 15 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 31c024a..ac17725 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -31,6 +31,26 @@ pub fn decompress_chunk( decompress_chunk_masked(compressed, pipeline, chunk_size, element_size, 0) } +/// Upper bound on the output of filter `filter_id` applied (in the write +/// direction) to `input` bytes. 0 means "unknown" and stays unknown. +/// +/// Shuffle preserves the size and Fletcher32 appends a 4-byte checksum. Any +/// other filter is a codec whose output can exceed its input on +/// incompressible data (deflate's stored blocks, LZ4's and zstd's literal +/// runs, codec headers); `n + n/8 + 64` covers every supported codec's worst +/// case while still bounding a decompression bomb to a small multiple of the +/// chunk. +fn filter_output_bound(filter_id: u16, input: usize) -> usize { + if input == 0 { + return 0; + } + match filter_id { + FILTER_SHUFFLE => input, + FILTER_FLETCHER32 => input.saturating_add(4), + _ => input.saturating_add(input / 8).saturating_add(64), + } +} + /// Whether bit `index` of a chunk's filter mask says filter `index` was /// skipped when the chunk was written. fn filter_skipped(filter_mask: u32, index: usize) -> bool { @@ -48,8 +68,12 @@ pub fn all_filters_skipped(pipeline: &FilterPipeline, filter_mask: u32) -> bool /// optional filter that declined, or a direct chunk write), so only that /// filter is skipped here; the others are still undone, in reverse order. /// -/// `chunk_size` is the chunk's decoded size (0 if unknown); it caps each -/// stage's output. +/// `chunk_size` is the chunk's decoded size (0 if unknown). Each stage's +/// output is capped at what the filters before it (in write order) can have +/// produced from `chunk_size` bytes — e.g. a Fletcher32 checksum placed +/// before deflate (NetCDF-4's ordering) makes deflate's output 4 bytes +/// larger than the chunk — so the decompression-bomb limit stays tight +/// without rejecting valid pipelines. pub fn decompress_chunk_masked( compressed: &[u8], pipeline: &FilterPipeline, @@ -57,12 +81,23 @@ pub fn decompress_chunk_masked( element_size: u32, filter_mask: u32, ) -> Result, FormatError> { + // bounds[i]: the most bytes that entered filter i on the write side, and + // so the most that undoing filter i may produce. + let mut bounds = Vec::with_capacity(pipeline.filters.len()); + let mut size = chunk_size; + for (i, filter) in pipeline.filters.iter().enumerate() { + bounds.push(size); + if !filter_skipped(filter_mask, i) { + size = filter_output_bound(filter.filter_id, size); + } + } + let mut data = compressed.to_vec(); for (i, filter) in pipeline.filters.iter().enumerate().rev() { if filter_skipped(filter_mask, i) { continue; } - let bound = chunk_size; + let bound = bounds[i]; data = match filter.filter_id { FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?, // `bound` caps the decoded size so these decoders can't be forced @@ -944,13 +979,13 @@ fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, Forma if element_size <= 1 { return Ok(data.to_vec()); } - if !data.len().is_multiple_of(element_size) { - return Err(FormatError::FilterError( - "shuffle: data length not a multiple of element size".into(), - )); - } + // Like libhdf5, only whole elements are shuffled; trailing bytes (e.g. a + // Fletcher32 checksum appended before the shuffle) are stored as-is. + let whole = data.len() - data.len() % element_size; + let (data, tail) = data.split_at(whole); let num_elements = data.len() / element_size; - let mut result = vec![0u8; data.len()]; + let mut result = vec![0u8; whole]; + result.reserve_exact(tail.len()); // The shuffled stream is `element_size` byte planes of `num_elements` // bytes each; un-shuffling interleaves them. This is on the read path of @@ -981,6 +1016,7 @@ fn shuffle_decompress(data: &[u8], element_size: usize) -> Result, Forma } } } + result.extend_from_slice(tail); Ok(result) } @@ -996,19 +1032,19 @@ fn shuffle_compress(data: &[u8], element_size: usize) -> Result, FormatE if element_size <= 1 { return Ok(data.to_vec()); } - if !data.len().is_multiple_of(element_size) { - return Err(FormatError::FilterError( - "shuffle: data length not a multiple of element size".into(), - )); - } + // Trailing bytes that don't make a whole element are left in place, as + // libhdf5 does. + let whole = data.len() - data.len() % element_size; + let (data, tail) = data.split_at(whole); let num_elements = data.len() / element_size; - let mut result = vec![0u8; data.len()]; + let mut result = vec![0u8; whole]; match element_size { 4 => shuffle_compress_4(data, num_elements, &mut result), 8 => shuffle_compress_general(data, num_elements, element_size, &mut result), _ => shuffle_compress_general(data, num_elements, element_size, &mut result), } + result.extend_from_slice(tail); Ok(result) } @@ -1503,6 +1539,51 @@ mod tests { ); } + #[test] + #[cfg(feature = "deflate")] + fn fletcher32_ahead_of_deflate_stays_bounded() { + // NetCDF-4 order: the checksum is appended before shuffle and deflate, + // so deflate decodes chunk + 4 bytes. + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + filter(FILTER_FLETCHER32), + filter(FILTER_SHUFFLE), + filter(FILTER_DEFLATE), + ], + }; + let data: Vec = (0..400).map(|i| (i * 13 % 251) as u8).collect(); + let n = data.len(); + let stored = compress_chunk(&data, &pipeline, 8).unwrap(); + assert_eq!(decompress_chunk(&stored, &pipeline, n, 8).unwrap(), data); + + // The cap still bites: a stream that inflates past chunk + 4 bytes + // is rejected rather than allocated. + let only_deflate = FilterPipeline { + version: 2, + filters: vec![filter(FILTER_DEFLATE)], + }; + let oversized = compress_chunk(&vec![0u8; n + 5], &only_deflate, 8).unwrap(); + let err = decompress_chunk(&oversized, &pipeline, n, 8).unwrap_err(); + assert!( + matches!(err, FormatError::DecompressionError(_)), + "expected a size-limit error, got {err:?}" + ); + // A bomb is still stopped near the chunk size. + let bomb = compress_chunk(&vec![0u8; 64 * n], &only_deflate, 8).unwrap(); + assert!(decompress_chunk(&bomb, &pipeline, n, 8).is_err()); + } + + #[test] + fn shuffle_leaves_a_partial_trailing_element_in_place() { + // libhdf5 shuffles whole elements and copies the remainder as-is. + let data: Vec = (0..20).collect(); + let shuffled = shuffle_compress(&data, 8).unwrap(); + assert_eq!(&shuffled[16..], &data[16..]); + assert_eq!(&shuffled[..4], &[0, 8, 1, 9]); + assert_eq!(shuffle_decompress(&shuffled, 8).unwrap(), data); + } + #[test] #[cfg(feature = "deflate")] fn pipeline_compress_decompress_roundtrip() { diff --git a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs index 58fe290..deda24f 100644 --- a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs +++ b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs @@ -173,3 +173,67 @@ with h5py.File("{p}", "w") as f: let want: Vec = (1006..1026i32).flat_map(i32::to_le_bytes).collect(); assert_eq!(part, want); } + +// --------------------------------------------------------------------------- +// Size-changing filters ahead of a codec +// --------------------------------------------------------------------------- + +/// Fletcher32 placed before the compressor (NetCDF-4's ordering) makes the +/// codec's decoded output 4 bytes larger than the chunk. The decompression +/// cap was the chunk size for every stage, so these files failed with +/// "deflate: output exceeds size limit". +#[test] +fn h5py_fletcher32_before_deflate_reads() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("fletcher_first.h5"); + let p = path.display().to_string(); + run_python(&format!( + r#" +import h5py, numpy as np +arr = np.sin(np.arange(5000) / 50.0) +grid = np.arange(37 * 21, dtype=" = (0..5000).map(|i| (f64::from(i) / 50.0).sin()).collect(); + for name in ["fl_shuf_gzip", "fl_gzip", "shuf_fl_gzip"] { + let values = file.dataset(name).unwrap().read_f64().unwrap(); + assert_eq!(values.len(), arr.len(), "{name}"); + for (i, (v, w)) in values.iter().zip(&arr).enumerate() { + assert!((v - w).abs() < 1e-12, "{name}[{i}]: {v} vs {w}"); + } + } + let grid: Vec = (0..37 * 21).collect(); + assert_eq!( + file.dataset("grid_fl_shuf_gzip") + .unwrap() + .read_i32() + .unwrap(), + grid + ); +} From be88e3fec7a94f2dfd370b50d7ab0c349158facc Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:06:51 -0500 Subject: [PATCH 10/36] fix(format): encode Time, BitField, Opaque and Reference datatypes Datatype::serialize returned an empty message for these four classes, so any dataset or attribute of them (including a Raw attribute copied from another file) was unreadable by libhdf5 ("ran off end of input buffer while decoding"). They now encode exactly as libhdf5 does: legacy object and region references as datatype version 1, H5T_STD_REF kinds as version 4 with their encoding version, opaque tags NUL-padded to 8 bytes. Parsing an opaque tag now stops at its first NUL, so libhdf5's padding no longer becomes part of the tag. Datatype::check_encodable rejects what has no encoding (an opaque tag over 248 bytes); FileWriter::finish calls it for every dataset and attribute type. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/datatype.rs | 203 +++++++++++++++++- crates/clawhdf5-format/src/file_writer.rs | 11 + .../tests/writer_meta_tests.rs | 166 ++++++++++++++ 3 files changed, 377 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 4a1ba26..90c9657 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -4,7 +4,7 @@ //! for compound, enumeration, variable-length, and array types. #[cfg(not(feature = "std"))] -use alloc::{boxed::Box, string::String, vec, vec::Vec}; +use alloc::{boxed::Box, format, string::String, vec, vec::Vec}; use byteorder::{ByteOrder, LittleEndian}; @@ -137,6 +137,17 @@ pub enum Datatype { }, } +/// Longest opaque tag that can be stored: its NUL-padded length must fit +/// the 8-bit length in the datatype's class bits. +pub const MAX_OPAQUE_TAG_LEN: usize = 248; + +/// An opaque tag up to (not including) its first NUL. +fn opaque_tag_text(tag: &[u8]) -> &[u8] { + tag.iter() + .position(|&b| b == 0) + .map_or(tag, |end| &tag[..end]) +} + fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> { match offset.checked_add(needed) { Some(end) if end <= data.len() => Ok(()), @@ -361,7 +372,10 @@ impl Datatype { // Opaque let tag_len = bf0 as usize; ensure_len(data, pos, tag_len)?; - let tag = data[pos..pos + tag_len].to_vec(); + // The stored tag is NUL-padded to a multiple of 8 bytes; the + // tag itself ends at the first NUL (libhdf5 reads it with + // `strndup`). + let tag = opaque_tag_text(&data[pos..pos + tag_len]).to_vec(); // Tags are padded to multiple of 8 bytes let padded = (tag_len + 7) & !7; let pos = 8 + padded; // from start of properties @@ -767,7 +781,74 @@ impl Datatype { buf.extend_from_slice(&base_type.serialize()); buf } - _ => Vec::new(), + Datatype::Time { + size, + bit_precision, + } => { + // Byte order is not modelled for time types; write little-endian. + let mut buf = Self::build_header(2, 1, [0, 0, 0], *size); + buf.extend_from_slice(&bit_precision.to_le_bytes()); + buf + } + Datatype::BitField { + size, + byte_order, + bit_offset, + bit_precision, + } => { + let bf0 = u8::from(matches!(byte_order, DatatypeByteOrder::BigEndian)); + let mut buf = Self::build_header(4, 1, [bf0, 0, 0], *size); + buf.extend_from_slice(&bit_offset.to_le_bytes()); + buf.extend_from_slice(&bit_precision.to_le_bytes()); + buf + } + Datatype::Opaque { size, tag } => { + // The tag is stored NUL-padded to a multiple of 8 bytes and the + // padded length goes in the class bits, as libhdf5 writes it. + // A tag longer than MAX_OPAQUE_TAG_LEN cannot be encoded; + // `check_encodable` rejects it before a file is written. + let tag = opaque_tag_text(tag); + let tag = &tag[..tag.len().min(MAX_OPAQUE_TAG_LEN)]; + let padded = tag.len().div_ceil(8) * 8; + let mut buf = Self::build_header(5, 1, [padded as u8, 0, 0], *size); + buf.extend_from_slice(tag); + buf.resize(8 + padded, 0); + buf + } + Datatype::Reference { size, ref_type } => { + // Legacy references are datatype version 1; the H5T_STD_REF + // kinds only exist from version 4, which also carries their + // encoding version (1) in the high nibble. + let (version, bf0) = match ref_type { + ReferenceType::Object => (1, 0), + ReferenceType::DatasetRegion => (1, 1), + ReferenceType::Object2 => (4, 0x12), + ReferenceType::DatasetRegion2 => (4, 0x13), + ReferenceType::Attribute => (4, 0x14), + }; + Self::build_header(7, version, [bf0, 0, 0], *size) + } + } + } + + /// Check that this datatype can be written: every part of it has an + /// on-disk encoding. [`Self::serialize`] cannot report errors, so the + /// writer calls this first. + pub fn check_encodable(&self) -> Result<(), FormatError> { + match self { + Datatype::Opaque { tag, .. } if opaque_tag_text(tag).len() > MAX_OPAQUE_TAG_LEN => { + Err(FormatError::SerializationError(format!( + "opaque tag is {} bytes; at most {MAX_OPAQUE_TAG_LEN} can be stored", + opaque_tag_text(tag).len() + ))) + } + Datatype::Compound { members, .. } => members + .iter() + .try_for_each(|m| m.datatype.check_encodable()), + Datatype::Enumeration { base_type, .. } + | Datatype::VariableLength { base_type, .. } + | Datatype::Array { base_type, .. } => base_type.check_encodable(), + _ => Ok(()), } } @@ -1625,6 +1706,122 @@ mod tests { ); } + fn hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// `serialize` used to return an empty message for these four classes, + /// which libhdf5 rejects ("ran off end of input buffer while decoding"). + /// Expected bytes are libhdf5's own encoding (HDF5 2.0 `H5Tencode`, or the + /// datatype message of an HDF5 2.0 file for `H5T_STD_REF`). + #[test] + fn serialize_matches_libhdf5_for_time_bitfield_opaque_reference() { + let cases = [ + ( + Datatype::Reference { + size: 8, + ref_type: ReferenceType::Object, + }, + "1700000008000000", + ), + ( + Datatype::Reference { + size: 12, + ref_type: ReferenceType::DatasetRegion, + }, + "170100000c000000", + ), + ( + Datatype::Reference { + size: 18, + ref_type: ReferenceType::Object2, + }, + "4712000012000000", + ), + ( + Datatype::BitField { + size: 1, + byte_order: DatatypeByteOrder::LittleEndian, + bit_offset: 0, + bit_precision: 8, + }, + "140000000100000000000800", + ), + ( + Datatype::BitField { + size: 2, + byte_order: DatatypeByteOrder::BigEndian, + bit_offset: 0, + bit_precision: 16, + }, + "140100000200000000001000", + ), + ( + Datatype::Opaque { + size: 4, + tag: b"mytag".to_vec(), + }, + "15080000040000006d79746167000000", + ), + ( + Datatype::Opaque { + size: 4, + tag: b"12345678".to_vec(), + }, + "15080000040000003132333435363738", + ), + ( + Datatype::Opaque { + size: 4, + tag: vec![], + }, + "1500000004000000", + ), + ( + Datatype::Time { + size: 4, + bit_precision: 32, + }, + "12000000040000002000", + ), + ]; + for (dt, expected) in cases { + let bytes = dt.serialize(); + assert_eq!(bytes, hex(expected), "{dt:?}"); + let (parsed, consumed) = Datatype::parse(&bytes).unwrap(); + assert_eq!(parsed, dt); + assert_eq!(consumed, bytes.len()); + } + } + + #[test] + fn opaque_tag_padding_is_not_part_of_the_tag() { + // libhdf5 pads "mytag" to 8 bytes; parsing must not return the NULs, + // or copying the type would grow the tag. + let (dt, _) = Datatype::parse(&hex("15080000040000006d79746167000000")).unwrap(); + assert_eq!( + dt, + Datatype::Opaque { + size: 4, + tag: b"mytag".to_vec() + } + ); + let long = Datatype::Opaque { + size: 1, + tag: vec![b'x'; MAX_OPAQUE_TAG_LEN + 1], + }; + assert!(long.check_encodable().is_err()); + let ok = Datatype::Opaque { + size: 1, + tag: vec![b'x'; MAX_OPAQUE_TAG_LEN], + }; + assert!(ok.check_encodable().is_ok()); + assert_eq!(Datatype::parse(&ok.serialize()).unwrap().0, ok); + } + #[test] fn test_error_invalid_reference_type() { let buf = build_dt_header(7, 1, [5, 0, 0], 8); diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index d5260c2..b8d0edb 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -1125,6 +1125,17 @@ impl FileWriter { root_attrs.push(build_attr_message(n, v)); } + // Every datatype must have an on-disk encoding before anything is laid + // out: `Datatype::serialize` itself cannot report a failure. + let group_attrs = groups.iter().flat_map(|g| &g.attrs); + let ds_attrs = all_ds.iter().flat_map(|d| &d.attrs); + for a in root_attrs.iter().chain(group_attrs).chain(ds_attrs) { + a.datatype.check_encodable()?; + } + for d in &all_ds { + d.dt.check_encodable()?; + } + let is_vds: Vec = all_ds.iter().map(|d| d.virtual_sources.is_some()).collect(); let is_chunked: Vec = all_ds .iter() diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index 1175ab4..22db1c5 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -6,6 +6,7 @@ //! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it. use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType}; use clawhdf5_format::file_writer::{AttrValue, FileWriter}; use clawhdf5_format::group_v2::resolve_path_any; use clawhdf5_format::message_type::MessageType; @@ -151,3 +152,168 @@ fn h5py_reads_compact_datasets_at_the_limit() { h5dump_ok(&path); } } + +// ---- 2. Time / BitField / Opaque / Reference datatypes ---- + +fn exotic_types() -> Vec<(&'static str, Datatype, Vec)> { + // Four elements each. The object references point at the root group, + // which a v3-superblock file without an extension puts at address 48. + let refs: Vec = (0..4).flat_map(|_| 48u64.to_le_bytes()).collect(); + vec![ + ( + "bits", + Datatype::BitField { + size: 1, + byte_order: DatatypeByteOrder::LittleEndian, + bit_offset: 0, + bit_precision: 8, + }, + vec![1, 2, 4, 8], + ), + ( + "opaque", + Datatype::Opaque { + size: 4, + tag: b"mytag".to_vec(), + }, + (0..16).collect(), + ), + ( + "ref", + Datatype::Reference { + size: 8, + ref_type: ReferenceType::Object, + }, + refs, + ), + ( + "time", + Datatype::Time { + size: 4, + bit_precision: 32, + }, + (0..16).collect(), + ), + ] +} + +fn exotic_file() -> Vec { + let mut fw = FileWriter::new(); + for (name, dt, raw) in exotic_types() { + fw.create_dataset(name) + .with_compound_data(dt.clone(), raw.clone(), 4); + fw.set_root_attr( + name, + AttrValue::Raw { + datatype: dt, + shape: vec![4], + data: raw, + }, + ); + } + fw.finish().unwrap() +} + +#[test] +fn exotic_datatypes_are_written_not_emptied() { + let bytes = exotic_file(); + let (sb, root) = header_at(&bytes, "/"); + assert_eq!(sb.root_group_address, 48); + let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap(); + for (name, dt, raw) in exotic_types() { + let (_, oh) = header_at(&bytes, name); + let msg = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::Datatype) + .unwrap(); + assert_eq!(msg.data, dt.serialize(), "{name}"); + assert_eq!(Datatype::parse(&msg.data).unwrap().0, dt, "{name}"); + let attr = attrs.iter().find(|a| a.name == name).unwrap(); + assert_eq!(attr.datatype, dt, "{name}"); + assert_eq!(attr.raw_data, raw, "{name}"); + } +} + +#[test] +#[ignore = "requires Python h5py module and h5dump"] +fn h5py_reads_exotic_datatypes() { + let path = write_tmp("exotic", &exotic_file()); + let out = h5py( + &path, + "from h5py import h5t, h5s\n\ + f = h5py.File(path, 'r')\n\ + r = {}\n\ + buf = np.zeros(4, dtype='V4')\n\ + f['opaque'].id.read(h5s.ALL, h5s.ALL, buf, mtype=f['opaque'].id.get_type())\n\ + r['bits'] = f['bits'][()].tolist(), f.attrs['bits'].tolist()\n\ + r['opaque'] = (f['opaque'].id.get_type().get_tag().decode(),\n\ + \x20 f.attrs.get_id('opaque').get_type().get_tag().decode(),\n\ + \x20 buf.tobytes().hex())\n\ + r['ref'] = [f[x].name for x in f['ref'][()]] + [f[x].name for x in f.attrs['ref']]\n\ + r['time'] = (f['time'].id.get_type().get_class() == h5t.TIME,\n\ + \x20 f.attrs.get_id('time').get_type().get_class() == h5t.TIME)\n\ + print(json.dumps(r))", + ); + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!(v["bits"], serde_json::json!([[1, 2, 4, 8], [1, 2, 4, 8]])); + assert_eq!( + v["opaque"], + serde_json::json!(["mytag", "mytag", "000102030405060708090a0b0c0d0e0f"]) + ); + assert_eq!(v["ref"], serde_json::json!(vec!["/"; 8])); + assert_eq!(v["time"], serde_json::json!([true, true])); + h5dump_ok(&path); +} + +#[test] +#[ignore = "requires Python h5py module and h5dump"] +fn raw_attributes_copied_from_h5py_survive_a_rewrite() { + // Read Raw attributes of the exotic classes out of an h5py file and write + // them back: this used to emit empty datatype messages. + let src = std::env::temp_dir().join("clawhdf5_writer_meta_exotic_src.h5"); + h5py( + &src, + "from h5py import h5t, h5s, h5a\n\ + f = h5py.File(path, 'w')\n\ + f.attrs['ref'] = np.array([f.ref, f.ref], dtype=h5py.ref_dtype)\n\ + f.attrs.create('opaque', np.frombuffer(b'abcdefgh', dtype='V4'))\n\ + t = h5t.STD_B16BE.copy()\n\ + a = h5a.create(f.id, b'bits', t, h5s.create_simple((2,)))\n\ + a.write(np.array([0x0102, 0x0304], dtype='>u2'), mtype=t)\n\ + a.close()\n\ + f.close()", + ); + let src_bytes = std::fs::read(&src).unwrap(); + let (sb, root) = header_at(&src_bytes, "/"); + let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap(); + assert_eq!(attrs.len(), 3); + let mut fw = FileWriter::new(); + for a in &attrs { + let data = if a.name == "ref" { + // Re-target the references at our root group. + 48u64.to_le_bytes().repeat(2) + } else { + a.raw_data.clone() + }; + fw.set_root_attr( + &a.name, + AttrValue::Raw { + datatype: a.datatype.clone(), + shape: a.dataspace.dimensions.clone(), + data, + }, + ); + } + let path = write_tmp("exotic_copy", &fw.finish().unwrap()); + let out = h5py( + &path, + "f = h5py.File(path, 'r')\n\ + print(json.dumps([[f[x].name for x in f.attrs['ref']],\n\ + \x20 f.attrs['opaque'].tobytes().decode(),\n\ + \x20 f.attrs.get_id('bits').get_type().get_order(),\n\ + \x20 f.attrs['bits'].tolist()]))", + ); + assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#); + h5dump_ok(&path); +} From 4a1876faf2dd865ed20948ce10d2d647b75d237b Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:07:08 -0500 Subject: [PATCH 11/36] fix(format): page Fixed Array data blocks past 1024 chunks The Fixed Array writer always packed every element into one data block behind one checksum. Past 2^10 elements libhdf5 (and our reader) expect a paged block: a page-init bitmap after the prefix, then one checksummed page per 1024 elements. Any dataset with more than 1024 chunks and no unlimited dimension failed with "incorrect metadata checksum" in h5py, h5dump and our own reader. build_fixed_array_at now takes one Option per array slot so later fixes can leave unallocated slots. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_write.rs | 172 +++++++++++-------- crates/clawhdf5/tests/chunk_index_interop.rs | 138 ++++++++++++++- 2 files changed, 240 insertions(+), 70 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 6816546..c0d02e1 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -499,108 +499,140 @@ fn serialize_v4_fixed_array( buf } +/// log2 of the elements per Fixed Array data block page (the library's +/// default, `H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS`). +const FA_PAGE_BITS: u8 = 10; + +fn push_addr(buf: &mut Vec, addr: u64, offset_size: u8) { + match offset_size { + 4 => buf.extend_from_slice(&(addr as u32).to_le_bytes()), + _ => buf.extend_from_slice(&addr.to_le_bytes()), + } +} + +/// Width of the chunk-size field of a filtered chunk index element. Must +/// match the library's `H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN` (the EA and +/// B-tree v2 indexes use the same formula): +/// `1 + ((log2(unfiltered chunk bytes) + 8) / 8)`, capped at 8. +pub(crate) fn filtered_chunk_size_len(slots: &[Option]) -> usize { + let max_raw = slots + .iter() + .flatten() + .map(|c| c.raw_size) + .max() + .unwrap_or(1); + let log2_val = if max_raw <= 1 { + 0 + } else { + 63 - max_raw.leading_zeros() + }; + (1 + ((log2_val + 8) / 8) as usize).min(8) +} + +/// Append one chunk index element: the chunk's address, plus its stored size +/// and filter mask when the dataset is filtered. `None` is an unallocated +/// chunk (undefined address, zero size and mask). +pub(crate) fn push_index_element( + buf: &mut Vec, + slot: Option<&WrittenChunk>, + offset_size: u8, + chunk_size_bytes: Option, +) { + match slot { + Some(c) => { + push_addr(buf, c.address, offset_size); + if let Some(n) = chunk_size_bytes { + buf.extend_from_slice(&c.compressed_size.to_le_bytes()[..n]); + buf.extend_from_slice(&c.filter_mask.to_le_bytes()); + } + } + None => { + buf.extend(core::iter::repeat_n(0xFF, offset_size as usize)); + if let Some(n) = chunk_size_bytes { + buf.extend(core::iter::repeat_n(0x00, n + 4)); + } + } + } +} + /// Build a complete Fixed Array at a known absolute address. +/// +/// `slots` holds one entry per element of the array, i.e. per chunk of the +/// dataset's *maximum* extent in the order [`crate::chunk_grid`] defines; +/// `None` marks a chunk that is not allocated. An array with more elements +/// than fit in one page (`2^FA_PAGE_BITS`) gets a paged data block: a +/// page-init bitmap after the prefix, then one checksummed page per +/// `2^FA_PAGE_BITS` elements, the last one short (`H5FA__dblock_create`). pub fn build_fixed_array_at( - chunks: &[WrittenChunk], + slots: &[Option], offset_size: u8, length_size: u8, has_filters: bool, fa_base_address: u64, ) -> Vec { let os = offset_size as usize; - let num_elements = chunks.len(); - - // For filtered chunks, compute chunk_size encoding width. - // Must match the HDF5 C library's H5D_FARRAY_FILT_COMPUTE_CHUNK_SIZE_LEN macro: - // chunk_size_len = 1 + ((H5VM_log2_gen(chunk.size) + 8) / 8) - // where chunk.size is the unfiltered chunk size in bytes (product of all chunk dims). - let chunk_size_bytes: usize = if has_filters { - let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1); - let log2_val = if max_raw <= 1 { - 0 - } else { - 63 - max_raw.leading_zeros() - }; - let len = 1 + ((log2_val + 8) / 8) as usize; - len.min(8) - } else { - 0 - }; - - let elem_size = if has_filters { - os + chunk_size_bytes + 4 - } else { - os - }; + let num_elements = slots.len(); + let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots)); + let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4); let client_id: u8 = if has_filters { 1 } else { 0 }; // FAHD total size - let nelmts_field_size = length_size as usize; - let fahd_total_size = 4 + 1 + 1 + 1 + 1 + nelmts_field_size + os + 4; + let fahd_total_size = 4 + 1 + 1 + 1 + 1 + length_size as usize + os + 4; let fadb_address = fa_base_address + fahd_total_size as u64; - // Build FAHD let mut fahd = Vec::with_capacity(fahd_total_size); fahd.extend_from_slice(b"FAHD"); fahd.push(0); // version fahd.push(client_id); fahd.push(elem_size as u8); - - // max_nelmts_bits: use 10 as default (page_size = 1024), matching h5py convention - let max_bits: u8 = 10; - fahd.push(max_bits); - + fahd.push(FA_PAGE_BITS); match length_size { 4 => fahd.extend_from_slice(&(num_elements as u32).to_le_bytes()), - 8 => fahd.extend_from_slice(&(num_elements as u64).to_le_bytes()), _ => fahd.extend_from_slice(&(num_elements as u64).to_le_bytes()), } - - match offset_size { - 4 => fahd.extend_from_slice(&(fadb_address as u32).to_le_bytes()), - 8 => fahd.extend_from_slice(&fadb_address.to_le_bytes()), - _ => fahd.extend_from_slice(&fadb_address.to_le_bytes()), - } - - // Checksum + push_addr(&mut fahd, fadb_address, offset_size); let checksum = jenkins_lookup3(&fahd); fahd.extend_from_slice(&checksum.to_le_bytes()); - assert_eq!(fahd.len(), fahd_total_size); - // Build FADB + // FADB prefix let mut fadb = Vec::new(); fadb.extend_from_slice(b"FADB"); fadb.push(0); // version fadb.push(client_id); + push_addr(&mut fadb, fa_base_address, offset_size); - // header address - match offset_size { - 4 => fadb.extend_from_slice(&(fa_base_address as u32).to_le_bytes()), - 8 => fadb.extend_from_slice(&fa_base_address.to_le_bytes()), - _ => fadb.extend_from_slice(&fa_base_address.to_le_bytes()), - } - - // Element data - for chunk in chunks { - match offset_size { - 4 => fadb.extend_from_slice(&(chunk.address as u32).to_le_bytes()), - 8 => fadb.extend_from_slice(&chunk.address.to_le_bytes()), - _ => fadb.extend_from_slice(&chunk.address.to_le_bytes()), + let page_nelmts = 1usize << FA_PAGE_BITS; + if num_elements <= page_nelmts { + // Unpaged: the elements follow the prefix, one checksum over both. + for slot in slots { + push_index_element(&mut fadb, slot.as_ref(), offset_size, chunk_size_bytes); } - if has_filters { - // Write compressed size using chunk_size_bytes (variable width) - let cs_bytes = chunk.compressed_size.to_le_bytes(); - fadb.extend_from_slice(&cs_bytes[..chunk_size_bytes]); - fadb.extend_from_slice(&chunk.filter_mask.to_le_bytes()); + let fadb_checksum = jenkins_lookup3(&fadb); + fadb.extend_from_slice(&fadb_checksum.to_le_bytes()); + } else { + // Paged: every page is written, so every page-init bit is set + // (MSB-first, as `H5VM_bit_set` packs them). The prefix and bitmap + // share a checksum; each page carries its own. + let npages = num_elements.div_ceil(page_nelmts); + let mut bitmap = vec![0u8; npages.div_ceil(8)]; + for p in 0..npages { + bitmap[p / 8] |= 0x80 >> (p % 8); + } + fadb.extend_from_slice(&bitmap); + let prefix_checksum = jenkins_lookup3(&fadb); + fadb.extend_from_slice(&prefix_checksum.to_le_bytes()); + for page in slots.chunks(page_nelmts) { + let start = fadb.len(); + for slot in page { + push_index_element(&mut fadb, slot.as_ref(), offset_size, chunk_size_bytes); + } + let page_checksum = jenkins_lookup3(&fadb[start..]); + fadb.extend_from_slice(&page_checksum.to_le_bytes()); } } - // FADB checksum - let fadb_checksum = jenkins_lookup3(&fadb); - fadb.extend_from_slice(&fadb_checksum.to_le_bytes()); - let mut combined = fahd; combined.extend_from_slice(&fadb); combined @@ -734,8 +766,9 @@ pub fn build_chunked_data_from_precompressed( ) } else { let fa_address = base_address + data_buf.len() as u64; + let slots: Vec> = written_chunks.iter().cloned().map(Some).collect(); let fa_bytes = build_fixed_array_at( - &written_chunks, + &slots, offset_size, length_size, pre.has_filters, @@ -747,7 +780,7 @@ pub fn build_chunked_data_from_precompressed( fa_address, offset_size, element_size as u32, - 10, // max_nelmts_bits — matches h5py convention + FA_PAGE_BITS, ) }; @@ -1382,7 +1415,8 @@ mod tests { filter_mask: 0, }, ]; - let fa = build_fixed_array_at(&chunks, 8, 8, false, 0x2000); + let slots: Vec<_> = chunks.into_iter().map(Some).collect(); + let fa = build_fixed_array_at(&slots, 8, 8, false, 0x2000); // Should start with FAHD assert_eq!(&fa[0..4], b"FAHD"); // FAHD size = 4+1+1+1+1+8+8+4 = 28 diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index 9c030a3..cce2df6 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -11,7 +11,7 @@ use std::process::Command; -use clawhdf5::File; +use clawhdf5::{File, FileBuilder}; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) @@ -254,3 +254,139 @@ fn h5py_fixed_array_partial_extent_reads_correctly() { }, ]); } + +// =========================================================================== +// Files we write, read back by libhdf5 (h5py and h5dump) and by us +// =========================================================================== + +/// One `i4` dataset we write, filled with `arange` over `shape`. +struct WriteCase { + name: String, + shape: Vec, + chunks: Vec, + maxshape: Option>, + deflate: bool, +} + +fn wcase(name: &str, shape: &[u64], chunks: &[u64], maxshape: Option<&[u64]>) -> WriteCase { + WriteCase { + name: name.to_string(), + shape: shape.to_vec(), + chunks: chunks.to_vec(), + maxshape: maxshape.map(<[u64]>::to_vec), + deflate: false, + } +} + +fn h5dump_available() -> bool { + Command::new("h5dump") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Write every case into one file with our writer, then check that our own +/// reader, h5py and h5dump (when installed) all return every value. Only the +/// libhdf5 half is skipped without h5py. +fn check_we_write(cases: &[WriteCase]) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ours_chunk_index.h5"); + let path_str = path.display().to_string(); + + let mut b = FileBuilder::new(); + for c in cases { + let n: u64 = c.shape.iter().product(); + let data: Vec = (0..n as i32).collect(); + let ds = b.create_dataset(&c.name); + ds.with_i32_data(&data) + .with_shape(&c.shape) + .with_chunks(&c.chunks); + if let Some(ms) = &c.maxshape { + ds.with_maxshape(ms); + } + if c.deflate { + ds.with_deflate(4); + } + } + b.write(&path).unwrap(); + + // Our reader. + let file = File::open(&path).unwrap(); + for c in cases { + let got = file.dataset(&c.name).unwrap().read_i32().unwrap(); + let n: u64 = c.shape.iter().product(); + let bad = got + .iter() + .enumerate() + .filter(|&(i, &v)| v != i as i32) + .count(); + assert!( + got.len() == n as usize && bad == 0, + "{}: our reader: {bad} of {n} values wrong", + c.name + ); + } + + // libhdf5 via h5py. + skip_if_no_python!(); + let mut script = + format!("import h5py, numpy as np\nbad = []\nf = h5py.File(r'{path_str}', 'r')\n"); + for c in cases { + let shape: Vec = c.shape.iter().map(u64::to_string).collect(); + let maxshape: Vec = c + .maxshape + .as_ref() + .unwrap_or(&c.shape) + .iter() + .map(|&d| { + if d == u64::MAX { + "None".to_string() + } else { + d.to_string() + } + }) + .collect(); + script += &format!( + "d = f['{name}']\n\ + want = np.arange({n}, dtype='i4').reshape(({shape},))\n\ + got = d[()]\n\ + if d.maxshape != ({maxshape},): bad.append(('{name}', 'maxshape', d.maxshape))\n\ + elif not np.array_equal(got, want): \ + bad.append(('{name}', int((got != want).sum()), 'of', got.size))\n", + name = c.name, + n = c.shape.iter().product::(), + shape = shape.join(","), + maxshape = maxshape.join(","), + ); + } + script += "print(bad if bad else 'OK')\n"; + let out = run_python(&script); + assert_eq!(out, "OK", "h5py disagrees"); + + // libhdf5's own tool, when installed. + if h5dump_available() { + let o = Command::new("h5dump").arg(&path).output().unwrap(); + let stderr = String::from_utf8_lossy(&o.stderr); + assert!( + o.status.success() && !stderr.to_lowercase().contains("error"), + "h5dump failed: {stderr}" + ); + } +} + +/// A Fixed Array with more than 1024 elements must be paged, or libhdf5 +/// rejects the data block's checksum. +#[test] +fn we_write_paged_fixed_array() { + let mut cases: Vec = [1023u64, 1024, 1025, 2048, 5000] + .iter() + .map(|&n| wcase(&format!("fa_{n}"), &[n * 4], &[4], None)) + .collect(); + // Filtered elements are wider; a 2-D grid pages the same way. + let mut filtered = wcase("fa_1500_deflate", &[1500 * 4], &[4], None); + filtered.deflate = true; + cases.push(filtered); + cases.push(wcase("fa_2d_1100", &[110, 40], &[1, 4], None)); + check_we_write(&cases); +} From d074385944c8c49f1325a39de6208227e90e581b Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:08:36 -0500 Subject: [PATCH 12/36] fix(format): read partial edge chunks stored unfiltered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layout message v4 flag bit 0 (H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS, set with H5Pset_chunk_opts) makes libhdf5 store every chunk that extends past the dataset's extent without the filter pipeline, while its filter mask still reads 0. The parser ignored the flag, so readers tried to inflate raw bytes: libhdf5's own h5fc_edge_v3.h5 failed with "deflate: ... unknown compression method". DataLayout::Chunked gains dont_filter_partial_edge_chunks (always false for v3), and list_chunks — the one place every read path gets its chunk list from — marks such partial chunks as having skipped every filter, so the full, cached, indexed, parallel and selection readers all copy them as-is. chunked_write.rs gets `..` in one exhaustive test pattern for the new field. Regression: libhdf5_edge_chunk_fixture_reads (h5fc_edge_v3.h5 from the HDF5 tools test files, committed as a 2.5 KB fixture), and h5py_unfiltered_partial_edge_chunks_read (the flag set through h5py's bundled libhdf5 via ctypes, as h5py has no binding for it: fixed array, extensible array and B-tree v2 indexes, 1-D and 2-D, plus a hyperslab of the last chunk), and v4_chunked_dont_filter_partial_edge_chunks_flag. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 27 ++++- crates/clawhdf5-format/src/chunked_write.rs | 1 + crates/clawhdf5-format/src/data_layout.rs | 34 ++++++ .../clawhdf5/tests/fixtures/h5fc_edge_v3.h5 | Bin 0 -> 2478 bytes .../clawhdf5/tests/h5py_chunked_read_tests.rs | 106 ++++++++++++++++++ 5 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 crates/clawhdf5/tests/fixtures/h5fc_edge_v3.h5 diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 2dab147..bc006df 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -519,6 +519,7 @@ pub fn list_chunks( addr_opt, single_filtered_size, single_filter_mask, + unfiltered_edges, ) = match layout { DataLayout::Chunked { chunk_dimensions, @@ -527,6 +528,7 @@ pub fn list_chunks( chunk_index_type, single_chunk_filtered_size, single_chunk_filter_mask, + dont_filter_partial_edge_chunks, } => ( chunk_dimensions, *version, @@ -534,6 +536,7 @@ pub fn list_chunks( *btree_address, *single_chunk_filtered_size, *single_chunk_filter_mask, + *dont_filter_partial_edge_chunks, ), _ => { return Err(FormatError::ChunkedReadError( @@ -566,7 +569,7 @@ pub fn list_chunks( } // Collect chunks based on version and index type - let chunks = match (version, chunk_index_type) { + let mut chunks = match (version, chunk_index_type) { (3, _) => { let ndims = chunk_dimensions.len(); // rank+1 collect_chunk_info(file_data, addr, ndims, offset_size, length_size)? @@ -645,6 +648,23 @@ pub fn list_chunks( } }; + // With "don't filter partial edge chunks", a chunk that extends past the + // dataset's extent is stored raw while its filter mask still reads 0. + // Mark every filter skipped so all read paths copy it as-is. + if unfiltered_edges { + for chunk in &mut chunks { + let partial = chunk + .offsets + .iter() + .zip(&chunk_dims) + .zip(&ds_dims) + .any(|((&off, &cd), &dd)| off.saturating_add(cd as u64) > dd as u64); + if partial { + chunk.filter_mask = u32::MAX; + } + } + } + Ok((chunks, chunk_dims)) } @@ -1829,6 +1849,7 @@ mod tests { chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, }; let dataspace = Dataspace { @@ -1852,6 +1873,7 @@ mod tests { chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, }; let dataspace = Dataspace { space_type: DataspaceType::Simple, @@ -2009,6 +2031,7 @@ mod tests { chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, }; let dataspace = Dataspace { space_type: DataspaceType::Simple, @@ -2091,6 +2114,7 @@ mod tests { chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, }; let dataspace = Dataspace { space_type: DataspaceType::Simple, @@ -2253,6 +2277,7 @@ mod tests { chunk_index_type: Some(1), single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, }; let dataspace = Dataspace { space_type: DataspaceType::Simple, diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 6816546..04d2b3e 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -1314,6 +1314,7 @@ mod tests { chunk_index_type, single_chunk_filtered_size, single_chunk_filter_mask, + .. } => { assert_eq!(version, 4); assert_eq!(chunk_index_type, Some(1)); diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index dd931ff..8429c5d 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -53,6 +53,11 @@ pub enum DataLayout { single_chunk_filtered_size: Option, /// Filter mask for v4 single chunk with filters. single_chunk_filter_mask: Option, + /// Layout v4 flag bit 0 (`H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS`): + /// partial edge chunks — those extending past the dataset's current + /// extent in some dimension — are stored without the filter pipeline, + /// even though their filter mask is 0. Always `false` for v3. + dont_filter_partial_edge_chunks: bool, }, /// Virtual dataset layout (v4 only). Virtual { @@ -322,6 +327,7 @@ impl DataLayout { chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, }) } _ => Err(FormatError::InvalidLayoutClass(layout_class)), @@ -505,6 +511,7 @@ impl DataLayout { chunk_index_type: Some(chunk_index_type), single_chunk_filtered_size, single_chunk_filter_mask, + dont_filter_partial_edge_chunks: flags & 0x01 != 0, }) } 3 => { @@ -602,6 +609,7 @@ mod tests { chunk_index_type: None, single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, } ); } @@ -679,10 +687,35 @@ mod tests { chunk_index_type: Some(1), single_chunk_filtered_size: None, single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, } ); } + #[test] + fn v4_chunked_dont_filter_partial_edge_chunks_flag() { + let mut buf = vec![4u8, 2]; // version=4, class=2 + buf.push(0x01); // flags bit 0 = don't filter partial edge chunks + buf.push(2); // dimensionality=2 + buf.push(4); // dim_size_encoded_length=4 + buf.extend_from_slice(&5u32.to_le_bytes()); + buf.extend_from_slice(&4u32.to_le_bytes()); + buf.push(3); // Fixed Array + buf.push(10); // max_dblk_page_nelmts_bits + buf.extend_from_slice(&0x3000u64.to_le_bytes()); + match DataLayout::parse(&buf, 8, 8).unwrap() { + DataLayout::Chunked { + dont_filter_partial_edge_chunks, + btree_address, + .. + } => { + assert!(dont_filter_partial_edge_chunks); + assert_eq!(btree_address, Some(0x3000)); + } + other => panic!("expected Chunked, got {other:?}"), + } + } + #[test] fn v4_chunked_single_chunk_with_filters() { let mut buf = vec![4u8, 2]; // version=4, class=2 @@ -705,6 +738,7 @@ mod tests { chunk_index_type: Some(1), single_chunk_filtered_size: Some(1024), single_chunk_filter_mask: Some(0), + dont_filter_partial_edge_chunks: false, } ); } diff --git a/crates/clawhdf5/tests/fixtures/h5fc_edge_v3.h5 b/crates/clawhdf5/tests/fixtures/h5fc_edge_v3.h5 new file mode 100644 index 0000000000000000000000000000000000000000..6f92057cf653c4dc47cad6133e2ecdc425e861ae GIT binary patch literal 2478 zcmeD5aB<`1lHy|G;9!7(|4^`w6Cz>&mAE-AN6_EHC5TC3cZXdBGOl0}0;z#%gwb3~ z42%p6Y$6Omiqj?7H6-5E#ohHV)HntQ7=wKJL%-5Nu#*&#odaSsFfytzFfcJOGV(x; zXM@t{d}a{_MnMI61{Scq0)sdQgLplV%?flqGdDL610&E7pbvn40E%(|eagbb$jr#f z%EHWbALu_Ym;j{*5q0yk(ib;J4;KbTJ}#Kwe?#4S_Jl%}o1=>p10&1_DbV=mhxh=> zO9AtWQRJFXiz2}1{Kt`Q18UmvsFd71*A%GkLC!COZ5H@wiDsCkw pSUf!9kj%rtD=7IJjmyA5UE6S*2eO9{#taMMMTqqW@g}0H1ONf9YWx5I literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs index deda24f..2b9e902 100644 --- a/crates/clawhdf5/tests/h5py_chunked_read_tests.rs +++ b/crates/clawhdf5/tests/h5py_chunked_read_tests.rs @@ -237,3 +237,109 @@ with h5py.File("{p}", "w") as f: grid ); } + +// --------------------------------------------------------------------------- +// "Don't filter partial edge chunks" +// --------------------------------------------------------------------------- + +/// libhdf5's own test file for `H5Pset_chunk_opts(H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS)`: +/// a 12x6 f32 dataset in 5x5 gzip chunks whose edge chunks are stored raw +/// with a filter mask of 0. Reading it tried to inflate the raw edge chunks +/// ("deflate: ... unknown compression method"). +#[test] +fn libhdf5_edge_chunk_fixture_reads() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/h5fc_edge_v3.h5" + ); + let file = File::open(path).unwrap(); + let ds = file.dataset("DSET_EDGE").unwrap(); + assert_eq!(ds.shape().unwrap(), vec![12, 6]); + assert_eq!(ds.read_f32().unwrap(), vec![100.0f32; 72]); +} + +/// The same layout flag on datasets with varied contents, set through the +/// libhdf5 that h5py ships (h5py has no binding for `H5Pset_chunk_opts`). +#[test] +fn h5py_unfiltered_partial_edge_chunks_read() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("edge.h5"); + let p = path.display().to_string(); + let script = format!( + r#" +import ctypes, glob, os, sys +import h5py, numpy as np +here = os.path.dirname(h5py.__file__) +libs = glob.glob(os.path.join(here, "..", "h5py.libs", "libhdf5-*.so*")) +libs += glob.glob(os.path.join(here, ".dylibs", "libhdf5*.dylib")) +if not libs: + print("NO_LIBHDF5") + sys.exit(0) +lib = ctypes.CDLL(libs[0]) +lib.H5Pset_chunk_opts.argtypes = [ctypes.c_int64, ctypes.c_uint] +def make(f, name, data, chunk, maxshape, shuffle): + dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE) + dcpl.set_chunk(chunk) + if shuffle: + dcpl.set_shuffle() + dcpl.set_deflate(6) + assert lib.H5Pset_chunk_opts(dcpl.id, 0x0002) >= 0 + space = h5py.h5s.create_simple(data.shape, maxshape) + d = h5py.h5d.create(f.id, name.encode(), h5py.h5t.py_create(data.dtype), space, dcpl=dcpl) + d.write(h5py.h5s.ALL, h5py.h5s.ALL, np.ascontiguousarray(data)) +with h5py.File("{p}", "w") as f: + line = np.sin(np.arange(1000) / 7.0) + grid = np.arange(37 * 53, dtype=" = (0..1000).map(|i| (f64::from(i) / 7.0).sin()).collect(); + for name in ["fixed_1d", "ea_1d"] { + let values = file.dataset(name).unwrap().read_f64().unwrap(); + assert_eq!(values.len(), line.len(), "{name}"); + for (i, (v, w)) in values.iter().zip(&line).enumerate() { + assert!((v - w).abs() < 1e-12, "{name}[{i}]: {v} vs {w}"); + } + } + let grid: Vec = (0..37 * 53).map(|i| f64::from(i) * 0.5).collect(); + for name in ["bt2_2d", "fixed_2d"] { + assert_eq!( + file.dataset(name).unwrap().read_f64().unwrap(), + grid, + "{name}" + ); + } + // A selection touching only the last (partial, unfiltered) chunk. + let tail = file + .dataset("fixed_1d") + .unwrap() + .read_selection(&Selection::Hyperslab { + start: vec![990], + stride: vec![1], + count: vec![1], + block: vec![10], + }) + .unwrap(); + let want: Vec = line[990..].iter().flat_map(|v| v.to_le_bytes()).collect(); + assert_eq!(tail, want); +} From 081341b433b15319ab7f50cef7a6c1d414f9f07f Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:02:39 -0500 Subject: [PATCH 13/36] fix(format): convert float data read as integers instead of returning bit patterns read_i32/read_i64/read_u64 on a floating-point dataset reinterpreted the IEEE bits (1.5 read as i64 was 4609434218613702656). Convert like libhdf5's hard conversions instead: truncate toward zero and saturate at the target range; NaN reads as 0. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 120 ++++++++++++---- .../tests/numeric_conversion_interop.rs | 129 ++++++++++++++++++ 2 files changed, 226 insertions(+), 23 deletions(-) create mode 100644 crates/clawhdf5/tests/numeric_conversion_interop.rs diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index e3c1bbd..c998fad 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -982,7 +982,74 @@ fn convert_to_f64( } } +/// One numeric element as stored, before conversion to the caller's type. +#[derive(Debug, Clone, Copy, PartialEq)] +enum Scalar { + Signed(i64), + Unsigned(u64), + Float(f64), +} + +impl Scalar { + /// Float to integer conversions follow libhdf5's hard conversions: + /// truncate toward zero, and saturate a value outside the target range to + /// its minimum or maximum. NaN converts to 0 (libhdf5 leaves that case to + /// the C cast, whose result is platform-dependent). + fn to_i64(self) -> i64 { + match self { + Scalar::Signed(v) => v, + Scalar::Unsigned(v) => v as i64, + Scalar::Float(v) => v as i64, + } + } + + fn to_u64(self) -> u64 { + match self { + Scalar::Signed(v) => v as u64, + Scalar::Unsigned(v) => v, + Scalar::Float(v) => v as u64, + } + } + + fn to_i32(self) -> i32 { + match self { + Scalar::Signed(v) => v as i32, + Scalar::Unsigned(v) => v as i32, + Scalar::Float(v) => v as i32, + } + } +} + +/// Decode one element of a numeric datatype. +fn decode_scalar( + bytes: &[u8], + dt: &Datatype, + order: &DatatypeByteOrder, +) -> Result { + match dt { + Datatype::FixedPoint { + size, + signed, + bit_offset, + bit_precision, + .. + } => { + let full = read_unsigned_int(bytes, *size as usize, order); + let (off, prec) = effective_bits(*size as usize, *bit_offset, *bit_precision); + Ok(if *signed { + Scalar::Signed(extract_signed(full, off, prec)) + } else { + Scalar::Unsigned(extract_unsigned(full, off, prec)) + }) + } + _ => convert_to_f64(bytes, dt, order).map(Scalar::Float), + } +} + /// Convert raw bytes to `i64` values. +/// +/// Floating-point data is converted the way libhdf5 converts it: truncated +/// toward zero, saturating at the target type's range, with NaN read as 0. pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_i64(raw, base_type); @@ -1014,17 +1081,18 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let order = get_byte_order(datatype); - let (off, prec) = fixed_bits(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; - let full = read_unsigned_int(chunk, elem_size, &order); - result.push(extract_signed(full, off, prec)); + result.push(decode_scalar(chunk, datatype, &order)?.to_i64()); } Ok(result) } /// Convert raw bytes to `u64` values. +/// +/// Floating-point data is converted the way libhdf5 converts it: truncated +/// toward zero, saturating at the target type's range, with NaN read as 0. pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_u64(raw, base_type); @@ -1039,12 +1107,10 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let count = raw.len() / elem_size; let order = get_byte_order(datatype); - let (off, prec) = fixed_bits(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; - let full = read_unsigned_int(chunk, elem_size, &order); - result.push(extract_unsigned(full, off, prec)); + result.push(decode_scalar(chunk, datatype, &order)?.to_u64()); } Ok(result) } @@ -1140,6 +1206,9 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } /// Convert raw bytes to `i32` values. +/// +/// Floating-point data is converted the way libhdf5 converts it: truncated +/// toward zero, saturating at the target type's range, with NaN read as 0. pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_i32(raw, base_type); @@ -1170,12 +1239,10 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let order = get_byte_order(datatype); - let (off, prec) = fixed_bits(datatype); let mut result = Vec::with_capacity(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; - let full = read_unsigned_int(chunk, elem_size, &order); - result.push(extract_signed(full, off, prec) as i32); + result.push(decode_scalar(chunk, datatype, &order)?.to_i32()); } Ok(result) } @@ -1666,20 +1733,6 @@ fn effective_bits(size: usize, bit_offset: u16, bit_precision: u16) -> (u32, u32 (bit_offset as u32, prec) } -/// `(bit_offset, bit_precision)` for a fixed-point datatype, full width for -/// other types. -fn fixed_bits(datatype: &Datatype) -> (u32, u32) { - match datatype { - Datatype::FixedPoint { - size, - bit_offset, - bit_precision, - .. - } => effective_bits(*size as usize, *bit_offset, *bit_precision), - _ => (0, 0), - } -} - /// Whether a datatype occupies its full storage width (bit offset 0, precision /// == size·8), in which case the bulk-copy fast read paths apply. Non /// fixed-point types are treated as full width. @@ -1892,6 +1945,27 @@ mod tests { assert_eq!(read_as_u64(&raw, &dt).unwrap(), vec![4095, 1, 2048]); } + #[test] + fn float_to_int_truncates_and_saturates() { + // Values libhdf5 hands to an undefined C cast: NaN reads as 0 and + // exactly 2^63 saturates instead of wrapping to i64::MIN. + let dt = make_f64_le_type(); + let vals = [f64::NAN, 2f64.powi(63), -2.5, 2.0f64.powi(64)]; + let raw: Vec = vals.iter().flat_map(|v| v.to_le_bytes()).collect(); + assert_eq!( + read_as_i64(&raw, &dt).unwrap(), + vec![0, i64::MAX, -2, i64::MAX] + ); + assert_eq!( + read_as_u64(&raw, &dt).unwrap(), + vec![0, 1 << 63, 0, u64::MAX] + ); + assert_eq!( + read_as_i32(&raw, &dt).unwrap(), + vec![0, i32::MAX, -2, i32::MAX] + ); + } + #[test] fn full_width_signed_unchanged() { // Regression: full-width 32-bit signed must be unaffected. diff --git a/crates/clawhdf5/tests/numeric_conversion_interop.rs b/crates/clawhdf5/tests/numeric_conversion_interop.rs new file mode 100644 index 0000000..5c0fc3c --- /dev/null +++ b/crates/clawhdf5/tests/numeric_conversion_interop.rs @@ -0,0 +1,129 @@ +//! Numeric conversions on read, checked against h5py/libhdf5. +//! +//! h5py writes each file and prints what libhdf5 converts the data to +//! (`Dataset.astype`); the typed readers must return the same values. +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::collections::HashMap; +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Run `script` (which writes the file at `path`) and return its stdout as +/// `key -> values`, one `key v1 v2 ...` line per key. +fn run_python(script: &str) -> HashMap> { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| { + let mut words = line.split_whitespace().map(str::to_string); + Some((words.next()?, words.collect())) + }) + .collect() +} + +fn parse(values: &[String]) -> Vec +where + T::Err: std::fmt::Debug, +{ + values.iter().map(|v| v.parse().unwrap()).collect() +} + +/// Python prelude: `emit(key, array)` prints one line of integers. +const PRELUDE: &str = r#" +import h5py, numpy as np +def emit(key, arr): + print(key, *[int(v) for v in np.asarray(arr).ravel()]) +"#; + +#[test] +fn float_dataset_read_as_integers_converts_like_libhdf5() { + // read_i32/read_i64/read_u64 on a float dataset used to return the raw + // IEEE bit patterns (1.5 read as i64 was 4609434218613702656). + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("float_to_int.h5"); + let script = format!( + r#"{PRELUDE} +vals = [1.5, -2.75, 3e9, 1e300, -1e300, -0.5, 0.0, 7.99, np.inf, -np.inf, 1e19, -1e19] +with h5py.File("{path}", "w") as f: + for name, dt in (("f8", "f8"), ("f4", "(&expected[&format!("{name}:i32")]), + "{name} as i32" + ); + assert_eq!( + ds.read_i64().unwrap(), + parse::(&expected[&format!("{name}:i64")]), + "{name} as i64" + ); + if name != "f2" { + assert_eq!( + ds.read_u64().unwrap(), + parse::(&expected[&format!("{name}:u64")]), + "{name} as u64" + ); + } + } + // Negative values saturate at 0 rather than wrapping. + let f2 = file.dataset("f2").unwrap(); + assert_eq!(f2.read_u64().unwrap(), vec![1, 0, 0, 0, 7, 65504, 0]); +} From 53dbddb07b167639f8a6f3d5fdd596516fea6b4d Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:03:50 -0500 Subject: [PATCH 14/36] fix(format): saturate out-of-range integer reads instead of truncating Reading wider or differently-signed integers kept the low bits: i64 2^40+5 read as i32 was 5, u64::MAX read as i64 was -1, and -1 read as u64 was 4294967295. u32 data read as i32 also took the bulk-copy fast path meant for i32. Saturate at the target range like libhdf5's hard conversions (a negative value read as unsigned is 0), and keep the i32 fast path to signed data. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 38 +++++++----- .../tests/numeric_conversion_interop.rs | 61 +++++++++++++++++++ 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index c998fad..01060a6 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -991,21 +991,24 @@ enum Scalar { } impl Scalar { - /// Float to integer conversions follow libhdf5's hard conversions: - /// truncate toward zero, and saturate a value outside the target range to - /// its minimum or maximum. NaN converts to 0 (libhdf5 leaves that case to - /// the C cast, whose result is platform-dependent). + // Every conversion follows libhdf5's default (hard) conversions: a value + // outside the target type's range saturates to its minimum or maximum — + // including a negative value read as unsigned, which reads as 0 — rather + // than being truncated to its low bits. Floats truncate toward zero; NaN + // converts to 0 (libhdf5 leaves that case to the C cast, whose result is + // platform-dependent). + fn to_i64(self) -> i64 { match self { Scalar::Signed(v) => v, - Scalar::Unsigned(v) => v as i64, + Scalar::Unsigned(v) => i64::try_from(v).unwrap_or(i64::MAX), Scalar::Float(v) => v as i64, } } fn to_u64(self) -> u64 { match self { - Scalar::Signed(v) => v as u64, + Scalar::Signed(v) => u64::try_from(v).unwrap_or(0), Scalar::Unsigned(v) => v, Scalar::Float(v) => v as u64, } @@ -1013,8 +1016,8 @@ impl Scalar { fn to_i32(self) -> i32 { match self { - Scalar::Signed(v) => v as i32, - Scalar::Unsigned(v) => v as i32, + Scalar::Signed(v) => v.clamp(i32::MIN.into(), i32::MAX.into()) as i32, + Scalar::Unsigned(v) => i32::try_from(v).unwrap_or(i32::MAX), Scalar::Float(v) => v as i32, } } @@ -1048,8 +1051,10 @@ fn decode_scalar( /// Convert raw bytes to `i64` values. /// -/// Floating-point data is converted the way libhdf5 converts it: truncated -/// toward zero, saturating at the target type's range, with NaN read as 0. +/// Values are converted the way libhdf5 converts them: integers outside the +/// target range saturate at its minimum or maximum (a negative value read as +/// unsigned is 0), and floating-point data is truncated toward zero and +/// saturated, with NaN read as 0. pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_i64(raw, base_type); @@ -1091,8 +1096,10 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr /// Convert raw bytes to `u64` values. /// -/// Floating-point data is converted the way libhdf5 converts it: truncated -/// toward zero, saturating at the target type's range, with NaN read as 0. +/// Values are converted the way libhdf5 converts them: integers outside the +/// target range saturate at its minimum or maximum (a negative value read as +/// unsigned is 0), and floating-point data is truncated toward zero and +/// saturated, with NaN read as 0. pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_u64(raw, base_type); @@ -1207,8 +1214,10 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr /// Convert raw bytes to `i32` values. /// -/// Floating-point data is converted the way libhdf5 converts it: truncated -/// toward zero, saturating at the target type's range, with NaN read as 0. +/// Values are converted the way libhdf5 converts them: integers outside the +/// target range saturate at its minimum or maximum (a negative value read as +/// unsigned is 0), and floating-point data is truncated toward zero and +/// saturated, with NaN read as 0. pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { if let Datatype::Array { base_type, .. } = datatype { return read_as_i32(raw, base_type); @@ -1231,6 +1240,7 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr datatype, Datatype::FixedPoint { byte_order: DatatypeByteOrder::LittleEndian, + signed: true, .. } ) diff --git a/crates/clawhdf5/tests/numeric_conversion_interop.rs b/crates/clawhdf5/tests/numeric_conversion_interop.rs index 5c0fc3c..e75700a 100644 --- a/crates/clawhdf5/tests/numeric_conversion_interop.rs +++ b/crates/clawhdf5/tests/numeric_conversion_interop.rs @@ -127,3 +127,64 @@ with h5py.File("{path}", "r") as f: let f2 = file.dataset("f2").unwrap(); assert_eq!(f2.read_u64().unwrap(), vec![1, 0, 0, 0, 7, 65504, 0]); } + +#[test] +fn integer_reads_saturate_out_of_range_values_like_libhdf5() { + // Narrowing reads used to keep the low bits (i64 2^40+5 read as i32 was + // 5, u64::MAX read as i64 was -1) and signed-to-unsigned reads wrapped + // (-1 read as u64 was 4294967295). + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("int_narrowing.h5"); + let script = format!( + r#"{PRELUDE} +data = {{ + "i8": np.array([2**40 + 5, -(2**35), 7, -1, 2**63 - 1, -(2**63)], "i8"), + "u8": np.array([2**64 - 1, 2**63, 5, 0], "i2"), + "u1": np.array([255, 0, 128], "u1"), +}} +with h5py.File("{path}", "w") as f: + for name, arr in data.items(): + f.create_dataset(name, data=arr) +with h5py.File("{path}", "r") as f: + for name in data: + d = f[name] + emit(name + ":i32", d.astype("(&expected[&format!("{name}:i32")]), + "{name} as i32" + ); + assert_eq!( + ds.read_i64().unwrap(), + parse::(&expected[&format!("{name}:i64")]), + "{name} as i64" + ); + // libhdf5 wraps a negative big-endian i64 read as little-endian u64 + // (it only byte-swaps when the sizes match and the order differs); + // every other signed-to-unsigned read saturates at 0, so do that. + if name != "i8be" { + assert_eq!( + ds.read_u64().unwrap(), + parse::(&expected[&format!("{name}:u64")]), + "{name} as u64" + ); + } + } + let i8be = file.dataset("i8be").unwrap(); + assert_eq!(i8be.read_u64().unwrap(), vec![(1 << 40) + 5, 0, 7, 0]); +} From 417c9516ca69e9908c10d914b5adc4844a8ff299 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:06:19 -0500 Subject: [PATCH 15/36] fix(format): decode floats by their datatype fields, not their size Every 2-byte float was decoded as IEEE half, so bfloat16 (HDF5 2.0's H5T_FLOAT_BFLOAT16*, or any custom 8-bit-exponent type) read wrong: 1.5 as 1.9375, +inf as NaN. 1-byte FP8 floats were refused. Read the exponent/mantissa location and size and the bias from the datatype message: IEEE half/single/double keep their existing paths (half still through clawhdf5_format::float16), any other IEEE-style layout up to 64 bits whose values fit f64 (bfloat16, FP8 E4M3/E5M2, ...) is decoded generically, and the bulk-copy and zero-copy fast paths now require the IEEE layout rather than just the size. Datatypes with fields that describe no float still fall back to IEEE by size; layouts that cannot be represented in f64 (x87 80-bit, binary128) remain an error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 298 ++++++++++++++---- .../tests/numeric_conversion_interop.rs | 85 +++++ 2 files changed, 320 insertions(+), 63 deletions(-) diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 01060a6..1309080 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -773,14 +773,7 @@ pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&' // Only native LE f64 is eligible #[cfg(target_endian = "little")] { - if !matches!( - datatype, - Datatype::FloatingPoint { - size: 8, - byte_order: DatatypeByteOrder::LittleEndian, - .. - } - ) { + if !is_native_le_float(datatype, FloatFormat::Double) { return None; } if !raw.len().is_multiple_of(8) { @@ -809,14 +802,7 @@ pub fn read_as_f64_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&' pub fn read_as_f32_zerocopy<'a>(raw: &'a [u8], datatype: &Datatype) -> Option<&'a [f32]> { #[cfg(target_endian = "little")] { - if !matches!( - datatype, - Datatype::FloatingPoint { - size: 4, - byte_order: DatatypeByteOrder::LittleEndian, - .. - } - ) { + if !is_native_le_float(datatype, FloatFormat::Single) { return None; } if !raw.len().is_multiple_of(4) { @@ -919,20 +905,19 @@ pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr // Fast path: native-endian f64 — single bulk memcpy #[cfg(target_endian = "little")] - if matches!( - datatype, - Datatype::FloatingPoint { - size: 8, - byte_order: DatatypeByteOrder::LittleEndian, - .. - } - ) { + if is_native_le_float(datatype, FloatFormat::Double) { return Ok(native_le_to_vec::(raw, count)); } let order = get_byte_order(datatype); let mut result = Vec::with_capacity(count); - + if let Datatype::FloatingPoint { .. } = datatype { + let format = FloatFormat::of(datatype)?; + for chunk in raw.chunks_exact(elem_size) { + result.push(format.decode(chunk, &order)); + } + return Ok(result); + } for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; let val = convert_to_f64(chunk, datatype, &order)?; @@ -947,18 +932,7 @@ fn convert_to_f64( order: &DatatypeByteOrder, ) -> Result { match dt { - Datatype::FloatingPoint { size, .. } => match size { - 4 => { - let v = read_f32_bytes(bytes, order); - Ok(v as f64) - } - 8 => Ok(read_f64_bytes(bytes, order)), - 2 => Ok(read_f16_bytes(bytes, order) as f64), - _ => Err(FormatError::DataSizeMismatch { - expected: 8, - actual: *size as usize, - }), - }, + Datatype::FloatingPoint { .. } => Ok(FloatFormat::of(dt)?.decode(bytes, order)), Datatype::FixedPoint { size, signed, @@ -1139,25 +1113,11 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr // Fast path: native-endian f32 — single bulk memcpy #[cfg(target_endian = "little")] - if matches!( - datatype, - Datatype::FloatingPoint { - size: 4, - byte_order: DatatypeByteOrder::LittleEndian, - .. - } - ) { + if is_native_le_float(datatype, FloatFormat::Single) { return Ok(native_le_to_vec::(raw, count)); } - // Little-endian half precision (numpy float16): widen directly. - if matches!( - datatype, - Datatype::FloatingPoint { - size: 2, - byte_order: DatatypeByteOrder::LittleEndian, - .. - } - ) { + // Little-endian IEEE half precision (numpy float16): widen directly. + if is_native_le_float(datatype, FloatFormat::Half) { let (halves, _) = raw[..count * 2].as_chunks::<2>(); return Ok(halves .iter() @@ -1167,18 +1127,22 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr let order = get_byte_order(datatype); let mut result = Vec::with_capacity(count); + if let Datatype::FloatingPoint { .. } = datatype { + let format = FloatFormat::of(datatype)?; + for chunk in raw.chunks_exact(elem_size) { + result.push(match format { + FloatFormat::Single => read_f32_bytes(chunk, &order), + FloatFormat::Half => read_f16_bytes(chunk, &order), + // Double rounds; every other supported layout (bfloat16, FP8) + // is exact in f32. + _ => format.decode(chunk, &order) as f32, + }); + } + return Ok(result); + } for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; match datatype { - Datatype::FloatingPoint { size: 4, .. } => { - result.push(read_f32_bytes(chunk, &order)); - } - Datatype::FloatingPoint { size: 8, .. } => { - result.push(read_f64_bytes(chunk, &order) as f32); - } - Datatype::FloatingPoint { size: 2, .. } => { - result.push(read_f16_bytes(chunk, &order)); - } Datatype::FixedPoint { signed: true, size, @@ -1693,6 +1657,174 @@ fn reorder_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> [u8; 8] { buf } +/// How the bits of a floating-point datatype are laid out, read from the +/// datatype message's fields rather than assumed from its size (a 2-byte +/// float may be IEEE half or bfloat16). +#[derive(Debug, Clone, Copy, PartialEq)] +enum FloatFormat { + /// IEEE-754 binary16. + Half, + /// IEEE-754 binary32. + Single, + /// IEEE-754 binary64. + Double, + /// Any other IEEE-style layout (implied leading mantissa bit, all-ones + /// exponent for infinity/NaN) whose values are all exact in `f64`: + /// bfloat16, the FP8 formats, and similar. + Other(FloatLayout), +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct FloatLayout { + exponent_location: u32, + exponent_size: u32, + mantissa_location: u32, + mantissa_size: u32, + exponent_bias: u32, +} + +impl FloatFormat { + fn of(dt: &Datatype) -> Result { + let Datatype::FloatingPoint { + size, + exponent_location, + exponent_size, + mantissa_location, + mantissa_size, + exponent_bias, + .. + } = dt + else { + return Err(FormatError::TypeMismatch { + expected: "FloatingPoint", + actual: datatype_name(dt), + }); + }; + let layout = FloatLayout { + exponent_location: u32::from(*exponent_location), + exponent_size: u32::from(*exponent_size), + mantissa_location: u32::from(*mantissa_location), + mantissa_size: u32::from(*mantissa_size), + exponent_bias: *exponent_bias, + }; + let fields = ( + layout.exponent_location, + layout.exponent_size, + layout.mantissa_location, + layout.mantissa_size, + layout.exponent_bias, + ); + let bits = size.saturating_mul(8); + // The sign bit is not kept in `Datatype`; every standard layout has it + // directly above the exponent, with the mantissa below. + let well_formed = layout.exponent_size > 0 + && layout.mantissa_size > 0 + && layout.mantissa_location + layout.mantissa_size <= layout.exponent_location + && layout.exponent_location + layout.exponent_size < bits; + match (size, fields) { + (2, (10, 5, 0, 10, 15)) => Ok(FloatFormat::Half), + (4, (23, 8, 0, 23, 127)) => Ok(FloatFormat::Single), + (8, (52, 11, 0, 52, 1023)) => Ok(FloatFormat::Double), + _ if well_formed + && *size <= 8 + && layout.exponent_size <= 11 + && layout.mantissa_size <= 52 => + { + Ok(FloatFormat::Other(layout)) + } + // Fields that cannot describe any float (e.g. left zeroed by a + // hand-built datatype): fall back to the IEEE type of that size. + (2, _) if !well_formed => Ok(FloatFormat::Half), + (4, _) if !well_formed => Ok(FloatFormat::Single), + (8, _) if !well_formed => Ok(FloatFormat::Double), + // x87 80-bit extended, binary128, ...: not representable in f64. + _ => Err(FormatError::TypeMismatch { + expected: "floating point of at most 64 bits (IEEE-style layout)", + actual: "FloatingPoint", + }), + } + } + + fn decode(self, bytes: &[u8], order: &DatatypeByteOrder) -> f64 { + match self { + FloatFormat::Half => f64::from(read_f16_bytes(bytes, order)), + FloatFormat::Single => f64::from(read_f32_bytes(bytes, order)), + FloatFormat::Double => read_f64_bytes(bytes, order), + FloatFormat::Other(layout) => { + layout.decode(read_unsigned_int(bytes, bytes.len(), order)) + } + } + } +} + +impl FloatLayout { + /// Decode the value held in the low `size * 8` bits of `bits`. + fn decode(self, bits: u64) -> f64 { + let field = |location: u32, size: u32| (bits >> location) & ((1u64 << size) - 1); + let exponent = field(self.exponent_location, self.exponent_size); + let mantissa = field(self.mantissa_location, self.mantissa_size); + let negative = field(self.exponent_location + self.exponent_size, 1) == 1; + let max_exponent = (1u64 << self.exponent_size) - 1; + let magnitude = if exponent == max_exponent { + if mantissa == 0 { + f64::INFINITY + } else { + f64::NAN + } + } else { + let bias = i64::from(self.exponent_bias); + let msize = i64::from(self.mantissa_size); + // value = significand * 2^power, with an implied leading 1 unless + // the number is subnormal (exponent field 0). + let (significand, power) = if exponent == 0 { + (mantissa, 1 - bias - msize) + } else { + ( + mantissa | (1u64 << self.mantissa_size), + exponent as i64 - bias - msize, + ) + }; + scale_by_pow2(significand as f64, power) + }; + if negative { -magnitude } else { magnitude } + } +} + +/// `x * 2^power` without `std` (no `powi`/`libm`). `x` is a non-negative +/// integer below 2^53, so it is exact. +fn scale_by_pow2(x: f64, power: i64) -> f64 { + if x == 0.0 || power < -1200 { + return 0.0; + } + if power > 1100 { + return f64::INFINITY; + } + let pow2 = |p: i64| f64::from_bits(((p + 1023) as u64) << 52); + let mut x = x; + let mut power = power; + while power > 1023 { + x *= pow2(1023); + power -= 1023; + } + while power < -1022 { + x *= pow2(-1022); + power += 1022; + } + x * pow2(power) +} + +/// Whether `datatype` is the little-endian IEEE float `format`, whose bytes +/// can be copied straight into native values on a little-endian target. +fn is_native_le_float(datatype: &Datatype, format: FloatFormat) -> bool { + matches!( + datatype, + Datatype::FloatingPoint { + byte_order: DatatypeByteOrder::LittleEndian, + .. + } + ) && FloatFormat::of(datatype).is_ok_and(|f| f == format) +} + fn read_f64_bytes(bytes: &[u8], order: &DatatypeByteOrder) -> f64 { let buf = reorder_bytes(bytes, order); f64::from_le_bytes(buf) @@ -1976,6 +2108,46 @@ mod tests { ); } + #[test] + fn bfloat16_and_fp8_decode_by_fields() { + // bfloat16 is a 2-byte float that is not IEEE half. + let bf16 = Datatype::FloatingPoint { + size: 2, + byte_order: DatatypeByteOrder::LittleEndian, + bit_offset: 0, + bit_precision: 16, + exponent_location: 7, + exponent_size: 8, + mantissa_location: 0, + mantissa_size: 7, + exponent_bias: 127, + }; + let raw: Vec = [0x3FC0u16, 0xC010, 0x7F80, 0x0001] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(); + let got = read_as_f64(&raw, &bf16).unwrap(); + assert_eq!(&got[..3], &[1.5, -2.25, f64::INFINITY]); + assert_eq!(got[3], 2f64.powi(-133)); // smallest subnormal + assert_eq!(read_as_f32(&raw, &bf16).unwrap()[..2], [1.5, -2.25]); + + // FP8 E4M3: 1, -1, 2, 0, NaN (IEEE-style, as libhdf5 treats it). + let e4m3 = Datatype::FloatingPoint { + size: 1, + byte_order: DatatypeByteOrder::LittleEndian, + bit_offset: 0, + bit_precision: 8, + exponent_location: 3, + exponent_size: 4, + mantissa_location: 0, + mantissa_size: 3, + exponent_bias: 7, + }; + let got = read_as_f64(&[0x38, 0xB8, 0x40, 0x00, 0x7E], &e4m3).unwrap(); + assert_eq!(&got[..4], &[1.0, -1.0, 2.0, 0.0]); + assert!(got[4].is_nan()); + } + #[test] fn full_width_signed_unchanged() { // Regression: full-width 32-bit signed must be unaffected. diff --git a/crates/clawhdf5/tests/numeric_conversion_interop.rs b/crates/clawhdf5/tests/numeric_conversion_interop.rs index e75700a..556f2a9 100644 --- a/crates/clawhdf5/tests/numeric_conversion_interop.rs +++ b/crates/clawhdf5/tests/numeric_conversion_interop.rs @@ -188,3 +188,88 @@ with h5py.File("{path}", "r") as f: let i8be = file.dataset("i8be").unwrap(); assert_eq!(i8be.read_u64().unwrap(), vec![(1 << 40) + 5, 0, 7, 0]); } + +#[test] +fn floats_decode_by_their_datatype_fields() { + // Every 2-byte float used to decode as IEEE half, so bfloat16 1.5 read as + // 1.9375 and +inf as NaN; 1-byte FP8 floats were refused. + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("float_layouts.h5"); + let script = format!( + r#"{PRELUDE} +def custom(base, fields, bias, size): + # fields: (sign pos, exponent pos, exponent size, mantissa pos, mantissa size) + t = base.copy() + t.set_fields(*fields) + t.set_ebias(bias) + t.set_precision(size * 8) + t.set_size(size) + return t + +def write(f, name, ftype, raw): + raw = np.ascontiguousarray(raw) + space = h5py.h5s.create_simple(raw.shape) + ds = h5py.h5d.create(f.id, name.encode(), ftype, space) + ds.write(h5py.h5s.ALL, h5py.h5s.ALL, raw, mtype=ftype) + +# bfloat16: 1.5, -2.25, +inf, 0, 3.140625, 1, -0, smallest subnormal, +# largest finite, NaN +bf16 = np.array([0x3FC0, 0xC010, 0x7F80, 0x0000, 0x4049, 0x3F80, 0x8000, 0x0001, + 0x7F7F, 0x7FC1], "f2")) + f.create_dataset("f4_be", data=vals.astype(">f4")) + f.create_dataset("f8_le", data=vals.astype(" = parse(&expected[name]); + let got: Vec = ds + .read_f64() + .unwrap() + .into_iter() + .map(|v| canonical(v).to_bits()) + .collect(); + assert_eq!(got, want, "{name} as f64"); + // f32 reads agree too (every value here is exact in f32 except the + // f64 dataset, which rounds like `as f32`). + let got32: Vec = ds + .read_f32() + .unwrap() + .into_iter() + .map(|v| if v.is_nan() { f32::NAN } else { v }.to_bits()) + .collect(); + let want32: Vec = want + .iter() + .map(|&b| canonical(f64::from_bits(b)) as f32) + .map(|v| if v.is_nan() { f32::NAN } else { v }.to_bits()) + .collect(); + assert_eq!(got32, want32, "{name} as f32"); + } +} From c8c2930fc0827c218f6eedf1823ced258a36c5b1 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:07:02 -0500 Subject: [PATCH 16/36] fix(format): read enum and bool datasets through their base integer type read_i64/read_u64/read_i32/read_f64/read_f32 refused enumeration datatypes, including h5py's bool (an enum of int8), with a type mismatch. Read them as their base type's integer values, the way array datatypes already read through theirs. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/data_read.rs | 22 +++++++---- .../tests/numeric_conversion_interop.rs | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 1309080..82e6544 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -888,9 +888,9 @@ fn native_le_to_vec(raw: &[u8], count: usize) -> Vec { /// Convert raw bytes to `f64` values. pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { - // Array datatypes (e.g. an array-typed compound member) are read as a flat - // sequence of their base elements. - if let Datatype::Array { base_type, .. } = datatype { + // Array datatypes read as a flat sequence of their base elements, and + // enumerations (h5py's bool among them) as their integer values. + if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_f64(raw, base_type); } ensure_numeric(datatype, "FloatingPoint or FixedPoint")?; @@ -1030,7 +1030,9 @@ fn decode_scalar( /// unsigned is 0), and floating-point data is truncated toward zero and /// saturated, with NaN read as 0. pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { - if let Datatype::Array { base_type, .. } = datatype { + // Array datatypes read as a flat sequence of their base elements, and + // enumerations (h5py's bool among them) as their integer values. + if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_i64(raw, base_type); } ensure_numeric(datatype, "FixedPoint (signed)")?; @@ -1075,7 +1077,9 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr /// unsigned is 0), and floating-point data is truncated toward zero and /// saturated, with NaN read as 0. pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { - if let Datatype::Array { base_type, .. } = datatype { + // Array datatypes read as a flat sequence of their base elements, and + // enumerations (h5py's bool among them) as their integer values. + if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_u64(raw, base_type); } ensure_numeric(datatype, "FixedPoint (unsigned)")?; @@ -1098,7 +1102,9 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr /// Convert raw bytes to `f32` values. pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { - if let Datatype::Array { base_type, .. } = datatype { + // Array datatypes read as a flat sequence of their base elements, and + // enumerations (h5py's bool among them) as their integer values. + if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_f32(raw, base_type); } ensure_numeric(datatype, "FloatingPoint")?; @@ -1183,7 +1189,9 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr /// unsigned is 0), and floating-point data is truncated toward zero and /// saturated, with NaN read as 0. pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { - if let Datatype::Array { base_type, .. } = datatype { + // Array datatypes read as a flat sequence of their base elements, and + // enumerations (h5py's bool among them) as their integer values. + if let Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } = datatype { return read_as_i32(raw, base_type); } ensure_numeric(datatype, "FixedPoint")?; diff --git a/crates/clawhdf5/tests/numeric_conversion_interop.rs b/crates/clawhdf5/tests/numeric_conversion_interop.rs index 556f2a9..2777bd5 100644 --- a/crates/clawhdf5/tests/numeric_conversion_interop.rs +++ b/crates/clawhdf5/tests/numeric_conversion_interop.rs @@ -273,3 +273,41 @@ with h5py.File("{path}", "r") as f: assert_eq!(got32, want32, "{name} as f32"); } } + +#[test] +fn enum_and_bool_datasets_read_as_their_integer_values() { + // Enumerations (h5py stores bool as an enum of int8) were refused by the + // numeric readers with a type mismatch. + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("enums.h5"); + let script = format!( + r#"{PRELUDE} +with h5py.File("{path}", "w") as f: + f.create_dataset("bool", data=np.array([True, False, True])) + e = h5py.enum_dtype({{"RED": 0, "GREEN": 7, "BLUE": -3}}, basetype=">i2") + f.create_dataset("enum_i2be", data=np.array([0, 7, -3, 7], ">i2"), dtype=e) + e = h5py.enum_dtype({{"LOW": 0, "HIGH": 200}}, basetype="u1") + f.create_dataset("enum_u1", data=np.array([200, 0, 200], "u1"), dtype=e) + e = h5py.enum_dtype({{"A": -(2**40), "B": 2**40}}, basetype=" = parse(&expected[name]); + assert_eq!(ds.read_i64().unwrap(), want, "{name} as i64"); + let want_f64: Vec = want.iter().map(|&v| v as f64).collect(); + assert_eq!(ds.read_f64().unwrap(), want_f64, "{name} as f64"); + } + let bools = file.dataset("bool").unwrap(); + assert_eq!(bools.read_u64().unwrap(), vec![1, 0, 1]); + assert_eq!(bools.read_i32().unwrap(), vec![1, 0, 1]); +} From 2f252df0842c21e3e30bbe8fada5f9177dd1660d Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:07:57 -0500 Subject: [PATCH 17/36] fix(format): return whole VL sequences from read_vl_bytes read_vl_bytes cut each element to the reference's length field, which counts sequence elements, not bytes: a VL int32 [1, 2, 3] came back as 3 bytes. Return the whole global-heap object, which is element count x base size bytes. No in-tree caller depended on the old behaviour. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/vl_data.rs | 13 +++- .../tests/numeric_conversion_interop.rs | 61 +++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index 6b4dcfc..9a50d51 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -148,7 +148,12 @@ pub fn read_vl_strings( Ok(result) } -/// Resolve VL byte sequences from raw data. +/// Resolve VL sequences from raw data, returning each element's bytes. +/// +/// Each element is the sequence's full encoding — element count × base type +/// size bytes, in the base type's byte order — so a sequence of `i32` yields +/// four bytes per value. Decode it with the base type (e.g. +/// [`crate::data_read::read_as_i64`]). pub fn read_vl_bytes( file_data: &[u8], raw_data: &[u8], @@ -177,8 +182,10 @@ pub fn read_vl_bytes( }, )?; - let len = (vl.length as usize).min(obj.data.len()); - result.push(obj.data[..len].to_vec()); + // The heap object holds the whole sequence. `vl.length` counts + // elements, not bytes, so it is only the byte length when the base + // type is one byte wide. + result.push(obj.data.clone()); } Ok(result) diff --git a/crates/clawhdf5/tests/numeric_conversion_interop.rs b/crates/clawhdf5/tests/numeric_conversion_interop.rs index 2777bd5..357a37d 100644 --- a/crates/clawhdf5/tests/numeric_conversion_interop.rs +++ b/crates/clawhdf5/tests/numeric_conversion_interop.rs @@ -311,3 +311,64 @@ with h5py.File("{path}", "r") as f: assert_eq!(bools.read_u64().unwrap(), vec![1, 0, 1]); assert_eq!(bools.read_i32().unwrap(), vec![1, 0, 1]); } + +#[test] +fn vl_sequences_of_wide_base_types_read_whole() { + // read_vl_bytes took the sequence's element count as its byte length, so + // [1, 2, 3] as VL int32 came back as 3 bytes instead of 12. + use clawhdf5::Selection; + use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; + use clawhdf5_format::vl_data::read_vl_bytes; + + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("vlen.h5"); + let script = format!( + r#"{PRELUDE} +data = {{ + "i4": (h5py.vlen_dtype("> = (0..count) + .map(|i| parse(expected.get(&format!("{name}:{i}")).unwrap())) + .collect(); + assert_eq!(got, want, "{name}"); + if name == "i4" { + let i32_le = Datatype::FixedPoint { + size: 4, + byte_order: DatatypeByteOrder::LittleEndian, + signed: true, + bit_offset: 0, + bit_precision: 32, + }; + let values = clawhdf5_format::data_read::read_as_i64(&got[0], &i32_le).unwrap(); + assert_eq!(values, vec![1, 2, 3]); + assert_eq!(got[3].len(), 40 * 4); + } + } +} From 44f5f8b5c5bc0b4ba9960c2fa1aa286e2f630240 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:09:29 -0500 Subject: [PATCH 18/36] fix(format): index every Extensible Array chunk, not just the first 244 The Extensible Array writer only filled the index block's 4 inline elements and the 6 data blocks it addresses directly (240 elements); its super block addresses were always undefined. Chunks from index 244 on were written to the file but never indexed, so they read back as fill values in our reader and in libhdf5, without an error. The writer now lays out data blocks and super blocks for any element count as H5EA__hdr_init sizes them, pages data blocks larger than 1024 elements (page-init bits in the owning super block), leaves blocks with no defined element unallocated, and records real header statistics (max_idx_set is one past the highest defined index). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_write.rs | 8 +- crates/clawhdf5-format/src/ea_writer.rs | 513 +++++++++---------- crates/clawhdf5/tests/chunk_index_interop.rs | 21 + 3 files changed, 271 insertions(+), 271 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index c0d02e1..66bc82d 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -503,7 +503,7 @@ fn serialize_v4_fixed_array( /// default, `H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS`). const FA_PAGE_BITS: u8 = 10; -fn push_addr(buf: &mut Vec, addr: u64, offset_size: u8) { +pub(crate) fn push_addr(buf: &mut Vec, addr: u64, offset_size: u8) { match offset_size { 4 => buf.extend_from_slice(&(addr as u32).to_le_bytes()), _ => buf.extend_from_slice(&addr.to_le_bytes()), @@ -734,8 +734,9 @@ pub fn build_chunked_data_from_precompressed( let layout_message = if use_extensible { let ea_address = base_address + data_buf.len() as u64; + let slots: Vec> = written_chunks.iter().cloned().map(Some).collect(); let ea_bytes = ea_writer::build_extensible_array_at( - &written_chunks, + &slots, offset_size, length_size, pre.has_filters, @@ -1463,7 +1464,8 @@ mod tests { filter_mask: 0, }, ]; - let ea = ea_writer::build_extensible_array_at(&chunks, 8, 8, false, 0x2000); + let slots: Vec<_> = chunks.into_iter().map(Some).collect(); + let ea = ea_writer::build_extensible_array_at(&slots, 8, 8, false, 0x2000); assert_eq!(&ea[0..4], b"EAHD"); // Find EAIB after EAHD: 12 fixed + 6*8 stats + 8 addr + 4 checksum = 72 let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * 8 + 8 + 4; diff --git a/crates/clawhdf5-format/src/ea_writer.rs b/crates/clawhdf5-format/src/ea_writer.rs index d92fca2..e0f0375 100644 --- a/crates/clawhdf5-format/src/ea_writer.rs +++ b/crates/clawhdf5-format/src/ea_writer.rs @@ -7,7 +7,7 @@ extern crate alloc; use alloc::{vec, vec::Vec}; use crate::checksum::jenkins_lookup3; -use crate::chunked_write::WrittenChunk; +use crate::chunked_write::{WrittenChunk, filtered_chunk_size_len, push_addr, push_index_element}; /// Serialize a v4 Extensible Array layout message. pub(crate) fn serialize_v4_extensible_array( @@ -58,11 +58,11 @@ pub(crate) fn serialize_v4_extensible_array( buf.push(4); // EA creation parameters (must match AEHD and HDF5 C library defaults) - buf.push(32); // max_nelmts_bits - buf.push(4); // idx_blk_elmts - buf.push(4); // super_blk_min_data_ptrs - buf.push(16); // data_blk_min_elmts - buf.push(10); // max_dblk_page_nelmts_bits + buf.push(MAX_NELMTS_BITS); + buf.push(IDX_BLK_ELMTS); + buf.push(SUP_BLK_MIN_DATA_PTRS); + buf.push(DATA_BLK_MIN_ELMTS); + buf.push(MAX_DBLK_PAGE_NELMTS_BITS); // EA header address match offset_size { @@ -74,304 +74,281 @@ pub(crate) fn serialize_v4_extensible_array( buf } +// EA creation parameters — the HDF5 library's defaults for chunk indexes +// (`H5D_EARRAY_*`); the layout message above and the header must agree. +const MAX_NELMTS_BITS: u8 = 32; +const IDX_BLK_ELMTS: u8 = 4; +const SUP_BLK_MIN_DATA_PTRS: u8 = 4; +const DATA_BLK_MIN_ELMTS: u8 = 16; +const MAX_DBLK_PAGE_NELMTS_BITS: u8 = 10; + +/// One data block of the array: its first element (relative to the end of +/// the index block's own elements), element count, and address when it is +/// allocated. +struct DataBlock { + start: usize, + nelmts: usize, + addr: Option, +} + /// Build a complete Extensible Array at a known absolute address. /// -/// For simplicity, we put all elements inline in the index block when the -/// number of chunks is small (up to idx_blk_elmts), otherwise use inline + -/// direct data blocks. +/// `slots[i]` is the element at linear index `i` (see `chunk_grid`); `None` +/// marks an unallocated chunk. The first `IDX_BLK_ELMTS` elements live in +/// the index block, the rest in data blocks grouped by super block level +/// exactly as `H5EA__hdr_init` sizes them: level `u` has `2^(u/2)` data +/// blocks of `DATA_BLK_MIN_ELMTS * 2^ceil(u/2)` elements. The data blocks of +/// the first levels are addressed straight from the index block; later +/// levels go through a super block (EASB). Data blocks larger than a page +/// (`2^MAX_DBLK_PAGE_NELMTS_BITS` elements) are paged, with their page-init +/// bits kept in the owning super block. Only blocks holding a defined element +/// are allocated; the rest keep the undefined address, as in a file the +/// library wrote. pub fn build_extensible_array_at( - chunks: &[WrittenChunk], + slots: &[Option], offset_size: u8, length_size: u8, has_filters: bool, ea_base_address: u64, ) -> Vec { let os = offset_size as usize; - let num_elements = chunks.len(); - - // Compute element encoding size (same logic as Fixed Array) - let chunk_size_bytes: usize = if has_filters { - let max_raw = chunks.iter().map(|c| c.raw_size).max().unwrap_or(1); - let log2_val = if max_raw <= 1 { - 0 - } else { - 63 - max_raw.leading_zeros() - }; - let len = 1 + ((log2_val + 8) / 8) as usize; - len.min(8) - } else { - 0 - }; - - let elem_size = if has_filters { - os + chunk_size_bytes + 4 - } else { - os - }; - + let chunk_size_bytes = has_filters.then(|| filtered_chunk_size_len(slots)); + let elem_size = os + chunk_size_bytes.map_or(0, |n| n + 4); let client_id: u8 = if has_filters { 1 } else { 0 }; + let arr_off_size = (MAX_NELMTS_BITS as usize).div_ceil(8); + let page_nelmts = 1usize << MAX_DBLK_PAGE_NELMTS_BITS; + let idx_blk = IDX_BLK_ELMTS as usize; - // EA creation parameters — must match HDF5 C library defaults exactly - let max_nelmts_bits: u8 = 32; - let idx_blk_elmts: u8 = 4; - let min_dblk_nelmts: u8 = 16; - let super_blk_min_nelmts: u8 = 4; - let max_dblk_nelmts_bits: u8 = 10; + // Elements past the last defined one are never realised + // (`max_idx_set` is one past the highest index ever set). + let max_idx_set = slots.iter().rposition(Option::is_some).map_or(0, |i| i + 1); + let slots = &slots[..max_idx_set]; + let defined_in = |start: usize, n: usize| -> bool { + let lo = idx_blk.saturating_add(start).min(slots.len()); + let hi = idx_blk + .saturating_add(start) + .saturating_add(n) + .min(slots.len()); + slots[lo..hi].iter().any(Option::is_some) + }; - // EAHD size: fixed(12) + 6 stats(6*length_size) + addr(offset_size) + checksum(4) + // Super block levels: (ndblks, dblk_nelmts, first element). + let log2_dmin = (DATA_BLK_MIN_ELMTS as u32).trailing_zeros() as usize; + let nsblks = 1 + MAX_NELMTS_BITS as usize - log2_dmin; + let ndblk_addrs = 2 * (SUP_BLK_MIN_DATA_PTRS as usize - 1); + let mut levels: Vec<(usize, usize, usize)> = Vec::with_capacity(nsblks); + let mut start = 0usize; + for u in 0..nsblks { + let ndblks = 1usize << (u / 2); + let nelmts = (DATA_BLK_MIN_ELMTS as usize) << u.div_ceil(2); + levels.push((ndblks, nelmts, start)); + // Saturate: on 32-bit targets the last levels only need to compare + // as "beyond the end". + start = start.saturating_add(ndblks.saturating_mul(nelmts)); + } + // Levels whose data blocks the index block addresses directly. + let mut direct_levels = 0; + let mut n = 0; + while n < ndblk_addrs { + n += levels[direct_levels].0; + direct_levels += 1; + } + let nsblk_addrs = nsblks - direct_levels; + + let dblk_size = |nelmts: usize| -> usize { + let prefix = 4 + 1 + 1 + os + arr_off_size + 4; + if nelmts > page_nelmts { + prefix + (nelmts / page_nelmts) * (page_nelmts * elem_size + 4) + } else { + prefix + nelmts * elem_size + } + }; + let sblk_bitmap_len = |ndblks: usize, nelmts: usize| -> usize { + if nelmts > page_nelmts { + ndblks * (nelmts / page_nelmts).div_ceil(8) + } else { + 0 + } + }; + + // Plan addresses: header, index block, the direct data blocks, then each + // allocated super block followed by its allocated data blocks. let aehd_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * length_size as usize + os + 4; let aeib_address = ea_base_address + aehd_size as u64; + let aeib_size = 4 + 1 + 1 + os + idx_blk * elem_size + ndblk_addrs * os + nsblk_addrs * os + 4; + let mut cursor = aeib_address + aeib_size as u64; - // Determine how many elements go inline vs data blocks - let n_inline = (idx_blk_elmts as usize).min(num_elements); - let remaining_after_inline = num_elements.saturating_sub(n_inline); + let mut ndata_blks = 0u64; + let mut data_blk_size = 0u64; + let mut nsuper_blks = 0u64; + let mut super_blk_size = 0u64; + let mut realized = idx_blk as u64; - // Compute super block layout per HDF5 spec - let sblk_min = super_blk_min_nelmts as usize; - let log2_dblk_min = if min_dblk_nelmts <= 1 { - 0 - } else { - (min_dblk_nelmts as u32).trailing_zeros() as usize + let mut plan_dblk = |cursor: &mut u64, start: usize, nelmts: usize| -> DataBlock { + let addr = defined_in(start, nelmts).then(|| { + let a = *cursor; + let size = dblk_size(nelmts) as u64; + *cursor += size; + ndata_blks += 1; + data_blk_size += size; + realized += nelmts as u64; + a + }); + DataBlock { + start, + nelmts, + addr, + } }; - let nsblks = (max_nelmts_bits as usize).saturating_sub(log2_dblk_min) + 1; - // Direct data block addresses (from super blocks 0..sblk_min-1) - let mut dblk_sizes: Vec = Vec::new(); - for sblk_idx in 0..sblk_min.min(nsblks) { - let ndblks = 1usize << (sblk_idx / 2); - let dblk_nelmts = (min_dblk_nelmts as usize) * (1 << sblk_idx.div_ceil(2)); - for _ in 0..ndblks { - dblk_sizes.push(dblk_nelmts); + let mut direct: Vec = Vec::with_capacity(ndblk_addrs); + for &(ndblks, nelmts, first) in &levels[..direct_levels] { + for k in 0..ndblks { + direct.push(plan_dblk(&mut cursor, first + k * nelmts, nelmts)); } } - let n_direct_dblks = dblk_sizes.len(); - - // Super block addresses (for super blocks sblk_min..nsblks-1) - let n_sblk_addrs = nsblks.saturating_sub(sblk_min); - - // EAIB size - let aeib_size = 4 - + 1 - + 1 - + os - + idx_blk_elmts as usize * elem_size - + n_direct_dblks * os - + n_sblk_addrs * os - + 4; - - // Build AEHD - let mut aehd = Vec::with_capacity(aehd_size); - aehd.extend_from_slice(b"EAHD"); - aehd.push(0); // version - aehd.push(client_id); - aehd.push(elem_size as u8); - aehd.push(max_nelmts_bits); - aehd.push(idx_blk_elmts); - aehd.push(min_dblk_nelmts); - aehd.push(super_blk_min_nelmts); - aehd.push(max_dblk_nelmts_bits); - - // Count data blocks that will have chunks - let n_active_dblks: u64 = if remaining_after_inline > 0 { - let mut count = 0u64; - let mut ci = n_inline; - for &sz in &dblk_sizes { - if ci < num_elements { - count += 1; - ci += sz; - } + // (super block address, level, its data blocks) + let mut supers: Vec<(Option, usize, Vec)> = Vec::with_capacity(nsblk_addrs); + for (u, &(ndblks, nelmts, first)) in levels.iter().enumerate().skip(direct_levels) { + if !defined_in(first, ndblks.saturating_mul(nelmts)) { + supers.push((None, u, Vec::new())); + continue; } - count - } else { - 0 - }; - let blk_off_size = (max_nelmts_bits as usize).div_ceil(8); - let aedb_header_overhead = 4 + 1 + 1 + os + blk_off_size + 4; - let data_blk_total_size: u64 = if remaining_after_inline > 0 { - let mut total = 0u64; - let mut ci = n_inline; - for &sz in &dblk_sizes { - if ci < num_elements { - total += (aedb_header_overhead + sz * elem_size) as u64; - ci += sz; - } - } - total - } else { - 0 - }; - let max_idx_set: u64 = if remaining_after_inline > 0 { - let mut max_set = idx_blk_elmts as u64; - let mut ci = n_inline; - for &sz in &dblk_sizes { - if ci < num_elements { - max_set += sz as u64; - ci += sz; - } - } - max_set - } else { - idx_blk_elmts as u64 - }; + let sb_size = + 4 + 1 + 1 + os + arr_off_size + sblk_bitmap_len(ndblks, nelmts) + ndblks * os + 4; + let sb_addr = cursor; + cursor += sb_size as u64; + nsuper_blks += 1; + super_blk_size += sb_size as u64; + let dblks = (0..ndblks) + .map(|k| plan_dblk(&mut cursor, first + k * nelmts, nelmts)) + .collect(); + supers.push((Some(sb_addr), u, dblks)); + } + let slot = |i: usize| slots.get(i).and_then(Option::as_ref); let write_length = |buf: &mut Vec, val: u64| match length_size { 4 => buf.extend_from_slice(&(val as u32).to_le_bytes()), _ => buf.extend_from_slice(&val.to_le_bytes()), }; - let write_addr = |buf: &mut Vec, val: u64| match offset_size { - 4 => buf.extend_from_slice(&(val as u32).to_le_bytes()), - _ => buf.extend_from_slice(&val.to_le_bytes()), + let write_addr_opt = |buf: &mut Vec, addr: Option| match addr { + Some(a) => push_addr(buf, a, offset_size), + None => buf.extend(core::iter::repeat_n(0xFF, os)), + }; + let block_prefix = |buf: &mut Vec, sig: &[u8; 4], block_off: usize| { + buf.extend_from_slice(sig); + buf.push(0); // version + buf.push(client_id); + push_addr(buf, ea_base_address, offset_size); + buf.extend_from_slice(&(block_off as u64).to_le_bytes()[..arr_off_size]); + }; + // Serialise one data block (paged or not) onto `out`. + let write_dblk = |out: &mut Vec, db: &DataBlock| { + let at = out.len(); + block_prefix(out, b"EADB", db.start); + let first = idx_blk + db.start; + if db.nelmts > page_nelmts { + // Paged: the prefix carries only its own checksum; each page + // follows with one of its own. + let sum = jenkins_lookup3(&out[at..]); + out.extend_from_slice(&sum.to_le_bytes()); + for p in 0..db.nelmts / page_nelmts { + let page_at = out.len(); + for e in 0..page_nelmts { + let i = first + p * page_nelmts + e; + push_index_element(out, slot(i), offset_size, chunk_size_bytes); + } + let sum = jenkins_lookup3(&out[page_at..]); + out.extend_from_slice(&sum.to_le_bytes()); + } + } else { + for i in first..first + db.nelmts { + push_index_element(out, slot(i), offset_size, chunk_size_bytes); + } + let sum = jenkins_lookup3(&out[at..]); + out.extend_from_slice(&sum.to_le_bytes()); + } + debug_assert_eq!(out.len() - at, dblk_size(db.nelmts)); }; - write_length(&mut aehd, 0); - write_length(&mut aehd, 0); - write_length(&mut aehd, n_active_dblks); - write_length(&mut aehd, data_blk_total_size); - write_length(&mut aehd, num_elements as u64); - write_length(&mut aehd, max_idx_set); + // Header (EAHD). The six statistics are, in order: super blocks, their + // bytes, data blocks, their bytes, max index set, elements realised. + let mut out = Vec::with_capacity((cursor - ea_base_address) as usize); + out.extend_from_slice(b"EAHD"); + out.push(0); // version + out.push(client_id); + out.push(elem_size as u8); + out.push(MAX_NELMTS_BITS); + out.push(IDX_BLK_ELMTS); + out.push(DATA_BLK_MIN_ELMTS); + out.push(SUP_BLK_MIN_DATA_PTRS); + out.push(MAX_DBLK_PAGE_NELMTS_BITS); + write_length(&mut out, nsuper_blks); + write_length(&mut out, super_blk_size); + write_length(&mut out, ndata_blks); + write_length(&mut out, data_blk_size); + write_length(&mut out, max_idx_set as u64); + write_length(&mut out, realized); + push_addr(&mut out, aeib_address, offset_size); + let sum = jenkins_lookup3(&out); + out.extend_from_slice(&sum.to_le_bytes()); + debug_assert_eq!(out.len(), aehd_size); - write_addr(&mut aehd, aeib_address); - - let aehd_checksum = jenkins_lookup3(&aehd); - aehd.extend_from_slice(&aehd_checksum.to_le_bytes()); - debug_assert_eq!(aehd.len(), aehd_size); - - // Build AEIB - let mut aeib = Vec::with_capacity(aeib_size); - aeib.extend_from_slice(b"EAIB"); - aeib.push(0); - aeib.push(client_id); - - match offset_size { - 4 => aeib.extend_from_slice(&(ea_base_address as u32).to_le_bytes()), - 8 => aeib.extend_from_slice(&ea_base_address.to_le_bytes()), - _ => aeib.extend_from_slice(&ea_base_address.to_le_bytes()), + // Index block (EAIB): inline elements, data block and super block + // addresses. + let ib_start = out.len(); + out.extend_from_slice(b"EAIB"); + out.push(0); + out.push(client_id); + push_addr(&mut out, ea_base_address, offset_size); + for i in 0..idx_blk { + push_index_element(&mut out, slot(i), offset_size, chunk_size_bytes); } - - // Inline elements - #[allow(clippy::needless_range_loop)] - for i in 0..idx_blk_elmts as usize { - if i < n_inline { - write_chunk_element( - &mut aeib, - &chunks[i], - offset_size, - has_filters, - chunk_size_bytes, - ); - } else { - write_undefined_element(&mut aeib, offset_size, has_filters, chunk_size_bytes); - } + for db in &direct { + write_addr_opt(&mut out, db.addr); } + for (sb_addr, _, _) in &supers { + write_addr_opt(&mut out, *sb_addr); + } + let sum = jenkins_lookup3(&out[ib_start..]); + out.extend_from_slice(&sum.to_le_bytes()); + debug_assert_eq!(out.len() - ib_start, aeib_size); - // Data block addresses + build data blocks - let mut data_blocks_buf = Vec::new(); - let dblks_base = aeib_address + aeib_size as u64; - let mut dblk_cursor = dblks_base; - let mut chunk_idx = n_inline; - - for &nelmts in &dblk_sizes { - if chunk_idx >= num_elements { - match offset_size { - 4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()), - 8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()), - _ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()), - } + for db in direct.iter().filter(|d| d.addr.is_some()) { + write_dblk(&mut out, db); + } + for (sb_addr, u, dblks) in &supers { + if sb_addr.is_none() { continue; } - - match offset_size { - 4 => aeib.extend_from_slice(&(dblk_cursor as u32).to_le_bytes()), - 8 => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()), - _ => aeib.extend_from_slice(&dblk_cursor.to_le_bytes()), - } - - // Build EADB - let mut aedb = Vec::new(); - aedb.extend_from_slice(b"EADB"); - aedb.push(0); - aedb.push(client_id); - match offset_size { - 4 => aedb.extend_from_slice(&(ea_base_address as u32).to_le_bytes()), - 8 => aedb.extend_from_slice(&ea_base_address.to_le_bytes()), - _ => aedb.extend_from_slice(&ea_base_address.to_le_bytes()), - } - - let blk_off_size = (max_nelmts_bits as usize).div_ceil(8); - let blk_off_val = (chunk_idx - n_inline) as u64; - aedb.extend_from_slice(&blk_off_val.to_le_bytes()[..blk_off_size]); - - for slot in 0..nelmts { - if chunk_idx + slot < num_elements { - write_chunk_element( - &mut aedb, - &chunks[chunk_idx + slot], - offset_size, - has_filters, - chunk_size_bytes, - ); - } else { - write_undefined_element(&mut aedb, offset_size, has_filters, chunk_size_bytes); + let (ndblks, nelmts, first) = levels[*u]; + let sb_start = out.len(); + block_prefix(&mut out, b"EASB", first); + if nelmts > page_nelmts { + // Page-init bits, `npages` per data block, packed MSB-first + // (`H5VM_bit_set`): every page of an allocated data block is + // written. + let npages = nelmts / page_nelmts; + let mut bitmap = vec![0u8; sblk_bitmap_len(ndblks, nelmts)]; + for (k, db) in dblks.iter().enumerate() { + if db.addr.is_some() { + for p in 0..npages { + let bit = k * npages + p; + bitmap[bit / 8] |= 0x80 >> (bit % 8); + } + } } + out.extend_from_slice(&bitmap); } - - let aedb_checksum = jenkins_lookup3(&aedb); - aedb.extend_from_slice(&aedb_checksum.to_le_bytes()); - - dblk_cursor += aedb.len() as u64; - data_blocks_buf.extend_from_slice(&aedb); - chunk_idx += nelmts; - } - - // Super block addresses (all undefined) - for _ in 0..n_sblk_addrs { - match offset_size { - 4 => aeib.extend_from_slice(&u32::MAX.to_le_bytes()), - 8 => aeib.extend_from_slice(&u64::MAX.to_le_bytes()), - _ => aeib.extend_from_slice(&u64::MAX.to_le_bytes()), + for db in dblks { + write_addr_opt(&mut out, db.addr); + } + let sum = jenkins_lookup3(&out[sb_start..]); + out.extend_from_slice(&sum.to_le_bytes()); + for db in dblks.iter().filter(|d| d.addr.is_some()) { + write_dblk(&mut out, db); } } - - let aeib_checksum = jenkins_lookup3(&aeib); - aeib.extend_from_slice(&aeib_checksum.to_le_bytes()); - debug_assert_eq!(aeib.len(), aeib_size); - - let mut combined = aehd; - combined.extend_from_slice(&aeib); - combined.extend_from_slice(&data_blocks_buf); - combined -} - -fn write_chunk_element( - buf: &mut Vec, - chunk: &WrittenChunk, - offset_size: u8, - has_filters: bool, - chunk_size_bytes: usize, -) { - match offset_size { - 4 => buf.extend_from_slice(&(chunk.address as u32).to_le_bytes()), - 8 => buf.extend_from_slice(&chunk.address.to_le_bytes()), - _ => buf.extend_from_slice(&chunk.address.to_le_bytes()), - } - if has_filters { - let cs_bytes = chunk.compressed_size.to_le_bytes(); - buf.extend_from_slice(&cs_bytes[..chunk_size_bytes]); - buf.extend_from_slice(&chunk.filter_mask.to_le_bytes()); - } -} - -fn write_undefined_element( - buf: &mut Vec, - offset_size: u8, - has_filters: bool, - chunk_size_bytes: usize, -) { - let os = offset_size as usize; - // Use extend with repeat to avoid heap-allocating a temporary Vec on each call. - buf.extend(core::iter::repeat_n(0xFF, os)); - if has_filters { - buf.extend(core::iter::repeat_n(0x00, chunk_size_bytes)); - buf.extend_from_slice(&0u32.to_le_bytes()); - } + debug_assert_eq!(out.len() as u64, cursor - ea_base_address); + out } diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index cce2df6..0e7c845 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -390,3 +390,24 @@ fn we_write_paged_fixed_array() { cases.push(wcase("fa_2d_1100", &[110, 40], &[1, 4], None)); check_we_write(&cases); } + +/// An Extensible Array holds 4 elements in its index block and 240 in the +/// data blocks the index block addresses; everything after that lives under +/// super blocks, and from ~131K elements on in paged data blocks. Chunks past +/// index 243 used to be written but never indexed (read back as fill by us +/// and by libhdf5). +#[test] +fn we_write_extensible_array_past_index_block() { + let unl: &[u64] = &[u64::MAX]; + let mut cases: Vec = [1u64, 4, 5, 243, 244, 245, 300, 1000, 5000] + .iter() + .map(|&n| wcase(&format!("ea_{n}"), &[n * 4], &[4], Some(unl))) + .collect(); + let mut filtered = wcase("ea_300_deflate", &[300 * 4], &[4], Some(unl)); + filtered.deflate = true; + cases.push(filtered); + // Several super blocks and paged data blocks (level 13, the first with + // data blocks over 1024 elements, starts at element 4 + 131056). + cases.push(wcase("ea_140000", &[140_000], &[1], Some(unl))); + check_we_write(&cases); +} From 74fdf0582b55ebd2daeed053c1c7a74541070bfb Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:09:31 -0500 Subject: [PATCH 19/36] fix(format): write paged files libhdf5 can open FileWriter::with_page_size wrote a "version 4" superblock with an extra page-size field. HDF5 has no superblock version 4, so libhdf5 refused every such file ("bad superblock version number"). A paged file is now what libhdf5 itself writes for fs_strategy="page": a v3 superblock whose extension object header holds a File Space Info message (strategy PAGE, the page size, free space not persisted; same bytes and flags as HDF5 2.0), with the file padded to a whole page. h5py opens it, reports the strategy and page size, and can modify it in r+ mode. Page sizes outside libhdf5's 512 B..1 GiB are an error. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 115 +++++++++++++++--- crates/clawhdf5-format/src/superblock.rs | 13 +- .../tests/writer_meta_tests.rs | 56 +++++++++ 3 files changed, 163 insertions(+), 21 deletions(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index b8d0edb..045ad9b 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -4,7 +4,7 @@ //! link messages, contiguous datasets, inline and dense attributes. #[cfg(not(feature = "std"))] -use alloc::{string::String, string::ToString, vec, vec::Vec}; +use alloc::{format, string::String, string::ToString, vec, vec::Vec}; use crate::attribute::AttributeMessage; use crate::chunked_write::{ @@ -38,6 +38,31 @@ const SUPERBLOCK_SIZE: usize = 48; /// field is 2 bytes. Bigger "compact" requests fall back to contiguous storage. const MAX_COMPACT_DATA_SIZE: usize = crate::object_header_writer::MAX_MESSAGE_SIZE - 4; +/// libhdf5's bounds on a file space page size (`H5F_FILE_SPACE_PAGE_SIZE_MIN` +/// and `_MAX`). +const MIN_FILE_SPACE_PAGE_SIZE: u32 = 512; +const MAX_FILE_SPACE_PAGE_SIZE: u32 = 1024 * 1024 * 1024; + +/// Superblock extension object header for a file using the paged file-space +/// strategy: a single File Space Info message (0x0017), as libhdf5 writes it +/// for `fs_strategy="page"` without persisted free space. +fn build_paged_superblock_extension(page_size: u32) -> Result, FormatError> { + let mut fsinfo = Vec::new(); + fsinfo.push(1); // version + fsinfo.push(1); // strategy: H5F_FSPACE_STRATEGY_PAGE + fsinfo.push(0); // persisting free space: no + write_length(&mut fsinfo, 1, LENGTH_SIZE); // free-space section threshold + write_length(&mut fsinfo, u64::from(page_size), LENGTH_SIZE); + fsinfo.extend_from_slice(&0u16.to_le_bytes()); // page end metadata threshold + write_undef_offset(&mut fsinfo, OFFSET_SIZE); // EOA before free-space info + let mut w = ObjectHeaderWriter::new(); + // Flags as libhdf5 sets them: bit 2 (never share) and bit 4 (mark if + // unknown). Not constant: libhdf5 rewrites the message when it closes a + // file it opened for writing. + w.add_message_with_flags(MessageType::Unknown(0x0017), fsinfo, 0x14); + w.serialize() +} + /// Threshold for switching from compact (inline) to dense attribute storage. const DENSE_ATTR_THRESHOLD: usize = 8; @@ -961,7 +986,9 @@ pub struct FileWriter { alignment_threshold: usize, /// Global alignment boundary in bytes (0 = disabled). alignment_bytes: usize, - /// Page size for page-buffer mode. When set, a v4 superblock is written. + /// File space page size. When set, the file uses libhdf5's paged + /// file-space strategy (a File Space Info message in the superblock + /// extension). page_size: Option, } @@ -993,9 +1020,16 @@ impl FileWriter { self } - /// Enable page-buffer mode with the given page size. Writing this causes - /// the file to be written with a v4 superblock (page_size field) instead - /// of the default v3. + /// Write the file with libhdf5's *paged* file-space strategy and the given + /// page size, as `H5Pset_file_space_strategy(H5F_FSPACE_STRATEGY_PAGE)` + + /// `H5Pset_file_space_page_size` (h5py: `fs_strategy="page"`, + /// `fs_page_size=...`) do: a v3 superblock with an extension holding a + /// File Space Info message, and the file padded to a whole number of + /// pages. Readers with a page buffer can then fetch metadata page by page. + /// + /// `page_size` must be between 512 bytes and 1 GiB (libhdf5's limits); + /// [`Self::finish`] fails otherwise. This used to write a "version 4" + /// superblock, which does not exist and no HDF5 library can open. pub fn with_page_size(&mut self, page_size: u32) -> &mut Self { self.page_size = Some(page_size); self @@ -1020,6 +1054,14 @@ impl FileWriter { pub fn finish(self) -> Result, FormatError> { let page_size = self.page_size; + if let Some(ps) = page_size + && !(MIN_FILE_SPACE_PAGE_SIZE..=MAX_FILE_SPACE_PAGE_SIZE).contains(&ps) + { + return Err(FormatError::SerializationError(format!( + "file space page size {ps} is outside libhdf5's \ + {MIN_FILE_SPACE_PAGE_SIZE}..={MAX_FILE_SPACE_PAGE_SIZE} bytes" + ))); + } struct DsFlat { name: String, dt: Datatype, @@ -1328,12 +1370,12 @@ impl FileWriter { let actual_ds_oh_sizes: Vec = dummy_blobs.iter().map(|b| b.oh_bytes.len()).collect(); // Pass 2: compute real addresses - // v4 superblocks add a 4-byte page_size field before the checksum. - let superblock_size = if page_size.is_some() { - SUPERBLOCK_SIZE + 4 - } else { - SUPERBLOCK_SIZE - }; + // A paged file carries its File Space Info in a superblock extension + // object header, placed right after the superblock. + let sb_ext = page_size + .map(build_paged_superblock_extension) + .transpose()?; + let superblock_size = SUPERBLOCK_SIZE + sb_ext.as_ref().map_or(0, Vec::len); let root_group_addr = superblock_size as u64; let mut cursor2 = superblock_size + root_oh_size; @@ -1507,11 +1549,16 @@ impl FileWriter { let actual_ds_oh_sizes2: Vec = ds_blobs2.iter().map(|b| b.oh_bytes.len()).collect(); debug_assert_eq!(actual_ds_oh_sizes, actual_ds_oh_sizes2); + // libhdf5 ends a paged file on a page boundary. + let data_end = cursor2; + if let Some(ps) = page_size { + cursor2 = cursor2.next_multiple_of(ps as usize); + } let eof_addr2 = cursor2 as u64; let mut buf = Vec::with_capacity(cursor2); let sb = Superblock { - version: if page_size.is_some() { 4 } else { 3 }, + version: 3, offset_size: OFFSET_SIZE, length_size: LENGTH_SIZE, base_address: 0, @@ -1523,11 +1570,18 @@ impl FileWriter { free_space_address: None, driver_info_address: None, consistency_flags: 0, - superblock_extension_address: Some(u64::MAX), + superblock_extension_address: Some(if sb_ext.is_some() { + SUPERBLOCK_SIZE as u64 + } else { + u64::MAX + }), checksum: None, - page_size, + page_size: None, }; buf.extend_from_slice(&sb.serialize()); + if let Some(ref ext) = sb_ext { + buf.extend_from_slice(ext); + } // Root group OH let mut root_links: Vec = Vec::new(); @@ -1595,7 +1649,8 @@ impl FileWriter { buf.extend_from_slice(&blob.data); } - debug_assert_eq!(buf.len(), cursor2); + debug_assert_eq!(buf.len(), data_end); + buf.resize(cursor2, 0); Ok(buf) } } @@ -2169,7 +2224,8 @@ mod tests { } #[test] - fn file_writer_v4_superblock() { + fn file_writer_paged_file_uses_v3_superblock_and_fsinfo_extension() { + // This used to write superblock "version 4", which does not exist. let mut fw = FileWriter::new(); fw.with_page_size(4096); fw.create_dataset("data").with_f64_data(&[1.0, 2.0]); @@ -2177,8 +2233,31 @@ mod tests { let sig = signature::find_signature(&bytes).unwrap(); let sb = Superblock::parse(&bytes, sig).unwrap(); - assert_eq!(sb.version, 4, "expected superblock v4"); - assert_eq!(sb.page_size, Some(4096)); + assert_eq!(sb.version, 3); + assert_eq!(sb.superblock_extension_address, Some(48)); + assert_eq!(bytes.len() % 4096, 0); + assert_eq!(sb.eof_address, bytes.len() as u64); + let ext = ObjectHeader::parse(&bytes, 48, 8, 8).unwrap(); + let fsinfo = &ext.messages[0]; + assert_eq!(fsinfo.msg_type, MessageType::Unknown(0x0017)); + // Byte-for-byte what HDF5 2.0 writes for fs_strategy="page", + // fs_page_size=4096. + let mut expected = vec![1u8, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0]; + expected.extend_from_slice(&4096u64.to_le_bytes()); + expected.extend_from_slice(&[0, 0]); + expected.extend_from_slice(&[0xff; 8]); + assert_eq!(fsinfo.data, expected); + assert_eq!(fsinfo.flags, 0x14); + assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0]); + } + + #[test] + fn file_writer_rejects_page_sizes_libhdf5_would() { + for ps in [0u32, 511, MAX_FILE_SPACE_PAGE_SIZE + 1] { + let mut fw = FileWriter::new(); + fw.with_page_size(ps); + assert!(fw.finish().is_err(), "page size {ps}"); + } } #[test] diff --git a/crates/clawhdf5-format/src/superblock.rs b/crates/clawhdf5-format/src/superblock.rs index aedcc55..e971495 100644 --- a/crates/clawhdf5-format/src/superblock.rs +++ b/crates/clawhdf5-format/src/superblock.rs @@ -39,7 +39,13 @@ pub struct Superblock { pub superblock_extension_address: Option, /// CRC32C checksum (v2/v3 only). pub checksum: Option, - /// Page size for page-buffer mode (v4 only). `None` for v0–v3. + /// Page size of the non-standard "version 4" superblock layout (v4 only). + /// `None` for v0–v3. + /// + /// HDF5 has no superblock version 4 — libhdf5 refuses it. A real paged + /// file is a v2/v3 superblock whose extension holds a File Space Info + /// message (what `FileWriter::with_page_size` writes). This field is kept + /// only so such files written by older clawhdf5 versions still parse. pub page_size: Option, } @@ -127,8 +133,9 @@ impl Superblock { /// Serialize this superblock to bytes. /// - /// Writes v2/v3 format, or v4 (with `page_size`) when `self.version == 4`. - /// Computes and appends Jenkins lookup3 checksum. + /// Writes v2/v3 format, or the non-standard v4 (with `page_size`) when + /// `self.version == 4` — which no HDF5 library opens; see + /// [`Self::page_size`]. Computes and appends Jenkins lookup3 checksum. pub fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity(48); buf.extend_from_slice(&HDF5_SIGNATURE); diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index 22db1c5..48f5c12 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -317,3 +317,59 @@ fn raw_attributes_copied_from_h5py_survive_a_rewrite() { assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#); h5dump_ok(&path); } + +// ---- 3. paged file-space strategy ---- + +fn paged_file(page_size: u32) -> Vec { + let mut fw = FileWriter::new(); + fw.with_page_size(page_size); + fw.create_dataset("d").with_f64_data(&[1.0, 2.0, 3.0]); + fw.create_dataset("c") + .with_i32_data(&(0..100).collect::>()) + .with_chunks(&[10]); + fw.set_root_attr("a", AttrValue::I64(7)); + let mut g = fw.create_group("g"); + g.create_dataset("e").with_u8_data(&[9; 5000]); + fw.add_group(g.finish()); + fw.finish().unwrap() +} + +#[test] +fn paged_file_has_a_real_superblock() { + // Measured: `with_page_size` wrote superblock version 4, which does not + // exist ("bad superblock version number" in libhdf5). + for ps in [512u32, 4096, 65536] { + let bytes = paged_file(ps); + let (sb, _) = header_at(&bytes, "/"); + assert_eq!(sb.version, 3); + assert_eq!(bytes.len() % ps as usize, 0); + let (_, e) = header_at(&bytes, "g/e"); + assert!( + e.messages + .iter() + .any(|m| m.msg_type == MessageType::Dataspace) + ); + } +} + +#[test] +#[ignore = "requires Python h5py module and h5dump"] +fn h5py_opens_paged_files() { + for ps in [512u32, 4096, 65536] { + let path = write_tmp(&format!("paged_{ps}"), &paged_file(ps)); + let out = h5py( + &path, + "f = h5py.File(path, 'r')\n\ + p = f.id.get_create_plist()\n\ + print(json.dumps([p.get_file_space_strategy()[0], p.get_file_space_page_size(),\n\ + \x20 f['d'][()].tolist(), int(f['c'][()].sum()), int(f.attrs['a']),\n\ + \x20 int(f['g/e'][()].sum())]))", + ); + assert_eq!( + out, + format!("[1, {ps}, [1.0, 2.0, 3.0], 4950, 7, 45000]"), + "page size {ps}" + ); + h5dump_ok(&path); + } +} From 8c3ef996ea671331403438cbccea8972b33c7a12 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:11:07 -0500 Subject: [PATCH 20/36] fix(format): write fill times with libhdf5's codes; add fill values FillTime::to_byte had the fill-time field rotated against libhdf5 (H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2): Never was written as ALLOC, Alloc as IFSET and IfSet as NEVER, as h5py reported. The flags byte is now late allocation plus the right code, and FillTime::from_byte decodes it. The default becomes IfSet, which is libhdf5's default and exactly the byte (0x0a) every dataset was already written with, so default output does not change; `Alloc` was documented as the C library's default but never was. DatasetCreateProps follows. DatasetBuilder::with_fill_value sets a user-defined fill value (one element's stored bytes, checked against the datatype size), written as a defined value in the fill value message. h5py reports it, and extending the dataset in h5py fills the new elements with it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 40 ++++---- crates/clawhdf5-format/src/property_list.rs | 4 +- crates/clawhdf5-format/src/type_builders.rs | 91 +++++++++++++++--- .../tests/writer_meta_tests.rs | 93 ++++++++++++++++++- 4 files changed, 193 insertions(+), 35 deletions(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 045ad9b..d432f5e 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -19,7 +19,7 @@ use crate::metadata_index::{DatasetMetadata, MetadataBlock, MetadataIndex}; use crate::object_header_writer::ObjectHeaderWriter; use crate::superblock::Superblock; use crate::type_builders::{ - DatasetBuilder, FillTime, FinishedGroup, GroupBuilder, build_attr_message, + DatasetBuilder, FinishedGroup, GroupBuilder, build_attr_message, fill_value_message, }; // Re-export public types that moved to type_builders for API compatibility. @@ -80,12 +80,12 @@ pub(crate) fn build_chunked_dataset_oh( pipeline_message: Option<&[u8]>, attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, - fill_time: FillTime, + fill_message: &[u8], ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); - w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01); + w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01); w.add_message(MessageType::DataLayout, layout_message.to_vec()); if let Some(pm) = pipeline_message { w.add_message(MessageType::FilterPipeline, pm.to_vec()); @@ -107,12 +107,12 @@ pub(crate) fn build_dataset_oh( data_size: u64, attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, - fill_time: FillTime, + fill_message: &[u8], ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); - w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01); + w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01); let mut dl = Vec::new(); dl.push(4); // version dl.push(1); // class = contiguous @@ -142,12 +142,12 @@ pub(crate) fn build_compact_dataset_oh( data: &[u8], attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, - fill_time: FillTime, + fill_message: &[u8], ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); - w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01); + w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01); // Compact layout message: version=4, class=0, u16 size, inline data let mut dl = Vec::new(); dl.push(4); // version @@ -932,12 +932,12 @@ pub(crate) fn build_vds_dataset_oh( global_heap_addr: u64, attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, - fill_time: FillTime, + fill_message: &[u8], ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE)); - w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01); + w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01); // VDS layout message: version=4, class=3, global_heap_address(8), global_heap_index=1(4) let mut dl = Vec::new(); dl.push(4u8); // version @@ -1070,7 +1070,8 @@ impl FileWriter { attrs: Vec, chunk_options: ChunkOptions, maxshape: Option>, - fill_time: FillTime, + /// Serialized Fill Value message. + fill_message: Vec, compact: bool, alignment: usize, /// VDS source mappings (set for Virtual datasets). @@ -1120,6 +1121,7 @@ impl FileWriter { }; attrs.extend(p.build_attrs(&raw)); } + let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?; Ok(DsFlat { name: db.name, dt, @@ -1128,7 +1130,7 @@ impl FileWriter { attrs, chunk_options: db.chunk_options, maxshape: db.maxshape, - fill_time: db.fill_time, + fill_message, compact: db.compact, alignment: db.alignment, virtual_sources: db.virtual_sources, @@ -1274,7 +1276,7 @@ impl FileWriter { 0, // dummy address &d.attrs, dense_blob.as_ref(), - d.fill_time, + &d.fill_message, )?; // Global heap blob size is address-independent; compute it now // so pass 2 can place it correctly. @@ -1318,7 +1320,7 @@ impl FileWriter { result.pipeline_message.as_deref(), &d.attrs, dense_blob.as_ref(), - d.fill_time, + &d.fill_message, )?; dummy_blobs.push(DataBlob { data: result.data_bytes, @@ -1337,7 +1339,7 @@ impl FileWriter { &d.raw, &d.attrs, dense_blob.as_ref(), - d.fill_time, + &d.fill_message, )?; dummy_blobs.push(DataBlob { data: vec![], @@ -1357,7 +1359,7 @@ impl FileWriter { d.raw.len() as u64, &d.attrs, dense_blob.as_ref(), - d.fill_time, + &d.fill_message, )?; dummy_blobs.push(DataBlob { data: d.raw.clone(), @@ -1466,7 +1468,7 @@ impl FileWriter { heap_addr, &d.attrs, ds_dense_blobs[i].as_ref(), - d.fill_time, + &d.fill_message, )?; ds_blobs2.push(DataBlob { data: gcol_bytes.clone(), @@ -1493,7 +1495,7 @@ impl FileWriter { result.pipeline_message.as_deref(), &d.attrs, ds_dense_blobs[i].as_ref(), - d.fill_time, + &d.fill_message, )?; ds_blobs2.push(DataBlob { data: result.data_bytes, @@ -1508,7 +1510,7 @@ impl FileWriter { &d.raw, &d.attrs, ds_dense_blobs[i].as_ref(), - d.fill_time, + &d.fill_message, )?; ds_blobs2.push(DataBlob { data: vec![], @@ -1533,7 +1535,7 @@ impl FileWriter { d.raw.len() as u64, &d.attrs, ds_dense_blobs[i].as_ref(), - d.fill_time, + &d.fill_message, )?; let mut data = vec![0u8; padding]; data.extend_from_slice(&d.raw); diff --git a/crates/clawhdf5-format/src/property_list.rs b/crates/clawhdf5-format/src/property_list.rs index 15ea6b9..adf21d4 100644 --- a/crates/clawhdf5-format/src/property_list.rs +++ b/crates/clawhdf5-format/src/property_list.rs @@ -43,7 +43,7 @@ impl Default for DatasetCreateProps { fletcher32: false, lz4: false, zstd_level: None, - fill_time: FillTime::Alloc, + fill_time: FillTime::IfSet, compact: false, alignment: 0, } @@ -335,7 +335,7 @@ mod tests { fn dcpl_defaults() { let dcpl = DatasetCreateProps::new(); assert!(dcpl.chunk_dims.is_none()); - assert_eq!(dcpl.fill_time, FillTime::Alloc); + assert_eq!(dcpl.fill_time, FillTime::IfSet); assert!(!dcpl.compact); } diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 9ee2520..b47cdba 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -15,29 +15,81 @@ use crate::datatype::{ /// Controls when fill values are written to dataset storage. /// -/// Corresponds to the HDF5 fill value message's "fill time" field. +/// Corresponds to the HDF5 fill value message's "fill time" field +/// (`H5D_fill_time_t`). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum FillTime { - /// Never write fill values (0x02). Avoids initialization overhead - /// for datasets that will be fully written before any read. + /// Never write fill values (`H5D_FILL_TIME_NEVER`). Avoids + /// initialization overhead for datasets that will be fully written + /// before any read. Never, - /// Write fill values at allocation time (0x0a). This is the default - /// and matches the HDF5 C library's behavior. - #[default] + /// Write fill values when storage is allocated (`H5D_FILL_TIME_ALLOC`). Alloc, - /// Write fill values only when the fill value has been explicitly set (0x06). + /// Write fill values at allocation only if one was set explicitly + /// (`H5D_FILL_TIME_IFSET`). The default, as in the HDF5 C library. + #[default] IfSet, } +/// Space allocation time written with every fill value message: late +/// (`H5D_ALLOC_TIME_LATE`), bits 0-1 of the flags byte. +const ALLOC_TIME_LATE: u8 = 2; + impl FillTime { - /// Serialize to the byte used in the fill value message (version 3). + /// Serialize to the flags byte of a version 3 fill value message: the + /// space allocation time (late) in bits 0-1 and the fill time in bits + /// 2-3 (`H5D_FILL_TIME_ALLOC` = 0, `NEVER` = 1, `IFSET` = 2). + /// + /// This used to put `Never` in the ALLOC slot, `Alloc` in IFSET and + /// `IfSet` in NEVER, so libhdf5 saw every choice as a different one. pub fn to_byte(self) -> u8 { - match self { - FillTime::Never => 0x02, - FillTime::Alloc => 0x0a, - FillTime::IfSet => 0x06, + ALLOC_TIME_LATE | (self.code() << 2) + } + + /// Decode the fill time from a version 3 fill value message's flags. + pub fn from_byte(flags: u8) -> Option { + match (flags >> 2) & 0x03 { + 0 => Some(FillTime::Alloc), + 1 => Some(FillTime::Never), + 2 => Some(FillTime::IfSet), + _ => None, } } + + fn code(self) -> u8 { + match self { + FillTime::Alloc => 0, + FillTime::Never => 1, + FillTime::IfSet => 2, + } + } +} + +/// Serialize a version 3 Fill Value message for a dataset of `dt`: the fill +/// time, and the user-defined fill value if there is one (bit 5). +pub(crate) fn fill_value_message( + fill_time: FillTime, + value: Option<&[u8]>, + dt: &Datatype, +) -> Result, crate::error::FormatError> { + let mut msg = vec![3, fill_time.to_byte()]; + if let Some(value) = value { + if matches!(dt, Datatype::VariableLength { .. }) { + return Err(crate::error::FormatError::SerializationError( + "a fill value for a variable-length datatype is not supported".into(), + )); + } + if value.len() != dt.type_size() as usize { + return Err(crate::error::FormatError::DataSizeMismatch { + expected: dt.type_size() as usize, + actual: value.len(), + }); + } + msg[1] |= 0x20; // fill value defined + msg.extend_from_slice(&(value.len() as u32).to_le_bytes()); + msg.extend_from_slice(value); + } + Ok(msg) } // ---- Datatype constructors ---- @@ -431,8 +483,10 @@ pub struct DatasetBuilder { pub(crate) data: Option>, pub(crate) attrs: Vec<(String, AttrValue)>, pub(crate) chunk_options: ChunkOptions, - /// Controls when fill values are written. Default is `FillTime::Alloc`. + /// Controls when fill values are written. Default is `FillTime::IfSet`. pub(crate) fill_time: FillTime, + /// User-defined fill value: one element's bytes, as stored. + pub(crate) fill_value: Option>, /// Use compact (inline) storage: data is stored in the object header. /// Only valid when raw data is <= 65536 bytes and dataset is not chunked. pub(crate) compact: bool, @@ -459,6 +513,7 @@ impl DatasetBuilder { attrs: Vec::new(), chunk_options: ChunkOptions::default(), fill_time: FillTime::default(), + fill_value: None, compact: false, alignment: 0, virtual_sources: None, @@ -715,6 +770,16 @@ impl DatasetBuilder { self } + /// Set the dataset's fill value: what readers return for storage that + /// was never written (e.g. after the dataset is extended). `value` is one + /// element's bytes as stored — the dataset datatype's size and byte order + /// (`(-1i32).to_le_bytes()` for an `i32` dataset). A size mismatch, or a + /// variable-length datatype, makes `finish` fail. + pub fn with_fill_value(&mut self, value: &[u8]) -> &mut Self { + self.fill_value = Some(value.to_vec()); + self + } + /// Use compact (inline) storage for this dataset. /// /// The raw data is stored directly in the dataset's object header rather diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index 48f5c12..77da856 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -13,7 +13,7 @@ use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::type_builders::make_u8_type; +use clawhdf5_format::type_builders::{FillTime, make_u8_type}; // ---- helpers ---- @@ -373,3 +373,94 @@ fn h5py_opens_paged_files() { h5dump_ok(&path); } } + +// ---- 4. fill time and fill value ---- + +fn fill_message(bytes: &[u8], path: &str) -> clawhdf5_format::object_header::HeaderMessage { + let (_, oh) = header_at(bytes, path); + oh.messages + .into_iter() + .find(|m| m.msg_type == MessageType::FillValue) + .unwrap() +} + +fn fill_file() -> Vec { + let mut fw = FileWriter::new(); + fw.create_dataset("never") + .with_f64_data(&[1.0, 2.0]) + .fill_time(FillTime::Never); + fw.create_dataset("alloc") + .with_f64_data(&[1.0, 2.0]) + .fill_time(FillTime::Alloc); + fw.create_dataset("ifset") + .with_f64_data(&[1.0, 2.0]) + .fill_time(FillTime::IfSet); + fw.create_dataset("default").with_f64_data(&[1.0, 2.0]); + fw.create_dataset("filled") + .with_i32_data(&[1, 2, 3, 4]) + .with_chunks(&[2]) + .with_maxshape(&[u64::MAX]) + .with_fill_value(&(-1i32).to_le_bytes()); + fw.finish().unwrap() +} + +#[test] +fn fill_time_uses_libhdf5_codes() { + // H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2, in bits 2-3. Measured: + // h5py saw our Never as ALLOC, Alloc as IFSET and IfSet as NEVER. + let bytes = fill_file(); + for (path, code) in [("never", 1), ("alloc", 0), ("ifset", 2), ("default", 2)] { + let msg = fill_message(&bytes, path); + assert_eq!((msg.data[1] >> 2) & 3, code, "{path}"); + assert_eq!(msg.data[1] & 3, 2, "{path}: allocation time stays late"); + } + for ft in [FillTime::Never, FillTime::Alloc, FillTime::IfSet] { + assert_eq!(FillTime::from_byte(ft.to_byte()), Some(ft)); + } + assert_eq!(FillTime::default(), FillTime::IfSet); +} + +#[test] +fn fill_value_is_written_and_read_back() { + let bytes = fill_file(); + let msg = fill_message(&bytes, "filled"); + assert_eq!( + clawhdf5_format::fill_value::parse_fill_value(&msg).unwrap(), + Some((-1i32).to_le_bytes().to_vec()) + ); + assert_eq!( + clawhdf5_format::fill_value::parse_fill_value(&fill_message(&bytes, "ifset")).unwrap(), + None + ); + + // One element's bytes, no more, no less. + let mut fw = FileWriter::new(); + fw.create_dataset("d") + .with_f64_data(&[1.0]) + .with_fill_value(&[0; 4]); + assert!(fw.finish().is_err()); +} + +#[test] +#[ignore = "requires Python h5py module and h5dump"] +fn h5py_sees_our_fill_time_and_fill_value() { + let path = write_tmp("fill", &fill_file()); + let out = h5py( + &path, + "from h5py import h5d\n\ + f = h5py.File(path, 'r')\n\ + names = {h5d.FILL_TIME_NEVER: 'never', h5d.FILL_TIME_ALLOC: 'alloc', h5d.FILL_TIME_IFSET: 'ifset'}\n\ + t = [names[f[n].id.get_create_plist().get_fill_time()] for n in ('never', 'alloc', 'ifset', 'default')]\n\ + print(json.dumps([t, int(f['filled'].fillvalue), f['filled'][()].tolist()]))\n\ + f.close()\n\ + f = h5py.File(path, 'r+')\n\ + f['filled'].resize((7,))\n\ + f.close()\n\ + print(json.dumps(h5py.File(path, 'r')['filled'][()].tolist()))", + ); + assert_eq!( + out, + "[[\"never\", \"alloc\", \"ifset\", \"ifset\"], -1, [1, 2, 3, 4]]\n[1, 2, 3, 4, -1, -1, -1]" + ); + h5dump_ok(&path); +} From 5935e13866e30b9a8c362a7733f0fc90f916711a Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:11:24 -0500 Subject: [PATCH 21/36] fix(format): decode SZIP chunks the way libhdf5 does SZIP-filtered datasets from libhdf5 came back as garbage or zeros with no error (ref_szip.h5, h5repack_szip.h5, noencoder.h5, le_data/be_data Szip_float_data_*), and 64-bit ones failed with "invalid bits per sample" (h5wasm compressed.h5). The decoder called aec_buffer_decode directly, but libhdf5 goes through szlib's SZ_BufftoBuffDecompress (H5Zszip.c), which libaec implements with reshaping (sz_compat.c). Differences, all fixed: - H5Zszip.c prefixes the stream with the 4-byte LE uncompressed size; it was fed to libaec as data. - 32- and 64-bit samples are coded as byte planes of 8-bit samples and must be de-interleaved. - The reference sample interval is ceil(pixels_per_scanline / pixels_per_block), not a fixed 128. - Scanlines that are not a whole number of blocks are padded and must be unpadded. - Byte order comes from the MSB option bit; LE data was decoded as MSB. Test: szip_decodes_libhdf5_chunks_exactly compares chunks from HDF Group test files (noencoder.h5, le_data.h5) and an h5py-written file (64-bit, 16-bit, padded scanlines, NN and EC) byte for byte with h5py's values; it failed before on the first case. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters_szip.rs | 212 ++++++++++++++---- .../tests/fixtures/filters/README.md | 3 + .../tests/fixtures/filters/le_data.h5 | Bin 0 -> 72368 bytes .../tests/fixtures/filters/noencoder.h5 | Bin 0 -> 8088 bytes .../tests/fixtures/filters/szip_h5py.h5 | Bin 0 -> 11320 bytes 5 files changed, 177 insertions(+), 38 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/le_data.h5 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/noencoder.h5 create mode 100644 crates/clawhdf5-format/tests/fixtures/filters/szip_h5py.h5 diff --git a/crates/clawhdf5-format/src/filters_szip.rs b/crates/clawhdf5-format/src/filters_szip.rs index b05a5e4..4e7fd71 100644 --- a/crates/clawhdf5-format/src/filters_szip.rs +++ b/crates/clawhdf5-format/src/filters_szip.rs @@ -1,19 +1,39 @@ //! SZIP (libaec Adaptive Entropy Coding) decompression. //! //! Gated by the `szip` feature which links against the system libaec library. +//! +//! libhdf5's SZIP filter (`H5Zszip.c`) prefixes each chunk with its +//! uncompressed size and hands the rest to szlib's `SZ_BufftoBuffDecompress`. +//! libaec implements that call (`sz_compat.c`) on top of `aec_buffer_decode` +//! with some reshaping — 32/64-bit samples are coded as byte planes of 8-bit +//! samples, and scanlines that are not a whole number of blocks are padded — +//! which [`szip_decompress`] reproduces so its output matches libhdf5's. #[cfg(not(feature = "std"))] use alloc::vec::Vec; use crate::error::FormatError; -/// Decompress SZIP-compressed data using libaec. +/// `SZ_MSB_OPTION_MASK`: samples are big-endian. +#[cfg(feature = "szip")] +const SZ_MSB_OPTION_MASK: u32 = 16; +/// `SZ_NN_OPTION_MASK`: nearest-neighbour preprocessing. +#[cfg(feature = "szip")] +const SZ_NN_OPTION_MASK: u32 = 32; + +/// Decompress one SZIP-filtered chunk. /// -/// `cd` is the HDF5 SZIP filter client data (matches `H5Z_SZIP_PARM_*` indices): -/// cd[0] = options mask (`H5_SZIP_NN_OPTION_MASK = 0x20` enables NN preprocessing) -/// cd[1] = pixels per block (H5Z_SZIP_PARM_PPB; 8, 10, 16, or 32) -/// cd[2] = bits per sample (H5Z_SZIP_PARM_BPP; element bit width) -/// cd[3] = pixels per scan line (H5Z_SZIP_PARM_PPS; informational only) +/// `cd` is the HDF5 SZIP filter client data (`H5Z_SZIP_PARM_*` indices): +/// cd[0] = options mask (`SZ_*_OPTION_MASK`: 16 = MSB byte order, +/// 32 = nearest-neighbour preprocessing; K13/EC/LSB/RAW bits carry +/// no decoding information for libaec) +/// cd[1] = pixels per block +/// cd[2] = bits per pixel (sample precision, rounded up to 32 or 64 above +/// 24 by libhdf5) +/// cd[3] = pixels per scanline +/// +/// The chunk is a 4-byte little-endian uncompressed size followed by the +/// szlib stream. pub(crate) fn szip_decompress( _data: &[u8], _cd: &[u32], @@ -33,62 +53,174 @@ pub(crate) fn szip_decompress( #[cfg(feature = "szip")] fn szip_decode_impl(data: &[u8], cd: &[u32], chunk_size: usize) -> Result, FormatError> { - if cd.len() < 3 { - return Err(FormatError::ChunkedReadError( - "szip: missing client data".into(), - )); + let err = |m: &str| FormatError::ChunkedReadError(format!("szip: {m}")); + if cd.len() < 4 { + return Err(err("missing client data")); } let options = cd[0]; - let pixels_per_block = cd[1]; - let bits_per_sample = cd[2]; // H5Z_SZIP_PARM_BPP - if bits_per_sample == 0 || bits_per_sample > 32 { - return Err(FormatError::ChunkedReadError( - "szip: invalid bits per sample".into(), - )); + let pixels_per_block = cd[1] as usize; + let bits_per_pixel = cd[2]; + let pixels_per_scanline = cd[3] as usize; + if !(1..=32).contains(&bits_per_pixel) && bits_per_pixel != 64 { + return Err(err("invalid bits per sample")); } - if chunk_size == 0 { - return Err(FormatError::ChunkedReadError( - "szip: unknown output size".into(), - )); + if pixels_per_block == 0 || pixels_per_scanline == 0 { + return Err(err("invalid block or scanline size")); } - if data.is_empty() { - return Err(FormatError::ChunkedReadError("szip: empty input".into())); + if data.len() < 4 { + return Err(err("chunk too short")); } + // H5Zszip.c: UINT32DECODE of the uncompressed size, then the stream. + let dest_len = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; + let limit = if chunk_size != 0 { + chunk_size + } else { + crate::filters::MAX_DECOMPRESS_SIZE + }; + if dest_len > limit { + return Err(err("declared size exceeds chunk size")); + } + let stream = &data[4..]; - // Map HDF5 option mask to libaec flags. - // HDF5 always stores SZIP data in MSB order, so AEC_DATA_MSB is unconditional. - // H5_SZIP_NN_OPTION_MASK (0x20): NN differential preprocessing. - let mut flags: u32 = libaec_sys::AEC_DATA_MSB; - if options & 0x20 != 0 { + // --- libaec sz_compat.c: SZ_BufftoBuffDecompress --- + let rsi = pixels_per_scanline.div_ceil(pixels_per_block); + let mut flags = 0; + if options & SZ_MSB_OPTION_MASK != 0 { + flags |= libaec_sys::AEC_DATA_MSB; + } + if options & SZ_NN_OPTION_MASK != 0 { flags |= libaec_sys::AEC_DATA_PREPROCESS; } + let pad_scanline = !pixels_per_scanline.is_multiple_of(pixels_per_block); + let deinterleave = bits_per_pixel == 32 || bits_per_pixel == 64; + let bits_per_sample = if deinterleave { 8 } else { bits_per_pixel }; + let pixel_size = match bits_per_sample { + 17.. => 4, + 9.. => 2, + _ => 1, + }; + let scanlines = (dest_len / pixel_size).div_ceil(pixels_per_scanline); + let buf_size = if pad_scanline { + rsi.checked_mul(pixels_per_block) + .and_then(|n| n.checked_mul(pixel_size)) + .and_then(|n| n.checked_mul(scanlines)) + .filter(|&n| n <= crate::filters::MAX_DECOMPRESS_SIZE.max(limit)) + .ok_or_else(|| err("scanline padding too large"))? + } else { + dest_len + }; - let mut out = vec![0u8; chunk_size]; + let mut buf = vec![0u8; buf_size]; let mut strm = libaec_sys::AecStream::zeroed(); - strm.next_in = data.as_ptr(); - strm.avail_in = data.len(); - strm.next_out = out.as_mut_ptr(); - strm.avail_out = chunk_size; + strm.next_in = stream.as_ptr(); + strm.avail_in = stream.len(); + strm.next_out = buf.as_mut_ptr(); + strm.avail_out = buf_size; strm.bits_per_sample = bits_per_sample; - strm.block_size = pixels_per_block; - strm.rsi = 128; // HDF5 default: 128 blocks per reference sample interval + strm.block_size = pixels_per_block as u32; + strm.rsi = rsi as u32; strm.flags = flags; - + // SAFETY: next_in/avail_in and next_out/avail_out describe live buffers + // (`stream` and `buf`) that outlive the call. let result = unsafe { libaec_sys::aec_buffer_decode(&mut strm) }; if result != 0 { return Err(FormatError::DecompressionError(format!( "szip: libaec error {result}" ))); } - let decoded_len = chunk_size - strm.avail_out; - out.truncate(decoded_len); - Ok(out) + let mut total_out = strm.total_out; + if pad_scanline { + let line = pixels_per_scanline * pixel_size; + let padded_line = rsi * pixels_per_block * pixel_size; + // remove_padding: compact each padded line down to `line` bytes. + let mut i = line; + let mut j = padded_line; + while j < total_out { + let end = (j + line).min(buf.len()); + buf.copy_within(j..end, i); + i += line; + j += padded_line; + } + total_out = scanlines * line; + } + if total_out < dest_len { + return Err(err("stream decoded to fewer bytes than declared")); + } + buf.truncate(dest_len); + if deinterleave { + // deinterleave_buffer: byte planes back into words. + let w = (bits_per_pixel / 8) as usize; + let n = dest_len / w; + let mut out = vec![0u8; dest_len]; + for i in 0..n { + for j in 0..w { + out[i * w + j] = buf[j * n + i]; + } + } + Ok(out) + } else { + Ok(buf) + } } #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "szip")] + fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// SZIP chunks written by libhdf5, decoded exactly as libhdf5 decodes + /// them. Each case: fixture, chunk byte offset and size (from h5py's + /// `get_chunk_info`), the filter's cd_values, and the chunk's values as + /// h5py reads them (file byte order, hex). Before the fix every one of + /// these came back as garbage or zeros (or "invalid bits per sample" for + /// 64-bit): the 4-byte size prefix was fed to libaec, 32/64-bit samples + /// were not de-interleaved from byte planes, the reference sample + /// interval was fixed at 128 instead of derived from the scanline, padded + /// scanlines were not unpadded, and LE data was decoded as MSB. + #[cfg(feature = "szip")] + #[test] + fn szip_decodes_libhdf5_chunks_exactly() { + /// (name, file, chunk offset, chunk size, cd_values, decoded hex) + type Case<'a> = (&'a str, &'a [u8], usize, usize, [u32; 4], &'a str); + let noencoder: &[u8] = include_bytes!("../tests/fixtures/filters/noencoder.h5"); + let le_data: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5"); + let h5py: &[u8] = include_bytes!("../tests/fixtures/filters/szip_h5py.h5"); + #[rustfmt::skip] + let cases: &[Case] = &[ + // f4, MSB + NN. + ("le_data /Szip_float_data_be", le_data, 55396, 48, [177, 4, 32, 12], + "3eaaaaab3f2aaaab3f8000003f2aaaab3f8000003faaaaab3f8000003faaaaab3fd555553faaaaab3fd5555540000000"), + // %-xy%vXYjAc^Eb-%m&P4+hT z?j4sW>dbfH-oD-MXTQ7q$-8%-@4k2M9HJ*VW4jQ+{G}|xcd7&qjKv$Ie`g03&k9#Ruou$r$M`kb0Y^-(v3tjG6%Ur%=27v1XHaWy@+ z<0>B5wX2hpq90dS2V9LDSNku6GGmwaxI#X{>Nv>%eKw8H5mJ~<4Y8@)#XfwpsP|bx zFaY-By(MuljpH+iF0pO`+bqffHI5?DuAFUI-PF2v)4FwA)^3GnBEcbD!e7{4*2*&a z$2zZknEzlwqxuAfAwd^nI_CQtKCnZ!ok%~Em7m(M;Dw>b)Q2Y%pqNu*aqvhoK;oQe zi==tWc;=L-fssB<)JP}VUZWZ&&rhqS=OE{0;i}Vd2GOM@>vO{Kc0s`Nl+_>Ncz=Z! z2fOua;y5$}cyoMNaN_|F$>J|B93Mxiz`^+*@VVrXgl#u+aOAYg&+!732UK~`aXNX~ z;x9EhPqfn-&eWD|S6E<&96moV z=T(;T_N}|!aPX;@>2=S*N0-abA&dL6xo?@roE9q+9_mi?O`2NkII&0#e?|G340EW1 zGUN%{#+*B4?>zU<4!sO^sJAM#ocj@F$hq-tr}9^BxOhqL@l3CQd&=*(RC_TY?nmS; zPAn51>iSMBoY%hl_QNeZw>9_M_ZJlv6_5K?lKp0R_ZttsX-zL?t5cmgpicpmesGGNoH;Ka<6i8D_zS1H^#BwUImwXbKlx!_EHKT%Uh-i40bJErW+pmzV zKzf1<6%y234+PoD*8?Gqc@@v+)dxBHk*!!S$Tu!H7L<;>;(`wq&MPiBq~diGhk74} z{hkrmV;q1uEF+$BVt^PR28aP-fEXYKhyi~Xz__4VU7yeYJ^_#WCir~&YR zKY6Uf>-?F0P?;DY28aP-fEXYKhyh|Ce+GQh+g|r6E4>Z(?eWxM-lq(8FpIJ3ZCsBZ zgWf(gThf_TZ^OFqD2+;Y%JCbqv)^GT?~r;gX|MdGq@?td(0vuVCzc)_zvru!l~=I8 zUq5AoVXj}}b1HYJ4#L-D z@2Odt_nS9{7|rTb`59Ay8@Os!-ce*2zGOwW`)oX>&z?5o3Y@kXSL`=%ZgI-1E;!TQ z2Oe>XI9PBnloJEQ05L!e5Cg;jF+dFX#(-~e3cn5@HCyI5C8ZYX^*9CU#u}##UkAuP zPKmCTY~po*jB!f7`g+LF!}y0Y^>AgWw4h#h)7RIy;7otFDIE5P{t!p{8kwP-7$63S z0b+m{AO?s5e;M#iUx(BKC0SSuP`?fU+HQzHupJDjudyy)eGTo?uLDT`&_^ywyAIH| zQ8oaVAU`1=_4`-(>g#oe9tPi#Q0K}LV^ISq27yK`?{LAH{tkNRYd8N|SF}qE5Cg;j zF+dCu1H=F^kOu?4>1)1zEj3$MjKb8{TwaS6Y+YYN-B|VY$oh5O`kHN%yut8Y^|gdB zHn**<>E4Fcwdv(m!{vJT{mcgW1DXx%S~oRqZCKN^wQ0D8n;ku*+1G5kYZdpVJ8-ot z2Wj@z%}s4-tr>H%WB2LyEzO(SvW;J}qk}a6>o*#$jsA!2*xInSZrXTfgV+pHkmld8 zgO{6I*KS?iytd8cK2ts5=dE4W+O&183C~n723ddK`pu)j{9rKaaYpsccdc8;|AywF zkfB~QS-5Kb)=^;Y->gUf7-QmnSa@8TdA{-CidoVVUROcCW<1ws;N0T3pSj>ne=jH; z_J{ruhh?-=P7DwO!~iis3=jjv05RYX0~jBMmC(HS-ytXy*FoSpLi|5Zme}g?A=Jb5 z7fAZ^P_YggpsD+SJJM-?0;B-F-{cIar-_-o$9pm-t0Z#?>ijYkR7)} z`Rji&JFZ#b<~)=g_eHnn8?VHT_yXsNaRZi58929i;$s(_>2IQXST6cUe~6REK3XX! z28aP-fEXYKhyh}N81Rb$-{KX1JxgjfOAO77S7fx38q(txtb+z)jaNX%!)pxjO65)% zh`cNZGBjhnlCRFLR(jX#y1@4g9O_~*GHvq%7aZzmgx*a$8>yh27$63S0b+m{AO?tm zOUgii%So}+tk#<^{<=V=Xg@hTZTP;isUbFXyPAcrw?Q)=?=6XgNxy##>L#$T@Dotu zC}8sY)BK(raUaku^*khP9lx;?9F6E=AxX`3e8#FJD#xvxjyB}#C~qg!_aIE1se6MH zWmCkt>FDcRaHhW&g~I`%Kg3}f?UWM(!~iis3=jjv05L!e_``s2I(m9;Iyzp`dmKit z>u9LULr3q8NeVzJf@~C$@$=Hrzc6&K9NOsbF~4Ep+;ncyBc+CSO~RKKFY`0Z7HqTt`p$ zp!;zpcugGYS!}*h>E5uaaMQVuDcm#zXZq_V9gWmdP7DwO!~iis3=jjvz$InCHy!Qs zdUOstdd{CEQXTy@%l33DdIdWmiABMV972m9Q?#cQU zi%w7e3BZ#*H9a>i+gHYJTK4?1Jx=25tT>r^A5EMuwBcVd7uRjxi1s{~Vh)ORxm1U==kLHC7~IMJ$yh__#^l-)W7(G7BJt9-#Oq_z$E?pqpQAoDea!k?^f~Hd)5omO zMeM6F>toZ$tj|TC;|Ps{ z7!x21kbE;f7N2!v!K{%D6QH^F(lW+3Rin(GXWx8ol!yUhfEdV$0pHfq#(X_G`#Re1 z_-tTJD9m5RI@+;5UdI!dK~0FIqtV}!3UiZp%6A=Yjatw0T1R_I;k?$-UQsx&b+n|y z;mmN|4A(S}Y>k5kJhy~ad`8#=19P=q_ z*P&OOl@xgB=Zkw@SN+08^8)55jXUByV z?$G<$aUq4<+?yTew>95*r0g9z5?IvK6UGHtXBjxRcww^(&h*!*aOfZXAr8xEr<@od z28aP-fEXYKhyh~29|nAjN5;Iaf6?(s=XptiuklE}I{I}(_u^j%^(>azDY=Jw+fC=r zbitYa8Wj%xqd&x98SRu41H=F^KnxHA!~iis4EV!oKEs`e_};$Lc&8Fr&;eZ6^-YLEc~Rm}=_ zGMF8AOtoKpi;E*wy;tSuYO~|~wz|b5cwD&k_qd7+OdQ4q*sj~a;qicW%ul%BOn+x7 z9zn_|CkBWCVt^PR28aP-;F2=nTRbv7w|Jx|uqz4^)Z-DT%O@TI#hxJ^X)lvfKCgHr zUmacYF3gGzKkrSJWI%-bkY8=!+;sFUE;v(1uU9zi1N|Wm%V?*Z7$63S0b+m{AO?s5 JV!$5;{ts@6(^>!k literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/filters/noencoder.h5 b/crates/clawhdf5-format/tests/fixtures/filters/noencoder.h5 new file mode 100644 index 0000000000000000000000000000000000000000..84f87526ffe4943f03299168c5ff17fdbec94706 GIT binary patch literal 8088 zcmeHKF;Buk6n@uMQy?lD6OD^EFm!Uzn6QZn7+BP(0~;n#QjGy4xHz#mxH2(1@)L}^ zt8sR6)L)_ZdiS815E2px>3hlD>wEY0uJ_&b`t_lhKUhd^BmvEJ!GSc`VW6vTW4+DM zm>`c)FeM|tSlB-fL!pyJ)7CMbT%+kXs?C~zQ)%5@Tvuu>zr7Xg0{qIig4=ri%CD%> zt_BKCK+`=uQHTG;pPdv600M|-^7Qy0j=qwVU>_?61XyZ!07x_IW%Yhy+%X&e+~mm# zaKm<80n8b^WrHwVDTF!(M>*Tz(k`eo08NIqfOYMK9&j~`V6b0P;Dtk(W_2%)?*P8V z2<@wa{Yh2uka1Oi*pqEp-n{b4FinpED++;zn6J;596%PssqOUeD32WKUIE@FoLAC} zR9>8V$eFwm&ncl+h0Z7P=`pkIt&KM54@_jCN1Hif2ABb6fEi#0n1LB!;IDNk=?T&y zDJX}UQ5bzUNr$2$L_K4ECel;r)UBjAHF%tUB0a59 z!@AG9B7j0 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/fixtures/filters/szip_h5py.h5 b/crates/clawhdf5-format/tests/fixtures/filters/szip_h5py.h5 new file mode 100644 index 0000000000000000000000000000000000000000..f197d796871fe81bf8eaea6e29ab3e46b40009fb GIT binary patch literal 11320 zcmeHMO=uHA6n^`YW@)sW;!jJ7?N%cQS}0PXhhkTnlz8xm+Rz>f=1_zl1VN;+muy8e zR)kvhDm{3p0X>il$&sU zVeT4QJQo$`a7D#AEjn3-Zsa3kEDD|!DGG*}pxvqF()kN14UFI!(5FM|XyjkrCgrY1 z8j{~l;;q9!*ZWtbc;9KD*W7orO%_8Y1D?acvukK^6aEz5Dxk6HXW^?Ci=LR)F_{n6~B zwe-a7(u=cQ%iy?+x@pC02pTP|R}K`m_dPR{k5_K5JSji@(4YKtHCrr}-a>Zz{;Sgb z{M2ixYmUruu?iPDpckAr!~X^XtuT#%y(zQ+Rda*hNQWb0@t)|SUymMyKHo!aE{p&p zzz8q`jKCj4;Men!aljBFE|Q3tN0gaFfL8O64}>M=lw46mc}Smfc_^;RLk8(v!P?o< zD35lk$uu*R9~&b*eAs&BA?MNR!NNR7fDvE>7y(9r5nu#%9Rjs+w2WFc`tXj4AuNg- zB8e2l&4Q-~5*Zk_8aJ!&_h@}Y)F0pPbt@V84cKyuV4RcO-`qBeqmMV3_anu{10=__ fRF_I9ZdUTDg;lF_pAldL7y(9r5nu#%0RrCvf Date: Fri, 25 Sep 2026 21:12:12 -0500 Subject: [PATCH 22/36] fix(format): give empty string attributes a 1-byte type An empty AttrValue::String (or a StringArray of empty strings) was written with a size-0 fixed-length string type. libhdf5 rejects that ("invalid datatype size"), and the failure takes every attribute on the object with it. Strings are now at least 1 byte, NUL-padded, which is how h5py stores "" and reads back as "" in both h5py and our reader. check_encodable also refuses a size-0 string type passed in directly. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/datatype.rs | 3 + crates/clawhdf5-format/src/type_builders.rs | 11 +++- .../tests/writer_meta_tests.rs | 60 +++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 90c9657..436a6cf 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -842,6 +842,9 @@ impl Datatype { opaque_tag_text(tag).len() ))) } + Datatype::String { size: 0, .. } => Err(FormatError::SerializationError( + "fixed-length string datatype of size 0 (libhdf5 requires at least 1 byte)".into(), + )), Datatype::Compound { members, .. } => members .iter() .try_for_each(|m| m.datatype.check_encodable()), diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index b47cdba..09eaeba 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -384,7 +384,11 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess raw_data: data.clone(), }, AttrValue::String(s) => { - let bytes = s.as_bytes(); + // A fixed-length string type must be at least 1 byte: libhdf5 + // rejects size 0 ("invalid datatype size") and with it every + // attribute on the object. h5py stores "" as one NUL byte. + let mut bytes = s.as_bytes().to_vec(); + bytes.resize(bytes.len().max(1), 0); AttributeMessage { name: name.to_string(), datatype: Datatype::String { @@ -393,11 +397,12 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess charset: CharacterSet::Utf8, }, dataspace: scalar_ds(), - raw_data: bytes.to_vec(), + raw_data: bytes, } } AttrValue::StringArray(arr) => { - let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0); + // At least 1 byte per element, as for a single string. + let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0).max(1); let mut raw = Vec::new(); for s in arr { let mut b = s.as_bytes().to_vec(); diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index 77da856..4e78bbd 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -464,3 +464,63 @@ fn h5py_sees_our_fill_time_and_fill_value() { ); h5dump_ok(&path); } + +// ---- 5. empty string attributes ---- + +fn empty_string_file() -> Vec { + let mut fw = FileWriter::new(); + fw.set_root_attr("empty", AttrValue::String(String::new())); + fw.set_root_attr("x", AttrValue::String("héllo".into())); + fw.set_root_attr( + "empties", + AttrValue::StringArray(vec![String::new(), String::new()]), + ); + fw.set_root_attr("n", AttrValue::I64(3)); + fw.finish().unwrap() +} + +#[test] +fn empty_string_attribute_has_a_one_byte_type() { + // Measured: "" got a size-0 string type, and libhdf5 then refused every + // attribute on the object ("invalid datatype size"). + let bytes = empty_string_file(); + let (sb, root) = header_at(&bytes, "/"); + let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap(); + for name in ["empty", "empties"] { + let a = attrs.iter().find(|a| a.name == name).unwrap(); + assert_eq!(a.datatype.type_size(), 1, "{name}"); + let strings = a.read_as_strings().unwrap(); + assert!(strings.iter().all(String::is_empty), "{name}: {strings:?}"); + } + + // A size-0 string type handed in directly is refused, not written. + let mut fw = FileWriter::new(); + fw.set_root_attr( + "raw", + AttrValue::Raw { + datatype: Datatype::String { + size: 0, + padding: clawhdf5_format::datatype::StringPadding::NullPad, + charset: clawhdf5_format::datatype::CharacterSet::Ascii, + }, + shape: vec![], + data: vec![], + }, + ); + assert!(fw.finish().is_err()); +} + +#[test] +#[ignore = "requires Python h5py module and h5dump"] +fn h5py_reads_all_attributes_next_to_an_empty_string() { + let path = write_tmp("empty_str", &empty_string_file()); + let out = h5py( + &path, + "f = h5py.File(path, 'r')\n\ + d = lambda v: v.decode() if isinstance(v, bytes) else v\n\ + print(json.dumps([d(f.attrs['empty']), d(f.attrs['x']),\n\ + \x20 [d(s) for s in f.attrs['empties']], int(f.attrs['n'])], ensure_ascii=False))", + ); + assert_eq!(out, r#"["", "héllo", ["", ""], 3]"#); + h5dump_ok(&path); +} From 540fa08907913df9371e59d65f95dba0d357b2f0 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:12:47 -0500 Subject: [PATCH 23/36] fix(format): write chunk indexes over the max extent, swizzled for EA The writer indexed chunks by their position in the current shape, the same mistake the reader had. With a finite maxshape larger than the shape the Fixed Array was sized for the shape, so libhdf5 looked up chunks past its end ("addr overflow"); with the unlimited dimension anywhere but first, e.g. maxshape (20, None), libhdf5 swizzles that dimension to the slowest position and read our Extensible Array scrambled. Two unlimited dimensions produced a file libhdf5 refused to open ("already found unlimited dimension"). Chunks are now placed with the shared chunk_grid linearisation: Fixed Array slots cover every chunk of the maximum extent (unwritten ones undefined), Extensible Array indexes are swizzled, Single Chunk is only used when the maximum extent is one chunk, and a maxshape that is smaller than the shape, has more than one unlimited dimension, or would need an absurd Fixed Array is an error instead of a bad file. build_chunked_data_from_precompressed now returns a Result. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunk_grid.rs | 1 - crates/clawhdf5-format/src/chunked_write.rs | 137 +++++++++++++++++-- crates/clawhdf5-format/src/file_writer.rs | 4 +- crates/clawhdf5/tests/chunk_index_interop.rs | 117 ++++++++++++++++ 4 files changed, 243 insertions(+), 16 deletions(-) diff --git a/crates/clawhdf5-format/src/chunk_grid.rs b/crates/clawhdf5-format/src/chunk_grid.rs index c5a06fd..9e03b9d 100644 --- a/crates/clawhdf5-format/src/chunk_grid.rs +++ b/crates/clawhdf5-format/src/chunk_grid.rs @@ -147,7 +147,6 @@ impl ChunkGrid { /// Linear index of the chunk with scaled coordinates `scaled` /// (`offset / chunk_dim` per dimension, in dataset order). - #[allow(dead_code)] // used by the writer pub(crate) fn linear_index(&self, scaled: &[u64]) -> u64 { self.order .iter() diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 66bc82d..3136f58 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -8,6 +8,7 @@ use alloc::{vec, vec::Vec}; use crate::checksum::jenkins_lookup3; use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line}; +use crate::chunk_grid::ChunkGrid; use crate::ea_writer; use crate::error::FormatError; use crate::filter_pipeline::{ @@ -699,7 +700,8 @@ pub fn build_chunked_data_from_precompressed( pre: &PrecompressedChunks, base_address: u64, maxshape: Option<&[u64]>, -) -> ChunkedDataResult { +) -> Result { + let index = ChunkIndexPlan::new(&pre.shape, maxshape, &pre.chunk_dims)?; let offset_size: u8 = 8; let length_size: u8 = 8; let num_chunks = pre.chunks.len(); @@ -725,16 +727,15 @@ pub fn build_chunked_data_from_precompressed( } let chunk_dims_u32: Vec = pre.chunk_dims.iter().map(|&d| d as u32).collect(); - let use_extensible = maxshape.is_some_and(|ms| ms.contains(&u64::MAX)); let aligned_idx = align_to_cache_line(data_buf.len()); if aligned_idx > data_buf.len() { data_buf.resize(aligned_idx, 0u8); } - let layout_message = if use_extensible { + let layout_message = if let ChunkIndexPlan::ExtensibleArray(grid) = &index { let ea_address = base_address + data_buf.len() as u64; - let slots: Vec> = written_chunks.iter().cloned().map(Some).collect(); + let slots = index_slots(grid, &pre.shape, &pre.chunk_dims, &written_chunks, None)?; let ea_bytes = ea_writer::build_extensible_array_at( &slots, offset_size, @@ -749,7 +750,7 @@ pub fn build_chunked_data_from_precompressed( offset_size, element_size as u32, ) - } else if num_chunks == 1 { + } else if matches!(index, ChunkIndexPlan::SingleChunk) { let chunk_addr = written_chunks[0].address; let filtered_size = if pre.has_filters { Some(written_chunks[0].compressed_size) @@ -765,9 +766,15 @@ pub fn build_chunked_data_from_precompressed( offset_size, element_size as u32, ) - } else { + } else if let ChunkIndexPlan::FixedArray(grid, nslots) = &index { let fa_address = base_address + data_buf.len() as u64; - let slots: Vec> = written_chunks.iter().cloned().map(Some).collect(); + let slots = index_slots( + grid, + &pre.shape, + &pre.chunk_dims, + &written_chunks, + Some(*nslots), + )?; let fa_bytes = build_fixed_array_at( &slots, offset_size, @@ -783,15 +790,123 @@ pub fn build_chunked_data_from_precompressed( element_size as u32, FA_PAGE_BITS, ) + } else { + unreachable!("every chunk index plan is handled above") }; - ChunkedDataResult { + Ok(ChunkedDataResult { data_bytes: data_buf, layout_message, pipeline_message: pre.pipeline_message.clone(), + }) +} + +/// Most slots a Fixed Array index may have before we refuse to build it: its +/// data block holds one element per chunk of the *maximum* extent, so a huge +/// finite maxshape with small chunks would otherwise exhaust memory. +const MAX_FIXED_ARRAY_SLOTS: u64 = 1 << 26; + +/// Which chunk index a dataset gets, following the library's choice in +/// `H5D__layout_set_latest_indexing`: Extensible Array for exactly one +/// unlimited dimension, Fixed Array for a finite maxshape, Single Chunk when +/// the whole maximum extent is one chunk. +enum ChunkIndexPlan { + SingleChunk, + /// The grid and the number of array elements (chunks of the max extent). + FixedArray(ChunkGrid, usize), + ExtensibleArray(ChunkGrid), +} + +impl ChunkIndexPlan { + fn new( + shape: &[u64], + maxshape: Option<&[u64]>, + chunk_dims: &[u64], + ) -> Result { + let bad = |what: &str| FormatError::ChunkedReadError(format!("maxshape: {what}")); + if let Some(ms) = maxshape { + if ms.len() != shape.len() { + return Err(bad("rank differs from the shape")); + } + if ms.iter().zip(shape).any(|(&m, &s)| m < s) { + return Err(bad("smaller than the shape")); + } + } + let max = maxshape.unwrap_or(shape); + let nunlim = max.iter().filter(|&&d| d == u64::MAX).count(); + match nunlim { + 0 => { + let nslots = max + .iter() + .zip(chunk_dims) + .try_fold(1u64, |acc, (&m, &c)| acc.checked_mul(m.div_ceil(c.max(1)))) + .filter(|&n| n <= MAX_FIXED_ARRAY_SLOTS) + .ok_or_else(|| { + bad("too many chunks for a Fixed Array index; \ + use larger chunks or an unlimited dimension") + })?; + // A Single Chunk index needs that one chunk to exist; an + // empty dataset gets an all-unallocated Fixed Array instead. + let empty = shape.contains(&0); + if nslots == 1 && !empty { + Ok(Self::SingleChunk) + } else { + let grid = ChunkGrid::fixed_array(shape, Some(max), chunk_dims)?; + Ok(Self::FixedArray(grid, nslots as usize)) + } + } + 1 => Ok(Self::ExtensibleArray(ChunkGrid::extensible_array( + shape, + Some(max), + chunk_dims, + )?)), + _ => Err(bad( + "more than one unlimited dimension needs a B-tree v2 chunk index, \ + which the writer does not support", + )), + } } } +/// Place each written chunk at its linear index in `grid`. `chunks` are in +/// row-major order over the chunks of the current extent (`split_into_chunks`). +/// `len` fixes the slot count (Fixed Array); otherwise it is one past the +/// highest index used. +fn index_slots( + grid: &ChunkGrid, + shape: &[u64], + chunk_dims: &[u64], + chunks: &[WrittenChunk], + len: Option, +) -> Result>, FormatError> { + let rank = shape.len(); + let cur: Vec = shape + .iter() + .zip(chunk_dims) + .map(|(&s, &c)| s.div_ceil(c)) + .collect(); + let mut placed: Vec<(usize, &WrittenChunk)> = Vec::with_capacity(chunks.len()); + let mut scaled = vec![0u64; rank]; + for (i, chunk) in chunks.iter().enumerate() { + let mut rem = i as u64; + for d in (0..rank).rev() { + scaled[d] = rem % cur[d]; + rem /= cur[d]; + } + let idx = usize::try_from(grid.linear_index(&scaled)) + .map_err(|_| FormatError::Overflow("chunk index slot".into()))?; + placed.push((idx, chunk)); + } + let n = len.unwrap_or_else(|| placed.iter().map(|&(i, _)| i + 1).max().unwrap_or(0)); + let mut slots = vec![None; n]; + for (idx, chunk) in placed { + *slots + .get_mut(idx) + .ok_or_else(|| FormatError::Overflow("chunk index slot".into()))? = Some(chunk.clone()); + } + Ok(slots) +} + /// Build chunked data with absolute addresses. /// If `maxshape` has unlimited dims, uses Extensible Array index. pub fn build_chunked_data_at( @@ -824,11 +939,7 @@ pub fn build_chunked_data_at_ext( maxshape: Option<&[u64]>, ) -> Result { let pre = precompress_chunks(raw_data, shape, chunk_dims, element_size, options)?; - Ok(build_chunked_data_from_precompressed( - &pre, - base_address, - maxshape, - )) + build_chunked_data_from_precompressed(&pre, base_address, maxshape) } /// Write selected elements into an existing in-memory dataset buffer. diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index c262cd4..5350c88 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -1244,7 +1244,7 @@ impl FileWriter { &pre, dummy_cursor, d.maxshape.as_deref(), - ); + )?; dummy_cursor += result.data_bytes.len() as u64; let dense_blob = if ds_dense[i] { Some(build_dense_attrs(&d.attrs, 0)) @@ -1424,7 +1424,7 @@ impl FileWriter { .expect("chunked dataset missing precompressed cache"), base_address, d.maxshape.as_deref(), - ); + )?; cursor2 += result.data_bytes.len(); let oh = build_chunked_dataset_oh( &d.dt, diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index 0e7c845..41d7a5a 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -373,6 +373,73 @@ fn check_we_write(cases: &[WriteCase]) { "h5dump failed: {stderr}" ); } + + // Let libhdf5 grow every resizable dataset by two chunks per dimension + // (capped at the maxshape) and rewrite it, which updates our index in + // place and inserts new chunks into it. Then both readers must agree. + let script = format!( + r#" +import h5py, numpy as np +grown = {{}} +with h5py.File(r'{path_str}', 'r+') as f: + for name in f: + d = f[name] + if d.chunks is None: + continue + new = tuple(s + 2 * c if m is None else min(m, s + 2 * c) + for s, m, c in zip(d.shape, d.maxshape, d.chunks)) + if new == d.shape: + continue + old = d[()] + full = np.full(new, -7, 'i4') + full[tuple(slice(0, s) for s in old.shape)] = old + d.resize(new) + d[...] = full + grown[name] = (list(old.shape), list(new)) +with h5py.File(r'{path_str}', 'r') as f: + for name, (old, new) in grown.items(): + want = np.full(new, -7, 'i4') + want[tuple(slice(0, s) for s in old)] = np.arange(int(np.prod(old)), dtype='i4').reshape(old) + assert np.array_equal(f[name][()], want), name +for name, (old, new) in grown.items(): + print(name, ','.join(map(str, old)), ','.join(map(str, new))) +"# + ); + let out = run_python(&script); + let growable = cases + .iter() + .filter(|c| c.maxshape.as_ref().is_some_and(|m| *m != c.shape)) + .count(); + assert_eq!(out.lines().count(), growable, "libhdf5 grew: {out}"); + let dims = |s: &str| -> Vec { s.split(',').map(|x| x.parse().unwrap()).collect() }; + let file = File::open(&path).unwrap(); + for line in out.lines() { + let mut parts = line.split(' '); + let (name, old, new) = ( + parts.next().unwrap(), + dims(parts.next().unwrap()), + dims(parts.next().unwrap()), + ); + let got = file.dataset(name).unwrap().read_i32().unwrap(); + let n: usize = new.iter().product(); + let mut want = vec![-7i32; n]; + for (flat, w) in want.iter_mut().enumerate() { + let mut rem = flat; + let mut coords = vec![0usize; new.len()]; + for d in (0..new.len()).rev() { + coords[d] = rem % new[d]; + rem /= new[d]; + } + if coords.iter().zip(&old).all(|(c, o)| c < o) { + *w = coords.iter().zip(&old).fold(0, |acc, (c, o)| acc * o + c) as i32; + } + } + let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count(); + assert!( + got.len() == n && bad == 0, + "{name}: after libhdf5 grew it, our reader got {bad} of {n} values wrong" + ); + } } /// A Fixed Array with more than 1024 elements must be paged, or libhdf5 @@ -411,3 +478,53 @@ fn we_write_extensible_array_past_index_block() { cases.push(wcase("ea_140000", &[140_000], &[1], Some(unl))); check_we_write(&cases); } + +/// A maxshape larger than the shape: the index must be laid out over the +/// chunks of the maximum extent (libhdf5 read our Fixed Array past its end: +/// "addr overflow"), and an Extensible Array whose unlimited dimension is not +/// the first must swizzle it to the slowest position (libhdf5 read our +/// `(20, None)` dataset scrambled). +#[test] +fn we_write_maxshape_larger_than_shape() { + const U: u64 = u64::MAX; + let mut cases = vec![ + // Fixed Array over the maximum extent. + wcase("fa2d_finite_max", &[20, 30], &[5, 5], Some(&[40, 60])), + wcase("fa1d_finite_max", &[40], &[4], Some(&[100])), + wcase("fa3d_edges", &[6, 7, 8], &[4, 3, 5], Some(&[10, 9, 20])), + wcase("fa_paged_max", &[30, 50], &[1, 1], Some(&[40, 60])), + wcase("fa_one_chunk_now", &[5], &[5], Some(&[50])), + // Extensible Array, unlimited dimension first (no swizzle) ... + wcase("ea2d_unl_fin", &[20, 30], &[5, 5], Some(&[U, 30])), + wcase("ea2d_unl_fin_max", &[20, 30], &[5, 5], Some(&[U, 60])), + // ... and not first (swizzled). + wcase("ea2d_fin_unl", &[20, 30], &[5, 5], Some(&[20, U])), + wcase("ea2d_fin_max_unl", &[20, 30], &[5, 5], Some(&[40, U])), + wcase("ea3d_mid", &[6, 7, 8], &[4, 3, 5], Some(&[10, U, 20])), + // Past the index block and into super blocks, swizzled. + wcase("ea2d_many", &[3, 2000], &[1, 1], Some(&[4, U])), + ]; + let mut filtered = wcase( + "ea3d_last_deflate", + &[6, 7, 8], + &[4, 3, 5], + Some(&[6, 8, U]), + ); + filtered.deflate = true; + cases.push(filtered); + check_we_write(&cases); +} + +/// More than one unlimited dimension needs a B-tree v2 chunk index; the +/// writer must not produce a file libhdf5 cannot open. +#[test] +fn two_unlimited_dims_are_refused() { + let mut b = FileBuilder::new(); + b.create_dataset("d") + .with_i32_data(&(0..600).collect::>()) + .with_shape(&[20, 30]) + .with_chunks(&[5, 5]) + .with_maxshape(&[u64::MAX, u64::MAX]); + let dir = tempfile::tempdir().unwrap(); + assert!(b.write(dir.path().join("unl_unl.h5")).is_err()); +} From 9066d34eaa3032c4eca6be956c857a7296d1cdf4 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:12:52 -0500 Subject: [PATCH 24/36] fix(format): key the shared chunk cache by dataset A File is Send + Sync and keeps one ChunkCache for all its datasets. The cached readers bound that cache to "the current dataset" with ensure_dataset(addr), then checked, built and read its index and its decompressed chunks in separate lock acquisitions. Two threads reading two chunked datasets interleaved those steps, so one could store its chunk index under the other's binding, or get the other's decompressed chunk for the same coordinate: wrong data, or an index-out-of-bounds panic when the ranks differed (16 threads x 40 reads over 24 datasets panicked on every run). The cache now keeps per-dataset state keyed by chunk-index address: the chunk index, ChunkIndex and ChunkLayout per dataset (held as Arcs, built outside the lock, first writer wins), and decompressed chunks keyed by (address, coordinate). The chunked readers use the new addr-taking methods (chunks_for, chunk_layout_for, get/put_decompressed_in, prefetch_hint_in) exclusively. Memory stays bounded: decompressed data by the existing byte/slot budget across datasets, indexes by at most 64 datasets and 2^20 index entries in total, dropping the least recently used dataset's index first. Switching datasets no longer throws away the other datasets' cached chunks. The address-less methods remain and act on the dataset last bound with ensure_dataset; they are documented as not for concurrent readers. Regression: threads_reading_different_datasets_get_their_own_chunks (crates/clawhdf5/tests/concurrent_chunk_cache.rs), plus cache unit tests datasets_sharing_coordinates_stay_separate, dataset_indexes_are_bounded and concurrent_readers_of_different_datasets_see_their_own_chunks. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunk_cache.rs | 793 ++++++++++++------ crates/clawhdf5-format/src/chunked_read.rs | 131 ++- .../clawhdf5/tests/concurrent_chunk_cache.rs | 87 ++ 3 files changed, 660 insertions(+), 351 deletions(-) create mode 100644 crates/clawhdf5/tests/concurrent_chunk_cache.rs diff --git a/crates/clawhdf5-format/src/chunk_cache.rs b/crates/clawhdf5-format/src/chunk_cache.rs index 89703f4..aedb602 100644 --- a/crates/clawhdf5-format/src/chunk_cache.rs +++ b/crates/clawhdf5-format/src/chunk_cache.rs @@ -223,13 +223,32 @@ pub const DEFAULT_CACHE_BYTES: usize = 16 * 1024 * 1024; // 16 MiB /// coordinate map and reduces collision chains compared to power-of-two sizes. pub const DEFAULT_MAX_SLOTS: usize = 521; +/// Most datasets whose chunk index a [`ChunkCache`] keeps at once. +pub const MAX_INDEXED_DATASETS: usize = 64; + +/// Most chunk-index entries, summed over all datasets, a [`ChunkCache`] keeps. +/// Least-recently-used datasets' indexes are dropped past this (the dataset +/// being read is always kept), so a file with many or huge chunked datasets +/// cannot grow the cache without bound. +pub const MAX_INDEXED_CHUNKS: usize = 1 << 20; + +/// The dataset key the address-less (legacy) methods use when +/// [`ChunkCache::ensure_dataset`] has not been called. +#[cfg(feature = "std")] +const UNBOUND_DATASET: u64 = u64::MAX; + // --------------------------------------------------------------------------- // LRU entry // --------------------------------------------------------------------------- +/// Decompressed chunks are keyed by dataset *and* coordinate: every chunked +/// dataset has a chunk at (0, 0, ...), so the coordinate alone is ambiguous. +#[cfg(feature = "std")] +type SlotKey = (u64, ChunkCoord); + #[cfg(feature = "std")] struct CachedChunk { - coord: ChunkCoord, + key: SlotKey, /// Shared so a cache hit is a refcount bump, not a copy of the whole /// (potentially large) decompressed chunk. data: Arc, @@ -237,21 +256,48 @@ struct CachedChunk { last_access: u64, } +/// Per-dataset index state. +#[cfg(feature = "std")] +#[derive(Default)] +struct DatasetEntry { + /// Chunk coordinate -> ChunkInfo (offset + size in file). + index: Option>>, + /// Pre-built chunk index for O(1) coordinate lookups. + chunk_index: Option>, + /// Pre-computed chunk layout for fast assembly. + chunk_layout: Option>, + /// Tick of the last use, for dropping the least recently used dataset. + last_used: u64, +} + +#[cfg(feature = "std")] +impl DatasetEntry { + fn weight(&self) -> usize { + self.index.as_ref().map_or(0, |m| m.len()) + + self.chunk_index.as_ref().map_or(0, |c| c.num_chunks()) + } +} + // --------------------------------------------------------------------------- // ChunkCache // --------------------------------------------------------------------------- -/// A per-dataset chunk cache with hash-based index and LRU eviction. +/// A per-file chunk cache: chunk indexes per dataset, plus an LRU of +/// decompressed chunks, all keyed by dataset. /// -/// # Usage +/// A dataset is identified by the address of its chunk index (B-tree, fixed +/// or extensible array, ...), which is unique within a file. Every method +/// that takes an `addr` works on that dataset only, so threads reading +/// different datasets through one shared cache never see each other's +/// chunks. The address-less methods (`has_index`, `populate_index`, +/// `get_decompressed`, ...) act on the dataset last bound with +/// [`Self::ensure_dataset`]; that binding is shared state, so concurrent +/// readers must use the `*_in` / `*_for` methods instead (the chunked +/// readers in [`crate::chunked_read`] do). /// -/// ```ignore -/// let cache = ChunkCache::new(); -/// // Pass &cache to read_chunked_data — it will populate the index lazily. -/// ``` -/// -/// The cache is wrapped in `Mutex` internally so it can be mutated through -/// shared references (thread-safe). +/// Memory is bounded: decompressed data by `max_bytes`/`max_slots` across +/// all datasets, indexes by [`MAX_INDEXED_DATASETS`] and +/// [`MAX_INDEXED_CHUNKS`]. /// /// Only available with the `std` feature because it requires `std::sync::Mutex`. #[cfg(feature = "std")] @@ -261,26 +307,20 @@ pub struct ChunkCache { #[cfg(feature = "std")] struct CacheInner { - /// Hash index: chunk coordinate -> ChunkInfo (offset + size in file). - /// Populated once per dataset on first access. - index: Option>, + /// Per-dataset chunk indexes, keyed by chunk-index address. + datasets: HashMap, - /// Address of the dataset (its chunk-index base address) that the cached - /// index, chunk index, layout, and decompressed slots currently belong to. - /// The cache is shared per file across datasets, so every cached-read entry - /// checks this and resets the per-dataset state when the dataset changes — - /// otherwise one dataset's chunk index (with its own rank) would be reused - /// for another, corrupting reads. - index_addr: Option, + /// Dataset the address-less methods act on (see `ensure_dataset`). + current: Option, /// LRU cache of decompressed chunk data. slots: Vec, - /// Coordinate -> index into `slots`, for O(1) lookup instead of a linear + /// Key -> index into `slots`, for O(1) lookup instead of a linear /// scan. Kept in sync with `slots` on every insert/evict/clear — in /// particular, `slots.swap_remove(i)` moves the last element into slot /// `i`, so the moved element's index entry must be updated too. - slot_index: HashMap, + slot_index: HashMap, /// Current total bytes of cached decompressed data. current_bytes: usize, @@ -294,17 +334,145 @@ struct CacheInner { /// Monotonic counter for LRU ordering. tick: u64, - /// Last accessed chunk coordinate (for sequential detection). - last_coord: Option, + /// Last accessed chunk (for sequential detection). + last_coord: Option, /// Access pattern statistics. stats: AccessStats, +} - /// Pre-built chunk index for O(1) coordinate lookups. - chunk_index: Option, +#[cfg(feature = "std")] +impl CacheInner { + fn current(&self) -> u64 { + self.current.unwrap_or(UNBOUND_DATASET) + } - /// Pre-computed chunk layout for fast assembly. - chunk_layout: Option, + fn touch(&mut self, addr: u64) -> &mut DatasetEntry { + self.tick += 1; + let tick = self.tick; + let entry = self.datasets.entry(addr).or_default(); + entry.last_used = tick; + entry + } + + fn entry(&self, addr: u64) -> Option<&DatasetEntry> { + self.datasets.get(&addr) + } + + /// Drop least-recently-used datasets' indexes (never `keep`'s) until the + /// dataset and chunk-entry budgets hold. + fn trim_datasets(&mut self, keep: u64) { + loop { + let total: usize = self.datasets.values().map(DatasetEntry::weight).sum(); + if self.datasets.len() <= MAX_INDEXED_DATASETS && total <= MAX_INDEXED_CHUNKS { + return; + } + let victim = self + .datasets + .iter() + .filter(|(a, _)| **a != keep) + .min_by_key(|(_, e)| e.last_used) + .map(|(a, _)| *a); + match victim { + Some(a) => { + self.datasets.remove(&a); + } + None => return, + } + } + } + + fn get_decompressed(&mut self, addr: u64, coord: &[u64]) -> Option> { + self.tick += 1; + let tick = self.tick; + + // Track sequential vs random access + let is_sequential = self.last_coord.as_ref().is_some_and(|(prev_addr, prev)| { + // Sequential if exactly one dimension changed + let changes: usize = prev + .iter() + .zip(coord.iter()) + .filter(|(a, b)| a != b) + .count(); + *prev_addr == addr && changes <= 1 + }); + if is_sequential { + self.stats.sequential_count += 1; + } else if self.last_coord.is_some() { + self.stats.random_count += 1; + } + let key: SlotKey = (addr, coord.to_vec()); + let found = if let Some(&idx) = self.slot_index.get(&key) { + self.slots[idx].last_access = tick; + Some(Arc::clone(&self.slots[idx].data)) + } else { + None + }; + self.last_coord = Some(key); + if let Some(ref data) = found { + self.stats.hits += 1; + self.stats.bytes_read += data.len() as u64; + } else { + self.stats.misses += 1; + } + found + } + + fn put_decompressed( + &mut self, + key: SlotKey, + data: Arc, + ) -> Arc { + let data_len = data.len(); + + // Don't cache if single chunk exceeds budget — still return the data + // to the caller, just don't retain it. + if data_len > self.max_bytes { + return data; + } + + // Check if already present + self.tick += 1; + let tick = self.tick; + if let Some(&idx) = self.slot_index.get(&key) { + self.slots[idx].last_access = tick; + return Arc::clone(&self.slots[idx].data); // already cached + } + + // Evict until we have room + while self.slots.len() >= self.max_slots + || (self.current_bytes + data_len > self.max_bytes && !self.slots.is_empty()) + { + // Find LRU slot + let lru_idx = self + .slots + .iter() + .enumerate() + .min_by_key(|(_, s)| s.last_access) + .map(|(i, _)| i) + .unwrap(); + let removed = self.slots.swap_remove(lru_idx); + self.slot_index.remove(&removed.key); + // swap_remove moved the former last element into `lru_idx` (unless + // it *was* the last element) — fix up that element's index entry. + if lru_idx < self.slots.len() { + let moved_key = self.slots[lru_idx].key.clone(); + self.slot_index.insert(moved_key, lru_idx); + } + self.current_bytes -= removed.data.len(); + self.stats.evictions += 1; + } + + self.current_bytes += data_len; + let new_idx = self.slots.len(); + self.slot_index.insert(key.clone(), new_idx); + self.slots.push(CachedChunk { + key, + data: Arc::clone(&data), + last_access: tick, + }); + data + } } /// Access pattern statistics tracked by the chunk cache. @@ -356,8 +524,8 @@ impl ChunkCache { pub fn with_capacity(max_bytes: usize, max_slots: usize) -> Self { Self { inner: std::sync::Mutex::new(CacheInner { - index: None, - index_addr: None, + datasets: HashMap::new(), + current: None, slots: Vec::with_capacity(max_slots.min(64)), slot_index: HashMap::with_capacity(max_slots.min(64)), current_bytes: 0, @@ -366,340 +534,331 @@ impl ChunkCache { tick: 0, last_coord: None, stats: AccessStats::default(), - chunk_index: None, - chunk_layout: None, }), } } - // ----- Index operations ----- + fn lock(&self) -> std::sync::MutexGuard<'_, CacheInner> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } /// The most decompressed bytes this cache will hold. pub fn max_bytes(&self) -> usize { - self.inner.lock().map(|g| g.max_bytes).unwrap_or(0) + self.lock().max_bytes } - /// Bind the cache to the dataset at chunk-index address `addr`. + // ----- Dataset-keyed operations (safe to use concurrently) ----- + + /// The chunk list of the dataset whose chunk index is at `addr`. /// - /// The cache is shared per file across all of its datasets. If the cache - /// currently holds state for a different dataset, all per-dataset state - /// (chunk index, chunk-index map, layout, and decompressed slots) is - /// dropped so the next access rebuilds it for this dataset. Reading the - /// same dataset again is a no-op, preserving the cache's benefit for - /// repeated/sequential access. Returns `true` if a reset occurred. - pub fn ensure_dataset(&self, addr: u64) -> bool { - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - if inner.index_addr == Some(addr) { - return false; + /// On the first call for a dataset, `build` scans its chunk index; the + /// result is kept (offsets truncated to `rank` for the lookup key), so + /// later calls skip the scan. `build` runs without the cache lock held; + /// if two threads race to build the same dataset's index, the first + /// stored one wins and both return equivalent lists. + pub fn chunks_for( + &self, + addr: u64, + rank: usize, + build: impl FnOnce() -> Result, E>, + ) -> Result, E> { + Ok(self + .index_for(addr, rank, build)? + .values() + .cloned() + .collect()) + } + + fn index_for( + &self, + addr: u64, + rank: usize, + build: impl FnOnce() -> Result, E>, + ) -> Result>, E> { + if let Some(index) = self.lock().touch(addr).index.clone() { + return Ok(index); } - inner.index = None; - inner.chunk_index = None; - inner.chunk_layout = None; - inner.slots.clear(); - inner.slot_index.clear(); - inner.current_bytes = 0; - inner.last_coord = None; - inner.index_addr = Some(addr); - true + let chunks = build()?; + let map: HashMap = chunks + .into_iter() + .map(|ci| (ci.offsets.iter().take(rank).copied().collect(), ci)) + .collect(); + let mut inner = self.lock(); + let entry = inner.touch(addr); + let index = Arc::clone(entry.index.get_or_insert_with(|| Arc::new(map))); + inner.trim_datasets(addr); + Ok(index) } - /// Returns `true` if the chunk index has been built. + /// The pre-computed assembly layout of the dataset at `addr`, building + /// its chunk index (via `build`, as in [`Self::chunks_for`]) and layout on + /// first use. + pub fn chunk_layout_for( + &self, + addr: u64, + rank: usize, + build: impl FnOnce() -> Result, E>, + ds_dims: &[usize], + chunk_dims: &[usize], + elem_size: usize, + ) -> Result, E> { + let (layout, chunk_index) = { + let mut inner = self.lock(); + let entry = inner.touch(addr); + (entry.chunk_layout.clone(), entry.chunk_index.clone()) + }; + if let Some(layout) = layout { + return Ok(layout); + } + let chunk_index = match chunk_index { + Some(ci) => ci, + None => { + let index = self.index_for(addr, rank, build)?; + let chunks: Vec = index.values().cloned().collect(); + Arc::new(ChunkIndex::build(&chunks, rank)) + } + }; + let layout = ChunkLayout::build(&chunk_index, ds_dims, chunk_dims, elem_size); + let mut inner = self.lock(); + let entry = inner.touch(addr); + entry.chunk_index.get_or_insert(chunk_index); + let layout = Arc::clone(entry.chunk_layout.get_or_insert_with(|| Arc::new(layout))); + inner.trim_datasets(addr); + Ok(layout) + } + + /// Cached decompressed chunk at `coord` of the dataset at `addr`. + /// + /// O(1) lookup; the clone is an `Arc` refcount bump, not a copy of the + /// underlying decompressed data. + pub fn get_decompressed_in(&self, addr: u64, coord: &[u64]) -> Option> { + self.lock().get_decompressed(addr, coord) + } + + /// Cache decompressed chunk data for `coord` of the dataset at `addr`. + /// Returns the `Arc`-shared buffer now cached (or already cached). + pub fn put_decompressed_in( + &self, + addr: u64, + coord: ChunkCoord, + data: Vec, + ) -> Arc { + self.put_decompressed_aligned_in(addr, coord, CacheAlignedBuffer::from_vec(data)) + } + + /// [`Self::put_decompressed_in`] for an already-aligned buffer. + pub fn put_decompressed_aligned_in( + &self, + addr: u64, + coord: ChunkCoord, + data: CacheAlignedBuffer, + ) -> Arc { + let data = Arc::new(data); + self.lock().put_decompressed((addr, coord), data) + } + + /// Record that the given chunk coordinates of the dataset at `addr` are + /// predicted to be accessed soon (bookkeeping only). + /// + /// This does **not** prefetch or pre-decompress anything — it only + /// checks whether each coordinate is already in the chunk index and + /// updates access-pattern stats accordingly. + pub fn prefetch_hint_in(&self, addr: u64, next_coords: &[ChunkCoord]) { + let mut inner = self.lock(); + let Some(index) = inner.entry(addr).and_then(|e| e.index.clone()) else { + return; + }; + let known = next_coords + .iter() + .filter(|c| index.contains_key(*c)) + .count(); + inner.stats.sequential_count += known as u64; + } + + // ----- Address-less operations on the bound dataset ----- + + /// Bind the address-less methods to the dataset at chunk-index address + /// `addr`. Returns `true` if this changed the bound dataset. + /// + /// Each dataset's state is kept separately, so switching loses nothing + /// and never exposes one dataset's index or chunks to another. The + /// binding itself is shared, though: concurrent readers should use the + /// `addr`-taking methods rather than bind and then call these. + pub fn ensure_dataset(&self, addr: u64) -> bool { + let mut inner = self.lock(); + let changed = inner.current != Some(addr); + inner.current = Some(addr); + changed + } + + /// Returns `true` if the bound dataset's chunk index has been built. pub fn has_index(&self) -> bool { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .index - .is_some() + let inner = self.lock(); + inner + .entry(inner.current()) + .is_some_and(|e| e.index.is_some()) } - /// Build the chunk index from a pre-collected list of `ChunkInfo`. + /// Build the bound dataset's chunk index from a pre-collected list of + /// `ChunkInfo`. /// /// The `rank` parameter is used to truncate offsets to spatial dims only /// (B-tree v1 stores rank+1 offsets). pub fn populate_index(&self, chunks: &[ChunkInfo], rank: usize) { - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - if inner.index.is_some() { - return; // already populated - } - let mut map = HashMap::with_capacity(chunks.len()); - - for ci in chunks { - let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect(); - map.insert(coord, ci.clone()); - } - inner.index = Some(map); + let addr = self.lock().current(); + let _ = self.index_for::(addr, rank, || Ok(chunks.to_vec())); } - /// Look up a chunk by its spatial coordinate in the index. + /// Look up a chunk by its spatial coordinate in the bound dataset's index. pub fn lookup_index(&self, coord: &[u64]) -> Option { - let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - inner.index.as_ref()?.get(coord).cloned() + let inner = self.lock(); + inner + .entry(inner.current())? + .index + .as_ref()? + .get(coord) + .cloned() } - /// Return all indexed chunks as a `Vec` (order unspecified). + /// Return all of the bound dataset's indexed chunks (order unspecified). pub fn all_indexed_chunks(&self) -> Option> { - let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - inner.index.as_ref().map(|m| m.values().cloned().collect()) + let inner = self.lock(); + let index = inner.entry(inner.current())?.index.as_ref()?; + Some(index.values().cloned().collect()) } - // ----- Chunk index (pre-built coordinate → ChunkInfo map) ----- - - /// Returns `true` if the chunk B-tree index has been built. + /// Returns `true` if the bound dataset's `ChunkIndex` has been built. pub fn has_chunk_index(&self) -> bool { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .chunk_index - .is_some() + let inner = self.lock(); + inner + .entry(inner.current()) + .is_some_and(|e| e.chunk_index.is_some()) } - /// Build and store the chunk B-tree index from a pre-collected list of `ChunkInfo`. + /// Build and store the bound dataset's `ChunkIndex`. pub fn populate_chunk_index(&self, chunks: &[ChunkInfo], rank: usize) { - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - if inner.chunk_index.is_some() { - return; - } - inner.chunk_index = Some(ChunkIndex::build(chunks, rank)); + let built = Arc::new(ChunkIndex::build(chunks, rank)); + let mut inner = self.lock(); + let addr = inner.current(); + inner.touch(addr).chunk_index.get_or_insert(built); + inner.trim_datasets(addr); } - // ----- Chunk layout (pre-computed assembly plan) ----- - - /// Returns `true` if the chunk layout has been computed. + /// Returns `true` if the bound dataset's chunk layout has been computed. pub fn has_chunk_layout(&self) -> bool { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .chunk_layout - .is_some() + let inner = self.lock(); + inner + .entry(inner.current()) + .is_some_and(|e| e.chunk_layout.is_some()) } - /// Build and store the pre-computed chunk layout for fast assembly. + /// Build and store the bound dataset's chunk layout (needs its + /// `ChunkIndex`; does nothing without one). pub fn populate_chunk_layout(&self, ds_dims: &[usize], chunk_dims: &[usize], elem_size: usize) { - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - if inner.chunk_layout.is_some() { + let mut inner = self.lock(); + let addr = inner.current(); + let entry = inner.touch(addr); + if entry.chunk_layout.is_some() { return; } - if let Some(ref idx) = inner.chunk_index { - inner.chunk_layout = Some(ChunkLayout::build(idx, ds_dims, chunk_dims, elem_size)); + if let Some(idx) = entry.chunk_index.clone() { + entry.chunk_layout = Some(Arc::new(ChunkLayout::build( + &idx, ds_dims, chunk_dims, elem_size, + ))); } } - /// Execute a function with a reference to the chunk layout. - /// - /// Returns `None` if the layout hasn't been computed yet. + /// Execute a function with a reference to the bound dataset's chunk + /// layout. Returns `None` if the layout hasn't been computed yet. pub fn with_chunk_layout(&self, f: F) -> Option where F: FnOnce(&ChunkLayout) -> R, { - let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - inner.chunk_layout.as_ref().map(f) + let layout = { + let inner = self.lock(); + inner.entry(inner.current())?.chunk_layout.clone()? + }; + Some(f(&layout)) } - // ----- Decompressed data cache (LRU) ----- - - /// Try to get cached decompressed data for a chunk coordinate. + /// Try to get cached decompressed data for a chunk of the bound dataset. /// - /// O(1) lookup. Returns an owned copy for API compatibility with callers - /// that need a `Vec`; prefer [`Self::get_decompressed_aligned`] when - /// an `Arc`-shared buffer works for the caller, since that avoids the - /// copy entirely. + /// Returns an owned copy; prefer [`Self::get_decompressed_aligned`] when + /// an `Arc`-shared buffer works for the caller. pub fn get_decompressed(&self, coord: &[u64]) -> Option> { self.get_decompressed_aligned(coord) .map(|arc| arc.as_slice().to_vec()) } - /// Try to get a reference-counted clone of the aligned buffer for a chunk. - /// - /// O(1) index lookup; the clone is an `Arc` refcount bump, not a copy of - /// the underlying decompressed data. + /// Reference-counted cached buffer for a chunk of the bound dataset. pub fn get_decompressed_aligned(&self, coord: &[u64]) -> Option> { - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - inner.tick += 1; - let tick = inner.tick; - - // Track sequential vs random access - let is_sequential = inner.last_coord.as_ref().is_some_and(|prev| { - // Sequential if exactly one dimension changed - let changes: usize = prev - .iter() - .zip(coord.iter()) - .filter(|(a, b)| a != b) - .count(); - changes <= 1 - }); - if is_sequential { - inner.stats.sequential_count += 1; - } else if inner.last_coord.is_some() { - inner.stats.random_count += 1; - } - inner.last_coord = Some(coord.to_vec()); - - let found = if let Some(&idx) = inner.slot_index.get(coord) { - inner.slots[idx].last_access = tick; - Some(Arc::clone(&inner.slots[idx].data)) - } else { - None - }; - if let Some(ref data) = found { - inner.stats.hits += 1; - inner.stats.bytes_read += data.len() as u64; - } else { - inner.stats.misses += 1; - } - found + let mut inner = self.lock(); + let addr = inner.current(); + inner.get_decompressed(addr, coord) } - /// Insert decompressed chunk data into the LRU cache. - /// - /// The data is stored in a [`CacheAlignedBuffer`] so subsequent reads - /// return cache-line-aligned memory. Returns the `Arc`-shared buffer that - /// is now cached (or already was), so the caller can reuse it directly - /// instead of holding a separate copy of the same data. + /// Insert decompressed chunk data for the bound dataset into the LRU + /// cache, returning the `Arc`-shared buffer now cached. pub fn put_decompressed(&self, coord: ChunkCoord, data: Vec) -> Arc { - let aligned = CacheAlignedBuffer::from_vec(data); - self.put_decompressed_aligned(coord, aligned) + self.put_decompressed_aligned(coord, CacheAlignedBuffer::from_vec(data)) } - /// Insert an already-aligned buffer into the LRU cache. - /// - /// Returns the `Arc`-shared buffer now held by the cache (the one just - /// inserted, or the existing cached copy if `coord` was already present). + /// Insert an already-aligned buffer for the bound dataset. pub fn put_decompressed_aligned( &self, coord: ChunkCoord, data: CacheAlignedBuffer, ) -> Arc { let data = Arc::new(data); - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - let data_len = data.len(); - - // Don't cache if single chunk exceeds budget — still return the data - // to the caller, just don't retain it. - if data_len > inner.max_bytes { - return data; - } - - // Check if already present - inner.tick += 1; - let tick = inner.tick; - if let Some(&idx) = inner.slot_index.get(&coord) { - inner.slots[idx].last_access = tick; - return Arc::clone(&inner.slots[idx].data); // already cached - } - - // Evict until we have room - while inner.slots.len() >= inner.max_slots - || (inner.current_bytes + data_len > inner.max_bytes && !inner.slots.is_empty()) - { - // Find LRU slot - let lru_idx = inner - .slots - .iter() - .enumerate() - .min_by_key(|(_, s)| s.last_access) - .map(|(i, _)| i) - .unwrap(); - let removed = inner.slots.swap_remove(lru_idx); - inner.slot_index.remove(&removed.coord); - // swap_remove moved the former last element into `lru_idx` (unless - // it *was* the last element) — fix up that element's index entry. - if lru_idx < inner.slots.len() { - let moved_coord = inner.slots[lru_idx].coord.clone(); - inner.slot_index.insert(moved_coord, lru_idx); - } - inner.current_bytes -= removed.data.len(); - inner.stats.evictions += 1; - } - - inner.current_bytes += data_len; - let new_idx = inner.slots.len(); - inner.slot_index.insert(coord.clone(), new_idx); - inner.slots.push(CachedChunk { - coord, - data: Arc::clone(&data), - last_access: tick, - }); - data + let mut inner = self.lock(); + let addr = inner.current(); + inner.put_decompressed((addr, coord), data) } - /// Clear the entire cache (index + decompressed data). + /// [`Self::prefetch_hint_in`] for the bound dataset. + pub fn prefetch_hint(&self, next_coords: &[ChunkCoord]) { + let addr = self.lock().current(); + self.prefetch_hint_in(addr, next_coords); + } + + // ----- Whole-cache operations ----- + + /// Clear the entire cache (indexes + decompressed data + stats). pub fn clear(&self) { - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - inner.index = None; - inner.index_addr = None; + let mut inner = self.lock(); + inner.datasets.clear(); + inner.current = None; inner.slots.clear(); inner.slot_index.clear(); inner.current_bytes = 0; inner.tick = 0; inner.last_coord = None; inner.stats = AccessStats::default(); - inner.chunk_index = None; - inner.chunk_layout = None; - } - - /// Record that the given chunk coordinates are predicted to be accessed - /// soon (bookkeeping only). - /// - /// This does **not** prefetch or pre-decompress anything — it only - /// checks whether each coordinate is already in the chunk index and - /// updates access-pattern stats accordingly. Real prefetching (e.g. - /// background pre-decompression) is not implemented. - pub fn prefetch_hint(&self, next_coords: &[ChunkCoord]) { - let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - if inner.index.is_none() { - return; - } - drop(inner); - // For each predicted coordinate, verify it exists in the index. - // The index is already populated, so this is a no-op for known chunks. - // The purpose is to signal intent — callers can pre-decompress if needed. - // We touch the stats to record that prefetch hints were issued. - let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); - for coord in next_coords { - let exists = inner - .index - .as_ref() - .map(|idx| idx.contains_key(coord)) - .unwrap_or(false); - if exists { - inner.stats.sequential_count += 1; - } - } } /// Return the current access pattern statistics. pub fn access_stats(&self) -> AccessStats { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .stats - .clone() + self.lock().stats.clone() } /// Update the sweep direction label in the access stats. pub fn set_sweep_direction(&self, direction: &'static str) { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .stats - .sweep_direction = Some(direction); + self.lock().stats.sweep_direction = Some(direction); } - /// Number of decompressed chunks currently cached. + /// Number of decompressed chunks currently cached (all datasets). pub fn cached_chunk_count(&self) -> usize { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .slots - .len() + self.lock().slots.len() } - /// Total bytes of decompressed data currently cached. + /// Total bytes of decompressed data currently cached (all datasets). pub fn cached_bytes(&self) -> usize { - self.inner - .lock() - .unwrap_or_else(|e| e.into_inner()) - .current_bytes + self.lock().current_bytes + } + + /// Number of datasets whose chunk index is currently kept. + pub fn indexed_dataset_count(&self) -> usize { + self.lock().datasets.len() } } @@ -808,6 +967,92 @@ mod tests { assert_eq!(cache.cached_bytes(), 0); } + #[test] + fn datasets_sharing_coordinates_stay_separate() { + let cache = ChunkCache::new(); + let a = vec![make_chunk(vec![0, 0], 0x100, 8)]; + let b = vec![make_chunk(vec![0, 0], 0x900, 8)]; + let got_a = cache.chunks_for::<()>(1, 1, || Ok(a.clone())).unwrap(); + let got_b = cache.chunks_for::<()>(2, 1, || Ok(b.clone())).unwrap(); + assert_eq!(got_a[0].address, 0x100); + assert_eq!(got_b[0].address, 0x900); + // Built once per dataset: a second lookup doesn't call the builder. + let again = cache + .chunks_for::<()>(1, 1, || panic!("index rebuilt")) + .unwrap(); + assert_eq!(again[0].address, 0x100); + + cache.put_decompressed_in(1, vec![0], vec![1; 4]); + cache.put_decompressed_in(2, vec![0], vec![2; 4]); + assert_eq!( + cache.get_decompressed_in(1, &[0]).unwrap().as_slice(), + &[1; 4] + ); + assert_eq!( + cache.get_decompressed_in(2, &[0]).unwrap().as_slice(), + &[2; 4] + ); + assert!(cache.get_decompressed_in(3, &[0]).is_none()); + assert_eq!(cache.cached_chunk_count(), 2); + + // The bound-dataset methods see only the bound dataset. + cache.ensure_dataset(2); + assert_eq!(cache.lookup_index(&[0]).unwrap().address, 0x900); + assert_eq!(cache.get_decompressed(&[0]).unwrap(), vec![2; 4]); + } + + #[test] + fn dataset_indexes_are_bounded() { + let cache = ChunkCache::new(); + for addr in 0..(MAX_INDEXED_DATASETS as u64 + 10) { + cache + .chunks_for::<()>(addr, 1, || Ok(vec![make_chunk(vec![0], addr, 8)])) + .unwrap(); + } + assert_eq!(cache.indexed_dataset_count(), MAX_INDEXED_DATASETS); + + // One huge index evicts the others but is itself kept. + let huge: Vec = (0..MAX_INDEXED_CHUNKS as u64) + .map(|i| make_chunk(vec![i], i, 8)) + .collect(); + let got = cache.chunks_for::<()>(9999, 1, || Ok(huge)).unwrap(); + assert_eq!(got.len(), MAX_INDEXED_CHUNKS); + assert_eq!(cache.indexed_dataset_count(), 1); + } + + #[test] + fn concurrent_readers_of_different_datasets_see_their_own_chunks() { + let cache = std::sync::Arc::new(ChunkCache::with_capacity(1 << 20, 64)); + let handles: Vec<_> = (0..8u64) + .map(|t| { + let cache = std::sync::Arc::clone(&cache); + std::thread::spawn(move || { + for round in 0..500u64 { + let addr = (t + round) % 16; + let coord = vec![round % 4]; + let chunks = cache + .chunks_for::<()>(addr, 1, || { + Ok((0..4).map(|c| make_chunk(vec![c], addr, 8)).collect()) + }) + .unwrap(); + assert!(chunks.iter().all(|c| c.address == addr)); + let want = vec![addr as u8; 8]; + let got = match cache.get_decompressed_in(addr, &coord) { + Some(hit) => hit.to_vec(), + None => cache + .put_decompressed_in(addr, coord, want.clone()) + .to_vec(), + }; + assert_eq!(got, want); + } + }) + }) + .collect(); + for h in handles { + h.join().unwrap(); + } + } + #[test] fn duplicate_insert_is_noop() { let cache = ChunkCache::new(); diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index bc006df..90bc584 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -836,24 +836,20 @@ pub fn read_chunked_data_cached( ))); } - // The per-file cache is shared across datasets; bind it to this one so a - // different dataset's chunk index is never reused for this read. - cache.ensure_dataset(addr); - - // Populate chunk index on first access - if !cache.has_index() { - let (chunks, _) = list_chunks( + // The per-file cache is shared across datasets (and threads); every + // lookup is keyed by this dataset's chunk-index address, so another + // dataset's index or chunks are never used for this read. + let chunks = cache.chunks_for(addr, rank, || { + list_chunks( file_data, layout, dataspace, elem_size, offset_size, length_size, - )?; - cache.populate_index(&chunks, rank); - } - - let chunks = cache.all_indexed_chunks().unwrap_or_default(); + ) + .map(|(chunks, _)| chunks) + })?; // Assemble output let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?; @@ -920,7 +916,7 @@ pub fn read_chunked_data_cached( continue; } let coord: Vec = chunk_info.offsets.iter().take(rank).copied().collect(); - match cache.get_decompressed_aligned(&coord) { + match cache.get_decompressed_in(addr, &coord) { Some(cached) => place(&cached, chunk_info), None => misses.push(chunk_info), } @@ -957,7 +953,7 @@ pub fn read_chunked_data_cached( let data = data?; if cache_them { let coord: Vec = chunk_info.offsets.iter().take(rank).copied().collect(); - let cached = cache.put_decompressed(coord, data); + let cached = cache.put_decompressed_in(addr, coord, data); place(&cached, chunk_info); } else { place(&data, chunk_info); @@ -1161,24 +1157,20 @@ pub fn read_chunked_data_sweep( ))); } - // The per-file cache is shared across datasets; bind it to this one so a - // different dataset's chunk index is never reused for this read. - cache.ensure_dataset(addr); - - // Populate chunk index on first access - if !cache.has_index() { - let (chunks, _) = list_chunks( + // The per-file cache is shared across datasets (and threads); every + // lookup is keyed by this dataset's chunk-index address, so another + // dataset's index or chunks are never used for this read. + let chunks = cache.chunks_for(addr, rank, || { + list_chunks( file_data, layout, dataspace, elem_size, offset_size, length_size, - )?; - cache.populate_index(&chunks, rank); - } - - let chunks = cache.all_indexed_chunks().unwrap_or_default(); + ) + .map(|(chunks, _)| chunks) + })?; // Assemble output let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?; @@ -1209,12 +1201,12 @@ pub fn read_chunked_data_sweep( // Issue prefetch hint for predicted next chunks if !sweep.predicted_next.is_empty() { - cache.prefetch_hint(&sweep.predicted_next); + cache.prefetch_hint_in(addr, &sweep.predicted_next); cache.set_sweep_direction(sweep.direction); } // Try decompressed cache first - let decompressed = if let Some(cached) = cache.get_decompressed_aligned(&coord) { + let decompressed = if let Some(cached) = cache.get_decompressed_in(addr, &coord) { cached } else { // Decompress from file @@ -1233,7 +1225,7 @@ pub fn read_chunked_data_sweep( } else { raw_chunk.to_vec() }; - cache.put_decompressed(coord, dec) + cache.put_decompressed_in(addr, coord, dec) }; let chunk_offsets: Vec = chunk_info @@ -1317,48 +1309,34 @@ pub fn read_chunked_data_indexed( ))); } - // The per-file cache is shared across datasets; bind it to this one so a - // different dataset's chunk index is never reused for this read. - cache.ensure_dataset(addr); - - // Build chunk index on first access - if !cache.has_chunk_index() { - let (chunks, _) = list_chunks( - file_data, - layout, - dataspace, - elem_size, - offset_size, - length_size, - )?; - cache.populate_chunk_index(&chunks, rank); - // Also populate the legacy index for compatibility - if !cache.has_index() { - cache.populate_index(&chunks, rank); - } - } - - // Build chunk layout on first access - if !cache.has_chunk_layout() { - cache.populate_chunk_layout(&ds_dims, &chunk_dims, elem_size); - } - - // Get the layout info (mappings, output size, chunk total bytes) - let (mappings_info, output_bytes, chunk_total_bytes) = cache - .with_chunk_layout(|layout| { - let info: Vec<_> = layout - .mappings - .iter() - .map(|m| (m.coord.clone(), m.file_offset, m.file_size, m.filter_mask)) - .collect(); - (info, layout.output_bytes, layout.chunk_total_bytes) - }) - .ok_or_else(|| FormatError::ChunkedReadError("chunk layout not available".into()))?; + // Chunk index and assembly plan for this dataset, built on first access + // and kept per dataset (keyed by chunk-index address) in the shared cache. + let plan = cache.chunk_layout_for( + addr, + rank, + || { + list_chunks( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + ) + .map(|(chunks, _)| chunks) + }, + &ds_dims, + &chunk_dims, + elem_size, + )?; + let chunk_total_bytes = plan.chunk_total_bytes; // Decompress chunks (using LRU cache where possible) - let mut chunk_buffers: Vec> = Vec::with_capacity(mappings_info.len()); - for (coord, file_offset, file_size, filter_mask) in &mappings_info { - if let Some(cached) = cache.get_decompressed_aligned(coord) { + let mut chunk_buffers: Vec> = Vec::with_capacity(plan.mappings.len()); + for m in &plan.mappings { + let (coord, file_offset, file_size, filter_mask) = + (&m.coord, &m.file_offset, &m.file_size, &m.filter_mask); + if let Some(cached) = cache.get_decompressed_in(addr, coord) { chunk_buffers.push(cached); } else { let c_addr = *file_offset as usize; @@ -1377,17 +1355,15 @@ pub fn read_chunked_data_indexed( raw_chunk.to_vec() }; let aligned = CacheAlignedBuffer::from_vec(decompressed); - let arc = cache.put_decompressed_aligned(coord.clone(), aligned); + let arc = cache.put_decompressed_aligned_in(addr, coord.clone(), aligned); chunk_buffers.push(arc); } } // Assemble using pre-computed layout - let mut output = vec![0u8; output_bytes]; + let mut output = vec![0u8; plan.output_bytes]; let data_refs: Vec<&[u8]> = chunk_buffers.iter().map(|b| b.as_slice()).collect(); - cache.with_chunk_layout(|layout| { - layout.assemble(&data_refs, &mut output); - }); + plan.assemble(&data_refs, &mut output); Ok(output) } @@ -2307,12 +2283,12 @@ mod tests { let datatype = make_f64_type(); let cache = ChunkCache::new(); - assert!(!cache.has_index()); + assert_eq!(cache.indexed_dataset_count(), 0); let raw = read_chunked_data_cached( &file_data, &layout, &dataspace, &datatype, None, 8, 8, &cache, ) .unwrap(); - assert!(cache.has_index()); + assert_eq!(cache.indexed_dataset_count(), 1); assert_eq!(raw.len(), 20 * 8); for i in 0..20 { let val = f64::from_le_bytes(raw[i * 8..(i + 1) * 8].try_into().unwrap()); @@ -2334,7 +2310,7 @@ mod tests { &file_data, &layout, &dataspace, &datatype, None, 8, 8, &cache, ) .unwrap(); - assert!(cache.has_index()); + assert_eq!(cache.indexed_dataset_count(), 1); assert_eq!(cache.cached_chunk_count(), 0); // Second read — reuses the cached index @@ -2343,6 +2319,7 @@ mod tests { ) .unwrap(); assert_eq!(raw1, raw2); + assert_eq!(cache.indexed_dataset_count(), 1); } #[test] diff --git a/crates/clawhdf5/tests/concurrent_chunk_cache.rs b/crates/clawhdf5/tests/concurrent_chunk_cache.rs new file mode 100644 index 0000000..275e2eb --- /dev/null +++ b/crates/clawhdf5/tests/concurrent_chunk_cache.rs @@ -0,0 +1,87 @@ +//! A `File` is `Send + Sync` and keeps one chunk cache for all its datasets. +//! Threads reading different chunked datasets through the same `File` must +//! each get their own dataset's data. + +use std::sync::Arc; + +use clawhdf5::{File, FileBuilder}; + +const DATASETS: usize = 24; +const THREADS: usize = 16; +const ROUNDS: usize = 40; + +/// Contents of dataset `k`: distinct from every other dataset's, element for +/// element, so any chunk served from the wrong dataset shows. +fn values(k: usize, n: usize) -> Vec { + (0..n).map(|i| (k * 100_000 + i) as f64).collect() +} + +fn build() -> File { + let mut b = FileBuilder::new(); + for k in 0..DATASETS { + let ds = b.create_dataset(&format!("d{k:02}")); + match k % 3 { + // 1-D, compressed: chunk offsets 0, 8, 16, ... in every dataset. + 0 => { + ds.with_f64_data(&values(k, 64)).with_shape(&[64]); + ds.with_chunks(&[8]).with_deflate(1); + } + // 1-D, shuffle + compressed, a different length. + 1 => { + ds.with_f64_data(&values(k, 40)).with_shape(&[40]); + ds.with_chunks(&[8]).with_shuffle().with_deflate(1); + } + // 2-D, compressed: coordinates (0,0), (0,4), (4,0), ... overlap + // the other datasets' in the first dimension. + _ => { + ds.with_f64_data(&values(k, 64)).with_shape(&[8, 8]); + ds.with_chunks(&[4, 4]).with_deflate(1); + } + } + } + File::from_bytes(b.finish().unwrap()).unwrap() +} + +fn expected(k: usize) -> Vec { + values(k, if k % 3 == 1 { 40 } else { 64 }) +} + +#[test] +fn threads_reading_different_datasets_get_their_own_chunks() { + let file = Arc::new(build()); + // Sequential sanity check first. + for k in 0..DATASETS { + let got = file.dataset(&format!("d{k:02}")).unwrap().read_f64(); + assert_eq!(got.unwrap(), expected(k), "sequential d{k:02}"); + } + + let handles: Vec<_> = (0..THREADS) + .map(|t| { + let file = Arc::clone(&file); + std::thread::spawn(move || { + let mut wrong = Vec::new(); + for round in 0..ROUNDS { + let k = (t * 7 + round * 5) % DATASETS; + let name = format!("d{k:02}"); + match file.dataset(&name).unwrap().read_f64() { + Ok(v) if v == expected(k) => {} + Ok(v) => wrong.push(format!("{name}: wrong data, first {:?}", &v[..4])), + Err(e) => wrong.push(format!("{name}: {e}")), + } + } + wrong + }) + }) + .collect(); + let failures: Vec = handles + .into_iter() + .flat_map(|h| h.join().unwrap()) + .collect(); + assert!( + failures.is_empty(), + "{} of {} concurrent reads were wrong, e.g. {:?}", + failures.len(), + THREADS * ROUNDS, + &failures[..failures.len().min(5)] + ); +} From bc820fbd8c8a5c4704aa7098292f088bec332fb0 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:13:08 -0500 Subject: [PATCH 25/36] fix(format): refuse path-like group and dataset names FileWriter writes the root group plus one level of groups; it has no way to create intermediate groups. create_group("a/b") therefore stored a single link literally named "a/b", which no HDF5 reader can resolve (h5py: "component not found"). Nesting would mean restructuring the writer's layout around a group tree, so for now finish() rejects any group, dataset or external-link name that is empty, "." or contains '/'. Attribute names may still contain '/'. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 26 +++++++++++++ .../tests/writer_meta_tests.rs | 39 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index d432f5e..2294425 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -63,6 +63,19 @@ fn build_paged_superblock_extension(page_size: u32) -> Result, FormatErr w.serialize() } +/// A group or dataset name must be one path component: not empty, not ".", +/// and without '/'. `FileWriter` writes a root group plus one level of +/// groups, and cannot create intermediate groups for a path. +fn check_link_name(name: &str) -> Result<(), FormatError> { + if name.is_empty() || name == "." || name.contains('/') { + return Err(FormatError::SerializationError(format!( + "invalid object name {name:?}: names must be a single path component \ + (FileWriter does not create nested groups)" + ))); + } + Ok(()) +} + /// Threshold for switching from compact (inline) to dense attribute storage. const DENSE_ATTR_THRESHOLD: usize = 8; @@ -1137,6 +1150,19 @@ impl FileWriter { }) }; + // Every name becomes a single link in its parent group. The writer + // has no nested groups, so a path like "a/b" would be stored as one + // link literally named "a/b" — which no HDF5 reader can resolve. + let root_names = self.root_datasets.iter().map(|d| d.name.as_str()); + let group_names = self.groups.iter().flat_map(|g| { + core::iter::once(g.name.as_str()) + .chain(g.datasets.iter().map(|d| d.name.as_str())) + .chain(g.external_links.iter().map(|l| l.0.as_str())) + }); + for name in root_names.chain(group_names) { + check_link_name(name)?; + } + let mut all_ds: Vec = Vec::new(); let mut groups: Vec = Vec::new(); let mut root_ds_indices: Vec = Vec::new(); diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index 4e78bbd..587f65f 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -524,3 +524,42 @@ fn h5py_reads_all_attributes_next_to_an_empty_string() { assert_eq!(out, r#"["", "héllo", ["", ""], 3]"#); h5dump_ok(&path); } + +// ---- 6. path-like names ---- + +#[test] +fn slash_in_a_group_or_dataset_name_is_an_error() { + // Measured: create_group("a/b") wrote one link literally named "a/b", + // which h5py cannot reach ("component not found"). The writer has no + // nested groups, so such names are refused. + let mut fw = FileWriter::new(); + let mut g = fw.create_group("a/b"); + g.create_dataset("c").with_f64_data(&[1.0]); + fw.add_group(g.finish()); + assert!(fw.finish().is_err()); + + let mut fw = FileWriter::new(); + fw.create_dataset("x/y").with_f64_data(&[1.0]); + assert!(fw.finish().is_err()); + + let mut fw = FileWriter::new(); + let mut g = fw.create_group("g"); + g.create_dataset("x/y").with_f64_data(&[1.0]); + fw.add_group(g.finish()); + assert!(fw.finish().is_err()); + + for bad in ["", "."] { + let mut fw = FileWriter::new(); + fw.create_dataset(bad).with_f64_data(&[1.0]); + assert!(fw.finish().is_err(), "{bad:?}"); + } + + // One level of groups still works, and '/' stays legal in attribute names. + let mut fw = FileWriter::new(); + let mut g = fw.create_group("g"); + g.create_dataset("c").with_f64_data(&[1.0]); + g.set_attr("m/s", AttrValue::I64(1)); + fw.add_group(g.finish()); + let bytes = fw.finish().unwrap(); + header_at(&bytes, "g/c"); +} From 3000b40cf3d8187dbaec2d1d9aa664d693eccb8f Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:14:14 -0500 Subject: [PATCH 26/36] fix(format): N-Bit pass-through flag and no-op (enum) members - libhdf5 sets cd_values[1] ("need not compress") when every field is already full width and then stores the chunk unchanged (H5Z__filter_nbit: `if (cd_values[1]) HGOTO_DONE`). We ignored it and tried to unpack, so tfilters.h5 / h5stat_filters.h5 `/all` (shuffle + szip + deflate + fletcher32 + N-Bit) failed with "nbit: packed data too short". A type with no N-Bit parameters (cd = [3, 1, nelmts]) is now accepted the same way. - Class 4 (H5Z_NBIT_NOOPTYPE: enum, string, opaque, ... members) is stored whole, 8 bits per byte; it was UnsupportedFilter(5) (h5repack_nested_8bit_enum_deflated.h5). N-Bit on floats was not wrong in the filter: for le_data.h5 / Nbit_float_data_* our output equals libhdf5's decoded bytes in the file datatype (a 20-bit float, offset 7, bias 31). h5py's values differ because libhdf5 then converts that custom float layout to IEEE, which our datatype reader does not do; nbit_float_matches_libhdf5_file_type_bytes pins the filter output and the doc comment says where conversion belongs. Tests: nbit_need_not_compress_is_passthrough, nbit_in_multi_filter_pipeline_matches_libhdf5 (tfilters.h5 chunk, szip feature), nbit_compound_with_enum_member_matches_libhdf5 all failed before; nbit_float_matches_libhdf5_file_type_bytes (guard). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 193 +++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 6 deletions(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index c0bf701..54e56d4 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -385,6 +385,9 @@ enum NbitNode { count: usize, base_size: usize, }, + /// `H5Z_NBIT_NOOPTYPE`: a field N-Bit does not reduce (enum, string, + /// opaque, ...), stored as all `size` bytes, 8 bits each. + Noop { size: usize }, } impl NbitNode { @@ -395,6 +398,7 @@ impl NbitNode { NbitNode::Array { count, base_size, .. } => count * base_size, + NbitNode::Noop { size } => *size, } } } @@ -414,6 +418,7 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result NBIT_MAX_DEPTH { return Err(FormatError::ChunkedReadError( "nbit: type tree nested too deeply".into(), @@ -489,8 +494,17 @@ fn parse_nbit_node(cd: &[u32], idx: &mut usize, depth: u32) -> Result { + // class, size + let size = nbit_cd(cd, *idx + 1)? as usize; + *idx += 2; + if size == 0 { + return Err(FormatError::ChunkedReadError( + "nbit: invalid no-op type size".into(), + )); + } + Ok(NbitNode::Noop { size }) + } _ => Err(FormatError::UnsupportedFilter(FILTER_NBIT)), } } @@ -555,6 +569,11 @@ fn decode_nbit_node( decode_nbit_node(bnode, br, elem, base + i * base_size)?; } } + NbitNode::Noop { size } => { + for slot in &mut elem[base..base + size] { + *slot = br.read(8)? as u8; + } + } } Ok(()) } @@ -567,17 +586,28 @@ fn decode_nbit_node( /// type tree — atomic (`[1, size, order, precision, offset]`), array /// (`[2, total_size, ]`) and compound /// (`[3, total_size, nmembers, (offset, )*]`) — preceded by -/// `[nparms, flag, nelmts]`. Decompression walks the tree once per element, +/// `[nparms, need_not_compress, nelmts]`; when `need_not_compress` is set +/// (every field already uses its full width, e.g. a 32-bit int of precision +/// 32) libhdf5 stores the data unchanged and so do we. Fields N-Bit cannot +/// reduce (enums, strings, ...) are no-op nodes (`[4, size]`) copied whole. +/// Decompression walks the tree once per element, /// placing each field's bits at its byte/bit offset in a zero-filled element /// (HDF5's canonical reduced-precision layout). Sign-extension of reduced -/// precision signed integers is the datatype reader's job. Atomic floats are -/// encoded as full-precision atomics and handled transparently. +/// precision signed integers is the datatype reader's job, and so is +/// converting a reduced-precision float (its own sign/exponent/mantissa +/// layout, e.g. `le_data.h5`'s 20-bit `Nbit_float_data_*`) to IEEE: the +/// filter's output is the file type's bytes, as libhdf5's is before type +/// conversion. fn nbit_decompress(data: &[u8], cd: &[u32], expected_bytes: usize) -> Result, FormatError> { - if cd.len() < 4 { + if cd.len() < 3 { return Err(FormatError::ChunkedReadError( "nbit: missing filter client data".into(), )); } + // H5Z__filter_nbit: `if (cd_values[1]) HGOTO_DONE(*buf_size)`. + if cd[1] != 0 { + return Ok(data.to_vec()); + } let nelmts = cd[2] as usize; let mut idx = 3; let root = parse_nbit_node(cd, &mut idx, 0)?; @@ -1584,6 +1614,157 @@ mod tests { // --- LZ4 tests --- + fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + fn one_filter(filter_id: u16, client_data: Vec) -> FilterDescription { + FilterDescription { + filter_id, + name: None, + flags: 1, + client_data, + } + } + + /// `le_data.h5` `/Nbit_float_data_{le,be}` chunk (0,0): a 20-bit float + /// (offset 7) packed by N-Bit. The filter must reproduce libhdf5's + /// decoded bytes in the *file* datatype (h5py `DatasetID.read` with the + /// file type as memory type, so no conversion); converting that custom + /// float layout to IEEE is the datatype reader's job, not the filter's. + #[test] + fn nbit_float_matches_libhdf5_file_type_bytes() { + let file: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5"); + let cases = [ + ( + 55952, + 0, + "8055d5018055e5010000f0018055e5010000f0018055f5010000f0018055f50180aafa018055f50180aafa0100000002", + ), + ( + 56076, + 1, + "01d5558001e5558001f0000001e5558001f0000001f5558001f0000001f5558001faaa8001f5558001faaa8002000000", + ), + ]; + for (off, order, want) in cases { + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_NBIT, vec![8, 0, 12, 1, 4, order, 20, 7])], + }; + let got = decompress_chunk(&file[off..off + 31], &pipeline, 48, 4).unwrap(); + assert_eq!(got, unhex(want), "byte order {order}"); + } + } + + /// libhdf5 sets `cd_values[1]` ("need not compress") when every field is + /// already full width and then stores the data unchanged; we unpacked it + /// anyway and failed with "nbit: packed data too short". + #[test] + fn nbit_need_not_compress_is_passthrough() { + let data: Vec = (0..200u32).map(|i| (i * 7) as u8).collect(); + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_NBIT, vec![8, 1, 50, 1, 4, 0, 32, 0])], + }; + assert_eq!(decompress_chunk(&data, &pipeline, 200, 4).unwrap(), data); + // A top-level type N-Bit has no parameters for (e.g. an enum) carries only + // [nparms, need_not_compress, nelmts]. + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_NBIT, vec![3, 1, 50])], + }; + assert_eq!(decompress_chunk(&data, &pipeline, 200, 4).unwrap(), data); + } + + /// `tfilters.h5` `/all` chunk (0,0): shuffle, szip, deflate, fletcher32 + /// and a pass-through N-Bit in one pipeline; values from h5py. + #[test] + #[cfg(feature = "szip")] + fn nbit_in_multi_filter_pipeline_matches_libhdf5() { + let raw = unhex(concat!( + "785e3bc1c0c030cb6517ff039e556de1576c0f300b152c771070641170640b99ba879f8167d5cce82f760ccc4285d71b", + "20c2afc226329f60d60ab5636a3fc1905499c3c0a1d0c4a1d05c6a13d7f88271aaf6725fe7170c86363f1858c0ca0104", + "bf1e95c75f3eeb", + )); + let want = unhex(concat!( + "00000000010000000200000003000000040000000a0000000b0000000c0000000d0000000e0000001400000015000000", + "1600000017000000180000001e0000001f00000020000000210000002200000028000000290000002a0000002b000000", + "2c00000032000000330000003400000035000000360000003c0000003d0000003e0000003f0000004000000046000000", + "4700000048000000490000004a00000050000000510000005200000053000000540000005a0000005b0000005c000000", + "5d0000005e000000", + )); + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_SHUFFLE, vec![4]), + one_filter(FILTER_SZIP, vec![141, 4, 32, 5]), + one_filter(FILTER_DEFLATE, vec![5]), + one_filter(FILTER_FLETCHER32, vec![]), + one_filter(FILTER_NBIT, vec![8, 1, 50, 1, 4, 0, 32, 0]), + ], + }; + assert_eq!(decompress_chunk(&raw, &pipeline, 200, 4).unwrap(), want); + } + + /// `h5repack_nested_8bit_enum_deflated.h5` `/tracks/1/trace` chunk 0: a + /// 376-byte compound whose `u1` enum member N-Bit stores whole as a + /// no-op type (class 4), then deflate. Was `UnsupportedFilter(5)`. + /// Expected bytes: libhdf5's decode in the file datatype. + #[test] + #[cfg(feature = "deflate")] + fn nbit_compound_with_enum_member_matches_libhdf5() { + #[rustfmt::skip] + let cd: Vec = vec![ + 251, 0, 1, 3, 376, 38, + 0, 1, 4, 0, 32, 0, + 8, 2, 96, 1, 8, 0, 64, 0, + 104, 1, 8, 0, 64, 0, 112, 1, 8, 0, 64, 0, 120, 1, 8, 0, 64, 0, + 128, 1, 8, 0, 64, 0, 136, 1, 8, 0, 64, 0, 144, 1, 8, 0, 64, 0, + 152, 1, 8, 0, 64, 0, + 160, 2, 24, 1, 8, 0, 64, 0, 184, 2, 24, 1, 8, 0, 64, 0, + 208, 2, 16, 1, 8, 0, 64, 0, 224, 2, 32, 1, 8, 0, 64, 0, + 256, 2, 16, 1, 4, 0, 32, 0, 272, 2, 16, 1, 4, 0, 32, 0, + 288, 2, 32, 1, 8, 0, 64, 0, 320, 2, 16, 1, 8, 0, 64, 0, + 336, 2, 8, 1, 4, 0, 32, 0, + 344, 4, 1, + 346, 1, 2, 0, 16, 0, 348, 1, 4, 0, 32, 0, 352, 1, 1, 0, 4, 0, + 354, 1, 2, 0, 16, 0, 356, 1, 1, 0, 4, 0, 357, 1, 1, 0, 4, 0, + 358, 1, 1, 0, 4, 0, 359, 1, 1, 0, 4, 0, 360, 1, 1, 0, 4, 0, + 361, 1, 1, 0, 4, 0, 362, 1, 1, 0, 4, 0, 364, 1, 1, 0, 4, 0, + 363, 1, 1, 0, 4, 0, 365, 1, 1, 0, 4, 0, 366, 1, 1, 0, 4, 0, + 367, 1, 1, 0, 4, 0, 368, 1, 1, 0, 4, 0, 369, 1, 1, 0, 4, 0, + 370, 1, 1, 0, 4, 0, + ]; + assert_eq!(cd.len(), 251); + let raw = unhex(concat!( + "780163606078c930c880c3879573a60b2d7883eeac06a880835c47bda16cda7987a0c59ec9f74c4af6ff677e5df16445", + "adfd8493ce3ba592ddedab2a3bee87dd0faaff00d1808b66606006fa9d999b818125014837fd4703fba1fa71d150e720", + "512caa4073dc59212206500946060600604c37fc", + )); + let want = unhex(concat!( + "e90000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000eca012979ca9f040000000000000000000000000000000000000000000000080cf661d317f881e40", + "7434de6349a352407da8e478eb03ffbf47631ab943c9903f52df56df88797a3f000000000000f07f000000000000f07f", + "000000000000f07f000000000000f07fe90300000b0300006004000082030000ffffffffffffffffffffffffffffffff", + "000000000000f0bf000000000000f0bf000000000000f0bf000000000000f0bf00000000000000000000000000000000", + "25040000470300000500000000000000030000000000000000000000000000000100000000000000", + )); + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_NBIT, cd), + one_filter(FILTER_DEFLATE, vec![1]), + ], + }; + assert_eq!(decompress_chunk(&raw, &pipeline, 376, 376).unwrap(), want); + } + /// Chunk (0,0) of `/DS1` in the HDF Group's `h5ex_d_lz4.h5` example, /// written by libhdf5's registered LZ4 plugin with a 3-byte block size /// (so it has many blocks, some stored raw). Byte range from h5py's From 57e938c4383db3dfd7ae2ce9a7cab0d7dcb39d3c Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:14:30 -0500 Subject: [PATCH 27/36] fix(format): honour unknown-message flags the way libhdf5 does The object header parser failed on an unknown message with flag bit 3 set and ignored bit 7. Per the spec, bit 3 means "fail if unknown and the file is opened for writing" and bit 7 "fail if unknown, always". The parser only reads, so it now ignores bit 3 (as libhdf5 does for a read-only open) and refuses bit 7, in v1 headers, v2 headers and their continuation chunks. On libhdf5's conformance file tbogus.h5 (added as a fixture) we used to refuse Dataset2 and open Dataset3; we now match libhdf5: Dataset1, 2, 4 and 5 open, Dataset3 is refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/object_header.rs | 67 +++++++++++++----- .../clawhdf5-format/tests/fixtures/tbogus.h5 | Bin 0 -> 5056 bytes .../tests/writer_meta_tests.rs | 34 +++++++++ 3 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/tbogus.h5 diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 8306185..4b8067f 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -146,12 +146,7 @@ impl ObjectHeader { ensure_len(data, pos, msg_data_size)?; let msg_type = MessageType::from_u16(msg_type_raw); - // Check if unknown + must-understand (bit 3 of msg_flags) - if let MessageType::Unknown(id) = msg_type - && msg_flags & 0x08 != 0 - { - return Err(FormatError::UnsupportedMessage(id)); - } + check_unknown_message(msg_type, msg_flags)?; if msg_type != MessageType::Nil { messages.push(HeaderMessage { @@ -229,11 +224,7 @@ impl ObjectHeader { let msg_type = MessageType::from_u16(msg_type_raw); - if let MessageType::Unknown(id) = msg_type - && msg_flags & 0x08 != 0 - { - return Err(FormatError::UnsupportedMessage(id)); - } + check_unknown_message(msg_type, msg_flags)?; if msg_type != MessageType::Nil { messages.push(HeaderMessage { @@ -424,11 +415,7 @@ impl ObjectHeader { let msg_type = MessageType::from_u16(msg_type_raw); - if let MessageType::Unknown(id) = msg_type - && msg_flags & 0x08 != 0 - { - return Err(FormatError::UnsupportedMessage(id)); - } + check_unknown_message(msg_type, msg_flags)?; let msg_data = data[pos..pos + msg_data_size].to_vec(); @@ -509,6 +496,24 @@ impl ObjectHeader { } } +/// Header message flag bit 7: fail if the message is unknown, always. +const MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS: u8 = 0x80; + +/// Refuse an unknown message the file says no reader may skip. +/// +/// The parser only ever reads, so bit 3 (fail only when opened for writing) +/// is ignored, as libhdf5 ignores it for a read-only open; bit 7 fails +/// regardless of access mode. This had the two the wrong way round, failing +/// objects libhdf5 reads and reading ones it refuses (`tbogus.h5`). +fn check_unknown_message(msg_type: MessageType, msg_flags: u8) -> Result<(), FormatError> { + match msg_type { + MessageType::Unknown(id) if msg_flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 => { + Err(FormatError::UnsupportedMessage(id)) + } + _ => Ok(()), + } +} + #[cfg(test)] mod tests { use super::*; @@ -632,14 +637,38 @@ mod tests { } #[test] - fn parse_v1_unknown_must_understand_errors() { - // Bit 3 of msg_flags = must understand - let messages = [(0x00FFu16, &[0xAA][..], 0x08u8)]; + fn parse_v1_unknown_fail_always_errors() { + // Bit 7 of msg_flags = fail if unknown, whatever the access mode. + let messages = [(0x00FFu16, &[0xAA][..], 0x80u8)]; let data = build_v1_header(&messages, 8, 8); let err = ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(); assert_eq!(err, FormatError::UnsupportedMessage(0x00FF)); } + #[test] + fn parse_v1_unknown_fail_on_write_is_ignored_when_reading() { + // Bit 3 = fail if unknown *and the file is opened for writing*. This + // parser only reads, so libhdf5 (read-only) opens such an object and + // so must we. Bits 4/5 (mark if unknown / was unknown) never fail. + for flags in [0x08u8, 0x10, 0x20, 0x38] { + let messages = [(0x00FFu16, &[0xAA][..], flags)]; + let data = build_v1_header(&messages, 8, 8); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + assert_eq!(hdr.messages[0].msg_type, MessageType::Unknown(0x00FF)); + } + } + + #[test] + fn parse_v2_unknown_message_flags() { + let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x08)], None); + assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok()); + let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x80)], None); + assert_eq!( + ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(), + FormatError::UnsupportedMessage(0xF0) + ); + } + #[test] fn parse_v2_no_timestamps_one_message() { let data = build_v2_header(0x00, &[(0x01, &[10, 20], 0)], None); diff --git a/crates/clawhdf5-format/tests/fixtures/tbogus.h5 b/crates/clawhdf5-format/tests/fixtures/tbogus.h5 new file mode 100644 index 0000000000000000000000000000000000000000..f64229e9356628aa8995a04099f85a952e056805 GIT binary patch literal 5056 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf&;=35f~pPp8#brLg@}Dy@CnCU}OM61_lYJ zxFFPgbaf#?uC5F~l`!*RG*lad0Skl`0TURdM^p%SxH<-aJiGzw>jWrW!2@N`h+<@5 z2d7^M0ZO49V4Gm+of(*(L2Ln_FeHg8faO_%>OkU5OiW;<9MBxV%m_=_&;$)u&A`A3 zHTV6#wf8_mLP+);Y(A60z|a6yIj~f)pT7$u0~^$J3=9g)_}v4`_Z6)8)oDPbJJ|56 zvw%v^V8^e{11h}&7%%t$tUTGl2~h=$*9TBO1C7!bJ<}B^2nKt)qGxzCjD`m!u>(m^ zxdW>4N7Dx+NI>D?FeJhQd%Fs~+#=MjvfzXG8&+OIc%$S<2?1EU3RVxo>OTdvde0@X zB(XTP#1IxPP`(iw-x!T=g2p$6@nJNz%}p=LFD(EX4)X`N(Fn7Q0-9d+lQgttHQ38z zNIMYJ%7p+8Ui^UzYX>&)<5#Bvm7V~ql<)vpJ8*#@9z{SYSh==A2*0|4lBH+50>#x} fPgnE|kA~6kfG2xUxii`hga-!$C_Eg7K>7dxb&LV5 literal 0 HcmV?d00001 diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index 587f65f..88a08a1 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -563,3 +563,37 @@ fn slash_in_a_group_or_dataset_name_is_an_error() { let bytes = fw.finish().unwrap(); header_at(&bytes, "g/c"); } + +// ---- 7. unknown-message flags on read ---- + +#[test] +fn unknown_message_flags_follow_libhdf5_on_tbogus() { + // libhdf5's own test file (test/testfiles/tbogus.h5): datasets carrying + // an unknown message with various flags. libhdf5 (read-only) opens + // Dataset1, 2, 4 and 5 and refuses Dataset3 ("unknown message with 'fail + // if unknown' flag found"). We used to refuse Dataset2 (bit 3, which only + // applies when writing) and open Dataset3 (bit 7, fail always). + let bytes = include_bytes!("fixtures/tbogus.h5"); + let sig = signature::find_signature(bytes).unwrap(); + let sb = Superblock::parse(bytes, sig).unwrap(); + for (name, readable) in [ + ("Dataset1", true), + ("Dataset2", true), + ("Dataset3", false), + ("Dataset4", true), + ("Dataset5", true), + ] { + let addr = resolve_path_any(bytes, &sb, name).unwrap(); + let parsed = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size); + match parsed { + Ok(_) => assert!(readable, "{name} must be refused"), + Err(e) => { + assert!(!readable, "{name} must be readable, got {e:?}"); + assert!(matches!( + e, + clawhdf5_format::error::FormatError::UnsupportedMessage(_) + )); + } + } + } +} From 1dba7b465a95a8f2738b1a17241f99e6f17fbe6f Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:14:47 -0500 Subject: [PATCH 28/36] fix(format): index datasets with several unlimited dims by B-tree v2 A dataset with more than one unlimited dimension got an Extensible Array index, which libhdf5 refuses ("already found unlimited dimension"), so the whole file failed to open in h5py and h5dump. The previous commit turned that into a write error; this one writes what the library itself uses there: a version-2 B-tree chunk index (record type 10/11), as a single leaf of the library's 2048-byte node size, or a larger leaf when the records do not fit. The root's record count is 16-bit, so more than 65535 chunks is still refused rather than written wrong. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_write.rs | 334 ++++++++++++++----- crates/clawhdf5/tests/chunk_index_interop.rs | 33 +- 2 files changed, 268 insertions(+), 99 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 3136f58..6ff0c07 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -444,6 +444,27 @@ fn serialize_v4_fixed_array( element_size: u32, max_bits: u8, ) -> Vec { + let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size); + + // chunk index type = 3 (Fixed Array) + buf.push(3); + + // max_dblk_page_nelmts_bits — must match FAHD max_nelmts_bits + buf.push(max_bits); + + // Fixed Array header address + match offset_size { + 4 => buf.extend_from_slice(&(fixed_array_address as u32).to_le_bytes()), + 8 => buf.extend_from_slice(&fixed_array_address.to_le_bytes()), + _ => {} + } + + buf +} + +/// The part of a v4 chunked layout message before the chunk index type: +/// version, class, flags and the chunk dimensions (plus the element size). +fn layout_v4_chunked_prefix(chunk_dims: &[u32], element_size: u32) -> Vec { let mut buf = Vec::new(); buf.push(4); // version buf.push(2); // class = chunked @@ -483,20 +504,6 @@ fn serialize_v4_fixed_array( 4 => buf.extend_from_slice(&element_size.to_le_bytes()), _ => {} } - - // chunk index type = 3 (Fixed Array) - buf.push(3); - - // max_dblk_page_nelmts_bits — must match FAHD max_nelmts_bits - buf.push(max_bits); - - // Fixed Array header address - match offset_size { - 4 => buf.extend_from_slice(&(fixed_array_address as u32).to_le_bytes()), - 8 => buf.extend_from_slice(&fixed_array_address.to_le_bytes()), - _ => {} - } - buf } @@ -733,65 +740,91 @@ pub fn build_chunked_data_from_precompressed( data_buf.resize(aligned_idx, 0u8); } - let layout_message = if let ChunkIndexPlan::ExtensibleArray(grid) = &index { - let ea_address = base_address + data_buf.len() as u64; - let slots = index_slots(grid, &pre.shape, &pre.chunk_dims, &written_chunks, None)?; - let ea_bytes = ea_writer::build_extensible_array_at( - &slots, - offset_size, - length_size, - pre.has_filters, - ea_address, - ); - data_buf.extend_from_slice(&ea_bytes); - ea_writer::serialize_v4_extensible_array( - &chunk_dims_u32, - ea_address, - offset_size, - element_size as u32, - ) - } else if matches!(index, ChunkIndexPlan::SingleChunk) { - let chunk_addr = written_chunks[0].address; - let filtered_size = if pre.has_filters { - Some(written_chunks[0].compressed_size) - } else { - None - }; - let filter_mask = if pre.has_filters { Some(0u32) } else { None }; - serialize_v4_single_chunk( - &chunk_dims_u32, - chunk_addr, - filtered_size, - filter_mask, - offset_size, - element_size as u32, - ) - } else if let ChunkIndexPlan::FixedArray(grid, nslots) = &index { - let fa_address = base_address + data_buf.len() as u64; - let slots = index_slots( - grid, - &pre.shape, - &pre.chunk_dims, - &written_chunks, - Some(*nslots), - )?; - let fa_bytes = build_fixed_array_at( - &slots, - offset_size, - length_size, - pre.has_filters, - fa_address, - ); - data_buf.extend_from_slice(&fa_bytes); - serialize_v4_fixed_array( - &chunk_dims_u32, - fa_address, - offset_size, - element_size as u32, - FA_PAGE_BITS, - ) - } else { - unreachable!("every chunk index plan is handled above") + let layout_message = match &index { + ChunkIndexPlan::ExtensibleArray(grid) => { + let ea_address = base_address + data_buf.len() as u64; + let slots = index_slots(grid, &pre.shape, &pre.chunk_dims, &written_chunks, None)?; + let ea_bytes = ea_writer::build_extensible_array_at( + &slots, + offset_size, + length_size, + pre.has_filters, + ea_address, + ); + data_buf.extend_from_slice(&ea_bytes); + ea_writer::serialize_v4_extensible_array( + &chunk_dims_u32, + ea_address, + offset_size, + element_size as u32, + ) + } + ChunkIndexPlan::SingleChunk => { + let chunk_addr = written_chunks[0].address; + let filtered_size = if pre.has_filters { + Some(written_chunks[0].compressed_size) + } else { + None + }; + let filter_mask = if pre.has_filters { Some(0u32) } else { None }; + serialize_v4_single_chunk( + &chunk_dims_u32, + chunk_addr, + filtered_size, + filter_mask, + offset_size, + element_size as u32, + ) + } + ChunkIndexPlan::FixedArray(grid, nslots) => { + let fa_address = base_address + data_buf.len() as u64; + let slots = index_slots( + grid, + &pre.shape, + &pre.chunk_dims, + &written_chunks, + Some(*nslots), + )?; + let fa_bytes = build_fixed_array_at( + &slots, + offset_size, + length_size, + pre.has_filters, + fa_address, + ); + data_buf.extend_from_slice(&fa_bytes); + serialize_v4_fixed_array( + &chunk_dims_u32, + fa_address, + offset_size, + element_size as u32, + FA_PAGE_BITS, + ) + } + ChunkIndexPlan::BTreeV2 => { + let bt_address = base_address + data_buf.len() as u64; + let records: Vec<(Vec, &WrittenChunk)> = written_chunks + .iter() + .enumerate() + .map(|(i, c)| (scaled_coords(&pre.shape, &pre.chunk_dims, i), c)) + .collect(); + let (bt_bytes, node_size) = build_btree_v2_chunk_index_at( + pre.shape.len(), + &records, + offset_size, + length_size, + pre.has_filters, + bt_address, + )?; + data_buf.extend_from_slice(&bt_bytes); + serialize_v4_btree_v2( + &chunk_dims_u32, + bt_address, + offset_size, + element_size as u32, + node_size, + ) + } }; Ok(ChunkedDataResult { @@ -807,14 +840,15 @@ pub fn build_chunked_data_from_precompressed( const MAX_FIXED_ARRAY_SLOTS: u64 = 1 << 26; /// Which chunk index a dataset gets, following the library's choice in -/// `H5D__layout_set_latest_indexing`: Extensible Array for exactly one -/// unlimited dimension, Fixed Array for a finite maxshape, Single Chunk when -/// the whole maximum extent is one chunk. +/// `H5D__layout_set_latest_indexing`: version-2 B-tree for more than one +/// unlimited dimension, Extensible Array for exactly one, Fixed Array for a +/// finite maxshape, Single Chunk when the whole maximum extent is one chunk. enum ChunkIndexPlan { SingleChunk, /// The grid and the number of array elements (chunks of the max extent). FixedArray(ChunkGrid, usize), ExtensibleArray(ChunkGrid), + BTreeV2, } impl ChunkIndexPlan { @@ -860,10 +894,7 @@ impl ChunkIndexPlan { Some(max), chunk_dims, )?)), - _ => Err(bad( - "more than one unlimited dimension needs a B-tree v2 chunk index, \ - which the writer does not support", - )), + _ => Ok(Self::BTreeV2), } } } @@ -879,20 +910,9 @@ fn index_slots( chunks: &[WrittenChunk], len: Option, ) -> Result>, FormatError> { - let rank = shape.len(); - let cur: Vec = shape - .iter() - .zip(chunk_dims) - .map(|(&s, &c)| s.div_ceil(c)) - .collect(); let mut placed: Vec<(usize, &WrittenChunk)> = Vec::with_capacity(chunks.len()); - let mut scaled = vec![0u64; rank]; for (i, chunk) in chunks.iter().enumerate() { - let mut rem = i as u64; - for d in (0..rank).rev() { - scaled[d] = rem % cur[d]; - rem /= cur[d]; - } + let scaled = scaled_coords(shape, chunk_dims, i); let idx = usize::try_from(grid.linear_index(&scaled)) .map_err(|_| FormatError::Overflow("chunk index slot".into()))?; placed.push((idx, chunk)); @@ -907,6 +927,136 @@ fn index_slots( Ok(slots) } +/// Scaled coordinates (`offset / chunk_dim`) of the `i`-th chunk in the +/// row-major order `split_into_chunks` produces over the current extent. +fn scaled_coords(shape: &[u64], chunk_dims: &[u64], i: usize) -> Vec { + let rank = shape.len(); + let mut scaled = vec![0u64; rank]; + let mut rem = i as u64; + for d in (0..rank).rev() { + let n = shape[d].div_ceil(chunk_dims[d]); + scaled[d] = rem % n; + rem /= n; + } + scaled +} + +/// Node size the library gives a chunk index B-tree (`H5D_BT2_NODE_SIZE`), +/// with its split and merge percentages. +const BT2_NODE_SIZE: u32 = 2048; +const BT2_SPLIT_PERCENT: u8 = 100; +const BT2_MERGE_PERCENT: u8 = 40; +/// B-tree v2 record types for chunk indexes (`H5B2_CDSET_ID`, +/// `H5B2_CDSET_FILT_ID`). +const BT2_CHUNK_UNFILTERED: u8 = 10; +const BT2_CHUNK_FILTERED: u8 = 11; + +/// Build a version-2 B-tree chunk index (the library's index for datasets +/// with more than one unlimited dimension) at a known absolute address. +/// +/// `records` are `(scaled coordinates, chunk)` in lexicographic order of the +/// coordinates, which is the order the library's comparator +/// (`H5VM_vector_cmp_u`) keeps them in. The tree is a single leaf: the +/// library's 2048-byte node when the records fit, otherwise a leaf node +/// sized to hold them all (the root's record count is 16-bit, so at most +/// 65535 chunks). Returns the bytes and the node size the layout message +/// must record. +fn build_btree_v2_chunk_index_at( + rank: usize, + records: &[(Vec, &WrittenChunk)], + offset_size: u8, + length_size: u8, + has_filters: bool, + base_address: u64, +) -> Result<(Vec, u32), FormatError> { + let os = offset_size as usize; + let nrec = u16::try_from(records.len()).map_err(|_| { + FormatError::ChunkedReadError( + "more than 65535 chunks with more than one unlimited dimension: \ + use larger chunks" + .into(), + ) + })?; + let chunk_size_bytes = has_filters.then(|| { + let slots: Vec> = + records.iter().map(|(_, c)| Some((*c).clone())).collect(); + filtered_chunk_size_len(&slots) + }); + let record_size = os + chunk_size_bytes.map_or(0, |n| n + 4) + 8 * rank; + // Leaf: signature, version, type, records, checksum. + let leaf_len = 4 + 1 + 1 + records.len() * record_size + 4; + let node_size = u32::try_from(leaf_len) + .map_err(|_| FormatError::Overflow("B-tree v2 leaf size".into()))? + .max(BT2_NODE_SIZE); + let tree_type = if has_filters { + BT2_CHUNK_FILTERED + } else { + BT2_CHUNK_UNFILTERED + }; + + let hdr_len = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + length_size as usize + 4; + let leaf_address = base_address + hdr_len as u64; + + let mut out = Vec::with_capacity(hdr_len + node_size as usize); + out.extend_from_slice(b"BTHD"); + out.push(0); // version + out.push(tree_type); + out.extend_from_slice(&node_size.to_le_bytes()); + out.extend_from_slice(&(record_size as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // depth + out.push(BT2_SPLIT_PERCENT); + out.push(BT2_MERGE_PERCENT); + if records.is_empty() { + out.extend(core::iter::repeat_n(0xFF, os)); + } else { + push_addr(&mut out, leaf_address, offset_size); + } + out.extend_from_slice(&nrec.to_le_bytes()); + match length_size { + 4 => out.extend_from_slice(&(records.len() as u32).to_le_bytes()), + _ => out.extend_from_slice(&(records.len() as u64).to_le_bytes()), + } + let sum = jenkins_lookup3(&out); + out.extend_from_slice(&sum.to_le_bytes()); + debug_assert_eq!(out.len(), hdr_len); + if records.is_empty() { + return Ok((out, node_size)); + } + + let leaf_start = out.len(); + out.extend_from_slice(b"BTLF"); + out.push(0); // version + out.push(tree_type); + for (scaled, chunk) in records { + push_index_element(&mut out, Some(chunk), offset_size, chunk_size_bytes); + for &c in scaled { + out.extend_from_slice(&c.to_le_bytes()); + } + } + let sum = jenkins_lookup3(&out[leaf_start..]); + out.extend_from_slice(&sum.to_le_bytes()); + // The library reads whole nodes; pad the leaf out to the node size. + out.resize(leaf_start + node_size as usize, 0); + Ok((out, node_size)) +} + +/// Serialize a v4 layout message for a version-2 B-tree chunk index. +fn serialize_v4_btree_v2( + chunk_dims: &[u32], + btree_address: u64, + offset_size: u8, + element_size: u32, + node_size: u32, +) -> Vec { + let mut buf = layout_v4_chunked_prefix(chunk_dims, element_size); + buf.push(5); // chunk index type = 5 (version-2 B-tree) + buf.extend_from_slice(&node_size.to_le_bytes()); + buf.push(BT2_SPLIT_PERCENT); + buf.push(BT2_MERGE_PERCENT); + push_addr(&mut buf, btree_address, offset_size); + buf +} + /// Build chunked data with absolute addresses. /// If `maxshape` has unlimited dims, uses Extensible Array index. pub fn build_chunked_data_at( diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index 41d7a5a..2fb5aeb 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -515,16 +515,35 @@ fn we_write_maxshape_larger_than_shape() { check_we_write(&cases); } -/// More than one unlimited dimension needs a B-tree v2 chunk index; the -/// writer must not produce a file libhdf5 cannot open. +/// More than one unlimited dimension needs a version-2 B-tree chunk index, +/// as the library uses; an Extensible Array for `(None, None)` made libhdf5 +/// refuse the whole file ("already found unlimited dimension"). #[test] -fn two_unlimited_dims_are_refused() { +fn we_write_btree_v2_for_several_unlimited_dims() { + const U: u64 = u64::MAX; + let mut cases = vec![ + wcase("unl_unl", &[20, 30], &[5, 5], Some(&[U, U])), + wcase("unl_fin_unl", &[6, 7, 8], &[4, 3, 5], Some(&[U, 9, U])), + // More records than the library's 2048-byte node holds (84 here). + wcase("unl_unl_2400", &[40, 60], &[1, 1], Some(&[U, U])), + wcase("unl_unl_empty", &[0, 0], &[4, 4], Some(&[U, U])), + ]; + let mut filtered = wcase("unl_unl_deflate", &[6, 7, 8], &[4, 3, 5], Some(&[U, U, U])); + filtered.deflate = true; + cases.push(filtered); + check_we_write(&cases); +} + +/// A single-leaf B-tree has a 16-bit record count; beyond it the writer +/// refuses rather than writing a tree libhdf5 would misread. +#[test] +fn btree_v2_index_past_one_leaf_is_refused() { let mut b = FileBuilder::new(); b.create_dataset("d") - .with_i32_data(&(0..600).collect::>()) - .with_shape(&[20, 30]) - .with_chunks(&[5, 5]) + .with_i32_data(&vec![0i32; 70_000]) + .with_shape(&[70_000, 1]) + .with_chunks(&[1, 1]) .with_maxshape(&[u64::MAX, u64::MAX]); let dir = tempfile::tempdir().unwrap(); - assert!(b.write(dir.path().join("unl_unl.h5")).is_err()); + assert!(b.write(dir.path().join("too_many.h5")).is_err()); } From 95dcb0445452dcb936de616945866bacaebbf9f0 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:15:49 -0500 Subject: [PATCH 29/36] fix(format): scale-offset float decode with libhdf5's arithmetic D-scale floats were rebuilt as `minval + code / 10^D` in f64 and then rounded to f32 once, but libhdf5 (H5Z_scaleoffset_modify_3/4 with `float`/`powf`) computes `(float)(int)code / powf(10, D) + min` in single precision. The two differ by 1 ULP for some values: le_data.h5 /Scale_offset_float_data_{le,be} gave 1.6663332 (0x3fd54a69) where libhdf5 gives 1.6663333 (0x3fd54a6a). Use f32 arithmetic for 4-byte floats and `(double)(long)code / pow(10, D) + min` for 8-byte ones. Test: scaleoffset_float_dscale_matches_libhdf5_bits (le_data.h5 float LE/BE and double chunks, bit-exact against h5py); failed before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 54 ++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 54e56d4..94fa15f 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -246,8 +246,20 @@ fn scaleoffset_decompress( fill_value } else if is_escale { minval + code as f64 * powi_f64(2.0, scale_factor) + } else if elem_size == 4 { + // H5Z_scaleoffset_modify_3/4 for `float`: the code is + // read as an `int` and everything is single precision, + // `(float)code / powf(10, D) + min`. Doing it in f64 and + // rounding once at the end is off by 1 ULP at times. + let d = if scale_factor >= 0 { + powi_f64(10.0, scale_factor) as f32 + } else { + 1.0 / powi_f64(10.0, -scale_factor) as f32 + }; + ((code as u32 as i32) as f32 / d + minval as f32) as f64 } else { - minval + code as f64 / powi_f64(10.0, scale_factor) + // ... and for `double`: `(double)(long)code / pow(10, D) + min`. + (code as i64) as f64 / powi_f64(10.0, scale_factor) + minval } }) .collect(); @@ -1630,6 +1642,46 @@ mod tests { } } + /// `le_data.h5` scale-offset (D-scale, D = 3, fill -2.2) chunks, decoded + /// bit for bit as libhdf5 does: single-precision arithmetic for `float` + /// (we computed in f64 and rounded once, which was 1 ULP off for e.g. + /// 1.6663333: `694ad53f` instead of `6a4ad53f`), double for `double`. + #[test] + fn scaleoffset_float_dscale_matches_libhdf5_bits() { + let file: &[u8] = include_bytes!("../tests/fixtures/filters/le_data.h5"); + let cd = |size: u32, order: u32, fill_lo: u32, fill_hi: u32| { + let mut cd = vec![0, 3, 12, 1, size, 0, order, 1, fill_lo, fill_hi]; + cd.resize(20, 0); + cd + }; + let f32_le = cd(4, 0, 0xC00C_CCCD, 0); + let f32_be = cd(4, 1, 0xC00C_CCCD, 0); + let f64_le = cd(8, 0, 2576980378, 3221330329); + #[rustfmt::skip] + let cases: [(usize, usize, &[u32], &str); 6] = [ + (2816, 38, &f32_le, "abaaaa3ed2942a3fec0a803fd2942a3fec0a803fabaaaa3fec0a803fabaaaa3f694ad53fabaaaa3f694ad53f76050040"), + (2854, 38, &f32_le, "abaaaa3f6a4ad53f760500406a4ad53f7605004056551540760500405655154034a52a405655154034a52a4076054040"), + (712, 38, &f32_be, "3eaaaaab3f2a94d23f800aec3f2a94d23f800aec3faaaaab3f800aec3faaaaab3fd54a693faaaaab3fd54a6940000576"), + (750, 38, &f32_be, "3faaaaab3fd54a6a400005763fd54a6a40000576401555564000057640155556402aa53440155556402aa53440400576"), + (2050, 38, &f64_le, concat!( + "555555555555d53fb9d75c489a52e53fce3e7c865d01f03fb9d75c489a52e53fce3e7c865d01f03f555555555555f53f", + "ce3e7c865d01f03f555555555555f53fdc6b2e244da9fa3f555555555555f53fdc6b2e244da9fa3f671f3ec3ae000040")), + (2088, 38, &f64_le, concat!( + "555555555555f53fdc6b2e244da9fa3f671f3ec3ae000040dc6b2e244da9fa3f671f3ec3ae000040aaaaaaaaaaaa0240", + "671f3ec3ae000040aaaaaaaaaaaa0240ee351792a6540540aaaaaaaaaaaa0240ee351792a6540540671f3ec3ae000840")), + ]; + for (off, len, cd, want) in cases { + let pipeline = FilterPipeline { + version: 2, + filters: vec![one_filter(FILTER_SCALEOFFSET, cd.to_vec())], + }; + let want = unhex(want); + let got = + decompress_chunk(&file[off..off + len], &pipeline, want.len(), cd[4]).unwrap(); + assert_eq!(got, want, "chunk at {off}"); + } + } + /// `le_data.h5` `/Nbit_float_data_{le,be}` chunk (0,0): a 20-bit float /// (offset 7) packed by N-Bit. The filter must reproduce libhdf5's /// decoded bytes in the *file* datatype (h5py `DatasetID.read` with the From f5505fb03dce4944c2b9aa32a2618e4d6f485ae8 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:15:55 -0500 Subject: [PATCH 30/36] fix(format): keep maxshape == shape datasets contiguous Any maxshape forced chunked storage, even one equal to the shape, which cannot grow. h5py and the library store such a dataset contiguously; we now do too unless chunks (or a filter) are requested. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 7 +++- crates/clawhdf5/tests/chunk_index_interop.rs | 44 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 5350c88..644b64e 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -1124,7 +1124,12 @@ impl FileWriter { let is_chunked: Vec = all_ds .iter() .enumerate() - .map(|(i, d)| !is_vds[i] && (d.chunk_options.is_chunked() || d.maxshape.is_some())) + .map(|(i, d)| { + // Only a dataset that can grow needs chunks; a maxshape equal + // to the shape is as fixed as no maxshape at all. + let resizable = d.maxshape.as_ref().is_some_and(|m| *m != d.ds.dimensions); + !is_vds[i] && (d.chunk_options.is_chunked() || resizable) + }) .collect(); // Determine which datasets use compact storage let is_compact: Vec = all_ds diff --git a/crates/clawhdf5/tests/chunk_index_interop.rs b/crates/clawhdf5/tests/chunk_index_interop.rs index 2fb5aeb..b72b7e2 100644 --- a/crates/clawhdf5/tests/chunk_index_interop.rs +++ b/crates/clawhdf5/tests/chunk_index_interop.rs @@ -547,3 +547,47 @@ fn btree_v2_index_past_one_leaf_is_refused() { let dir = tempfile::tempdir().unwrap(); assert!(b.write(dir.path().join("too_many.h5")).is_err()); } + +/// A maxshape equal to the shape cannot grow, so it needs no chunks: the +/// dataset stays contiguous (as h5py makes it) unless chunks are requested. +#[test] +fn maxshape_equal_to_shape_stays_contiguous() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("ms_eq.h5"); + let data: Vec = (0..40).collect(); + let mut b = FileBuilder::new(); + b.create_dataset("plain") + .with_i32_data(&data) + .with_shape(&[40]) + .with_maxshape(&[40]); + b.create_dataset("chunked") + .with_i32_data(&data) + .with_shape(&[40]) + .with_maxshape(&[40]) + .with_chunks(&[8]); + b.write(&path).unwrap(); + + let file = File::open(&path).unwrap(); + let plain = file.dataset("plain").unwrap(); + assert_eq!(plain.read_i32().unwrap(), data); + assert_eq!(plain.max_dimensions().unwrap(), Some(vec![40])); + assert!( + plain.read_raw_ref().unwrap().is_some(), + "maxshape == shape should be contiguous" + ); + let chunked = file.dataset("chunked").unwrap(); + assert_eq!(chunked.read_i32().unwrap(), data); + assert!(chunked.read_raw_ref().unwrap().is_none()); + + skip_if_no_python!(); + let out = run_python(&format!( + "import h5py, numpy as np\n\ + f = h5py.File(r'{}', 'r')\n\ + for n in ('plain', 'chunked'):\n\ + \x20 d = f[n]\n\ + \x20 assert np.array_equal(d[()], np.arange(40, dtype='i4')), n\n\ + \x20 print(n, d.chunks, d.maxshape)\n", + path.display() + )); + assert_eq!(out, "plain None (40,)\nchunked (8,) (40,)"); +} From d99426be94216456eed2c3627e3c2d5744799b74 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:16:48 -0500 Subject: [PATCH 31/36] fix(format): allow Fletcher32 ahead of a compressor in the pipeline libhdf5 applies filters in pipeline order, so with Fletcher32 before deflate (h5repack_filters.h5 /dset_all: shuffle, fletcher32, deflate; or h5py's set_fletcher32() then set_deflate()) the compressor holds the chunk plus a 4-byte checksum. decompress_chunk bounded every stage by the chunk size and rejected it: "deflate: output exceeds size limit". Bound each stage by the chunk size plus 4 bytes per Fletcher32 that precedes it in the pipeline. Test: fletcher32_before_deflate_decodes (h5py-written chunk, and our own shuffle + fletcher32 + deflate round trip); failed before. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/filters.rs | 49 ++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 94fa15f..5d9fded 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -28,7 +28,20 @@ pub fn decompress_chunk( ) -> Result, FormatError> { let mut data = compressed.to_vec(); - for filter in pipeline.filters.iter().rev() { + for (i, filter) in pipeline.filters.iter().enumerate().rev() { + // Fletcher32 appends a 4-byte checksum on write, so a filter that + // follows it in the (forward) pipeline decodes to the chunk plus one + // checksum per earlier Fletcher32 — e.g. libhdf5's + // shuffle + fletcher32 + deflate (h5repack_filters.h5 `/dset_all`). + let chunk_size = if chunk_size == 0 { + 0 + } else { + let checksums = pipeline.filters[..i] + .iter() + .filter(|f| f.filter_id == FILTER_FLETCHER32) + .count(); + chunk_size.saturating_add(4 * checksums) + }; data = match filter.filter_id { FILTER_SHUFFLE => shuffle_decompress(&data, element_size as usize)?, // `chunk_size` is the expected decompressed size (shuffle/fletcher32 @@ -1642,6 +1655,40 @@ mod tests { } } + /// Fletcher32 before a compressor (libhdf5 applies filters in pipeline + /// order, so the compressor sees chunk + checksum): deflate's output is 4 + /// bytes over the chunk size, which we rejected as "deflate: output + /// exceeds size limit". Chunks from h5py/libhdf5, values from h5py. + #[test] + #[cfg(feature = "deflate")] + fn fletcher32_before_deflate_decodes() { + // h5py: set_fletcher32(); set_deflate(4); i32 0..100, chunks of 10. + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_FLETCHER32, vec![]), + one_filter(FILTER_DEFLATE, vec![4]), + ], + }; + let raw = + unhex("785e936360609007620520560462252056066215205605623520560762c6483e3d00234501f0"); + let want: Vec = (30..40i32).flat_map(i32::to_le_bytes).collect(); + assert_eq!(decompress_chunk(&raw, &pipeline, 40, 4).unwrap(), want); + + // And our own writer's round trip through the same pipeline order. + let data: Vec = (0..400u32).map(|i| (i % 13) as u8).collect(); + let pipeline = FilterPipeline { + version: 2, + filters: vec![ + one_filter(FILTER_SHUFFLE, vec![4]), + one_filter(FILTER_FLETCHER32, vec![]), + one_filter(FILTER_DEFLATE, vec![9]), + ], + }; + let c = compress_chunk(&data, &pipeline, 4).unwrap(); + assert_eq!(decompress_chunk(&c, &pipeline, 400, 4).unwrap(), data); + } + /// `le_data.h5` scale-offset (D-scale, D = 3, fill -2.2) chunks, decoded /// bit for bit as libhdf5 does: single-precision arithmetic for `float` /// (we computed in f64 and rounded once, which was 1 ULP off for e.g. From 6db13c60b857be3e236849cc19c996463e19b1d1 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:17:08 -0500 Subject: [PATCH 32/36] docs: changelog for the filter interop fixes Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1ece78..cbe22bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -179,6 +179,31 @@ `float16` rounding matches numpy's bit for bit on 4 020 probe values, including ties, subnormals and the overflow boundary), and an agent store — `f32` and `float16` — opened by h5py with every dataset decoded. +- `clawhdf5-format` filters, checked against libhdf5 + hdf5plugin: + - **LZ4 (32004) now uses the registered HDF5 LZ4 format** (8-byte BE size, + 4-byte BE block size, BE-length-prefixed blocks). Our old framing (4-byte + LE size + one block) was readable only by clawhdf5, and we could not read + libhdf5's (`h5ex_d_lz4.h5`). Old clawhdf5 LZ4 chunks still read; they are + told apart unambiguously (a registered chunk starts with four zero bytes). + - **Zstd (32015) frames now record the content size**, which libhdf5's zstd + plugin needs; h5py could not read our zstd datasets. + - **Pcodec moved from filter ID 32023 to 480.** 32023 is registered to + Granular BitRound, whose decode is a pass-through — libhdf5 with that + plugin would have returned compressed bytes as data. Pcodec has no + registered ID; 480 is in the registry's private range (256–511) and only + clawhdf5 can read it. Chunks written under 32023 with the filter name + `pcodec` (clawhdf5 ≤ 2.7.0) still read. + - **SZIP decode matches libhdf5.** It returned garbage or zeros with no + error for libhdf5-written files (the 4-byte size prefix, 32/64-bit + byte-plane interleaving, reference interval, scanline padding and byte + order were all handled wrongly) and rejected 64-bit data. + - N-Bit honours libhdf5's "need not compress" flag (multi-filter pipelines + such as `tfilters.h5` failed) and reads enum/no-op members. + - Scale-offset `float` decode uses libhdf5's single-precision arithmetic + (was 1 ULP off for some values). + - A pipeline with Fletcher32 ahead of the compressor (h5py + `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate: + output exceeds size limit". ### Storage - `clawhdf5-format`: **half-precision datasets.** From 7c1968a34a5795cf716481d76a777e342ba55e14 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:17:53 -0500 Subject: [PATCH 33/36] fix(format): resolve shared fill value messages instead of zero-filling dataset_fill_value treated a shared Fill Value message as "no fill value", so unwritten storage of a dataset whose fill value lives in the file's shared-message (SOHM) heap read as zeros rather than its fill value. libhdf5 shares fill values whenever the file has a SOHM index for them. - fill_value::dataset_fill_value_in follows the reference (another object header, or the SOHM heap); read_full_with_fill and the facade's selection read use it. - dataset_fill_value, which has no file to follow a reference into, now returns UnresolvedSharedMessage for a shared message instead of None. - shared_message::load_sohm_table / message_data_with_sohm load the SOHM table from the superblock extension on demand. - parse_sohm_table skipped each index's leading version byte, reading every field one byte off; SOHM references could never resolve. Fixture shared_fill_value.h5 (HDF5 2.0, gen_shared_fill.py): sohm_b read [0,1,2,3,0,0,0,0] and now reads [0,1,2,3,-7,-7,-7,-7], as h5py does. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/fill_value.rs | 49 +++++++++-- crates/clawhdf5-format/src/shared_message.rs | 79 +++++++++++++++++- .../tests/fixtures/gen_shared_fill.py | 49 +++++++++++ .../tests/fixtures/shared_fill_value.h5 | Bin 0 -> 5717 bytes .../tests/writer_meta_tests.rs | 47 +++++++++++ crates/clawhdf5/src/reader.rs | 7 +- crates/clawhdf5/tests/shared_fill_value.rs | 49 +++++++++++ 7 files changed, 268 insertions(+), 12 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/gen_shared_fill.py create mode 100644 crates/clawhdf5-format/tests/fixtures/shared_fill_value.h5 create mode 100644 crates/clawhdf5/tests/shared_fill_value.rs diff --git a/crates/clawhdf5-format/src/fill_value.rs b/crates/clawhdf5-format/src/fill_value.rs index d10ba47..3b18790 100644 --- a/crates/clawhdf5-format/src/fill_value.rs +++ b/crates/clawhdf5-format/src/fill_value.rs @@ -98,15 +98,50 @@ pub fn parse_fill_value(msg: &HeaderMessage) -> Result>, FormatEr /// The fill value that applies to a dataset given its header messages. The new /// message wins over the old one when both are present. +/// +/// A *shared* fill value message holds only a reference to the real message, +/// which cannot be followed without the file: this returns +/// [`FormatError::UnresolvedSharedMessage`] for one (it used to answer "zeros"). +/// Use [`dataset_fill_value_in`] when the file bytes are at hand. pub fn dataset_fill_value(messages: &[HeaderMessage]) -> Result>, FormatError> { + fill_value_from(messages, |_| Err(FormatError::UnresolvedSharedMessage)) +} + +/// [`dataset_fill_value`] for a dataset in `file_data`, following a shared +/// fill value message to where it lives: another object header, or the +/// file's shared-message (SOHM) heap, as libhdf5 writes it when the file has +/// a SOHM index for fill values. +pub fn dataset_fill_value_in( + file_data: &[u8], + messages: &[HeaderMessage], + offset_size: u8, + length_size: u8, +) -> Result>, FormatError> { + fill_value_from(messages, |msg| { + crate::shared_message::message_data_with_sohm(file_data, msg, offset_size, length_size) + .map(|data| data.into_owned()) + }) +} + +fn fill_value_from( + messages: &[HeaderMessage], + resolve_shared: impl Fn(&HeaderMessage) -> Result, FormatError>, +) -> Result>, FormatError> { for wanted in [MessageType::FillValue, MessageType::FillValueOld] { if let Some(msg) = messages.iter().find(|m| m.msg_type == wanted) { - if crate::shared_message::is_shared(msg.flags) { - // A shared fill value is legal but vanishingly rare; treat it - // as the default rather than misparsing the reference. - return Ok(None); - } - if let Some(value) = parse_fill_value(msg)? { + let value = if crate::shared_message::is_shared(msg.flags) { + let data = resolve_shared(msg)?; + parse_fill_value(&HeaderMessage { + msg_type: msg.msg_type, + size: data.len(), + flags: msg.flags & !0x02, + creation_order: msg.creation_order, + data, + })? + } else { + parse_fill_value(msg)? + }; + if let Some(value) = value { return Ok(Some(value)); } } @@ -174,7 +209,7 @@ pub fn read_full_with_fill>( { return Err(FormatError::ExternalDataFilesUnsupported.into()); } - let fill = dataset_fill_value(messages)?; + let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?; if !has_storage(layout) { return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?); } diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 954618d..33334fa 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -225,9 +225,12 @@ pub fn parse_sohm_table_message( /// Parse the SOHM table structure (signature "SMTB") from the file. /// -/// Each index entry: index_type(1) + mesg_types(2) + min_mesg_size(4) + -/// list_max(2) + btree_min(2) + num_messages(2) + index_addr(offset_size) + -/// heap_addr(offset_size) +/// Each index entry: version(1) + index_type(1) + mesg_types(2) + +/// min_mesg_size(4) + list_max(2) + btree_min(2) + num_messages(2) + +/// index_addr(offset_size) + heap_addr(offset_size) +/// +/// The leading per-index version byte (0) was missing here, so every field +/// after it was read one byte off — verified against an HDF5 2.0 file. pub fn parse_sohm_table( file_data: &[u8], table_addr: usize, @@ -240,11 +243,16 @@ pub fn parse_sohm_table( } let mut pos = table_addr + 4; let os = offset_size as usize; - let entry_size = 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 13 + 2*offset_size + let entry_size = 1 + 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 14 + 2*offset_size let mut indexes = Vec::with_capacity(nindexes as usize); for _ in 0..nindexes { ensure_len(file_data, pos, entry_size)?; + let version = file_data[pos]; + if version != 0 { + return Err(FormatError::InvalidSohmTableVersion(version)); + } + pos += 1; let index_type = file_data[pos]; pos += 1; let mesg_types = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); @@ -381,6 +389,68 @@ pub fn parse_sohm_btree_entries( // ---- SOHM resolution ---- /// Find the SOHM index that handles the given message type. +/// Load a file's SOHM table: superblock → superblock extension → Shared +/// Message Table message → SMTB. `Ok(None)` when the file has no superblock +/// extension or no shared-message table. +pub fn load_sohm_table( + file_data: &[u8], + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let sig = crate::signature::find_signature(file_data)?; + let sb = crate::superblock::Superblock::parse(file_data, sig)?; + let Some(ext_addr) = sb + .superblock_extension_address + .filter(|&a| !is_undefined(a, offset_size)) + else { + return Ok(None); + }; + let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?; + let Some(msg) = ext + .messages + .iter() + .find(|m| m.msg_type == MessageType::SharedMessageTable) + else { + return Ok(None); + }; + let table_msg = parse_sohm_table_message(&msg.data, offset_size)?; + parse_sohm_table( + file_data, + table_msg.table_address as usize, + table_msg.nindexes, + offset_size, + ) + .map(Some) +} + +/// Like [`message_data`], but also follows references into the file's SOHM +/// heap (shared object header messages), loading the SOHM table on demand. +pub fn message_data_with_sohm<'a>( + file_data: &[u8], + msg: &'a crate::object_header::HeaderMessage, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + if !is_shared(msg.flags) { + return Ok(Cow::Borrowed(&msg.data)); + } + let shared_ref = parse_shared_ref(&msg.data, offset_size)?; + let table = if shared_ref.heap_id.is_some() { + load_sohm_table(file_data, offset_size, length_size)? + } else { + None + }; + resolve_shared_message_with_sohm( + file_data, + &shared_ref, + msg.msg_type, + offset_size, + length_size, + table.as_ref(), + ) + .map(Cow::Owned) +} + fn find_index_for_msg_type(table: &SohmTable, msg_type: MessageType) -> Option<&SohmIndex> { let type_bit = 1u16 << msg_type.to_u16(); table @@ -707,6 +777,7 @@ mod tests { let mut buf = Vec::new(); buf.extend_from_slice(b"SMTB"); for idx in indexes { + buf.push(0); // version buf.push(idx.index_type); buf.extend_from_slice(&idx.mesg_types.to_le_bytes()); buf.extend_from_slice(&idx.min_mesg_size.to_le_bytes()); diff --git a/crates/clawhdf5-format/tests/fixtures/gen_shared_fill.py b/crates/clawhdf5-format/tests/fixtures/gen_shared_fill.py new file mode 100644 index 0000000..431719c --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/gen_shared_fill.py @@ -0,0 +1,49 @@ +"""Generate shared_fill_value.h5: datasets whose Fill Value message is +*shared*, in the two ways libhdf5 can share one. + +- /sohm_a, /sohm_b: the file has a shared-object-header-message (SOHM) index + for fill values, so libhdf5 stores the fill value (-7, int32) in the SOHM + heap and /sohm_b's header holds only a reference to it. Chunked, with only + the first chunk written, so the rest reads as the fill value. +- /unwritten_a, /unwritten_b: the same, never written: no storage at all, + read entirely as the fill value. + +h5py has no API for SOHM indexes, so the file creation property list is +configured by calling the libhdf5 bundled in the h5py wheel through ctypes. +Written with h5py 3.16.0 / HDF5 2.0.0. Re-run only to regenerate: + + python gen_shared_fill.py shared_fill_value.h5 +""" +import ctypes +import glob +import os +import sys + +import h5py +import numpy as np + +libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs") +libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)] +lib = ctypes.CDLL(libs[0]) +lib.H5open() + +H5O_SHMESG_FILL_FLAG = 1 << 0x0005 + +fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE) +lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint] +lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint] +assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0 +assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, H5O_SHMESG_FILL_FLAG, 0) >= 0 + +fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) +fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST) +fid = h5py.h5f.create(sys.argv[1].encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl) +with h5py.File(fid) as f: + # Chunked, with only the first chunk written: the rest reads as fill. + # libhdf5 keeps the first copy of a message in its own header; the second + # identical one (the `_b` datasets) is the SOHM reference. + for name in ("sohm_a", "sohm_b"): + d = f.create_dataset(name, shape=(8,), chunks=(4,), dtype="ATUAhHdt5c=_)wC~uBtzZ%YDfy2ExR@9i85r0E z8Gsa9aehW_e4-Rk00fW)ll0i2f&w)afStt1z{n`Vz`(@F$OsD`7|qPXz$gIZK>Vt}z{L*d#lWza!#p1>paAjb98ZHV1QQ^(253@ z7M8Jn+K(PL5>V52YQEz4clPjRL$B$Qc!2c~7<64&0cBxOAh&~J8=Ba_en949Xc!e3 z4S~@R7!85Z5Eu=C(GVEYA%NbZgS7$RG`3!@7N`*k?&Sj8AIyxf777n6BsRbpLx(o- zGdV#nl Dataset<'f> { // sparse) dataset — select from a fill-aware full read instead. (The // selection reader currently decodes the full dataset too, so this // costs nothing extra.) - let fill = clawhdf5_format::fill_value::dataset_fill_value(&self.header.messages)?; + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + self.file.data.as_bytes(), + &self.header.messages, + self.file.offset_size(), + self.file.length_size(), + )?; let fill_matters = !clawhdf5_format::fill_value::has_storage(&dl) || (matches!(dl, DataLayout::Chunked { .. }) && !clawhdf5_format::fill_value::is_default(fill.as_deref())); diff --git a/crates/clawhdf5/tests/shared_fill_value.rs b/crates/clawhdf5/tests/shared_fill_value.rs new file mode 100644 index 0000000..9f1c50b --- /dev/null +++ b/crates/clawhdf5/tests/shared_fill_value.rs @@ -0,0 +1,49 @@ +//! Datasets whose Fill Value message is shared through the file's SOHM heap +//! (fixture written by HDF5 2.0, see `gen_shared_fill.py`). Their unwritten +//! storage must read as the fill value (-7), not as zeros. + +use clawhdf5::File; +use clawhdf5_format::selection::Selection; + +const FIXTURE: &[u8] = include_bytes!("../../clawhdf5-format/tests/fixtures/shared_fill_value.h5"); + +#[test] +fn shared_fill_value_applies_to_unwritten_storage() { + let file = File::from_bytes(FIXTURE.to_vec()).unwrap(); + // `_a` keeps its fill value in its own header, `_b` references the SOHM + // heap; both must read the same. + for name in ["sohm_a", "sohm_b"] { + assert_eq!( + file.dataset(name).unwrap().read_i32().unwrap(), + [0, 1, 2, 3, -7, -7, -7, -7], + "{name}" + ); + } + for name in ["unwritten_a", "unwritten_b"] { + assert_eq!( + file.dataset(name).unwrap().read_i32().unwrap(), + [-7, -7, -7], + "{name}" + ); + } + + // The selection path decides on its own whether the fill value matters. + let slab = Selection::Hyperslab { + start: vec![2], + stride: vec![1], + count: vec![4], + block: vec![1], + }; + let raw = file + .dataset("sohm_b") + .unwrap() + .read_selection(&slab) + .unwrap(); + let values: Vec = raw + .as_chunks::<4>() + .0 + .iter() + .map(|b| i32::from_le_bytes(*b)) + .collect(); + assert_eq!(values, [2, 3, -7, -7]); +} From e7f2d8575da92c275dbdd30a0d0c12905e4c34a6 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:18:13 -0500 Subject: [PATCH 34/36] fix(format): import format! for the no_std chunk index planner The maxshape checks added to chunked_write use format!, which a no_std build has to import from alloc (scripts/check-nostd.sh). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 6ff0c07..682a1c6 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -4,7 +4,7 @@ extern crate alloc; #[cfg(not(feature = "std"))] -use alloc::{vec, vec::Vec}; +use alloc::{format, vec, vec::Vec}; use crate::checksum::jenkins_lookup3; use crate::chunk_cache::{CACHE_LINE_SIZE, align_to_cache_line}; From 650f355219a5de3ea8943a4f050c9c2f88d445d1 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:24:50 -0500 Subject: [PATCH 35/36] ci: run the hdf5plugin LZ4/Zstd interop tests The interop step built writer_h5py_tests without the lz4/zstd features, so the hdf5plugin round-trips added with the registered LZ4 framing and the Zstd content-size fix never compiled in CI, and CI never installed hdf5plugin. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitea/workflows/ci.yml | 4 ++-- scripts/ci-test.sh | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7b0b5c3..1a7cdde 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -33,10 +33,10 @@ jobs: # build (pure-Rust zlib-rs) does not need it. apt-get install -y --no-install-recommends python3 python3-venv cmake python3 -m venv /opt/interop - /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray + /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin echo "/opt/interop/bin" >> "$GITHUB_PATH" - name: Show interop library versions - run: /opt/interop/bin/python -c "import h5py, netCDF4; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__)" + run: /opt/interop/bin/python -c "import h5py, netCDF4, hdf5plugin; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__, 'hdf5plugin', hdf5plugin.version)" - name: Run CI script env: # Name the interpreter outright rather than relying on $GITHUB_PATH diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index b52c88b..1d7a7f9 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -145,8 +145,10 @@ run_step "cargo test (fast-deflate / zlib-ng)" cargo test \ # skipping — the tests read the same variable. PYTHON="${CLAWHDF5_PYTHON:-python3}" if "$PYTHON" -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then + # lz4/zstd so the hdf5plugin round-trips (our LZ4 and Zstd output read by + # libhdf5's registered plugins) compile and run too. run_step "h5py interop (format, ignored tests)" cargo test \ - -p clawhdf5-format --test writer_h5py_tests -- --include-ignored + -p clawhdf5-format --features lz4,zstd --test writer_h5py_tests -- --include-ignored else echo "" echo "==> [h5py interop] SKIPPED: no h5py in $PYTHON" From 72b9cfb1e185ab473d701bd86c4a28d707be8a10 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:26:56 -0500 Subject: [PATCH 36/36] docs: record the 2026-09-25 HDF5 audit fixes and open gaps CHANGELOG: upgrade notes (changed read results for max-shape files, saturating conversions, new writer errors, format-crate API changes) and the reader/writer correctness fixes. known-issues: the silent-wrong-data table with before/after sweep numbers, the gaps still open, and a correction to the Extensible Array entry, which said files we wrote were unaffected. CLAUDE.md: clawhdf5-gpu is vector distance computation, not I/O, and clawhdf5-filters holds only deflate backends (no Blosc). Also a facade test that libhdf5's 20-bit N-Bit float test data reads as libhdf5's values. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 79 +++++++++++++++ CLAUDE.md | 8 +- .../tests/numeric_conversion_interop.rs | 26 +++++ docs/known-issues.md | 98 ++++++++++++++++++- 4 files changed, 206 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbe22bb..8b87415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,38 @@ ## Unreleased ### Upgrade Notes +- **HDF5 correctness audit (2026-09-25).** A sweep of 686 public files (the + libhdf5 test files, the HDF Group's CVE reproducers, pyfive, netcdf-c, + netcdf4-python, h5wasm, h5py and xarray corpora), a 567-case read matrix and + a 96-case write matrix against HDF5 1.10–2.0 found bugs that returned wrong + values with no error, and files we wrote that libhdf5 rejects. The fixes are + listed under Correctness and Interop. What changes for callers: + - **Chunked datasets whose max shape is larger than their current shape**, + or whose unlimited dimension is not the first, were indexed by the current + shape instead of the max shape, both when read and when written. Files from + libhdf5 now read correctly. Files clawhdf5 wrote with such a max shape were + laid out wrongly and now read the way libhdf5 always read them — rewrite + them. Agent stores and ClawBrainHub files have no max shape and are + unaffected. + - Integer reads (`read_i32`/`read_i64`/`read_u64`/...) of float data now + convert (truncate toward zero, saturate at the type's range, NaN reads as + 0) instead of returning the IEEE bit pattern, and out-of-range integers + saturate instead of keeping the low bits. + - `FileWriter::finish()` now returns an error instead of writing a corrupt + file for: a header message over 64 KiB (e.g. an attribute larger than + ~64 KiB), a group/dataset/link name that is empty, `.` or contains `/` + (nested paths were written as one literal link), a max shape smaller than + the shape, a page size outside 512 B–1 GiB, and more than 65 535 chunks in + a dataset with several unlimited dimensions. + - **Breaking (format crate):** `ObjectHeaderWriter::serialize`, + `BatchObjectHeaderWriter::compute_sizes`/`serialize_all` and + `build_chunked_data_from_precompressed` return `Result`; + `read_fixed_array_chunks`/`read_extensible_array_chunks` take `max_dims`; + `build_fixed_array_at`/`ea_writer::build_extensible_array_at` take one + `Option` per index slot; `fill_value::dataset_fill_value` + returns `UnresolvedSharedMessage` for a shared message it cannot resolve + instead of `None`. `FillTime::default()` is `IfSet` (libhdf5's default; + default files are byte-identical). - **ZeroClaw does not use clawhdf5.** The project described itself as ZeroClaw's memory backend ("imported as a `clawhdf5` Cargo feature"). Checked against ZeroClaw v0.8.5 (the latest release), the `osobh/zeroclaw` fork and @@ -241,6 +273,53 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` reader — **values returned wrong with no error:** + - Fixed Array and Extensible Array chunk indexes were laid out by the + dataset's current shape instead of its max shape (23 libhdf5 test files, + and any h5py file with e.g. `maxshape=(10, None)` or `(20, 10)` under + `libver='latest'`). + - Files with 4-byte offsets: unfiltered chunked datasets read as zeros. + Chunk B-tree keys store offsets in 8 bytes whatever the file's offset + size. + - A chunk's filter mask skipped the whole pipeline when any bit was set; + only the flagged filters are skipped now. + - Float data read as an integer returned the bit pattern; narrowing integer + reads kept the low bits; bfloat16 was decoded as IEEE half. Floats are now + decoded from their datatype fields (bf16, FP8 E4M3/E5M2, IEEE half, single + and double). + - `vl_data::read_vl_bytes` truncated sequences of non-byte base types. + - A shared fill-value message read as zero fill; it is resolved now, + including from the file's shared-message (SOHM) table, which could never + resolve because its index version byte was skipped. + - Two threads reading two chunked datasets through one `File` could get each + other's chunks (the shared chunk cache was switched between datasets + across separate lock acquisitions). The cache is now keyed by dataset. +- `clawhdf5-format` reader — errors on valid files: enum and bool datasets + through the numeric readers; the "don't filter partial edge chunks" layout + flag; Fletcher32 ahead of deflate (NetCDF-4's order). Unknown-message flags + follow libhdf5 (`tbogus.h5`): "fail if unknown" is refused, "fail if unknown + and writing" is ignored by a reader. +- `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** + - Extensible Array (one unlimited dimension): chunks from index 244 on were + written but never indexed and read as 0, by libhdf5 and by us. + - Fixed Array: more than 1 024 chunks gave checksum errors (data blocks + were never paged). + - A finite max shape larger than the shape gave libhdf5 "addr overflow"; an + unlimited dimension that is not the first scrambled the data; several + unlimited dimensions (`(None, None)`) broke the whole file. These now + write the index libhdf5 writes (swizzled Extensible Array, or a B-tree v2 + index for several unlimited dimensions). + - Header messages over 64 KiB (the size field is 16 bits) and compact + datasets at 65 534–65 535 bytes produced corrupt files. + - Reference, Opaque, BitField and Time datatypes were written as empty + messages; they now encode as HDF5 2.0 does. + - `with_page_size` wrote a nonexistent superblock version 4; it now writes + the v3 superblock and File Space Info message libhdf5 writes. + - `FillTime` values were rotated on disk (NEVER was written as ALLOC, and so + on). New `DatasetBuilder::with_fill_value`. + - An empty-string attribute got a zero-size datatype, which made every + attribute on the object unreadable in libhdf5. + - `maxshape` equal to the shape no longer forces chunked layout. - `clawhdf5-format`: **a truncated deflate chunk read back short, with no error.** The deflate filter used flate2's streaming reader, which returns the bytes it has when the input runs out before the end-of-stream marker. It now diff --git a/CLAUDE.md b/CLAUDE.md index 1fa4581..3f8d9c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,7 @@ # clawhdf5 ## Purpose -Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persistence, agent memory storage, and GPU-accelerated I/O. A standalone library. Its one verified consumer is ClawBrainHub (`.brain` files); no agent framework integrates it (OpenClaw and ZeroClaw claims were withdrawn on 2026-09-25 — neither was ever true). +Pure-Rust HDF5 format implementation with HNSW vector search, WAL-backed persistence, agent memory storage, and GPU-accelerated vector search. A standalone library. Its one verified consumer is ClawBrainHub (`.brain` files); no agent framework integrates it (OpenClaw and ZeroClaw claims were withdrawn on 2026-09-25 — neither was ever true). ## Architecture @@ -11,13 +11,13 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F |-------|------| | `clawhdf5-format` | HDF5 binary spec parser (superblock, B-tree, heap) — also holds shared type definitions and physical constants | | `clawhdf5-io` | Read/write implementation | -| `clawhdf5-filters` | Compression filters (gzip, LZ4, Zstd, Blosc) | +| `clawhdf5-filters` | Deflate backends (zlib-rs, zlib-ng, Apple Compression); the HDF5 filter pipeline and the other codecs (LZ4, Zstd, SZIP, N-Bit, scale-offset, pcodec) live in `clawhdf5-format`. No Blosc. | | `clawhdf5-derive` | Proc-macro derive for HDF5-serializable structs | | `clawhdf5` | Main facade crate | | `clawhdf5-netcdf4` | NetCDF-4 compatibility layer | | `clawhdf5-ann` | HNSW approximate nearest-neighbor vector index | | `clawhdf5-agent` | Agent memory, session history, knowledge graph storage | -| `clawhdf5-gpu` | GPU-accelerated I/O via wgpu (hand-written WGSL compute shaders) | +| `clawhdf5-gpu` | GPU vector distance computation via wgpu (hand-written WGSL compute shaders) — not dataset I/O | | `clawhdf5-accel` | CPU SIMD acceleration path | | `clawhdf5-migrate` | SQLite → HDF5 agent-memory migration | | `clawhdf5-android` | Android JNI bindings | @@ -148,7 +148,7 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F Alerts never block a save — drain them with `HDF5Memory::take_anomaly_alerts`. `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). -- GPU-accelerated batch I/O for large dataset processing +- GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Python and Node.js bindings for cross-language use - NetCDF-4 compatibility for scientific data interop diff --git a/crates/clawhdf5/tests/numeric_conversion_interop.rs b/crates/clawhdf5/tests/numeric_conversion_interop.rs index 357a37d..22310ed 100644 --- a/crates/clawhdf5/tests/numeric_conversion_interop.rs +++ b/crates/clawhdf5/tests/numeric_conversion_interop.rs @@ -372,3 +372,29 @@ with h5py.File("{path}", "r") as f: } } } + +/// libhdf5's N-Bit float test data is stored as a 20-bit custom float +/// (`le_data.h5` from the HDF5 test suite). The N-Bit filter restores the +/// file type's bytes; the typed reader must then decode that layout the way +/// libhdf5 converts it (h5py reads 0.3333435, 0.666687, 1, ...). +#[test] +fn nbit_custom_float_decodes_like_libhdf5() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../clawhdf5-format/tests/fixtures/filters/le_data.h5" + ); + let f = File::open(path).unwrap(); + // Exactly representable in the 20-bit type, so exact in f32 and f64. + let expected = [ + 0.333343505859375, + 0.66668701171875, + 1.0, + 1.3333740234375, + 1.6666259765625, + 2.0, + ]; + for name in ["Nbit_float_data_le", "Nbit_float_data_be"] { + let got = f.dataset(name).unwrap().read_f64().unwrap(); + assert_eq!(&got[..6], &expected, "{name}"); + } +} diff --git a/docs/known-issues.md b/docs/known-issues.md index f99dbe6..6dfa369 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,99 @@ deleting it. --- +## Silent wrong data found by the 2026-09-25 HDF5 audit + +**Status:** fixed after v2.7.0 (2026-09-25). **Every release up +to and including v2.7.0 is affected.** + +An audit on tank checked clawhdf5 against libhdf5 in three ways: +- a sweep of 686 public files: the libhdf5 test files, the HDF Group's + `cve_hdf5` reproducers, and the pyfive, netcdf-c, netcdf4-python, h5wasm, + h5py and xarray corpora; +- 567 read cases generated with h5py 3.16 / HDF5 2.0; +- 96 write cases checked with h5py builds linking HDF5 1.10, 1.12, 1.14 and + 2.0, plus h5dump 1.14.6. + +It found these cases where a value came back wrong **without an error**: + +| Area | What happened | Who is affected | +|---|---|---| +| Chunk index (read) | Fixed/Extensible Array indexes laid out by the current shape, not the max shape: chunks returned from the wrong place | any file with a max shape larger than its shape and `libver='latest'` (h5py `maxshape=(10, None)`, `(20, 10)`) | +| Chunk index (write) | Extensible Array chunks from index 244 on never indexed (read as 0); unlimited dimension not first: data scrambled | files we wrote with one unlimited dimension and > 244 chunks, or e.g. `maxshape=(20, None)` | +| 4-byte offsets | unfiltered chunked datasets read as zeros | files created with `sizeof_addr = 4` | +| Filter mask | any skipped filter skipped the whole pipeline | files with partially filtered chunks (optional filters, direct chunk writes) | +| Numeric reads | float read as integer returned the bit pattern; narrowing integer reads kept the low bits; bfloat16 decoded as IEEE half | `read_i32`/`read_i64`/`read_u64` callers on float or wider data; HDF5 2.0 bf16 data | +| SZIP | garbage or zeros | every libhdf5-written SZIP dataset | +| Scale-offset | float values 1 ULP off | libhdf5 D-scale float data | +| Shared fill value | read as zero fill | fill values stored as shared messages | +| VL sequences | `read_vl_bytes` truncated non-byte base types | VL int/float sequences | +| Chunk cache | two threads reading two chunked datasets through one `File` could get each other's chunks | multi-threaded readers, including Python with the GIL released | + +The audit also found files we wrote that libhdf5 **refuses**, now fixed: +- Fixed Array datasets with more than 1 024 chunks. +- Header messages over 64 KiB (large attributes). +- Reference, Opaque, BitField and Time datatypes. +- Files written with `with_page_size`. +- Several unlimited dimensions. +- A finite max shape larger than the shape. +- An empty-string attribute, which broke every attribute on its object. +- `FillTime` codes, which were rotated. + +Our LZ4 and Zstd output could not be read by libhdf5's registered plugins, and +our pcodec filter used Granular BitRound's ID. The details are in +`CHANGELOG.md` under Correctness and Interop. + +Before the fix, 419 of the 686 files read correctly and 43 differed from h5py. +After it, 448 read correctly and 23 differ. Of those 23: +- 17 are N-Bit float files. The probe compares raw file-type bytes; the typed + reader returns libhdf5's values (`nbit_custom_float_decodes_like_libhdf5`). +- 2 are an h5py bug: VL data with a big-endian base type comes back + byte-swapped in h5py, and h5dump agrees with us. +- The rest are object or attribute listing differences. + +There were no panics, hangs or crashes before or after, including on all 147 +CVE and fuzzer files. On some of those files, h5dump 1.14.6 and h5py/HDF5 2.0 +segfault or abort. + +## Gaps found by the 2026-09-25 HDF5 audit (open) + +**Status:** open. These fail with an error; none returns wrong data, except +the VDS item, which is marked. + +- **Layout message versions 1 and 2** (HDF5 1.6-era files): 84 of the 686 + sweep files, `InvalidLayoutVersion`. This is the largest single gap. +- **Virtual datasets:** + - **Wrong data:** unmapped regions read as 0 instead of the fill value. + - `%b` printf-style source names are not expanded. + - Hyperslab selection versions 1 and 2 are refused. +- **Files with a user block:** the base address is not applied. +- **Old-style shared messages (version 1)** read the wrong address. +- **Groups and links:** + - Groups with a user-defined link type (e.g. 187) cannot be listed. + - Dense groups with more than about 22 000 links cannot be listed. + - Soft links are left out of `datasets()`. +- **Dense attributes:** a large attribute stored as a fractal-heap "huge" + object makes every attribute on the object fail. This affects real NetCDF + files (`issue671.nc`). +- **Other readers:** + - VL-string datasets are not readable through `File`. + - Metadata cache images are not supported. + - x87 long double and binary128 are refused. + - N-Bit on 64-bit scale-offset data and some N-Bit parameter layouts fail. +- **Filters:** blosc, blosc2, bitshuffle, bzip2, LZF and zfp are not + implemented. +- **Header checks:** on 12 CVE datasets libhdf5 rejects a corrupt header and + we read data anyway. We need stricter header checks. +- **Writer:** + - Nested groups beyond one level: path-like names are now refused, not + created. + - Dense attribute storage for attributes over 64 KiB. + - Output that HDF5 1.8 can read. + - A B-tree v2 chunk index larger than one leaf, so datasets with several + unlimited dimensions are limited to 65 535 chunks. + +--- + ## Compound datatype message version 5 is not parsed (HDF5 2.0) **Status:** fixed on `main` in `a13ff51` (2026-06-03); **not in the v2.1.0 @@ -226,7 +319,10 @@ block-offset field in the super block, and a page-init bitmap read from the wrong structure. All four are fixed and covered by interop tests against HDF5 2.0 at sizes that cross each boundary, including paged data blocks. -Files written by this crate are unaffected — this was purely a read-path bug. +Files written by this crate were not affected by *this* read bug, but the +writer had its own: it indexed only the first 244 chunks, so later chunks +read back as 0 in libhdf5 and in clawhdf5. See "Silent wrong data found by +the 2026-09-25 HDF5 audit" below. ## Every `f32` dataset we wrote was unreadable by h5py / libhdf5