From d668e45ab54f1db82a8f538119761371ae87f72c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 13:59:18 -0700 Subject: [PATCH] feat(format): read chunked datasets indexed by a version-2 B-tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With libver='latest', a chunked dataset with two or more unlimited dimensions indexes its chunks with a v2 B-tree (layout v4, index type 5). Reading one failed with "unsupported chunked layout version=4, index_type=Some(5)". read_btree_v2_chunks decodes record types 10 (address + scaled offsets) and 11 (address, stored size, filter mask, scaled offsets). The width of the stored-size field is taken from the record size the tree header declares rather than re-deriving the library's formula. Scaled offsets are multiplied back by the chunk dimensions with overflow checks. The chunk-index dispatch existed four times (uncached, cached, sweep and indexed readers). The three copies outside list_chunks now call it, so every read path — and fill-value handling and partial reads — supports every index type from one place. h5py interop test: plain, gzip+shuffle, a 2500-chunk tree with internal nodes, a sparse dataset with a fill value, and a strided hyperslab. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + crates/clawhdf5-format/src/chunked_read.rs | 400 ++++++++------------ crates/clawhdf5/tests/h5py_interop_tests.rs | 66 ++++ docs/known-issues.md | 15 +- 4 files changed, 229 insertions(+), 256 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41950b1..c1eed95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ 2.7 ms, one column 5.2 ms. Results are identical to the full-read path (equivalence-tested over random hyperslabs and point lists, ranks 1-3, contiguous / chunked / deflate). New `read_harness` bench binary. +- **Datasets indexed by a version-2 B-tree now read** (layout v4, chunk index + type 5 — what `libver='latest'` uses for two or more unlimited dimensions; + previously "unsupported chunked layout"). The four copies of the chunk-index + dispatch are now one shared function, so every read path gets it. - **Out-of-range selections are errors.** They used to return data: a hyperslab past an edge came back padded with zeros, and a point whose column was out of range wrapped into the next row and returned that element. Now diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 8ba9332..0dbf6d4 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -362,6 +362,112 @@ pub fn generate_implicit_chunks( } /// Read a chunked dataset, decompressing chunks as needed. +/// B-tree v2 record types used for chunk indexing. +const BT2_CHUNK_UNFILTERED: u8 = 10; +const BT2_CHUNK_FILTERED: u8 = 11; + +/// Chunks indexed by a version-2 B-tree (layout v4, index type 5). +/// +/// Record layouts (all little endian): +/// * type 10, unfiltered: address, then one 8-byte *scaled* offset per +/// dimension (offset / chunk dimension); +/// * type 11, filtered: address, stored chunk size (a variable number of +/// bytes), 4-byte filter mask, then the scaled offsets. +/// +/// The width of the stored-size field depends on the largest possible chunk; +/// rather than re-derive the library's formula it is taken from the record +/// size the tree header declares, which is what actually governs the bytes. +fn read_btree_v2_chunks( + file_data: &[u8], + addr: u64, + chunk_dims: &[usize], + elem_size: usize, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; + + let bad = |what: &str| FormatError::ChunkedReadError(format!("B-tree v2 chunk index: {what}")); + let header = BTreeV2Header::parse(file_data, addr as usize, offset_size, length_size)?; + let rank = chunk_dims.len(); + let os = offset_size as usize; + let record_size = header.record_size as usize; + let size_len = match header.tree_type { + BT2_CHUNK_UNFILTERED => { + if record_size != os + 8 * rank { + return Err(bad("unexpected record size for unfiltered chunks")); + } + 0 + } + BT2_CHUNK_FILTERED => { + let fixed = os + 4 + 8 * rank; + let size_len = record_size + .checked_sub(fixed) + .ok_or_else(|| bad("record too small"))?; + if !(1..=8).contains(&size_len) { + return Err(bad("implausible chunk-size field width")); + } + size_len + } + _ => return Err(bad("tree is not a chunk index")), + }; + let unfiltered_bytes = checked_chunk_byte_len(chunk_dims, elem_size)?; + let unfiltered_bytes = + u32::try_from(unfiltered_bytes).map_err(|_| bad("chunk larger than 4 GiB"))?; + + let records = collect_btree_v2_records(file_data, &header, offset_size, length_size)?; + let mut chunks = Vec::with_capacity(records.len()); + for record in &records { + let data = record.data.as_slice(); + if data.len() < record_size { + return Err(bad("truncated record")); + } + let address = read_offset(data, 0, offset_size)?; + let mut pos = os; + let (chunk_size, filter_mask) = if size_len == 0 { + (unfiltered_bytes, 0) + } else { + let mut size = 0u64; + for (i, &b) in data[pos..pos + size_len].iter().enumerate() { + size |= u64::from(b) << (8 * i); + } + pos += size_len; + let mask = u32::from_le_bytes([data[pos], data[pos + 1], data[pos + 2], data[pos + 3]]); + pos += 4; + ( + u32::try_from(size).map_err(|_| bad("stored chunk larger than 4 GiB"))?, + mask, + ) + }; + let mut offsets = Vec::with_capacity(rank); + for &dim in chunk_dims { + let scaled = u64::from_le_bytes([ + data[pos], + data[pos + 1], + data[pos + 2], + data[pos + 3], + data[pos + 4], + data[pos + 5], + data[pos + 6], + data[pos + 7], + ]); + pos += 8; + offsets.push( + scaled + .checked_mul(dim as u64) + .ok_or_else(|| bad("chunk offset overflows"))?, + ); + } + chunks.push(ChunkInfo { + chunk_size, + filter_mask, + offsets, + address, + }); + } + Ok(chunks) +} + /// Every allocated chunk of a chunked dataset, for any supported chunk index, /// plus the spatial chunk dimensions. Chunks the file never allocated (sparse /// datasets) are simply absent from the list. @@ -487,6 +593,18 @@ pub fn list_chunks( length_size, )? } + (4, Some(5)) => { + // Version-2 B-tree: what the library uses for a dataset with two + // or more unlimited dimensions. + read_btree_v2_chunks( + file_data, + addr, + &chunk_dims, + elem_size, + offset_size, + length_size, + )? + } (v, idx) => { return Err(FormatError::ChunkedReadError(format!( "unsupported chunked layout version={v}, index_type={idx:?}" @@ -629,29 +747,12 @@ pub fn read_chunked_data_cached( length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { - let ( - chunk_dimensions, - version, - chunk_index_type, - addr_opt, - single_filtered_size, - single_filter_mask, - ) = match layout { + let (chunk_dimensions, addr_opt) = match layout { DataLayout::Chunked { chunk_dimensions, btree_address, - version, - chunk_index_type, - single_chunk_filtered_size, - single_chunk_filter_mask, - } => ( - chunk_dimensions, - *version, - *chunk_index_type, - *btree_address, - *single_chunk_filtered_size, - *single_chunk_filter_mask, - ), + .. + } => (chunk_dimensions, *btree_address), _ => { return Err(FormatError::ChunkedReadError( "expected chunked layout".into(), @@ -688,69 +789,14 @@ pub fn read_chunked_data_cached( // Populate chunk index on first access if !cache.has_index() { - let chunks = match (version, chunk_index_type) { - (3, _) => collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?, - (4, Some(1)) => { - let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?; - let (csize, fmask) = if let Some(fs) = single_filtered_size { - (fs as u32, single_filter_mask.unwrap_or(0)) - } else { - (chunk_byte_size as u32, 0) - }; - vec![ChunkInfo { - chunk_size: csize, - filter_mask: fmask, - offsets: vec![0u64; rank], - address: addr, - }] - } - (4, Some(2)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - generate_implicit_chunks( - addr, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - ) - } - (4, Some(3)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; - read_fixed_array_chunks( - file_data, - &header, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - offset_size, - length_size, - )? - } - (4, Some(4)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = ExtensibleArrayHeader::parse( - file_data, - addr as usize, - offset_size, - length_size, - )?; - read_extensible_array_chunks( - file_data, - &header, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - offset_size, - length_size, - )? - } - (v, idx) => { - return Err(FormatError::ChunkedReadError(format!( - "unsupported chunked layout version={v}, index_type={idx:?}" - ))); - } - }; + let (chunks, _) = list_chunks( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + )?; cache.populate_index(&chunks, rank); } @@ -985,29 +1031,12 @@ pub fn read_chunked_data_sweep( cache: &ChunkCache, sweep: &mut SweepContext, ) -> Result, FormatError> { - let ( - chunk_dimensions, - version, - chunk_index_type, - addr_opt, - single_filtered_size, - single_filter_mask, - ) = match layout { + let (chunk_dimensions, addr_opt) = match layout { DataLayout::Chunked { chunk_dimensions, btree_address, - version, - chunk_index_type, - single_chunk_filtered_size, - single_chunk_filter_mask, - } => ( - chunk_dimensions, - *version, - *chunk_index_type, - *btree_address, - *single_chunk_filtered_size, - *single_chunk_filter_mask, - ), + .. + } => (chunk_dimensions, *btree_address), _ => { return Err(FormatError::ChunkedReadError( "expected chunked layout".into(), @@ -1044,69 +1073,14 @@ pub fn read_chunked_data_sweep( // Populate chunk index on first access if !cache.has_index() { - let chunks = match (version, chunk_index_type) { - (3, _) => collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?, - (4, Some(1)) => { - let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?; - let (csize, fmask) = if let Some(fs) = single_filtered_size { - (fs as u32, single_filter_mask.unwrap_or(0)) - } else { - (chunk_byte_size as u32, 0) - }; - vec![ChunkInfo { - chunk_size: csize, - filter_mask: fmask, - offsets: vec![0u64; rank], - address: addr, - }] - } - (4, Some(2)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - generate_implicit_chunks( - addr, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - ) - } - (4, Some(3)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; - read_fixed_array_chunks( - file_data, - &header, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - offset_size, - length_size, - )? - } - (4, Some(4)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = ExtensibleArrayHeader::parse( - file_data, - addr as usize, - offset_size, - length_size, - )?; - read_extensible_array_chunks( - file_data, - &header, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - offset_size, - length_size, - )? - } - (v, idx) => { - return Err(FormatError::ChunkedReadError(format!( - "unsupported chunked layout version={v}, index_type={idx:?}" - ))); - } - }; + let (chunks, _) = list_chunks( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + )?; cache.populate_index(&chunks, rank); } @@ -1211,29 +1185,12 @@ pub fn read_chunked_data_indexed( length_size: u8, cache: &ChunkCache, ) -> Result, FormatError> { - let ( - chunk_dimensions, - version, - chunk_index_type, - addr_opt, - single_filtered_size, - single_filter_mask, - ) = match layout { + let (chunk_dimensions, addr_opt) = match layout { DataLayout::Chunked { chunk_dimensions, btree_address, - version, - chunk_index_type, - single_chunk_filtered_size, - single_chunk_filter_mask, - } => ( - chunk_dimensions, - *version, - *chunk_index_type, - *btree_address, - *single_chunk_filtered_size, - *single_chunk_filter_mask, - ), + .. + } => (chunk_dimensions, *btree_address), _ => { return Err(FormatError::ChunkedReadError( "expected chunked layout".into(), @@ -1270,69 +1227,14 @@ pub fn read_chunked_data_indexed( // Build chunk index on first access if !cache.has_chunk_index() { - let chunks = match (version, chunk_index_type) { - (3, _) => collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?, - (4, Some(1)) => { - let chunk_byte_size = checked_chunk_byte_len(&chunk_dims, elem_size)?; - let (csize, fmask) = if let Some(fs) = single_filtered_size { - (fs as u32, single_filter_mask.unwrap_or(0)) - } else { - (chunk_byte_size as u32, 0) - }; - vec![ChunkInfo { - chunk_size: csize, - filter_mask: fmask, - offsets: vec![0u64; rank], - address: addr, - }] - } - (4, Some(2)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - generate_implicit_chunks( - addr, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - ) - } - (4, Some(3)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = - FixedArrayHeader::parse(file_data, addr as usize, offset_size, length_size)?; - read_fixed_array_chunks( - file_data, - &header, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - offset_size, - length_size, - )? - } - (4, Some(4)) => { - let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - let header = ExtensibleArrayHeader::parse( - file_data, - addr as usize, - offset_size, - length_size, - )?; - read_extensible_array_chunks( - file_data, - &header, - &dataspace.dimensions, - spatial_chunk_dims, - elem_size as u32, - offset_size, - length_size, - )? - } - (v, idx) => { - return Err(FormatError::ChunkedReadError(format!( - "unsupported chunked layout version={v}, index_type={idx:?}" - ))); - } - }; + let (chunks, _) = list_chunks( + file_data, + layout, + dataspace, + elem_size, + offset_size, + length_size, + )?; cache.populate_chunk_index(&chunks, rank); // Also populate the legacy index for compatibility if !cache.has_index() { diff --git a/crates/clawhdf5/tests/h5py_interop_tests.rs b/crates/clawhdf5/tests/h5py_interop_tests.rs index 2402a07..b5692d3 100644 --- a/crates/clawhdf5/tests/h5py_interop_tests.rs +++ b/crates/clawhdf5/tests/h5py_interop_tests.rs @@ -913,3 +913,69 @@ with h5py.File("{dst_str}", "r") as f: "[(1, 2.5), (3, 4.5)] ('a', 'b') [18446744073709551615, 0, 9223372036854775808] uint64" ); } + +// --------------------------------------------------------------------------- +// h5py writes datasets indexed by a version-2 B-tree -> clawhdf5 reads +// --------------------------------------------------------------------------- + +/// With `libver='latest'`, a chunked dataset with two or more unlimited +/// dimensions indexes its chunks with a version-2 B-tree (layout v4, index +/// type 5). These used to fail with "unsupported chunked layout". +#[test] +fn h5py_btree_v2_chunk_index_clawhdf5_reads() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("bt2.h5"); + let path_str = path.display().to_string(); + let script = format!( + r#" +import h5py, numpy as np +with h5py.File("{path_str}", "w", libver="latest") as f: + a = np.arange(60 * 45, dtype="> = out + .lines() + .map(|l| { + let (name, list) = l.split_once(' ').unwrap(); + (name, parse_int_list(list)) + }) + .collect(); + + let file = File::open(&path).unwrap(); + let small: Vec = (0..60 * 45).collect(); + assert_eq!(file.dataset("plain").unwrap().read_i32().unwrap(), small); + assert_eq!(file.dataset("gz").unwrap().read_i32().unwrap(), small); + let deep: Vec = (0..200 * 200).collect(); + assert_eq!(file.dataset("deep").unwrap().read_i32().unwrap(), deep); + assert_eq!( + file.dataset("sparse").unwrap().read_i32().unwrap(), + expected["sparse"] + ); + // Partial read through the same index: rows 37,50,..,128 x cols 5,36,..,160. + let slab = clawhdf5_format::selection::Selection::Hyperslab { + start: vec![37, 5], + stride: vec![13, 31], + count: vec![8, 6], + block: vec![1, 1], + }; + assert_eq!( + file.dataset("deep") + .unwrap() + .read_i32_selection(&slab) + .unwrap(), + expected["slab"] + ); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 5eecaac..d3c8993 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -116,16 +116,17 @@ not reported). ## B-tree v2 chunk index (layout v4, index type 5) is not supported -**Status:** open. +**Status:** fixed 2026-09-19. **Summary:** a chunked dataset with **two or more unlimited dimensions** written -with `libver='latest'` indexes its chunks with a version-2 B-tree. Reading it -fails with `ChunkedReadError("unsupported chunked layout version=4, -index_type=Some(5)")`. Single-chunk, implicit, fixed-array and -extensible-array indexes (and the v3 B-tree v1) are supported. +with `libver='latest'` indexes its chunks with a version-2 B-tree, and reading it +failed with `unsupported chunked layout version=4, index_type=Some(5)`. -**Repro:** `f.create_dataset("d", shape=(5, 7), chunks=(2, 3), maxshape=(None, None))` -with `h5py.File(..., libver='latest')`. +**Fix:** record types 10 (unfiltered) and 11 (filtered) are decoded — address, +stored size, filter mask, scaled offsets — through the shared chunk-listing +function, so full reads, cached reads, partial reads and fill-value handling +all work. Covered by an h5py interop test (plain, gzip+shuffle, a 2500-chunk +tree with internal nodes, a sparse dataset with a fill value, a hyperslab). ## External links and external raw data are not followed