quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): limit chunks to 4 GiB only under a v1 B-tree index

Browse Source 
libhdf5 refuses a chunk of 4 GiB or more only when a version-1 B-tree
indexes it (H5D__chunk_init: "chunk size must be < 4GB with v1 b-tree
index"). HDF5 2.0 writes larger chunks with layout version 5, and h5py
reads them; these were refused. chunk_geometry now takes the layout
version and applies the limit to layout version 3 and earlier only.

The interop test is ignored by default: h5py writes a 4 GiB chunk and
both libraries hold it in memory.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:17:36 -05:00
co-authored by Claude Opus 5.5
parent 6a8ee3ec7f
commit a14ccc36bf
5 changed files with 104 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_read.rs
+55 -39
View File
@@ -152,12 +152,15 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
/// the layout message's list: one per dataspace dimension, then the element
/// size), after the checks libhdf5 makes when it opens a chunked dataset
/// (`H5D__chunk_init` / `H5D__chunk_set_sizes`): the chunk rank must match
/// the dataspace's, no chunk dimension may be 0, and a chunk may not be 4 GiB
/// or more (a v1 B-tree records chunk sizes in 32 bits; libhdf5 before
/// layout version 5 refuses larger ones). A zero chunk dimension used to read
/// as all fill values, and a huge one to hang the reader.
/// the dataspace's, no chunk dimension may be 0, and a chunk indexed by a
/// version-1 B-tree (`layout_version` below 4) may not be 4 GiB or more (the
/// B-tree records chunk sizes in 32 bits; libhdf5: "chunk size must be < 4GB
/// with v1 b-tree index"). The other chunk indexes allow larger chunks:
/// HDF5 2.0 writes them with layout version 5. A zero chunk dimension used
/// to read as all fill values, and a huge one to hang the reader.
pub(crate) fn chunk_geometry(
chunk_dimensions: &[u32],
layout_version: u8,
dataspace: &Dataspace,
elem_size: usize,
) -> Result<(usize, Vec<usize>), FormatError> {
@@ -180,9 +183,9 @@ pub(crate) fn chunk_geometry(
let bytes = spatial
.iter()
.fold(elem_size as u128, |acc, &c| acc * u128::from(c));
if bytes > u128::from(u32::MAX) {
if layout_version < 4 && bytes > u128::from(u32::MAX) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be < 4GB (chunk {spatial:?} of {elem_size}-byte elements)"
"chunk size must be < 4GB with v1 b-tree index (chunk {spatial:?} of {elem_size}-byte elements)"
)));
}
Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
@@ -658,7 +661,7 @@ pub fn list_chunks(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
// Both v3 and v4 include element size as last dim (rank+1)
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// Collect chunks based on version and index type
@@ -894,12 +897,13 @@ pub fn read_chunked_data_cached(
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
let (chunk_dimensions, addr_opt) = match layout {
let (chunk_dimensions, version, addr_opt) = match layout {
DataLayout::Chunked {
chunk_dimensions,
version,
btree_address,
..
} => (chunk_dimensions, *btree_address),
} => (chunk_dimensions, *version, *btree_address),
_ => {
return Err(FormatError::ChunkedReadError(
"expected chunked layout".into(),
@@ -911,7 +915,7 @@ pub fn read_chunked_data_cached(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// The per-file cache is shared across datasets (and threads); every
@@ -1199,12 +1203,13 @@ pub fn read_chunked_data_sweep(
cache: &ChunkCache,
sweep: &mut SweepContext,
) -> Result<Vec<u8>, FormatError> {
let (chunk_dimensions, addr_opt) = match layout {
let (chunk_dimensions, version, addr_opt) = match layout {
DataLayout::Chunked {
chunk_dimensions,
version,
btree_address,
..
} => (chunk_dimensions, *btree_address),
} => (chunk_dimensions, *version, *btree_address),
_ => {
return Err(FormatError::ChunkedReadError(
"expected chunked layout".into(),
@@ -1216,7 +1221,7 @@ pub fn read_chunked_data_sweep(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// The per-file cache is shared across datasets (and threads); every
@@ -1335,12 +1340,13 @@ pub fn read_chunked_data_indexed(
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
let (chunk_dimensions, addr_opt) = match layout {
let (chunk_dimensions, version, addr_opt) = match layout {
DataLayout::Chunked {
chunk_dimensions,
version,
btree_address,
..
} => (chunk_dimensions, *btree_address),
} => (chunk_dimensions, *version, *btree_address),
_ => {
return Err(FormatError::ChunkedReadError(
"expected chunked layout".into(),
@@ -1352,7 +1358,7 @@ pub fn read_chunked_data_indexed(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// Chunk index and assembly plan for this dataset, built on first access
@@ -1845,31 +1851,41 @@ mod tests {
dimensions: dims.to_vec(),
max_dimensions: None,
};
for v in [3, 4] {
assert_eq!(
chunk_geometry(&[4, 5, 8], v, &space(&[10, 10]), 8).unwrap(),
(2, vec![4, 5])
);
// Rank mismatch.
assert!(matches!(
chunk_geometry(&[4, 8], v, &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
));
// Zero dimension (a layout built in memory, bypassing the parser).
assert!(matches!(
chunk_geometry(&[4, 0, 8], v, &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
));
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], v, &space(&[10]), 1).is_ok());
}
// With a v1 B-tree index (layout version 3) the largest chunk is
// 4 GiB - 1 bytes: 0x80000000 x 4-byte elements (8 GiB) is refused.
// These dims used to hang the reader.
assert!(matches!(
chunk_geometry(&[0x8000_0000, 4], 3, &space(&[10]), 4),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB with v1 b-tree")
));
assert!(matches!(
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], 3, &space(&[10, 10]), 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB with v1 b-tree")
));
// The other chunk indexes (layout version 4, and 5, which is read as
// 4) allow chunks of 4 GiB and more; HDF5 2.0 writes them.
assert_eq!(
chunk_geometry(&[4, 5, 8], &space(&[10, 10]), 8).unwrap(),
(2, vec![4, 5])
chunk_geometry(&[0x2000_0001, 8], 4, &space(&[10]), 8).unwrap(),
(1, vec![0x2000_0001])
);
// Rank mismatch.
assert!(matches!(
chunk_geometry(&[4, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
));
// Zero dimension (a layout built in memory, bypassing the parser).
assert!(matches!(
chunk_geometry(&[4, 0, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
));
// 0x80000000 x 4-byte elements is 8 GiB; the largest allowed chunk
// is 4 GiB - 1 bytes. These dims used to hang the reader.
assert!(matches!(
chunk_geometry(&[0x8000_0000, 4], &space(&[10]), 4),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(matches!(
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], &space(&[10, 10]), 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], &space(&[10]), 1).is_ok());
assert!(chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], 4, &space(&[10, 10]), 1).is_ok());
}
fn make_f64_type() -> Datatype {