quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

fix(format): limit chunks to 4 GiB only under a v1 B-tree index

Browse Source 
libhdf5 refuses a chunk of 4 GiB or more only when a version-1 B-tree
indexes it (H5D__chunk_init: "chunk size must be < 4GB with v1 b-tree
index"). HDF5 2.0 writes larger chunks with layout version 5, and h5py
reads them; these were refused. chunk_geometry now takes the layout
version and applies the limit to layout version 3 and earlier only.

The interop test is ignored by default: h5py writes a 4 GiB chunk and
both libraries hold it in memory.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 01:17:36 -05:00
co-authored by Claude Opus 5.5
parent 6a8ee3ec7f
commit a14ccc36bf
5 changed files with 104 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3 -1
View File
@@ -323,7 +323,9 @@
VAX byte order; libhdf5 ignores it before version 3, and so does this.
- chunked layouts (`FormatError::InvalidChunkDimensions`): a zero chunk
dimension, a chunk rank that does not match the dataspace, a chunk of
4 GiB or more (0x80000000-sized chunks hung the reader), and v1 B-tree
4 GiB or more indexed by a v1 B-tree (layout version 3 or earlier;
0x80000000-sized chunks hung the reader — layout versions 4 and 5 allow
larger chunks, and HDF5 2.0 writes them), and v1 B-tree
chunk keys whose offsets are not multiples of the chunk dimensions,
including the keys that only bound a node
(`chunked_read::collect_chunk_info_checked`).
crates/clawhdf5-format/src/chunked_read.rs
+55 -39
View File
@@ -152,12 +152,15 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
/// the layout message's list: one per dataspace dimension, then the element
/// size), after the checks libhdf5 makes when it opens a chunked dataset
/// (`H5D__chunk_init` / `H5D__chunk_set_sizes`): the chunk rank must match
/// the dataspace's, no chunk dimension may be 0, and a chunk may not be 4 GiB
/// or more (a v1 B-tree records chunk sizes in 32 bits; libhdf5 before
/// layout version 5 refuses larger ones). A zero chunk dimension used to read
/// as all fill values, and a huge one to hang the reader.
/// the dataspace's, no chunk dimension may be 0, and a chunk indexed by a
/// version-1 B-tree (`layout_version` below 4) may not be 4 GiB or more (the
/// B-tree records chunk sizes in 32 bits; libhdf5: "chunk size must be < 4GB
/// with v1 b-tree index"). The other chunk indexes allow larger chunks:
/// HDF5 2.0 writes them with layout version 5. A zero chunk dimension used
/// to read as all fill values, and a huge one to hang the reader.
pub(crate) fn chunk_geometry(
chunk_dimensions: &[u32],
layout_version: u8,
dataspace: &Dataspace,
elem_size: usize,
) -> Result<(usize, Vec<usize>), FormatError> {
@@ -180,9 +183,9 @@ pub(crate) fn chunk_geometry(
let bytes = spatial
.iter()
.fold(elem_size as u128, |acc, &c| acc * u128::from(c));
if bytes > u128::from(u32::MAX) {
if layout_version < 4 && bytes > u128::from(u32::MAX) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be < 4GB (chunk {spatial:?} of {elem_size}-byte elements)"
"chunk size must be < 4GB with v1 b-tree index (chunk {spatial:?} of {elem_size}-byte elements)"
)));
}
Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
@@ -658,7 +661,7 @@ pub fn list_chunks(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
// Both v3 and v4 include element size as last dim (rank+1)
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// Collect chunks based on version and index type
@@ -894,12 +897,13 @@ pub fn read_chunked_data_cached(
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
let (chunk_dimensions, addr_opt) = match layout {
let (chunk_dimensions, version, addr_opt) = match layout {
DataLayout::Chunked {
chunk_dimensions,
version,
btree_address,
..
} => (chunk_dimensions, *btree_address),
} => (chunk_dimensions, *version, *btree_address),
_ => {
return Err(FormatError::ChunkedReadError(
"expected chunked layout".into(),
@@ -911,7 +915,7 @@ pub fn read_chunked_data_cached(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// The per-file cache is shared across datasets (and threads); every
@@ -1199,12 +1203,13 @@ pub fn read_chunked_data_sweep(
cache: &ChunkCache,
sweep: &mut SweepContext,
) -> Result<Vec<u8>, FormatError> {
let (chunk_dimensions, addr_opt) = match layout {
let (chunk_dimensions, version, addr_opt) = match layout {
DataLayout::Chunked {
chunk_dimensions,
version,
btree_address,
..
} => (chunk_dimensions, *btree_address),
} => (chunk_dimensions, *version, *btree_address),
_ => {
return Err(FormatError::ChunkedReadError(
"expected chunked layout".into(),
@@ -1216,7 +1221,7 @@ pub fn read_chunked_data_sweep(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// The per-file cache is shared across datasets (and threads); every
@@ -1335,12 +1340,13 @@ pub fn read_chunked_data_indexed(
length_size: u8,
cache: &ChunkCache,
) -> Result<Vec<u8>, FormatError> {
let (chunk_dimensions, addr_opt) = match layout {
let (chunk_dimensions, version, addr_opt) = match layout {
DataLayout::Chunked {
chunk_dimensions,
version,
btree_address,
..
} => (chunk_dimensions, *btree_address),
} => (chunk_dimensions, *version, *btree_address),
_ => {
return Err(FormatError::ChunkedReadError(
"expected chunked layout".into(),
@@ -1352,7 +1358,7 @@ pub fn read_chunked_data_indexed(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, version, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
// Chunk index and assembly plan for this dataset, built on first access
@@ -1845,31 +1851,41 @@ mod tests {
dimensions: dims.to_vec(),
max_dimensions: None,
};
for v in [3, 4] {
assert_eq!(
chunk_geometry(&[4, 5, 8], v, &space(&[10, 10]), 8).unwrap(),
(2, vec![4, 5])
);
// Rank mismatch.
assert!(matches!(
chunk_geometry(&[4, 8], v, &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
));
// Zero dimension (a layout built in memory, bypassing the parser).
assert!(matches!(
chunk_geometry(&[4, 0, 8], v, &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
));
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], v, &space(&[10]), 1).is_ok());
}
// With a v1 B-tree index (layout version 3) the largest chunk is
// 4 GiB - 1 bytes: 0x80000000 x 4-byte elements (8 GiB) is refused.
// These dims used to hang the reader.
assert!(matches!(
chunk_geometry(&[0x8000_0000, 4], 3, &space(&[10]), 4),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB with v1 b-tree")
));
assert!(matches!(
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], 3, &space(&[10, 10]), 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB with v1 b-tree")
));
// The other chunk indexes (layout version 4, and 5, which is read as
// 4) allow chunks of 4 GiB and more; HDF5 2.0 writes them.
assert_eq!(
chunk_geometry(&[4, 5, 8], &space(&[10, 10]), 8).unwrap(),
(2, vec![4, 5])
chunk_geometry(&[0x2000_0001, 8], 4, &space(&[10]), 8).unwrap(),
(1, vec![0x2000_0001])
);
// Rank mismatch.
assert!(matches!(
chunk_geometry(&[4, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
));
// Zero dimension (a layout built in memory, bypassing the parser).
assert!(matches!(
chunk_geometry(&[4, 0, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
));
// 0x80000000 x 4-byte elements is 8 GiB; the largest allowed chunk
// is 4 GiB - 1 bytes. These dims used to hang the reader.
assert!(matches!(
chunk_geometry(&[0x8000_0000, 4], &space(&[10]), 4),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(matches!(
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], &space(&[10, 10]), 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], &space(&[10]), 1).is_ok());
assert!(chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], 4, &space(&[10, 10]), 1).is_ok());
}
fn make_f64_type() -> Datatype {
crates/clawhdf5-format/src/data_read.rs
+1 -1
View File
@@ -360,7 +360,7 @@ pub fn read_raw_data_selection(
chunk_index_type,
..
} => {
crate::chunked_read::chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?;
// For chunked data, only read chunks that intersect the selection
let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let rank = dims.len();
crates/clawhdf5-format/src/error.rs
+1 -1
View File
@@ -212,7 +212,7 @@ pub enum FormatError {
InvalidDatatype(String),
/// A chunked layout whose chunk dimensions libhdf5 refuses: a zero
/// dimension, a rank that does not match the dataspace, or a chunk of
/// 4 GiB or more.
/// 4 GiB or more indexed by a version-1 B-tree.
InvalidChunkDimensions(String),
/// The superblock's end-of-file address lies past the end of the file:
/// the file was truncated (libhdf5 refuses to open it).
crates/clawhdf5/tests/header_validation_interop.rs
+44
View File
@@ -269,3 +269,47 @@ save("float_overlap", bad)
],
);
}
/// Runs a Python script (with `h5py`, `numpy as np` and `struct` imported,
/// `d` the output directory) and fails the test if it fails.
fn run_python(dir: &Path, body: &str) {
let script = format!(
"import h5py, numpy as np, struct, os\nd = \"{}\"\n{body}",
dir.display()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
}
/// libhdf5 limits a chunk to under 4 GiB only when a version-1 B-tree
/// indexes it; HDF5 2.0 writes larger chunks with layout version 5 (libver
/// v200), and h5py reads them. These were refused as "chunk size must be <
/// 4GB". Ignored by default: h5py writes a 4 GiB chunk and both libraries
/// hold it in memory (about 9 GiB in all).
#[test]
#[ignore = "writes and reads a 4 GiB chunk (about 9 GiB of memory)"]
fn chunks_of_4_gib_and_more_read_with_layout_v5() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
with h5py.File(os.path.join(d, "big.h5"), "w", libver=("v200", "v200")) as f:
ds = f.create_dataset("d", shape=(10,), maxshape=(None,), chunks=(2**29 + 1,),
dtype="<f8", compression="gzip", compression_opts=1)
ds[:] = np.arange(10.0)
with h5py.File(os.path.join(d, "big.h5"), "r") as f:
assert list(f["d"][:]) == list(np.arange(10.0))
"#,
);
let file = File::open(dir.path().join("big.h5")).unwrap();
let values = file.dataset("d").unwrap().read_f64().unwrap();
assert_eq!(values, (0..10).map(f64::from).collect::<Vec<_>>());
}