quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(format): validate chunk dimensions and chunk index offsets like libhdf5

Browse Source 
A chunk dimension of 0 read a dataset as all fill values, 0x80000000 made
an 8 GiB chunk, and a chunk dimension the chunk index's offsets are not
multiples of read chunks at the wrong place (cve-2018-11205). libhdf5
refuses all of these; now so does clawhdf5:

- DataLayout::parse (H5O__layout_decode): no chunk dimension 0 ("bad chunk
  dimension value"), at most 33 dimensions, and before layout v4 at least
  2 ("bad dimensions for chunked storage"). New
  FormatError::InvalidChunkDimensions.
- Reading a chunked dataset (H5D__chunk_init / H5D__chunk_set_sizes): the
  chunk rank must match the dataspace's and a chunk must be under 4 GiB.
  One chunked_read::chunk_geometry replaces the four copies of the rank
  check.
- v1 B-tree chunk index (H5D__btree_decode_key): every key's offsets must
  be multiples of the chunk dimensions, including the keys that only bound
  a node, which is where cve-2018-11205's bad dimension shows. New
  chunked_read::collect_chunk_info_checked; the chunked read and selection
  paths use it.

New interop test header_validation_interop.rs: h5py writes chunked files
(layout v3 and v4), the script corrupts the chunk dimension, and
clawhdf5 must read exactly the copies h5py reads.

Conformance (cached corpus, tank): 570 ok, unchanged; cve-2018-11205 now
refuses the dataset h5py refuses; six more objects that already failed now
fail with libhdf5's reason.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 00:21:58 -05:00
co-authored by Claude Opus 5.5
parent a5ca970015
commit e73ac2af09
5 changed files with 455 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_read.rs
+205 -83
View File
@@ -148,6 +148,46 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
})
}
/// The spatial chunk dimensions of a chunked layout (`chunk_dimensions` is
/// the layout message's list: one per dataspace dimension, then the element
/// size), after the checks libhdf5 makes when it opens a chunked dataset
/// (`H5D__chunk_init` / `H5D__chunk_set_sizes`): the chunk rank must match
/// the dataspace's, no chunk dimension may be 0, and a chunk may not be 4 GiB
/// or more (a v1 B-tree records chunk sizes in 32 bits; libhdf5 before
/// layout version 5 refuses larger ones). A zero chunk dimension used to read
/// as all fill values, and a huge one to hang the reader.
pub(crate) fn chunk_geometry(
chunk_dimensions: &[u32],
dataspace: &Dataspace,
elem_size: usize,
) -> Result<(usize, Vec<usize>), FormatError> {
let rank = chunk_dimensions.len().checked_sub(1).ok_or_else(|| {
FormatError::InvalidChunkDimensions("chunked layout has no dimensions".into())
})?;
if dataspace.dimensions.len() != rank {
return Err(FormatError::InvalidChunkDimensions(format!(
"dimensionality of chunks doesn't match the dataspace (chunk rank {rank}, \
dataspace rank {})",
dataspace.dimensions.len()
)));
}
let spatial = &chunk_dimensions[..rank];
if let Some(d) = spatial.iter().position(|&c| c == 0) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be > 0, dim = {d}"
)));
}
let bytes = spatial
.iter()
.fold(elem_size as u128, |acc, &c| acc * u128::from(c));
if bytes > u128::from(u32::MAX) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be < 4GB (chunk {spatial:?} of {elem_size}-byte elements)"
)));
}
Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
}
/// Product of chunk dimensions times the element size, overflow-checked.
pub(crate) fn checked_chunk_byte_len(
chunk_dims: &[usize],
@@ -222,7 +262,76 @@ pub fn collect_chunk_info(
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0)
collect_chunk_info_inner(
file_data,
btree_address,
ndims,
None,
offset_size,
length_size,
0,
)
}
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
/// layout message's list, element size last), checking every key of the
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
/// must be a multiple of its chunk dimension. That includes the keys that
/// only bound a node (internal-node keys and each node's final key), which
/// is where a corrupt chunk dimension shows when the chunks themselves all
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
/// chunks were read at the wrong place, or the dataset read as fill values.
pub fn collect_chunk_info_checked(
file_data: &[u8],
btree_address: u64,
chunk_dimensions: &[u32],
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(
file_data,
btree_address,
chunk_dimensions.len(),
Some(chunk_dimensions),
offset_size,
length_size,
0,
)
}
/// Check one v1 B-tree chunk key's offsets (see
/// [`collect_chunk_info_checked`]).
fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), FormatError> {
for (&offset, &dim) in offsets.iter().zip(chunk_dimensions) {
if dim == 0 || offset % u64::from(dim) != 0 {
return Err(FormatError::ChunkedReadError(format!(
"bad coordinate offset {offsets:?} for chunk dimensions {chunk_dimensions:?}"
)));
}
}
Ok(())
}
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
/// chunk size and filter mask) and check them when `chunk_dimensions` is
/// given.
fn read_key_offsets(
file_data: &[u8],
pos: usize,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
) -> Result<Vec<u64>, FormatError> {
let mut offsets = Vec::with_capacity(ndims);
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
if let Some(dims) = chunk_dimensions {
check_key_offsets(&offsets, dims)?;
}
Ok(offsets)
}
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
@@ -237,6 +346,7 @@ fn collect_chunk_info_inner(
file_data: &[u8],
btree_address: u64,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
offset_size: u8,
_length_size: u8,
depth: usize,
@@ -296,12 +406,7 @@ fn collect_chunk_info_inner(
file_data[pos + 6],
file_data[pos + 7],
]);
let mut offsets = Vec::with_capacity(ndims);
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
// Parse child address
@@ -315,7 +420,8 @@ fn collect_chunk_info_inner(
address,
});
}
// Skip final key
// The final key only bounds the node; libhdf5 still checks it.
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
Ok(chunks)
} else {
// Internal node: recurse into children
@@ -324,11 +430,13 @@ fn collect_chunk_info_inner(
let mut child_addrs = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
pos += key_size; // skip key
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
let child_addr = read_offset(file_data, pos, offset_size)?;
child_addrs.push(child_addr);
pos += os;
}
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
let mut all_chunks = Vec::new();
for child_addr in child_addrs {
@@ -336,6 +444,7 @@ fn collect_chunk_info_inner(
file_data,
child_addr,
ndims,
chunk_dimensions,
offset_size,
_length_size,
depth + 1,
@@ -549,30 +658,13 @@ pub fn list_chunks(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
// Both v3 and v4 include element size as last dim (rank+1)
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// Collect chunks based on version and index type
let mut chunks = match (version, chunk_index_type) {
(3, _) => {
let ndims = chunk_dimensions.len(); // rank+1
collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?
collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)?
}
(4, Some(1)) => {
// Single chunk — one chunk covering the entire dataset
@@ -819,24 +911,8 @@ pub fn read_chunked_data_cached(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// The per-file cache is shared across datasets (and threads); every
// lookup is keyed by this dataset's chunk-index address, so another
@@ -1140,24 +1216,8 @@ pub fn read_chunked_data_sweep(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// The per-file cache is shared across datasets (and threads); every
// lookup is keyed by this dataset's chunk-index address, so another
@@ -1292,24 +1352,8 @@ pub fn read_chunked_data_indexed(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// Chunk index and assembly plan for this dataset, built on first access
// and kept per dataset (keyed by chunk-index address) in the shared cache.
@@ -1620,11 +1664,12 @@ mod tests {
write_offset(&mut buf, chunk.address, offset_size);
}
// Final key (dummy)
// Final key (its offsets must be on the chunk grid, as libhdf5
// checks; 0 always is)
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
for _ in 0..ndims {
write_offset(&mut buf, u64::MAX, 8);
write_offset(&mut buf, 0, 8);
}
buf
@@ -1632,6 +1677,48 @@ mod tests {
// --- ChunkInfo collection tests ---
#[test]
fn checked_collection_refuses_keys_off_the_chunk_grid() {
let chunk = |offsets: Vec<u64>, address| ChunkInfo {
chunk_size: 80,
filter_mask: 0,
offsets,
address,
};
let good =
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
assert_eq!(
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
.unwrap()
.len(),
2
);
// A chunk key off the grid.
let bad =
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![7, 0], 0x200)], 2, 8);
assert!(collect_chunk_info(&bad, 0, 2, 8, 8).is_ok());
assert!(matches!(
collect_chunk_info_checked(&bad, 0, &[10, 8], 8, 8),
Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset")
));
// cve-2018-11205: the chunks all start at 0 in dimension 1, and only
// the node's final key shows the chunk dimension is wrong.
let mut two_d = build_chunk_btree_leaf(
&[chunk(vec![0, 0, 0], 0x100), chunk(vec![10, 0, 0], 0x200)],
3,
8,
);
// Final key: (20, 20, 0), the end of a 20 x 20 dataset.
let final_key = two_d.len() - 24;
two_d[final_key..final_key + 8].copy_from_slice(&20u64.to_le_bytes());
two_d[final_key + 8..final_key + 16].copy_from_slice(&20u64.to_le_bytes());
assert!(collect_chunk_info_checked(&two_d, 0, &[10, 20, 4], 8, 8).is_ok());
assert!(matches!(
collect_chunk_info_checked(&two_d, 0, &[10, 32788, 4], 8, 8),
Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset [20, 20, 0]")
));
}
#[test]
fn collect_two_chunks_from_leaf() {
let ndims = 2; // rank+1 for 1D dataset
@@ -1750,6 +1837,41 @@ mod tests {
use crate::dataspace::{Dataspace, DataspaceType};
use crate::datatype::{Datatype, DatatypeByteOrder};
#[test]
fn chunk_geometry_matches_libhdf5_open_checks() {
let space = |dims: &[u64]| Dataspace {
space_type: DataspaceType::Simple,
rank: dims.len() as u8,
dimensions: dims.to_vec(),
max_dimensions: None,
};
assert_eq!(
chunk_geometry(&[4, 5, 8], &space(&[10, 10]), 8).unwrap(),
(2, vec![4, 5])
);
// Rank mismatch.
assert!(matches!(
chunk_geometry(&[4, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
));
// Zero dimension (a layout built in memory, bypassing the parser).
assert!(matches!(
chunk_geometry(&[4, 0, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
));
// 0x80000000 x 4-byte elements is 8 GiB; the largest allowed chunk
// is 4 GiB - 1 bytes. These dims used to hang the reader.
assert!(matches!(
chunk_geometry(&[0x8000_0000, 4], &space(&[10]), 4),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(matches!(
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], &space(&[10, 10]), 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], &space(&[10]), 1).is_ok());
}
fn make_f64_type() -> Datatype {
Datatype::FloatingPoint {
size: 8,
@@ -1863,8 +1985,8 @@ mod tests {
let file_data = vec![0u8; 64];
let result = read_chunked_data(&file_data, &layout, &dataspace, &datatype, None, 8, 8);
assert!(
matches!(result, Err(FormatError::ChunkedReadError(_))),
"expected a clean ChunkedReadError, got {result:?}"
matches!(result, Err(FormatError::InvalidChunkDimensions(_))),
"expected a clean InvalidChunkDimensions, got {result:?}"
);
}