quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

fix(format): validate chunk dimensions and chunk index offsets like libhdf5

Browse Source 
A chunk dimension of 0 read a dataset as all fill values, 0x80000000 made
an 8 GiB chunk, and a chunk dimension the chunk index's offsets are not
multiples of read chunks at the wrong place (cve-2018-11205). libhdf5
refuses all of these; now so does clawhdf5:

- DataLayout::parse (H5O__layout_decode): no chunk dimension 0 ("bad chunk
  dimension value"), at most 33 dimensions, and before layout v4 at least
  2 ("bad dimensions for chunked storage"). New
  FormatError::InvalidChunkDimensions.
- Reading a chunked dataset (H5D__chunk_init / H5D__chunk_set_sizes): the
  chunk rank must match the dataspace's and a chunk must be under 4 GiB.
  One chunked_read::chunk_geometry replaces the four copies of the rank
  check.
- v1 B-tree chunk index (H5D__btree_decode_key): every key's offsets must
  be multiples of the chunk dimensions, including the keys that only bound
  a node, which is where cve-2018-11205's bad dimension shows. New
  chunked_read::collect_chunk_info_checked; the chunked read and selection
  paths use it.

New interop test header_validation_interop.rs: h5py writes chunked files
(layout v3 and v4), the script corrupts the chunk dimension, and
clawhdf5 must read exactly the copies h5py reads.

Conformance (cached corpus, tank): 570 ok, unchanged; cve-2018-11205 now
refuses the dataset h5py refuses; six more objects that already failed now
fail with libhdf5's reason.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 00:21:58 -05:00
co-authored by Claude Opus 5.5
parent a5ca970015
commit e73ac2af09
5 changed files with 455 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/chunked_read.rs
+205 -83
View File
@@ -148,6 +148,46 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
})
}
/// The spatial chunk dimensions of a chunked layout (`chunk_dimensions` is
/// the layout message's list: one per dataspace dimension, then the element
/// size), after the checks libhdf5 makes when it opens a chunked dataset
/// (`H5D__chunk_init` / `H5D__chunk_set_sizes`): the chunk rank must match
/// the dataspace's, no chunk dimension may be 0, and a chunk may not be 4 GiB
/// or more (a v1 B-tree records chunk sizes in 32 bits; libhdf5 before
/// layout version 5 refuses larger ones). A zero chunk dimension used to read
/// as all fill values, and a huge one to hang the reader.
pub(crate) fn chunk_geometry(
chunk_dimensions: &[u32],
dataspace: &Dataspace,
elem_size: usize,
) -> Result<(usize, Vec<usize>), FormatError> {
let rank = chunk_dimensions.len().checked_sub(1).ok_or_else(|| {
FormatError::InvalidChunkDimensions("chunked layout has no dimensions".into())
})?;
if dataspace.dimensions.len() != rank {
return Err(FormatError::InvalidChunkDimensions(format!(
"dimensionality of chunks doesn't match the dataspace (chunk rank {rank}, \
dataspace rank {})",
dataspace.dimensions.len()
)));
}
let spatial = &chunk_dimensions[..rank];
if let Some(d) = spatial.iter().position(|&c| c == 0) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be > 0, dim = {d}"
)));
}
let bytes = spatial
.iter()
.fold(elem_size as u128, |acc, &c| acc * u128::from(c));
if bytes > u128::from(u32::MAX) {
return Err(FormatError::InvalidChunkDimensions(format!(
"chunk size must be < 4GB (chunk {spatial:?} of {elem_size}-byte elements)"
)));
}
Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
}
/// Product of chunk dimensions times the element size, overflow-checked.
pub(crate) fn checked_chunk_byte_len(
chunk_dims: &[usize],
@@ -222,7 +262,76 @@ pub fn collect_chunk_info(
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0)
collect_chunk_info_inner(
file_data,
btree_address,
ndims,
None,
offset_size,
length_size,
0,
)
}
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
/// layout message's list, element size last), checking every key of the
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
/// must be a multiple of its chunk dimension. That includes the keys that
/// only bound a node (internal-node keys and each node's final key), which
/// is where a corrupt chunk dimension shows when the chunks themselves all
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
/// chunks were read at the wrong place, or the dataset read as fill values.
pub fn collect_chunk_info_checked(
file_data: &[u8],
btree_address: u64,
chunk_dimensions: &[u32],
offset_size: u8,
length_size: u8,
) -> Result<Vec<ChunkInfo>, FormatError> {
collect_chunk_info_inner(
file_data,
btree_address,
chunk_dimensions.len(),
Some(chunk_dimensions),
offset_size,
length_size,
0,
)
}
/// Check one v1 B-tree chunk key's offsets (see
/// [`collect_chunk_info_checked`]).
fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), FormatError> {
for (&offset, &dim) in offsets.iter().zip(chunk_dimensions) {
if dim == 0 || offset % u64::from(dim) != 0 {
return Err(FormatError::ChunkedReadError(format!(
"bad coordinate offset {offsets:?} for chunk dimensions {chunk_dimensions:?}"
)));
}
}
Ok(())
}
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
/// chunk size and filter mask) and check them when `chunk_dimensions` is
/// given.
fn read_key_offsets(
file_data: &[u8],
pos: usize,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
) -> Result<Vec<u64>, FormatError> {
let mut offsets = Vec::with_capacity(ndims);
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
if let Some(dims) = chunk_dimensions {
check_key_offsets(&offsets, dims)?;
}
Ok(offsets)
}
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
@@ -237,6 +346,7 @@ fn collect_chunk_info_inner(
file_data: &[u8],
btree_address: u64,
ndims: usize,
chunk_dimensions: Option<&[u32]>,
offset_size: u8,
_length_size: u8,
depth: usize,
@@ -296,12 +406,7 @@ fn collect_chunk_info_inner(
file_data[pos + 6],
file_data[pos + 7],
]);
let mut offsets = Vec::with_capacity(ndims);
let mut kp = pos + 8;
for _ in 0..ndims {
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
kp += CHUNK_KEY_OFFSET_SIZE as usize;
}
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
// Parse child address
@@ -315,7 +420,8 @@ fn collect_chunk_info_inner(
address,
});
}
// Skip final key
// The final key only bounds the node; libhdf5 still checks it.
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
Ok(chunks)
} else {
// Internal node: recurse into children
@@ -324,11 +430,13 @@ fn collect_chunk_info_inner(
let mut child_addrs = Vec::with_capacity(entries_used);
for _ in 0..entries_used {
pos += key_size; // skip key
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
pos += key_size;
let child_addr = read_offset(file_data, pos, offset_size)?;
child_addrs.push(child_addr);
pos += os;
}
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
let mut all_chunks = Vec::new();
for child_addr in child_addrs {
@@ -336,6 +444,7 @@ fn collect_chunk_info_inner(
file_data,
child_addr,
ndims,
chunk_dimensions,
offset_size,
_length_size,
depth + 1,
@@ -549,30 +658,13 @@ pub fn list_chunks(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
// Both v3 and v4 include element size as last dim (rank+1)
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// Collect chunks based on version and index type
let mut chunks = match (version, chunk_index_type) {
(3, _) => {
let ndims = chunk_dimensions.len(); // rank+1
collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?
collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)?
}
(4, Some(1)) => {
// Single chunk — one chunk covering the entire dataset
@@ -819,24 +911,8 @@ pub fn read_chunked_data_cached(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// The per-file cache is shared across datasets (and threads); every
// lookup is keyed by this dataset's chunk-index address, so another
@@ -1140,24 +1216,8 @@ pub fn read_chunked_data_sweep(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// The per-file cache is shared across datasets (and threads); every
// lookup is keyed by this dataset's chunk-index address, so another
@@ -1292,24 +1352,8 @@ pub fn read_chunked_data_indexed(
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
let elem_size = datatype.type_size() as usize;
let ndims = chunk_dimensions.len();
let rank = ndims
.checked_sub(1)
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
.iter()
.map(|&d| d as usize)
.collect();
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
if ds_dims.len() != rank {
return Err(FormatError::ChunkedReadError(format!(
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
ds_dims.len(),
chunk_dimensions.len(),
rank
)));
}
// Chunk index and assembly plan for this dataset, built on first access
// and kept per dataset (keyed by chunk-index address) in the shared cache.
@@ -1620,11 +1664,12 @@ mod tests {
write_offset(&mut buf, chunk.address, offset_size);
}
// Final key (dummy)
// Final key (its offsets must be on the chunk grid, as libhdf5
// checks; 0 always is)
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
for _ in 0..ndims {
write_offset(&mut buf, u64::MAX, 8);
write_offset(&mut buf, 0, 8);
}
buf
@@ -1632,6 +1677,48 @@ mod tests {
// --- ChunkInfo collection tests ---
#[test]
fn checked_collection_refuses_keys_off_the_chunk_grid() {
let chunk = |offsets: Vec<u64>, address| ChunkInfo {
chunk_size: 80,
filter_mask: 0,
offsets,
address,
};
let good =
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
assert_eq!(
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
.unwrap()
.len(),
2
);
// A chunk key off the grid.
let bad =
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![7, 0], 0x200)], 2, 8);
assert!(collect_chunk_info(&bad, 0, 2, 8, 8).is_ok());
assert!(matches!(
collect_chunk_info_checked(&bad, 0, &[10, 8], 8, 8),
Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset")
));
// cve-2018-11205: the chunks all start at 0 in dimension 1, and only
// the node's final key shows the chunk dimension is wrong.
let mut two_d = build_chunk_btree_leaf(
&[chunk(vec![0, 0, 0], 0x100), chunk(vec![10, 0, 0], 0x200)],
3,
8,
);
// Final key: (20, 20, 0), the end of a 20 x 20 dataset.
let final_key = two_d.len() - 24;
two_d[final_key..final_key + 8].copy_from_slice(&20u64.to_le_bytes());
two_d[final_key + 8..final_key + 16].copy_from_slice(&20u64.to_le_bytes());
assert!(collect_chunk_info_checked(&two_d, 0, &[10, 20, 4], 8, 8).is_ok());
assert!(matches!(
collect_chunk_info_checked(&two_d, 0, &[10, 32788, 4], 8, 8),
Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset [20, 20, 0]")
));
}
#[test]
fn collect_two_chunks_from_leaf() {
let ndims = 2; // rank+1 for 1D dataset
@@ -1750,6 +1837,41 @@ mod tests {
use crate::dataspace::{Dataspace, DataspaceType};
use crate::datatype::{Datatype, DatatypeByteOrder};
#[test]
fn chunk_geometry_matches_libhdf5_open_checks() {
let space = |dims: &[u64]| Dataspace {
space_type: DataspaceType::Simple,
rank: dims.len() as u8,
dimensions: dims.to_vec(),
max_dimensions: None,
};
assert_eq!(
chunk_geometry(&[4, 5, 8], &space(&[10, 10]), 8).unwrap(),
(2, vec![4, 5])
);
// Rank mismatch.
assert!(matches!(
chunk_geometry(&[4, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
));
// Zero dimension (a layout built in memory, bypassing the parser).
assert!(matches!(
chunk_geometry(&[4, 0, 8], &space(&[10, 10]), 8),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
));
// 0x80000000 x 4-byte elements is 8 GiB; the largest allowed chunk
// is 4 GiB - 1 bytes. These dims used to hang the reader.
assert!(matches!(
chunk_geometry(&[0x8000_0000, 4], &space(&[10]), 4),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(matches!(
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], &space(&[10, 10]), 1),
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
));
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], &space(&[10]), 1).is_ok());
}
fn make_f64_type() -> Datatype {
Datatype::FloatingPoint {
size: 8,
@@ -1863,8 +1985,8 @@ mod tests {
let file_data = vec![0u8; 64];
let result = read_chunked_data(&file_data, &layout, &dataspace, &datatype, None, 8, 8);
assert!(
matches!(result, Err(FormatError::ChunkedReadError(_))),
"expected a clean ChunkedReadError, got {result:?}"
matches!(result, Err(FormatError::InvalidChunkDimensions(_))),
"expected a clean InvalidChunkDimensions, got {result:?}"
);
}
crates/clawhdf5-format/src/data_layout.rs
+85 -2
View File
@@ -24,6 +24,34 @@ pub struct VdsMapping {
pub virtual_selection: Vec<u8>,
}
/// Most dimensions a layout message can list (libhdf5 `H5O_LAYOUT_NDIMS`):
/// 32 dataspace dimensions plus the element size.
const MAX_LAYOUT_NDIMS: usize = 33;
/// libhdf5's checks on a chunked layout message's dimensions
/// (`H5O__layout_decode`): at most [`MAX_LAYOUT_NDIMS`], no dimension 0, and
/// before version 4 at least one dataspace dimension plus the element size.
/// A zero chunk dimension used to read the dataset as all fill values.
fn check_chunk_dims(dims: Vec<u32>, layout_version: u8) -> Result<Vec<u32>, FormatError> {
if dims.len() > MAX_LAYOUT_NDIMS {
return Err(FormatError::InvalidChunkDimensions(
"dimensionality is too large".into(),
));
}
if layout_version < 4 && dims.len() < 2 {
return Err(FormatError::InvalidChunkDimensions(
"bad dimensions for chunked storage".into(),
));
}
if let Some(u) = dims.iter().position(|&d| d == 0) {
return Err(FormatError::InvalidChunkDimensions(format!(
"bad chunk dimension value when parsing layout message - chunk dimension must be \
positive: mesg->u.chunk.dim[{u}] = 0"
)));
}
Ok(dims)
}
/// Parsed HDF5 data layout message.
#[derive(Debug, Clone, PartialEq)]
pub enum DataLayout {
@@ -394,7 +422,7 @@ impl DataLayout {
Ok(DataLayout::Contiguous { address, size })
}
_ => Ok(DataLayout::Chunked {
chunk_dimensions: dims,
chunk_dimensions: check_chunk_dims(dims, 2)?,
btree_address: address,
version: 3,
chunk_index_type: None,
@@ -457,7 +485,7 @@ impl DataLayout {
p += 4;
}
Ok(DataLayout::Chunked {
chunk_dimensions,
chunk_dimensions: check_chunk_dims(chunk_dimensions, 3)?,
btree_address,
version: 3,
chunk_index_type: None,
@@ -506,6 +534,11 @@ impl DataLayout {
let dimensionality = data[pos + 1] as usize;
let dim_size_encoded_length = data[pos + 2] as usize;
let mut p = pos + 3;
if dimensionality > MAX_LAYOUT_NDIMS {
return Err(FormatError::InvalidChunkDimensions(
"dimensionality is too large".into(),
));
}
// dimension sizes
ensure_len(data, p, dimensionality * dim_size_encoded_length)?;
@@ -547,6 +580,7 @@ impl DataLayout {
chunk_dimensions.push(val);
p += dim_size_encoded_length;
}
let chunk_dimensions = check_chunk_dims(chunk_dimensions, 4)?;
// chunk index type
ensure_len(data, p, 1)?;
@@ -755,6 +789,55 @@ mod tests {
);
}
/// A v3 chunked layout message with these dims (element size last).
fn v3_chunked_msg(dims: &[u32]) -> Vec<u8> {
let mut buf = vec![3u8, 2, dims.len() as u8];
buf.extend_from_slice(&0x1000u64.to_le_bytes());
for d in dims {
buf.extend_from_slice(&d.to_le_bytes());
}
buf
}
#[test]
fn chunk_dimensions_are_checked_when_the_layout_is_parsed() {
assert!(DataLayout::parse(&v3_chunked_msg(&[4, 4, 8]), 8, 8).is_ok());
// A zero chunk dimension used to read as all fill values.
let err = DataLayout::parse(&v3_chunked_msg(&[4, 0, 8]), 8, 8).unwrap_err();
assert!(
matches!(&err, FormatError::InvalidChunkDimensions(m) if m.contains("dim[1] = 0")),
"{err:?}"
);
// Only the element-size dimension: libhdf5 "bad dimensions".
assert_eq!(
DataLayout::parse(&v3_chunked_msg(&[8]), 8, 8).unwrap_err(),
FormatError::InvalidChunkDimensions("bad dimensions for chunked storage".into())
);
assert_eq!(
DataLayout::parse(&v3_chunked_msg(&[1; 34]), 8, 8).unwrap_err(),
FormatError::InvalidChunkDimensions("dimensionality is too large".into())
);
// v1/v2 and v4 messages get the zero check too.
let mut v1 = v1v2_header(1, 2, 2);
v1.extend_from_slice(&0x1000u64.to_le_bytes());
v1.extend_from_slice(&0u32.to_le_bytes());
v1.extend_from_slice(&8u32.to_le_bytes());
assert!(matches!(
DataLayout::parse(&v1, 8, 8),
Err(FormatError::InvalidChunkDimensions(_))
));
let mut v4 = vec![4u8, 2, 0, 2, 4];
v4.extend_from_slice(&0u32.to_le_bytes());
v4.extend_from_slice(&8u32.to_le_bytes());
v4.push(3); // fixed array index
v4.push(0); // page bits
v4.extend_from_slice(&0x1000u64.to_le_bytes());
assert!(matches!(
DataLayout::parse(&v4, 8, 8),
Err(FormatError::InvalidChunkDimensions(_))
));
}
#[test]
fn v1v2_rejects_bad_class_dimensionality_and_truncation() {
assert_eq!(
crates/clawhdf5-format/src/data_read.rs
+3 -2
View File
@@ -360,6 +360,7 @@ pub fn read_raw_data_selection(
chunk_index_type,
..
} => {
crate::chunked_read::chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
// For chunked data, only read chunks that intersect the selection
let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
let rank = dims.len();
@@ -400,10 +401,10 @@ pub fn read_raw_data_selection(
} else {
// v3: B-tree v1
if let Some(addr) = btree_address {
crate::chunked_read::collect_chunk_info(
crate::chunked_read::collect_chunk_info_checked(
file_data,
*addr,
rank + 1,
chunk_dimensions,
offset_size,
length_size,
)?
crates/clawhdf5-format/src/error.rs
+7
View File
@@ -210,6 +210,10 @@ pub enum FormatError {
/// libhdf5's own error text): size 0, bit fields outside the type,
/// an empty enum name, a compound member outside its compound, …
InvalidDatatype(String),
/// A chunked layout whose chunk dimensions libhdf5 refuses: a zero
/// dimension, a rank that does not match the dataspace, or a chunk of
/// 4 GiB or more.
InvalidChunkDimensions(String),
}
impl fmt::Display for FormatError {
@@ -460,6 +464,9 @@ impl fmt::Display for FormatError {
FormatError::InvalidDatatype(why) => {
write!(f, "invalid datatype: {why}")
}
FormatError::InvalidChunkDimensions(why) => {
write!(f, "invalid chunk dimensions: {why}")
}
}
}
}
crates/clawhdf5/tests/header_validation_interop.rs
+155
View File
@@ -0,0 +1,155 @@
//! Corrupt files that libhdf5 refuses must be refused here too, not read.
//!
//! h5py writes a valid file, the script corrupts a copy the way a damaged or
//! malicious file would be, and records whether h5py (libhdf5) still opens
//! and reads the object. clawhdf5 must agree: read the valid file, refuse
//! each corrupt one. Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
use std::path::Path;
use std::process::Command;
use clawhdf5::File;
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn interop_required() -> bool {
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py, numpy"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
macro_rules! skip_if_no_python {
() => {
if !python_available() {
assert!(
!interop_required(),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
return;
}
};
}
/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
/// `d` the output directory) and then, for every `NAME.h5` it wrote,
/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR`
/// otherwise. Returns those lines, sorted.
fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
let script = format!(
r#"
import h5py, numpy as np, struct, os, glob
d = "{dir}"
{body}
for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
name = os.path.basename(path)[:-3]
try:
with h5py.File(path, "r") as f:
f["d"][()]
print(name, "ok")
except Exception:
print(name, "ERROR")
"#,
dir = dir.display()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
let mut lines: Vec<String> = String::from_utf8_lossy(&out.stdout)
.lines()
.map(str::to_owned)
.collect();
lines.sort();
lines
}
/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of
/// whatever type it has).
fn clawhdf5_reads(path: &Path) -> Result<(), String> {
let file = File::open(path).map_err(|e| format!("open: {e}"))?;
let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?;
ds.dtype().map_err(|e| format!("dtype: {e}"))?;
ds.shape().map_err(|e| format!("shape: {e}"))?;
ds.read_i32().map(|_| ()).map_err(|e| format!("read: {e}"))
}
/// h5py's verdict for each file must be `expected`, and clawhdf5 must read
/// exactly the files h5py reads.
fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
assert_eq!(verdicts, expected, "h5py's view changed");
for line in verdicts {
let (name, verdict) = line.split_once(' ').unwrap();
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")));
match verdict {
"ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
_ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
}
}
}
#[test]
fn chunk_dimensions_libhdf5_refuses_are_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
// A chunked int32 dataset (chunk 37, element size 4 after it in the
// layout message), with libver earliest (layout v3) and latest (v4).
// The corrupt copies set the chunk dimension to 0 (which read as all
// fill values), to 0x80000000 (an 8 GiB chunk) and to 38, which the
// chunk index's offsets 37 and 74 are not multiples of (libhdf5: "bad
// coordinate offset"; the chunks were read at the wrong place). A v4
// layout indexes chunks by position, not offset, but both libraries
// refuse the changed chunk grid there too.
let verdicts = h5py_verdicts(
dir.path(),
r#"
for libver in ("earliest", "latest"):
good = os.path.join(d, f"{libver}_good.h5")
with h5py.File(good, "w", libver=libver) as f:
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,))
data = bytearray(open(good, "rb").read())
if libver == "earliest":
at = data.find(struct.pack("<II", 37, 4))
width = 4
else:
# v4: flags, ndims, bytes per dim, then the dims
at = data.find(bytes([4, 2, 0, 2]))
width = data[at + 4]
at += 5
assert at > 0
for name, value in (("zero", 0), ("huge", 0x80000000), ("offgrid", 38)):
bad = bytearray(data)
if value >= 1 << (8 * width):
continue
bad[at:at + width] = value.to_bytes(width, "little")
open(os.path.join(d, f"{libver}_{name}.h5"), "wb").write(bad)
"#,
);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&[
"earliest_good ok",
"earliest_huge ERROR",
"earliest_offgrid ERROR",
"earliest_zero ERROR",
"latest_good ok",
"latest_offgrid ERROR",
"latest_zero ERROR",
],
);
}