From e73ac2af092fd3b7a85c6cedbe1955995fa12651 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 00:21:58 -0500 Subject: [PATCH] fix(format): validate chunk dimensions and chunk index offsets like libhdf5 A chunk dimension of 0 read a dataset as all fill values, 0x80000000 made an 8 GiB chunk, and a chunk dimension the chunk index's offsets are not multiples of read chunks at the wrong place (cve-2018-11205). libhdf5 refuses all of these; now so does clawhdf5: - DataLayout::parse (H5O__layout_decode): no chunk dimension 0 ("bad chunk dimension value"), at most 33 dimensions, and before layout v4 at least 2 ("bad dimensions for chunked storage"). New FormatError::InvalidChunkDimensions. - Reading a chunked dataset (H5D__chunk_init / H5D__chunk_set_sizes): the chunk rank must match the dataspace's and a chunk must be under 4 GiB. One chunked_read::chunk_geometry replaces the four copies of the rank check. - v1 B-tree chunk index (H5D__btree_decode_key): every key's offsets must be multiples of the chunk dimensions, including the keys that only bound a node, which is where cve-2018-11205's bad dimension shows. New chunked_read::collect_chunk_info_checked; the chunked read and selection paths use it. New interop test header_validation_interop.rs: h5py writes chunked files (layout v3 and v4), the script corrupts the chunk dimension, and clawhdf5 must read exactly the copies h5py reads. Conformance (cached corpus, tank): 570 ok, unchanged; cve-2018-11205 now refuses the dataset h5py refuses; six more objects that already failed now fail with libhdf5's reason. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 288 +++++++++++++----- crates/clawhdf5-format/src/data_layout.rs | 87 +++++- crates/clawhdf5-format/src/data_read.rs | 5 +- crates/clawhdf5-format/src/error.rs | 7 + .../tests/header_validation_interop.rs | 155 ++++++++++ 5 files changed, 455 insertions(+), 87 deletions(-) create mode 100644 crates/clawhdf5/tests/header_validation_interop.rs diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 3a3dc05..beac35f 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -148,6 +148,46 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result Result<(usize, Vec), FormatError> { + let rank = chunk_dimensions.len().checked_sub(1).ok_or_else(|| { + FormatError::InvalidChunkDimensions("chunked layout has no dimensions".into()) + })?; + if dataspace.dimensions.len() != rank { + return Err(FormatError::InvalidChunkDimensions(format!( + "dimensionality of chunks doesn't match the dataspace (chunk rank {rank}, \ + dataspace rank {})", + dataspace.dimensions.len() + ))); + } + let spatial = &chunk_dimensions[..rank]; + if let Some(d) = spatial.iter().position(|&c| c == 0) { + return Err(FormatError::InvalidChunkDimensions(format!( + "chunk size must be > 0, dim = {d}" + ))); + } + let bytes = spatial + .iter() + .fold(elem_size as u128, |acc, &c| acc * u128::from(c)); + if bytes > u128::from(u32::MAX) { + return Err(FormatError::InvalidChunkDimensions(format!( + "chunk size must be < 4GB (chunk {spatial:?} of {elem_size}-byte elements)" + ))); + } + Ok((rank, spatial.iter().map(|&c| c as usize).collect())) +} + /// Product of chunk dimensions times the element size, overflow-checked. pub(crate) fn checked_chunk_byte_len( chunk_dims: &[usize], @@ -222,7 +262,76 @@ pub fn collect_chunk_info( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0) + collect_chunk_info_inner( + file_data, + btree_address, + ndims, + None, + offset_size, + length_size, + 0, + ) +} + +/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the +/// layout message's list, element size last), checking every key of the +/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset +/// must be a multiple of its chunk dimension. That includes the keys that +/// only bound a node (internal-node keys and each node's final key), which +/// is where a corrupt chunk dimension shows when the chunks themselves all +/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails +/// ("bad coordinate offset") means a corrupt index or chunk dimension; the +/// chunks were read at the wrong place, or the dataset read as fill values. +pub fn collect_chunk_info_checked( + file_data: &[u8], + btree_address: u64, + chunk_dimensions: &[u32], + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + collect_chunk_info_inner( + file_data, + btree_address, + chunk_dimensions.len(), + Some(chunk_dimensions), + offset_size, + length_size, + 0, + ) +} + +/// Check one v1 B-tree chunk key's offsets (see +/// [`collect_chunk_info_checked`]). +fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), FormatError> { + for (&offset, &dim) in offsets.iter().zip(chunk_dimensions) { + if dim == 0 || offset % u64::from(dim) != 0 { + return Err(FormatError::ChunkedReadError(format!( + "bad coordinate offset {offsets:?} for chunk dimensions {chunk_dimensions:?}" + ))); + } + } + Ok(()) +} + +/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its +/// chunk size and filter mask) and check them when `chunk_dimensions` is +/// given. +fn read_key_offsets( + file_data: &[u8], + pos: usize, + ndims: usize, + chunk_dimensions: Option<&[u32]>, +) -> Result, FormatError> { + let mut offsets = Vec::with_capacity(ndims); + let mut kp = pos + 8; + for _ in 0..ndims { + offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?); + kp += CHUNK_KEY_OFFSET_SIZE as usize; + } + if let Some(dims) = chunk_dimensions { + check_key_offsets(&offsets, dims)?; + } + Ok(offsets) } /// Width of each chunk offset in a v1 chunk B-tree key, independent of the @@ -237,6 +346,7 @@ fn collect_chunk_info_inner( file_data: &[u8], btree_address: u64, ndims: usize, + chunk_dimensions: Option<&[u32]>, offset_size: u8, _length_size: u8, depth: usize, @@ -296,12 +406,7 @@ fn collect_chunk_info_inner( file_data[pos + 6], file_data[pos + 7], ]); - let mut offsets = Vec::with_capacity(ndims); - let mut kp = pos + 8; - for _ in 0..ndims { - offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?); - kp += CHUNK_KEY_OFFSET_SIZE as usize; - } + let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; pos += key_size; // Parse child address @@ -315,7 +420,8 @@ fn collect_chunk_info_inner( address, }); } - // Skip final key + // The final key only bounds the node; libhdf5 still checks it. + read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; Ok(chunks) } else { // Internal node: recurse into children @@ -324,11 +430,13 @@ fn collect_chunk_info_inner( let mut child_addrs = Vec::with_capacity(entries_used); for _ in 0..entries_used { - pos += key_size; // skip key + read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; + pos += key_size; let child_addr = read_offset(file_data, pos, offset_size)?; child_addrs.push(child_addr); pos += os; } + read_key_offsets(file_data, pos, ndims, chunk_dimensions)?; let mut all_chunks = Vec::new(); for child_addr in child_addrs { @@ -336,6 +444,7 @@ fn collect_chunk_info_inner( file_data, child_addr, ndims, + chunk_dimensions, offset_size, _length_size, depth + 1, @@ -549,30 +658,13 @@ pub fn list_chunks( .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?; // Both v3 and v4 include element size as last dim (rank+1) - let ndims = chunk_dimensions.len(); - let rank = ndims - .checked_sub(1) - .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?; - let chunk_dims: Vec = chunk_dimensions[..rank] - .iter() - .map(|&d| d as usize) - .collect(); - + let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?; let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); - if ds_dims.len() != rank { - return Err(FormatError::ChunkedReadError(format!( - "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})", - ds_dims.len(), - chunk_dimensions.len(), - rank - ))); - } // Collect chunks based on version and index type let mut chunks = match (version, chunk_index_type) { (3, _) => { - let ndims = chunk_dimensions.len(); // rank+1 - collect_chunk_info(file_data, addr, ndims, offset_size, length_size)? + collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)? } (4, Some(1)) => { // Single chunk — one chunk covering the entire dataset @@ -819,24 +911,8 @@ pub fn read_chunked_data_cached( .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?; let elem_size = datatype.type_size() as usize; - let ndims = chunk_dimensions.len(); - let rank = ndims - .checked_sub(1) - .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?; - let chunk_dims: Vec = chunk_dimensions[..rank] - .iter() - .map(|&d| d as usize) - .collect(); - + let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?; let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); - if ds_dims.len() != rank { - return Err(FormatError::ChunkedReadError(format!( - "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})", - ds_dims.len(), - chunk_dimensions.len(), - rank - ))); - } // The per-file cache is shared across datasets (and threads); every // lookup is keyed by this dataset's chunk-index address, so another @@ -1140,24 +1216,8 @@ pub fn read_chunked_data_sweep( .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?; let elem_size = datatype.type_size() as usize; - let ndims = chunk_dimensions.len(); - let rank = ndims - .checked_sub(1) - .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?; - let chunk_dims: Vec = chunk_dimensions[..rank] - .iter() - .map(|&d| d as usize) - .collect(); - + let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?; let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); - if ds_dims.len() != rank { - return Err(FormatError::ChunkedReadError(format!( - "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})", - ds_dims.len(), - chunk_dimensions.len(), - rank - ))); - } // The per-file cache is shared across datasets (and threads); every // lookup is keyed by this dataset's chunk-index address, so another @@ -1292,24 +1352,8 @@ pub fn read_chunked_data_indexed( .ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?; let elem_size = datatype.type_size() as usize; - let ndims = chunk_dimensions.len(); - let rank = ndims - .checked_sub(1) - .ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?; - let chunk_dims: Vec = chunk_dimensions[..rank] - .iter() - .map(|&d| d as usize) - .collect(); - + let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?; let ds_dims: Vec = dataspace.dimensions.iter().map(|&d| d as usize).collect(); - if ds_dims.len() != rank { - return Err(FormatError::ChunkedReadError(format!( - "rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})", - ds_dims.len(), - chunk_dimensions.len(), - rank - ))); - } // Chunk index and assembly plan for this dataset, built on first access // and kept per dataset (keyed by chunk-index address) in the shared cache. @@ -1620,11 +1664,12 @@ mod tests { write_offset(&mut buf, chunk.address, offset_size); } - // Final key (dummy) + // Final key (its offsets must be on the chunk grid, as libhdf5 + // checks; 0 always is) buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask for _ in 0..ndims { - write_offset(&mut buf, u64::MAX, 8); + write_offset(&mut buf, 0, 8); } buf @@ -1632,6 +1677,48 @@ mod tests { // --- ChunkInfo collection tests --- + #[test] + fn checked_collection_refuses_keys_off_the_chunk_grid() { + let chunk = |offsets: Vec, address| ChunkInfo { + chunk_size: 80, + filter_mask: 0, + offsets, + address, + }; + let good = + build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8); + assert_eq!( + collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8) + .unwrap() + .len(), + 2 + ); + // A chunk key off the grid. + let bad = + build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![7, 0], 0x200)], 2, 8); + assert!(collect_chunk_info(&bad, 0, 2, 8, 8).is_ok()); + assert!(matches!( + collect_chunk_info_checked(&bad, 0, &[10, 8], 8, 8), + Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset") + )); + // cve-2018-11205: the chunks all start at 0 in dimension 1, and only + // the node's final key shows the chunk dimension is wrong. + let mut two_d = build_chunk_btree_leaf( + &[chunk(vec![0, 0, 0], 0x100), chunk(vec![10, 0, 0], 0x200)], + 3, + 8, + ); + // Final key: (20, 20, 0), the end of a 20 x 20 dataset. + let final_key = two_d.len() - 24; + two_d[final_key..final_key + 8].copy_from_slice(&20u64.to_le_bytes()); + two_d[final_key + 8..final_key + 16].copy_from_slice(&20u64.to_le_bytes()); + assert!(collect_chunk_info_checked(&two_d, 0, &[10, 20, 4], 8, 8).is_ok()); + assert!(matches!( + collect_chunk_info_checked(&two_d, 0, &[10, 32788, 4], 8, 8), + Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset [20, 20, 0]") + )); + } + #[test] fn collect_two_chunks_from_leaf() { let ndims = 2; // rank+1 for 1D dataset @@ -1750,6 +1837,41 @@ mod tests { use crate::dataspace::{Dataspace, DataspaceType}; use crate::datatype::{Datatype, DatatypeByteOrder}; + #[test] + fn chunk_geometry_matches_libhdf5_open_checks() { + let space = |dims: &[u64]| Dataspace { + space_type: DataspaceType::Simple, + rank: dims.len() as u8, + dimensions: dims.to_vec(), + max_dimensions: None, + }; + assert_eq!( + chunk_geometry(&[4, 5, 8], &space(&[10, 10]), 8).unwrap(), + (2, vec![4, 5]) + ); + // Rank mismatch. + assert!(matches!( + chunk_geometry(&[4, 8], &space(&[10, 10]), 8), + Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match") + )); + // Zero dimension (a layout built in memory, bypassing the parser). + assert!(matches!( + chunk_geometry(&[4, 0, 8], &space(&[10, 10]), 8), + Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0") + )); + // 0x80000000 x 4-byte elements is 8 GiB; the largest allowed chunk + // is 4 GiB - 1 bytes. These dims used to hang the reader. + assert!(matches!( + chunk_geometry(&[0x8000_0000, 4], &space(&[10]), 4), + Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB") + )); + assert!(matches!( + chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], &space(&[10, 10]), 1), + Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB") + )); + assert!(chunk_geometry(&[0xFFFF_FFFF, 1], &space(&[10]), 1).is_ok()); + } + fn make_f64_type() -> Datatype { Datatype::FloatingPoint { size: 8, @@ -1863,8 +1985,8 @@ mod tests { let file_data = vec![0u8; 64]; let result = read_chunked_data(&file_data, &layout, &dataspace, &datatype, None, 8, 8); assert!( - matches!(result, Err(FormatError::ChunkedReadError(_))), - "expected a clean ChunkedReadError, got {result:?}" + matches!(result, Err(FormatError::InvalidChunkDimensions(_))), + "expected a clean InvalidChunkDimensions, got {result:?}" ); } diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 60a8243..54122fc 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -24,6 +24,34 @@ pub struct VdsMapping { pub virtual_selection: Vec, } +/// Most dimensions a layout message can list (libhdf5 `H5O_LAYOUT_NDIMS`): +/// 32 dataspace dimensions plus the element size. +const MAX_LAYOUT_NDIMS: usize = 33; + +/// libhdf5's checks on a chunked layout message's dimensions +/// (`H5O__layout_decode`): at most [`MAX_LAYOUT_NDIMS`], no dimension 0, and +/// before version 4 at least one dataspace dimension plus the element size. +/// A zero chunk dimension used to read the dataset as all fill values. +fn check_chunk_dims(dims: Vec, layout_version: u8) -> Result, FormatError> { + if dims.len() > MAX_LAYOUT_NDIMS { + return Err(FormatError::InvalidChunkDimensions( + "dimensionality is too large".into(), + )); + } + if layout_version < 4 && dims.len() < 2 { + return Err(FormatError::InvalidChunkDimensions( + "bad dimensions for chunked storage".into(), + )); + } + if let Some(u) = dims.iter().position(|&d| d == 0) { + return Err(FormatError::InvalidChunkDimensions(format!( + "bad chunk dimension value when parsing layout message - chunk dimension must be \ + positive: mesg->u.chunk.dim[{u}] = 0" + ))); + } + Ok(dims) +} + /// Parsed HDF5 data layout message. #[derive(Debug, Clone, PartialEq)] pub enum DataLayout { @@ -394,7 +422,7 @@ impl DataLayout { Ok(DataLayout::Contiguous { address, size }) } _ => Ok(DataLayout::Chunked { - chunk_dimensions: dims, + chunk_dimensions: check_chunk_dims(dims, 2)?, btree_address: address, version: 3, chunk_index_type: None, @@ -457,7 +485,7 @@ impl DataLayout { p += 4; } Ok(DataLayout::Chunked { - chunk_dimensions, + chunk_dimensions: check_chunk_dims(chunk_dimensions, 3)?, btree_address, version: 3, chunk_index_type: None, @@ -506,6 +534,11 @@ impl DataLayout { let dimensionality = data[pos + 1] as usize; let dim_size_encoded_length = data[pos + 2] as usize; let mut p = pos + 3; + if dimensionality > MAX_LAYOUT_NDIMS { + return Err(FormatError::InvalidChunkDimensions( + "dimensionality is too large".into(), + )); + } // dimension sizes ensure_len(data, p, dimensionality * dim_size_encoded_length)?; @@ -547,6 +580,7 @@ impl DataLayout { chunk_dimensions.push(val); p += dim_size_encoded_length; } + let chunk_dimensions = check_chunk_dims(chunk_dimensions, 4)?; // chunk index type ensure_len(data, p, 1)?; @@ -755,6 +789,55 @@ mod tests { ); } + /// A v3 chunked layout message with these dims (element size last). + fn v3_chunked_msg(dims: &[u32]) -> Vec { + let mut buf = vec![3u8, 2, dims.len() as u8]; + buf.extend_from_slice(&0x1000u64.to_le_bytes()); + for d in dims { + buf.extend_from_slice(&d.to_le_bytes()); + } + buf + } + + #[test] + fn chunk_dimensions_are_checked_when_the_layout_is_parsed() { + assert!(DataLayout::parse(&v3_chunked_msg(&[4, 4, 8]), 8, 8).is_ok()); + // A zero chunk dimension used to read as all fill values. + let err = DataLayout::parse(&v3_chunked_msg(&[4, 0, 8]), 8, 8).unwrap_err(); + assert!( + matches!(&err, FormatError::InvalidChunkDimensions(m) if m.contains("dim[1] = 0")), + "{err:?}" + ); + // Only the element-size dimension: libhdf5 "bad dimensions". + assert_eq!( + DataLayout::parse(&v3_chunked_msg(&[8]), 8, 8).unwrap_err(), + FormatError::InvalidChunkDimensions("bad dimensions for chunked storage".into()) + ); + assert_eq!( + DataLayout::parse(&v3_chunked_msg(&[1; 34]), 8, 8).unwrap_err(), + FormatError::InvalidChunkDimensions("dimensionality is too large".into()) + ); + // v1/v2 and v4 messages get the zero check too. + let mut v1 = v1v2_header(1, 2, 2); + v1.extend_from_slice(&0x1000u64.to_le_bytes()); + v1.extend_from_slice(&0u32.to_le_bytes()); + v1.extend_from_slice(&8u32.to_le_bytes()); + assert!(matches!( + DataLayout::parse(&v1, 8, 8), + Err(FormatError::InvalidChunkDimensions(_)) + )); + let mut v4 = vec![4u8, 2, 0, 2, 4]; + v4.extend_from_slice(&0u32.to_le_bytes()); + v4.extend_from_slice(&8u32.to_le_bytes()); + v4.push(3); // fixed array index + v4.push(0); // page bits + v4.extend_from_slice(&0x1000u64.to_le_bytes()); + assert!(matches!( + DataLayout::parse(&v4, 8, 8), + Err(FormatError::InvalidChunkDimensions(_)) + )); + } + #[test] fn v1v2_rejects_bad_class_dimensionality_and_truncation() { assert_eq!( diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 76c2b0e..91b739a 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -360,6 +360,7 @@ pub fn read_raw_data_selection( chunk_index_type, .. } => { + crate::chunked_read::chunk_geometry(chunk_dimensions, dataspace, elem_size)?; // For chunked data, only read chunks that intersect the selection let chunk_dims: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); let rank = dims.len(); @@ -400,10 +401,10 @@ pub fn read_raw_data_selection( } else { // v3: B-tree v1 if let Some(addr) = btree_address { - crate::chunked_read::collect_chunk_info( + crate::chunked_read::collect_chunk_info_checked( file_data, *addr, - rank + 1, + chunk_dimensions, offset_size, length_size, )? diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index 46f8123..814a5a7 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -210,6 +210,10 @@ pub enum FormatError { /// libhdf5's own error text): size 0, bit fields outside the type, /// an empty enum name, a compound member outside its compound, … InvalidDatatype(String), + /// A chunked layout whose chunk dimensions libhdf5 refuses: a zero + /// dimension, a rank that does not match the dataspace, or a chunk of + /// 4 GiB or more. + InvalidChunkDimensions(String), } impl fmt::Display for FormatError { @@ -460,6 +464,9 @@ impl fmt::Display for FormatError { FormatError::InvalidDatatype(why) => { write!(f, "invalid datatype: {why}") } + FormatError::InvalidChunkDimensions(why) => { + write!(f, "invalid chunk dimensions: {why}") + } } } } diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs new file mode 100644 index 0000000..a2f3e7a --- /dev/null +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -0,0 +1,155 @@ +//! Corrupt files that libhdf5 refuses must be refused here too, not read. +//! +//! h5py writes a valid file, the script corrupts a copy the way a damaged or +//! malicious file would be, and records whether h5py (libhdf5) still opens +//! and reads the object. clawhdf5 must agree: read the valid file, refuse +//! each corrupt one. Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and +/// `d` the output directory) and then, for every `NAME.h5` it wrote, +/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR` +/// otherwise. Returns those lines, sorted. +fn h5py_verdicts(dir: &Path, body: &str) -> Vec { + let script = format!( + r#" +import h5py, numpy as np, struct, os, glob +d = "{dir}" +{body} +for path in sorted(glob.glob(os.path.join(d, "*.h5"))): + name = os.path.basename(path)[:-3] + try: + with h5py.File(path, "r") as f: + f["d"][()] + print(name, "ok") + except Exception: + print(name, "ERROR") +"#, + dir = dir.display() + ); + let out = Command::new(python()) + .args(["-c", &script]) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let mut lines: Vec = String::from_utf8_lossy(&out.stdout) + .lines() + .map(str::to_owned) + .collect(); + lines.sort(); + lines +} + +/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of +/// whatever type it has). +fn clawhdf5_reads(path: &Path) -> Result<(), String> { + let file = File::open(path).map_err(|e| format!("open: {e}"))?; + let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?; + ds.dtype().map_err(|e| format!("dtype: {e}"))?; + ds.shape().map_err(|e| format!("shape: {e}"))?; + ds.read_i32().map(|_| ()).map_err(|e| format!("read: {e}")) +} + +/// h5py's verdict for each file must be `expected`, and clawhdf5 must read +/// exactly the files h5py reads. +fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) { + assert_eq!(verdicts, expected, "h5py's view changed"); + for line in verdicts { + let (name, verdict) = line.split_once(' ').unwrap(); + let ours = clawhdf5_reads(&dir.join(format!("{name}.h5"))); + match verdict { + "ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"), + _ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"), + } + } +} + +#[test] +fn chunk_dimensions_libhdf5_refuses_are_refused() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + // A chunked int32 dataset (chunk 37, element size 4 after it in the + // layout message), with libver earliest (layout v3) and latest (v4). + // The corrupt copies set the chunk dimension to 0 (which read as all + // fill values), to 0x80000000 (an 8 GiB chunk) and to 38, which the + // chunk index's offsets 37 and 74 are not multiples of (libhdf5: "bad + // coordinate offset"; the chunks were read at the wrong place). A v4 + // layout indexes chunks by position, not offset, but both libraries + // refuse the changed chunk grid there too. + let verdicts = h5py_verdicts( + dir.path(), + r#" +for libver in ("earliest", "latest"): + good = os.path.join(d, f"{libver}_good.h5") + with h5py.File(good, "w", libver=libver) as f: + f.create_dataset("d", data=np.arange(100, dtype=" 0 + for name, value in (("zero", 0), ("huge", 0x80000000), ("offgrid", 38)): + bad = bytearray(data) + if value >= 1 << (8 * width): + continue + bad[at:at + width] = value.to_bytes(width, "little") + open(os.path.join(d, f"{libver}_{name}.h5"), "wb").write(bad) +"#, + ); + assert_agrees_with_h5py( + dir.path(), + &verdicts, + &[ + "earliest_good ok", + "earliest_huge ERROR", + "earliest_offgrid ERROR", + "earliest_zero ERROR", + "latest_good ok", + "latest_offgrid ERROR", + "latest_zero ERROR", + ], + ); +}