fix(format): validate chunk dimensions and chunk index offsets like libhdf5
A chunk dimension of 0 read a dataset as all fill values, 0x80000000 made
an 8 GiB chunk, and a chunk dimension the chunk index's offsets are not
multiples of read chunks at the wrong place (cve-2018-11205). libhdf5
refuses all of these; now so does clawhdf5:
- DataLayout::parse (H5O__layout_decode): no chunk dimension 0 ("bad chunk
dimension value"), at most 33 dimensions, and before layout v4 at least
2 ("bad dimensions for chunked storage"). New
FormatError::InvalidChunkDimensions.
- Reading a chunked dataset (H5D__chunk_init / H5D__chunk_set_sizes): the
chunk rank must match the dataspace's and a chunk must be under 4 GiB.
One chunked_read::chunk_geometry replaces the four copies of the rank
check.
- v1 B-tree chunk index (H5D__btree_decode_key): every key's offsets must
be multiples of the chunk dimensions, including the keys that only bound
a node, which is where cve-2018-11205's bad dimension shows. New
chunked_read::collect_chunk_info_checked; the chunked read and selection
paths use it.
New interop test header_validation_interop.rs: h5py writes chunked files
(layout v3 and v4), the script corrupts the chunk dimension, and
clawhdf5 must read exactly the copies h5py reads.
Conformance (cached corpus, tank): 570 ok, unchanged; cve-2018-11205 now
refuses the dataset h5py refuses; six more objects that already failed now
fail with libhdf5's reason.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -148,6 +148,46 @@ pub(crate) fn checked_byte_len(elements: u64, elem_size: usize) -> Result<usize,
|
||||
})
|
||||
}
|
||||
|
||||
/// The spatial chunk dimensions of a chunked layout (`chunk_dimensions` is
|
||||
/// the layout message's list: one per dataspace dimension, then the element
|
||||
/// size), after the checks libhdf5 makes when it opens a chunked dataset
|
||||
/// (`H5D__chunk_init` / `H5D__chunk_set_sizes`): the chunk rank must match
|
||||
/// the dataspace's, no chunk dimension may be 0, and a chunk may not be 4 GiB
|
||||
/// or more (a v1 B-tree records chunk sizes in 32 bits; libhdf5 before
|
||||
/// layout version 5 refuses larger ones). A zero chunk dimension used to read
|
||||
/// as all fill values, and a huge one to hang the reader.
|
||||
pub(crate) fn chunk_geometry(
|
||||
chunk_dimensions: &[u32],
|
||||
dataspace: &Dataspace,
|
||||
elem_size: usize,
|
||||
) -> Result<(usize, Vec<usize>), FormatError> {
|
||||
let rank = chunk_dimensions.len().checked_sub(1).ok_or_else(|| {
|
||||
FormatError::InvalidChunkDimensions("chunked layout has no dimensions".into())
|
||||
})?;
|
||||
if dataspace.dimensions.len() != rank {
|
||||
return Err(FormatError::InvalidChunkDimensions(format!(
|
||||
"dimensionality of chunks doesn't match the dataspace (chunk rank {rank}, \
|
||||
dataspace rank {})",
|
||||
dataspace.dimensions.len()
|
||||
)));
|
||||
}
|
||||
let spatial = &chunk_dimensions[..rank];
|
||||
if let Some(d) = spatial.iter().position(|&c| c == 0) {
|
||||
return Err(FormatError::InvalidChunkDimensions(format!(
|
||||
"chunk size must be > 0, dim = {d}"
|
||||
)));
|
||||
}
|
||||
let bytes = spatial
|
||||
.iter()
|
||||
.fold(elem_size as u128, |acc, &c| acc * u128::from(c));
|
||||
if bytes > u128::from(u32::MAX) {
|
||||
return Err(FormatError::InvalidChunkDimensions(format!(
|
||||
"chunk size must be < 4GB (chunk {spatial:?} of {elem_size}-byte elements)"
|
||||
)));
|
||||
}
|
||||
Ok((rank, spatial.iter().map(|&c| c as usize).collect()))
|
||||
}
|
||||
|
||||
/// Product of chunk dimensions times the element size, overflow-checked.
|
||||
pub(crate) fn checked_chunk_byte_len(
|
||||
chunk_dims: &[usize],
|
||||
@@ -222,7 +262,76 @@ pub fn collect_chunk_info(
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
collect_chunk_info_inner(file_data, btree_address, ndims, offset_size, length_size, 0)
|
||||
collect_chunk_info_inner(
|
||||
file_data,
|
||||
btree_address,
|
||||
ndims,
|
||||
None,
|
||||
offset_size,
|
||||
length_size,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
|
||||
/// layout message's list, element size last), checking every key of the
|
||||
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
|
||||
/// must be a multiple of its chunk dimension. That includes the keys that
|
||||
/// only bound a node (internal-node keys and each node's final key), which
|
||||
/// is where a corrupt chunk dimension shows when the chunks themselves all
|
||||
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
|
||||
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
|
||||
/// chunks were read at the wrong place, or the dataset read as fill values.
|
||||
pub fn collect_chunk_info_checked(
|
||||
file_data: &[u8],
|
||||
btree_address: u64,
|
||||
chunk_dimensions: &[u32],
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
collect_chunk_info_inner(
|
||||
file_data,
|
||||
btree_address,
|
||||
chunk_dimensions.len(),
|
||||
Some(chunk_dimensions),
|
||||
offset_size,
|
||||
length_size,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
/// Check one v1 B-tree chunk key's offsets (see
|
||||
/// [`collect_chunk_info_checked`]).
|
||||
fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), FormatError> {
|
||||
for (&offset, &dim) in offsets.iter().zip(chunk_dimensions) {
|
||||
if dim == 0 || offset % u64::from(dim) != 0 {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"bad coordinate offset {offsets:?} for chunk dimensions {chunk_dimensions:?}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
|
||||
/// chunk size and filter mask) and check them when `chunk_dimensions` is
|
||||
/// given.
|
||||
fn read_key_offsets(
|
||||
file_data: &[u8],
|
||||
pos: usize,
|
||||
ndims: usize,
|
||||
chunk_dimensions: Option<&[u32]>,
|
||||
) -> Result<Vec<u64>, FormatError> {
|
||||
let mut offsets = Vec::with_capacity(ndims);
|
||||
let mut kp = pos + 8;
|
||||
for _ in 0..ndims {
|
||||
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
|
||||
kp += CHUNK_KEY_OFFSET_SIZE as usize;
|
||||
}
|
||||
if let Some(dims) = chunk_dimensions {
|
||||
check_key_offsets(&offsets, dims)?;
|
||||
}
|
||||
Ok(offsets)
|
||||
}
|
||||
|
||||
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
|
||||
@@ -237,6 +346,7 @@ fn collect_chunk_info_inner(
|
||||
file_data: &[u8],
|
||||
btree_address: u64,
|
||||
ndims: usize,
|
||||
chunk_dimensions: Option<&[u32]>,
|
||||
offset_size: u8,
|
||||
_length_size: u8,
|
||||
depth: usize,
|
||||
@@ -296,12 +406,7 @@ fn collect_chunk_info_inner(
|
||||
file_data[pos + 6],
|
||||
file_data[pos + 7],
|
||||
]);
|
||||
let mut offsets = Vec::with_capacity(ndims);
|
||||
let mut kp = pos + 8;
|
||||
for _ in 0..ndims {
|
||||
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
|
||||
kp += CHUNK_KEY_OFFSET_SIZE as usize;
|
||||
}
|
||||
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
pos += key_size;
|
||||
|
||||
// Parse child address
|
||||
@@ -315,7 +420,8 @@ fn collect_chunk_info_inner(
|
||||
address,
|
||||
});
|
||||
}
|
||||
// Skip final key
|
||||
// The final key only bounds the node; libhdf5 still checks it.
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
Ok(chunks)
|
||||
} else {
|
||||
// Internal node: recurse into children
|
||||
@@ -324,11 +430,13 @@ fn collect_chunk_info_inner(
|
||||
|
||||
let mut child_addrs = Vec::with_capacity(entries_used);
|
||||
for _ in 0..entries_used {
|
||||
pos += key_size; // skip key
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
pos += key_size;
|
||||
let child_addr = read_offset(file_data, pos, offset_size)?;
|
||||
child_addrs.push(child_addr);
|
||||
pos += os;
|
||||
}
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
|
||||
let mut all_chunks = Vec::new();
|
||||
for child_addr in child_addrs {
|
||||
@@ -336,6 +444,7 @@ fn collect_chunk_info_inner(
|
||||
file_data,
|
||||
child_addr,
|
||||
ndims,
|
||||
chunk_dimensions,
|
||||
offset_size,
|
||||
_length_size,
|
||||
depth + 1,
|
||||
@@ -549,30 +658,13 @@ pub fn list_chunks(
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
|
||||
|
||||
// Both v3 and v4 include element size as last dim (rank+1)
|
||||
let ndims = chunk_dimensions.len();
|
||||
let rank = ndims
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
|
||||
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
|
||||
.iter()
|
||||
.map(|&d| d as usize)
|
||||
.collect();
|
||||
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
if ds_dims.len() != rank {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
|
||||
ds_dims.len(),
|
||||
chunk_dimensions.len(),
|
||||
rank
|
||||
)));
|
||||
}
|
||||
|
||||
// Collect chunks based on version and index type
|
||||
let mut chunks = match (version, chunk_index_type) {
|
||||
(3, _) => {
|
||||
let ndims = chunk_dimensions.len(); // rank+1
|
||||
collect_chunk_info(file_data, addr, ndims, offset_size, length_size)?
|
||||
collect_chunk_info_checked(file_data, addr, chunk_dimensions, offset_size, length_size)?
|
||||
}
|
||||
(4, Some(1)) => {
|
||||
// Single chunk — one chunk covering the entire dataset
|
||||
@@ -819,24 +911,8 @@ pub fn read_chunked_data_cached(
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
|
||||
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let ndims = chunk_dimensions.len();
|
||||
let rank = ndims
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
|
||||
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
|
||||
.iter()
|
||||
.map(|&d| d as usize)
|
||||
.collect();
|
||||
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
if ds_dims.len() != rank {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
|
||||
ds_dims.len(),
|
||||
chunk_dimensions.len(),
|
||||
rank
|
||||
)));
|
||||
}
|
||||
|
||||
// The per-file cache is shared across datasets (and threads); every
|
||||
// lookup is keyed by this dataset's chunk-index address, so another
|
||||
@@ -1140,24 +1216,8 @@ pub fn read_chunked_data_sweep(
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
|
||||
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let ndims = chunk_dimensions.len();
|
||||
let rank = ndims
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
|
||||
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
|
||||
.iter()
|
||||
.map(|&d| d as usize)
|
||||
.collect();
|
||||
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
if ds_dims.len() != rank {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
|
||||
ds_dims.len(),
|
||||
chunk_dimensions.len(),
|
||||
rank
|
||||
)));
|
||||
}
|
||||
|
||||
// The per-file cache is shared across datasets (and threads); every
|
||||
// lookup is keyed by this dataset's chunk-index address, so another
|
||||
@@ -1292,24 +1352,8 @@ pub fn read_chunked_data_indexed(
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("no address for chunked layout".into()))?;
|
||||
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let ndims = chunk_dimensions.len();
|
||||
let rank = ndims
|
||||
.checked_sub(1)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("chunked layout has no dimensions".into()))?;
|
||||
let chunk_dims: Vec<usize> = chunk_dimensions[..rank]
|
||||
.iter()
|
||||
.map(|&d| d as usize)
|
||||
.collect();
|
||||
|
||||
let (rank, chunk_dims) = chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
|
||||
let ds_dims: Vec<usize> = dataspace.dimensions.iter().map(|&d| d as usize).collect();
|
||||
if ds_dims.len() != rank {
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"rank mismatch: dataspace has {} dims, layout has {} chunk dims (rank={})",
|
||||
ds_dims.len(),
|
||||
chunk_dimensions.len(),
|
||||
rank
|
||||
)));
|
||||
}
|
||||
|
||||
// Chunk index and assembly plan for this dataset, built on first access
|
||||
// and kept per dataset (keyed by chunk-index address) in the shared cache.
|
||||
@@ -1620,11 +1664,12 @@ mod tests {
|
||||
write_offset(&mut buf, chunk.address, offset_size);
|
||||
}
|
||||
|
||||
// Final key (dummy)
|
||||
// Final key (its offsets must be on the chunk grid, as libhdf5
|
||||
// checks; 0 always is)
|
||||
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
|
||||
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
|
||||
for _ in 0..ndims {
|
||||
write_offset(&mut buf, u64::MAX, 8);
|
||||
write_offset(&mut buf, 0, 8);
|
||||
}
|
||||
|
||||
buf
|
||||
@@ -1632,6 +1677,48 @@ mod tests {
|
||||
|
||||
// --- ChunkInfo collection tests ---
|
||||
|
||||
#[test]
|
||||
fn checked_collection_refuses_keys_off_the_chunk_grid() {
|
||||
let chunk = |offsets: Vec<u64>, address| ChunkInfo {
|
||||
chunk_size: 80,
|
||||
filter_mask: 0,
|
||||
offsets,
|
||||
address,
|
||||
};
|
||||
let good =
|
||||
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
|
||||
assert_eq!(
|
||||
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
|
||||
.unwrap()
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
// A chunk key off the grid.
|
||||
let bad =
|
||||
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![7, 0], 0x200)], 2, 8);
|
||||
assert!(collect_chunk_info(&bad, 0, 2, 8, 8).is_ok());
|
||||
assert!(matches!(
|
||||
collect_chunk_info_checked(&bad, 0, &[10, 8], 8, 8),
|
||||
Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset")
|
||||
));
|
||||
// cve-2018-11205: the chunks all start at 0 in dimension 1, and only
|
||||
// the node's final key shows the chunk dimension is wrong.
|
||||
let mut two_d = build_chunk_btree_leaf(
|
||||
&[chunk(vec![0, 0, 0], 0x100), chunk(vec![10, 0, 0], 0x200)],
|
||||
3,
|
||||
8,
|
||||
);
|
||||
// Final key: (20, 20, 0), the end of a 20 x 20 dataset.
|
||||
let final_key = two_d.len() - 24;
|
||||
two_d[final_key..final_key + 8].copy_from_slice(&20u64.to_le_bytes());
|
||||
two_d[final_key + 8..final_key + 16].copy_from_slice(&20u64.to_le_bytes());
|
||||
assert!(collect_chunk_info_checked(&two_d, 0, &[10, 20, 4], 8, 8).is_ok());
|
||||
assert!(matches!(
|
||||
collect_chunk_info_checked(&two_d, 0, &[10, 32788, 4], 8, 8),
|
||||
Err(FormatError::ChunkedReadError(m)) if m.starts_with("bad coordinate offset [20, 20, 0]")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_two_chunks_from_leaf() {
|
||||
let ndims = 2; // rank+1 for 1D dataset
|
||||
@@ -1750,6 +1837,41 @@ mod tests {
|
||||
use crate::dataspace::{Dataspace, DataspaceType};
|
||||
use crate::datatype::{Datatype, DatatypeByteOrder};
|
||||
|
||||
#[test]
|
||||
fn chunk_geometry_matches_libhdf5_open_checks() {
|
||||
let space = |dims: &[u64]| Dataspace {
|
||||
space_type: DataspaceType::Simple,
|
||||
rank: dims.len() as u8,
|
||||
dimensions: dims.to_vec(),
|
||||
max_dimensions: None,
|
||||
};
|
||||
assert_eq!(
|
||||
chunk_geometry(&[4, 5, 8], &space(&[10, 10]), 8).unwrap(),
|
||||
(2, vec![4, 5])
|
||||
);
|
||||
// Rank mismatch.
|
||||
assert!(matches!(
|
||||
chunk_geometry(&[4, 8], &space(&[10, 10]), 8),
|
||||
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("doesn't match")
|
||||
));
|
||||
// Zero dimension (a layout built in memory, bypassing the parser).
|
||||
assert!(matches!(
|
||||
chunk_geometry(&[4, 0, 8], &space(&[10, 10]), 8),
|
||||
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("must be > 0")
|
||||
));
|
||||
// 0x80000000 x 4-byte elements is 8 GiB; the largest allowed chunk
|
||||
// is 4 GiB - 1 bytes. These dims used to hang the reader.
|
||||
assert!(matches!(
|
||||
chunk_geometry(&[0x8000_0000, 4], &space(&[10]), 4),
|
||||
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
|
||||
));
|
||||
assert!(matches!(
|
||||
chunk_geometry(&[0xFFFF_FFFF, 0xFFFF_FFFF, 1], &space(&[10, 10]), 1),
|
||||
Err(FormatError::InvalidChunkDimensions(m)) if m.contains("4GB")
|
||||
));
|
||||
assert!(chunk_geometry(&[0xFFFF_FFFF, 1], &space(&[10]), 1).is_ok());
|
||||
}
|
||||
|
||||
fn make_f64_type() -> Datatype {
|
||||
Datatype::FloatingPoint {
|
||||
size: 8,
|
||||
@@ -1863,8 +1985,8 @@ mod tests {
|
||||
let file_data = vec![0u8; 64];
|
||||
let result = read_chunked_data(&file_data, &layout, &dataspace, &datatype, None, 8, 8);
|
||||
assert!(
|
||||
matches!(result, Err(FormatError::ChunkedReadError(_))),
|
||||
"expected a clean ChunkedReadError, got {result:?}"
|
||||
matches!(result, Err(FormatError::InvalidChunkDimensions(_))),
|
||||
"expected a clean InvalidChunkDimensions, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,34 @@ pub struct VdsMapping {
|
||||
pub virtual_selection: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Most dimensions a layout message can list (libhdf5 `H5O_LAYOUT_NDIMS`):
|
||||
/// 32 dataspace dimensions plus the element size.
|
||||
const MAX_LAYOUT_NDIMS: usize = 33;
|
||||
|
||||
/// libhdf5's checks on a chunked layout message's dimensions
|
||||
/// (`H5O__layout_decode`): at most [`MAX_LAYOUT_NDIMS`], no dimension 0, and
|
||||
/// before version 4 at least one dataspace dimension plus the element size.
|
||||
/// A zero chunk dimension used to read the dataset as all fill values.
|
||||
fn check_chunk_dims(dims: Vec<u32>, layout_version: u8) -> Result<Vec<u32>, FormatError> {
|
||||
if dims.len() > MAX_LAYOUT_NDIMS {
|
||||
return Err(FormatError::InvalidChunkDimensions(
|
||||
"dimensionality is too large".into(),
|
||||
));
|
||||
}
|
||||
if layout_version < 4 && dims.len() < 2 {
|
||||
return Err(FormatError::InvalidChunkDimensions(
|
||||
"bad dimensions for chunked storage".into(),
|
||||
));
|
||||
}
|
||||
if let Some(u) = dims.iter().position(|&d| d == 0) {
|
||||
return Err(FormatError::InvalidChunkDimensions(format!(
|
||||
"bad chunk dimension value when parsing layout message - chunk dimension must be \
|
||||
positive: mesg->u.chunk.dim[{u}] = 0"
|
||||
)));
|
||||
}
|
||||
Ok(dims)
|
||||
}
|
||||
|
||||
/// Parsed HDF5 data layout message.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum DataLayout {
|
||||
@@ -394,7 +422,7 @@ impl DataLayout {
|
||||
Ok(DataLayout::Contiguous { address, size })
|
||||
}
|
||||
_ => Ok(DataLayout::Chunked {
|
||||
chunk_dimensions: dims,
|
||||
chunk_dimensions: check_chunk_dims(dims, 2)?,
|
||||
btree_address: address,
|
||||
version: 3,
|
||||
chunk_index_type: None,
|
||||
@@ -457,7 +485,7 @@ impl DataLayout {
|
||||
p += 4;
|
||||
}
|
||||
Ok(DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
chunk_dimensions: check_chunk_dims(chunk_dimensions, 3)?,
|
||||
btree_address,
|
||||
version: 3,
|
||||
chunk_index_type: None,
|
||||
@@ -506,6 +534,11 @@ impl DataLayout {
|
||||
let dimensionality = data[pos + 1] as usize;
|
||||
let dim_size_encoded_length = data[pos + 2] as usize;
|
||||
let mut p = pos + 3;
|
||||
if dimensionality > MAX_LAYOUT_NDIMS {
|
||||
return Err(FormatError::InvalidChunkDimensions(
|
||||
"dimensionality is too large".into(),
|
||||
));
|
||||
}
|
||||
|
||||
// dimension sizes
|
||||
ensure_len(data, p, dimensionality * dim_size_encoded_length)?;
|
||||
@@ -547,6 +580,7 @@ impl DataLayout {
|
||||
chunk_dimensions.push(val);
|
||||
p += dim_size_encoded_length;
|
||||
}
|
||||
let chunk_dimensions = check_chunk_dims(chunk_dimensions, 4)?;
|
||||
|
||||
// chunk index type
|
||||
ensure_len(data, p, 1)?;
|
||||
@@ -755,6 +789,55 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A v3 chunked layout message with these dims (element size last).
|
||||
fn v3_chunked_msg(dims: &[u32]) -> Vec<u8> {
|
||||
let mut buf = vec![3u8, 2, dims.len() as u8];
|
||||
buf.extend_from_slice(&0x1000u64.to_le_bytes());
|
||||
for d in dims {
|
||||
buf.extend_from_slice(&d.to_le_bytes());
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_dimensions_are_checked_when_the_layout_is_parsed() {
|
||||
assert!(DataLayout::parse(&v3_chunked_msg(&[4, 4, 8]), 8, 8).is_ok());
|
||||
// A zero chunk dimension used to read as all fill values.
|
||||
let err = DataLayout::parse(&v3_chunked_msg(&[4, 0, 8]), 8, 8).unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, FormatError::InvalidChunkDimensions(m) if m.contains("dim[1] = 0")),
|
||||
"{err:?}"
|
||||
);
|
||||
// Only the element-size dimension: libhdf5 "bad dimensions".
|
||||
assert_eq!(
|
||||
DataLayout::parse(&v3_chunked_msg(&[8]), 8, 8).unwrap_err(),
|
||||
FormatError::InvalidChunkDimensions("bad dimensions for chunked storage".into())
|
||||
);
|
||||
assert_eq!(
|
||||
DataLayout::parse(&v3_chunked_msg(&[1; 34]), 8, 8).unwrap_err(),
|
||||
FormatError::InvalidChunkDimensions("dimensionality is too large".into())
|
||||
);
|
||||
// v1/v2 and v4 messages get the zero check too.
|
||||
let mut v1 = v1v2_header(1, 2, 2);
|
||||
v1.extend_from_slice(&0x1000u64.to_le_bytes());
|
||||
v1.extend_from_slice(&0u32.to_le_bytes());
|
||||
v1.extend_from_slice(&8u32.to_le_bytes());
|
||||
assert!(matches!(
|
||||
DataLayout::parse(&v1, 8, 8),
|
||||
Err(FormatError::InvalidChunkDimensions(_))
|
||||
));
|
||||
let mut v4 = vec![4u8, 2, 0, 2, 4];
|
||||
v4.extend_from_slice(&0u32.to_le_bytes());
|
||||
v4.extend_from_slice(&8u32.to_le_bytes());
|
||||
v4.push(3); // fixed array index
|
||||
v4.push(0); // page bits
|
||||
v4.extend_from_slice(&0x1000u64.to_le_bytes());
|
||||
assert!(matches!(
|
||||
DataLayout::parse(&v4, 8, 8),
|
||||
Err(FormatError::InvalidChunkDimensions(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v1v2_rejects_bad_class_dimensionality_and_truncation() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -360,6 +360,7 @@ pub fn read_raw_data_selection(
|
||||
chunk_index_type,
|
||||
..
|
||||
} => {
|
||||
crate::chunked_read::chunk_geometry(chunk_dimensions, dataspace, elem_size)?;
|
||||
// For chunked data, only read chunks that intersect the selection
|
||||
let chunk_dims: Vec<u64> = chunk_dimensions.iter().map(|&d| d as u64).collect();
|
||||
let rank = dims.len();
|
||||
@@ -400,10 +401,10 @@ pub fn read_raw_data_selection(
|
||||
} else {
|
||||
// v3: B-tree v1
|
||||
if let Some(addr) = btree_address {
|
||||
crate::chunked_read::collect_chunk_info(
|
||||
crate::chunked_read::collect_chunk_info_checked(
|
||||
file_data,
|
||||
*addr,
|
||||
rank + 1,
|
||||
chunk_dimensions,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?
|
||||
|
||||
@@ -210,6 +210,10 @@ pub enum FormatError {
|
||||
/// libhdf5's own error text): size 0, bit fields outside the type,
|
||||
/// an empty enum name, a compound member outside its compound, …
|
||||
InvalidDatatype(String),
|
||||
/// A chunked layout whose chunk dimensions libhdf5 refuses: a zero
|
||||
/// dimension, a rank that does not match the dataspace, or a chunk of
|
||||
/// 4 GiB or more.
|
||||
InvalidChunkDimensions(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for FormatError {
|
||||
@@ -460,6 +464,9 @@ impl fmt::Display for FormatError {
|
||||
FormatError::InvalidDatatype(why) => {
|
||||
write!(f, "invalid datatype: {why}")
|
||||
}
|
||||
FormatError::InvalidChunkDimensions(why) => {
|
||||
write!(f, "invalid chunk dimensions: {why}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user