Merge branch 'fix/p2b-remaining-conformance' into feat/p2b-scale
# Conflicts: # CHANGELOG.md # crates/clawhdf5-format/src/chunked_read.rs
This commit is contained in:
osobh
@@ -588,26 +588,41 @@ pub fn collect_chunk_info(
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
collect_chunk_info_inner(
|
||||
let _ = length_size;
|
||||
let mut chunks = Vec::new();
|
||||
parse_chunk_node(
|
||||
file_data,
|
||||
btree_address,
|
||||
ndims,
|
||||
None,
|
||||
offset_size,
|
||||
length_size,
|
||||
0,
|
||||
)
|
||||
&mut chunks,
|
||||
)?;
|
||||
Ok(chunks)
|
||||
}
|
||||
|
||||
/// [`collect_chunk_info`] for a layout with these `chunk_dimensions` (the
|
||||
/// layout message's list, element size last), checking every key of the
|
||||
/// B-tree as libhdf5 does (`H5D__btree_decode_key`): each coordinate offset
|
||||
/// must be a multiple of its chunk dimension. That includes the keys that
|
||||
/// only bound a node (internal-node keys and each node's final key), which
|
||||
/// is where a corrupt chunk dimension shows when the chunks themselves all
|
||||
/// start at offset 0 in that dimension (`cve-2018-11205`). A key that fails
|
||||
/// ("bad coordinate offset") means a corrupt index or chunk dimension; the
|
||||
/// chunks were read at the wrong place, or the dataset read as fill values.
|
||||
/// The chunks of a v1 B-tree chunk index as libhdf5 reads them, for a
|
||||
/// layout with these `chunk_dimensions` (the layout message's list, element
|
||||
/// size last).
|
||||
///
|
||||
/// Every key of the B-tree is checked as libhdf5 checks it
|
||||
/// (`H5D__btree_decode_key`): each coordinate offset must be a multiple of
|
||||
/// its chunk dimension. That includes the keys that only bound a node
|
||||
/// (internal-node keys and each node's final key), which is where a
|
||||
/// corrupt chunk dimension shows when the chunks themselves all start at
|
||||
/// offset 0 in that dimension (`cve-2018-11205`). A key that fails ("bad
|
||||
/// coordinate offset") means a corrupt index or chunk dimension.
|
||||
///
|
||||
/// libhdf5 does not read a chunk by walking the tree: it looks each chunk
|
||||
/// up (`H5B_find` with `H5D__btree_cmp3` and `H5D__btree_found`), comparing
|
||||
/// the element-size coordinate too, which it asks for as 0. So a chunk is
|
||||
/// returned only where that lookup finds it: a key whose element-size
|
||||
/// coordinate is not 0 is found in a 1-D dataset (the comparison looks at
|
||||
/// that coordinate only against the next key) but not in a dataset of rank
|
||||
/// 2 or more (`cve-2025-44905` `/Shuffle_float_data_le`), which then reads
|
||||
/// as fill values, and a tree whose keys are out of order loses the chunks
|
||||
/// libhdf5's binary search misses.
|
||||
pub fn collect_chunk_info_checked(
|
||||
file_data: &[u8],
|
||||
btree_address: u64,
|
||||
@@ -615,15 +630,141 @@ pub fn collect_chunk_info_checked(
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
collect_chunk_info_inner(
|
||||
let _ = length_size;
|
||||
let ndims = chunk_dimensions.len();
|
||||
if ndims == 0 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"chunk layout has no dimensions".into(),
|
||||
));
|
||||
}
|
||||
let mut stored = Vec::new();
|
||||
let mut root = parse_chunk_node(
|
||||
file_data,
|
||||
btree_address,
|
||||
chunk_dimensions.len(),
|
||||
ndims,
|
||||
Some(chunk_dimensions),
|
||||
offset_size,
|
||||
length_size,
|
||||
0,
|
||||
)
|
||||
&mut stored,
|
||||
)?;
|
||||
// Keys were checked to be multiples of non-zero dimensions.
|
||||
root.scale_keys(chunk_dimensions);
|
||||
// Look every stored chunk's position up. A lookup finds a chunk only at
|
||||
// that chunk's own position, so each is returned at most once.
|
||||
let mut returned = vec![false; stored.len()];
|
||||
let mut wanted = vec![0u64; ndims];
|
||||
for chunk in &stored {
|
||||
for (w, (&o, &d)) in wanted
|
||||
.iter_mut()
|
||||
.zip(chunk.offsets.iter().zip(chunk_dimensions))
|
||||
{
|
||||
*w = o / u64::from(d);
|
||||
}
|
||||
wanted[ndims - 1] = 0;
|
||||
if let Some(i) = root.find(&wanted) {
|
||||
returned[i] = true;
|
||||
}
|
||||
}
|
||||
Ok(stored
|
||||
.into_iter()
|
||||
.zip(returned)
|
||||
.filter_map(|(c, r)| r.then_some(c))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// A node of a v1 B-tree chunk index: its `n + 1` keys (`ndims`
|
||||
/// coordinates each, flattened; byte offsets as stored, or scaled by the
|
||||
/// chunk dimensions after [`ChunkNode::scale_keys`]) and its `n` children,
|
||||
/// a leaf's as indices into the list of stored chunks.
|
||||
struct ChunkNode {
|
||||
ndims: usize,
|
||||
keys: Vec<u64>,
|
||||
children: ChunkChildren,
|
||||
}
|
||||
|
||||
enum ChunkChildren {
|
||||
Nodes(Vec<ChunkNode>),
|
||||
Chunks(Vec<usize>),
|
||||
}
|
||||
|
||||
impl ChunkNode {
|
||||
fn key(&self, i: usize) -> &[u64] {
|
||||
&self.keys[i * self.ndims..(i + 1) * self.ndims]
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
match &self.children {
|
||||
ChunkChildren::Nodes(n) => n.len(),
|
||||
ChunkChildren::Chunks(c) => c.len(),
|
||||
}
|
||||
}
|
||||
|
||||
fn scale_keys(&mut self, dims: &[u32]) {
|
||||
for (k, &d) in self.keys.iter_mut().zip(dims.iter().cycle()) {
|
||||
*k /= u64::from(d);
|
||||
}
|
||||
if let ChunkChildren::Nodes(nodes) = &mut self.children {
|
||||
for n in nodes {
|
||||
n.scale_keys(dims);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `H5B_find_helper` over scaled keys: binary search for the child
|
||||
/// whose keys bracket `scaled`, then `H5D__btree_found` at the leaf.
|
||||
/// Returns the index of the chunk found.
|
||||
fn find(&self, scaled: &[u64]) -> Option<usize> {
|
||||
let (mut lt, mut rt) = (0, self.len());
|
||||
let mut idx = 0;
|
||||
let mut cmp = core::cmp::Ordering::Greater;
|
||||
while lt < rt && cmp != core::cmp::Ordering::Equal {
|
||||
idx = (lt + rt) / 2;
|
||||
cmp = btree_cmp3(self.key(idx), scaled, self.key(idx + 1));
|
||||
if cmp == core::cmp::Ordering::Less {
|
||||
rt = idx;
|
||||
} else {
|
||||
lt = idx + 1;
|
||||
}
|
||||
}
|
||||
if cmp != core::cmp::Ordering::Equal {
|
||||
return None;
|
||||
}
|
||||
match &self.children {
|
||||
ChunkChildren::Nodes(nodes) => nodes[idx].find(scaled),
|
||||
ChunkChildren::Chunks(chunks) => {
|
||||
// "Is this *really* the requested chunk?"
|
||||
let lt_key = self.key(idx);
|
||||
let found = scaled
|
||||
.iter()
|
||||
.zip(lt_key)
|
||||
.all(|(&s, &k)| s < k.wrapping_add(1));
|
||||
found.then_some(chunks[idx])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `H5D__btree_cmp3`: where `scaled` falls against a child's left and
|
||||
/// right keys. `Less` is left of the child, `Greater` right of it. With a
|
||||
/// rank-1 dataset (two coordinates, element size last) libhdf5 compares
|
||||
/// only the first coordinate, and the second against the right key.
|
||||
fn btree_cmp3(lt: &[u64], scaled: &[u64], rt: &[u64]) -> core::cmp::Ordering {
|
||||
use core::cmp::Ordering;
|
||||
if scaled.len() == 2 {
|
||||
if scaled[0] > rt[0] || (scaled[0] == rt[0] && scaled[1] >= rt[1]) {
|
||||
Ordering::Greater
|
||||
} else if scaled[0] < lt[0] {
|
||||
Ordering::Less
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
} else if scaled >= rt {
|
||||
Ordering::Greater
|
||||
} else if scaled < lt {
|
||||
Ordering::Less
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
}
|
||||
|
||||
/// Check one v1 B-tree chunk key's offsets (see
|
||||
@@ -640,24 +781,25 @@ fn check_key_offsets(offsets: &[u64], chunk_dimensions: &[u32]) -> Result<(), Fo
|
||||
}
|
||||
|
||||
/// Read the `ndims` 8-byte offsets of the chunk key at `pos` (after its
|
||||
/// chunk size and filter mask) and check them when `chunk_dimensions` is
|
||||
/// given.
|
||||
/// chunk size and filter mask) into `out`, checking them when
|
||||
/// `chunk_dimensions` is given.
|
||||
fn read_key_offsets(
|
||||
file_data: &[u8],
|
||||
pos: usize,
|
||||
ndims: usize,
|
||||
chunk_dimensions: Option<&[u32]>,
|
||||
) -> Result<Vec<u64>, FormatError> {
|
||||
let mut offsets = Vec::with_capacity(ndims);
|
||||
out: &mut Vec<u64>,
|
||||
) -> Result<(), FormatError> {
|
||||
let start = out.len();
|
||||
let mut kp = pos + 8;
|
||||
for _ in 0..ndims {
|
||||
offsets.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
|
||||
out.push(read_offset(file_data, kp, CHUNK_KEY_OFFSET_SIZE)?);
|
||||
kp += CHUNK_KEY_OFFSET_SIZE as usize;
|
||||
}
|
||||
if let Some(dims) = chunk_dimensions {
|
||||
check_key_offsets(&offsets, dims)?;
|
||||
check_key_offsets(&out[start..], dims)?;
|
||||
}
|
||||
Ok(offsets)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Width of each chunk offset in a v1 chunk B-tree key, independent of the
|
||||
@@ -668,15 +810,17 @@ const CHUNK_KEY_OFFSET_SIZE: u8 = 8;
|
||||
/// protection), matching `btree_v1.rs`'s `MAX_BTREE_DEPTH`.
|
||||
const MAX_CHUNK_BTREE_DEPTH: usize = 64;
|
||||
|
||||
fn collect_chunk_info_inner(
|
||||
/// Parse the v1 B-tree chunk index node at `btree_address` and its
|
||||
/// subtree, appending its chunks to `stored` in tree order.
|
||||
fn parse_chunk_node(
|
||||
file_data: &[u8],
|
||||
btree_address: u64,
|
||||
ndims: usize,
|
||||
chunk_dimensions: Option<&[u32]>,
|
||||
offset_size: u8,
|
||||
_length_size: u8,
|
||||
depth: usize,
|
||||
) -> Result<Vec<ChunkInfo>, FormatError> {
|
||||
stored: &mut Vec<ChunkInfo>,
|
||||
) -> Result<ChunkNode, FormatError> {
|
||||
if depth > MAX_CHUNK_BTREE_DEPTH {
|
||||
return Err(FormatError::NestingDepthExceeded);
|
||||
}
|
||||
@@ -711,74 +855,68 @@ fn collect_chunk_info_inner(
|
||||
.and_then(|n| n.checked_add(8))
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("chunk key too large".into()))?;
|
||||
|
||||
if node_level == 0 {
|
||||
// Leaf node: keys and children interleaved
|
||||
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
|
||||
let needed = entries_used * (key_size + os) + key_size;
|
||||
ensure_len(file_data, pos, needed)?;
|
||||
// key[0], child[0], key[1], child[1], ..., key[N-1], child[N-1], key[N]
|
||||
let needed = entries_used * (key_size + os) + key_size;
|
||||
ensure_len(file_data, pos, needed)?;
|
||||
|
||||
let mut chunks = Vec::with_capacity(entries_used);
|
||||
for _ in 0..entries_used {
|
||||
// Parse key
|
||||
let chunk_size = u32::from_le_bytes([
|
||||
file_data[pos],
|
||||
file_data[pos + 1],
|
||||
file_data[pos + 2],
|
||||
file_data[pos + 3],
|
||||
]);
|
||||
let filter_mask = u32::from_le_bytes([
|
||||
file_data[pos + 4],
|
||||
file_data[pos + 5],
|
||||
file_data[pos + 6],
|
||||
file_data[pos + 7],
|
||||
]);
|
||||
let offsets = read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
pos += key_size;
|
||||
|
||||
// Parse child address
|
||||
let address = read_offset(file_data, pos, offset_size)?;
|
||||
pos += os;
|
||||
|
||||
chunks.push(ChunkInfo {
|
||||
let mut keys = Vec::with_capacity((entries_used + 1) * ndims);
|
||||
let mut chunks = Vec::new();
|
||||
let mut child_addrs = Vec::new();
|
||||
for _ in 0..entries_used {
|
||||
let chunk_size = u32::from_le_bytes([
|
||||
file_data[pos],
|
||||
file_data[pos + 1],
|
||||
file_data[pos + 2],
|
||||
file_data[pos + 3],
|
||||
]);
|
||||
let filter_mask = u32::from_le_bytes([
|
||||
file_data[pos + 4],
|
||||
file_data[pos + 5],
|
||||
file_data[pos + 6],
|
||||
file_data[pos + 7],
|
||||
]);
|
||||
let k = keys.len();
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
|
||||
pos += key_size;
|
||||
let address = read_offset(file_data, pos, offset_size)?;
|
||||
pos += os;
|
||||
if node_level == 0 {
|
||||
chunks.push(stored.len());
|
||||
stored.push(ChunkInfo {
|
||||
chunk_size,
|
||||
filter_mask,
|
||||
offsets,
|
||||
offsets: keys[k..].to_vec(),
|
||||
address,
|
||||
});
|
||||
} else {
|
||||
child_addrs.push(address);
|
||||
}
|
||||
// The final key only bounds the node; libhdf5 still checks it.
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
Ok(chunks)
|
||||
}
|
||||
// The final key only bounds the node; libhdf5 still checks it.
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions, &mut keys)?;
|
||||
|
||||
let children = if node_level == 0 {
|
||||
ChunkChildren::Chunks(chunks)
|
||||
} else {
|
||||
// Internal node: recurse into children
|
||||
let needed = entries_used * (key_size + os) + key_size;
|
||||
ensure_len(file_data, pos, needed)?;
|
||||
|
||||
let mut child_addrs = Vec::with_capacity(entries_used);
|
||||
for _ in 0..entries_used {
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
pos += key_size;
|
||||
let child_addr = read_offset(file_data, pos, offset_size)?;
|
||||
child_addrs.push(child_addr);
|
||||
pos += os;
|
||||
}
|
||||
read_key_offsets(file_data, pos, ndims, chunk_dimensions)?;
|
||||
|
||||
let mut all_chunks = Vec::new();
|
||||
let mut nodes = Vec::with_capacity(child_addrs.len());
|
||||
for child_addr in child_addrs {
|
||||
let child_chunks = collect_chunk_info_inner(
|
||||
nodes.push(parse_chunk_node(
|
||||
file_data,
|
||||
child_addr,
|
||||
ndims,
|
||||
chunk_dimensions,
|
||||
offset_size,
|
||||
_length_size,
|
||||
depth + 1,
|
||||
)?;
|
||||
all_chunks.extend(child_chunks);
|
||||
stored,
|
||||
)?);
|
||||
}
|
||||
Ok(all_chunks)
|
||||
}
|
||||
ChunkChildren::Nodes(nodes)
|
||||
};
|
||||
Ok(ChunkNode {
|
||||
ndims,
|
||||
keys,
|
||||
children,
|
||||
})
|
||||
}
|
||||
|
||||
/// Generate ChunkInfo entries for an implicit index (v4 index type 2).
|
||||
@@ -1087,6 +1225,47 @@ pub fn list_chunks(
|
||||
Ok((chunks, chunk_dims))
|
||||
}
|
||||
|
||||
/// [`list_chunks`] for reading the chunks through `pipeline`: a dataset
|
||||
/// without filters stores every chunk at the chunk's full size, and a chunk
|
||||
/// the index records at another size is refused, as libhdf5 refuses it
|
||||
/// ("incorrect chunk size returned from index for unfiltered chunk"). Such
|
||||
/// a chunk was read at its recorded size, with the rest of the chunk left
|
||||
/// as zeros or fill values: `cve-2025-44904`'s `Scale_offset_float_data_le`
|
||||
/// has chunks of 38 and 37 bytes for 48-byte chunks, where HDF5 2.0 reads
|
||||
/// whatever its buffer held for the missing bytes.
|
||||
pub fn list_chunks_for_read(
|
||||
file_data: &[u8],
|
||||
layout: &DataLayout,
|
||||
dataspace: &Dataspace,
|
||||
elem_size: usize,
|
||||
pipeline: Option<&FilterPipeline>,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<(Vec<ChunkInfo>, Vec<usize>), FormatError> {
|
||||
let (chunks, chunk_dims) = list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
if pipeline.is_none_or(|p| p.filters.is_empty()) {
|
||||
let chunk_bytes = checked_chunk_byte_len(&chunk_dims, elem_size)?;
|
||||
if let Some(c) = chunks
|
||||
.iter()
|
||||
.find(|c| c.address != u64::MAX && c.chunk_size as usize != chunk_bytes)
|
||||
{
|
||||
return Err(FormatError::ChunkedReadError(format!(
|
||||
"incorrect chunk size returned from index for unfiltered chunk at {:?}: \
|
||||
{} bytes, expected {chunk_bytes}",
|
||||
c.offsets, c.chunk_size
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok((chunks, chunk_dims))
|
||||
}
|
||||
|
||||
/// The chunk cache a full read may use (`None` without `std`).
|
||||
#[cfg(feature = "std")]
|
||||
pub(crate) type CacheRef<'a> = Option<&'a ChunkCache>;
|
||||
@@ -1114,11 +1293,12 @@ pub(crate) fn read_chunked_full<O>(
|
||||
check_chunk_element_size(layout, datatype, offset_size)?;
|
||||
let elem_size = datatype.type_size() as usize;
|
||||
let list = || {
|
||||
list_chunks(
|
||||
list_chunks_for_read(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
pipeline,
|
||||
offset_size,
|
||||
length_size,
|
||||
)
|
||||
@@ -1429,11 +1609,12 @@ pub fn read_chunked_data_sweep(
|
||||
// lookup is keyed by this dataset's chunk-index address, so another
|
||||
// dataset's index or chunks are never used for this read.
|
||||
let chunks = cache.chunks_for(addr, rank, || {
|
||||
list_chunks(
|
||||
list_chunks_for_read(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
pipeline,
|
||||
offset_size,
|
||||
length_size,
|
||||
)
|
||||
@@ -1570,11 +1751,12 @@ pub fn read_chunked_data_indexed(
|
||||
addr,
|
||||
rank,
|
||||
|| {
|
||||
list_chunks(
|
||||
list_chunks_for_read(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
pipeline,
|
||||
offset_size,
|
||||
length_size,
|
||||
)
|
||||
@@ -1874,6 +2056,25 @@ mod tests {
|
||||
|
||||
/// Build a B-tree v1 type 1 leaf node with given chunk infos.
|
||||
fn build_chunk_btree_leaf(chunks: &[ChunkInfo], ndims: usize, offset_size: u8) -> Vec<u8> {
|
||||
build_chunk_btree_leaf_to(chunks, &vec![0; ndims], offset_size)
|
||||
}
|
||||
|
||||
/// A leaf whose final key is the one libhdf5 writes past the last chunk:
|
||||
/// each coordinate of the last chunk plus its chunk dimension (the
|
||||
/// element size last).
|
||||
fn build_chunk_btree_leaf_dims(chunks: &[ChunkInfo], dims: &[u32], offset_size: u8) -> Vec<u8> {
|
||||
let last = &chunks.last().expect("a chunk").offsets;
|
||||
let end: Vec<u64> = dims
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(d, &c)| last.get(d).copied().unwrap_or(0) + u64::from(c))
|
||||
.collect();
|
||||
build_chunk_btree_leaf_to(chunks, &end, offset_size)
|
||||
}
|
||||
|
||||
/// A leaf holding `chunks`, with final key `end`.
|
||||
fn build_chunk_btree_leaf_to(chunks: &[ChunkInfo], end: &[u64], offset_size: u8) -> Vec<u8> {
|
||||
let ndims = end.len();
|
||||
let _os = offset_size as usize;
|
||||
let entries_used = chunks.len() as u16;
|
||||
let mut buf = Vec::new();
|
||||
@@ -1915,8 +2116,8 @@ mod tests {
|
||||
// checks; 0 always is)
|
||||
buf.extend_from_slice(&0u32.to_le_bytes()); // chunk_size
|
||||
buf.extend_from_slice(&0u32.to_le_bytes()); // filter_mask
|
||||
for _ in 0..ndims {
|
||||
write_offset(&mut buf, 0, 8);
|
||||
for &e in end {
|
||||
write_offset(&mut buf, e, 8);
|
||||
}
|
||||
|
||||
buf
|
||||
@@ -1932,8 +2133,11 @@ mod tests {
|
||||
offsets,
|
||||
address,
|
||||
};
|
||||
let good =
|
||||
build_chunk_btree_leaf(&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)], 2, 8);
|
||||
let good = build_chunk_btree_leaf_dims(
|
||||
&[chunk(vec![0, 0], 0x100), chunk(vec![10, 0], 0x200)],
|
||||
&[10, 8],
|
||||
8,
|
||||
);
|
||||
assert_eq!(
|
||||
collect_chunk_info_checked(&good, 0, &[10, 8], 8, 8)
|
||||
.unwrap()
|
||||
@@ -2164,7 +2368,6 @@ mod tests {
|
||||
) -> (Vec<u8>, DataLayout, Dataspace) {
|
||||
let os: u8 = 8;
|
||||
let elem_size = 8usize;
|
||||
let ndims = 2; // rank(1) + 1
|
||||
let total = values.len();
|
||||
|
||||
// Place chunk data starting at offset 0x2000
|
||||
@@ -2195,12 +2398,13 @@ mod tests {
|
||||
}
|
||||
|
||||
// Build B-tree at offset 0x100
|
||||
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
|
||||
let dims = [chunk_size_elems as u32, elem_size as u32];
|
||||
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
|
||||
let btree_addr = 0x100usize;
|
||||
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
||||
|
||||
let layout = DataLayout::Chunked {
|
||||
chunk_dimensions: vec![chunk_size_elems as u32, elem_size as u32],
|
||||
chunk_dimensions: dims.to_vec(),
|
||||
btree_address: Some(btree_addr as u64),
|
||||
version: 3,
|
||||
chunk_index_type: None,
|
||||
@@ -2530,7 +2734,6 @@ mod tests {
|
||||
|
||||
let os: u8 = 8;
|
||||
let elem_size = 8usize;
|
||||
let ndims = 2;
|
||||
let chunk_elems = 10usize;
|
||||
let total = 20usize;
|
||||
|
||||
@@ -2569,12 +2772,13 @@ mod tests {
|
||||
data_offset += compressed.len() + 16; // some padding
|
||||
}
|
||||
|
||||
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
|
||||
let dims = [chunk_elems as u32, elem_size as u32];
|
||||
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
|
||||
let btree_addr = 0x100usize;
|
||||
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
||||
|
||||
let layout = DataLayout::Chunked {
|
||||
chunk_dimensions: vec![chunk_elems as u32, elem_size as u32],
|
||||
chunk_dimensions: dims.to_vec(),
|
||||
btree_address: Some(btree_addr as u64),
|
||||
version: 3,
|
||||
chunk_index_type: None,
|
||||
@@ -2612,7 +2816,6 @@ mod tests {
|
||||
// 4x6 dataset with chunk size 2x3 => 4 chunks
|
||||
let os: u8 = 8;
|
||||
let elem_size = 4usize; // f32
|
||||
let ndims = 3; // rank(2) + 1
|
||||
let ds_dims = [4usize, 6];
|
||||
let chunk_dims = [2usize, 3];
|
||||
|
||||
@@ -2652,12 +2855,13 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
let btree = build_chunk_btree_leaf(&chunk_infos, ndims, os);
|
||||
let dims = [chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32];
|
||||
let btree = build_chunk_btree_leaf_dims(&chunk_infos, &dims, os);
|
||||
let btree_addr = 0x100usize;
|
||||
file_data[btree_addr..btree_addr + btree.len()].copy_from_slice(&btree);
|
||||
|
||||
let layout = DataLayout::Chunked {
|
||||
chunk_dimensions: vec![chunk_dims[0] as u32, chunk_dims[1] as u32, elem_size as u32],
|
||||
chunk_dimensions: dims.to_vec(),
|
||||
btree_address: Some(btree_addr as u64),
|
||||
version: 3,
|
||||
chunk_index_type: None,
|
||||
|
||||
@@ -32,6 +32,80 @@ fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatErr
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The storage checks libhdf5 makes when it opens a dataset, before any
|
||||
/// data is read (`H5D__contig_check`, `H5D__compact_init`), so a dataset
|
||||
/// they refuse fails to open, as in libhdf5, instead of opening and
|
||||
/// reporting a shape nothing can be read from:
|
||||
///
|
||||
/// - the element count times the element size must not overflow 64 bits
|
||||
/// ("size of dataset's storage overflowed" — `cve-2024-32624`
|
||||
/// `/Dset_OBJREF`, 2^62 references of 8 bytes);
|
||||
/// - contiguous storage at a defined address must end within the file's
|
||||
/// `file_len` bytes (the HDF5 data up to the end of file the superblock
|
||||
/// records);
|
||||
/// - compact data must be exactly the dataset's size.
|
||||
///
|
||||
/// Deliberately not refused, unlike libhdf5: an empty contiguous dataset at
|
||||
/// a defined address (libhdf5's overflow test `addr + 0 <= addr` refuses
|
||||
/// it), which clawhdf5 up to v2.7.0 wrote. Chunked and virtual layouts are
|
||||
/// checked when their data is read.
|
||||
pub fn check_dataset_storage(
|
||||
layout: &DataLayout,
|
||||
dataspace: &Dataspace,
|
||||
datatype: &Datatype,
|
||||
file_len: u64,
|
||||
) -> Result<(), FormatError> {
|
||||
if !matches!(
|
||||
layout,
|
||||
DataLayout::Contiguous { .. } | DataLayout::Compact { .. }
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
const OVERFLOWED: &str = "size of dataset's storage overflowed";
|
||||
let n = dataspace
|
||||
.checked_num_elements()
|
||||
.map_err(|_| FormatError::InvalidDatasetStorage(OVERFLOWED))?;
|
||||
let data_size = n
|
||||
.checked_mul(u64::from(datatype.type_size()))
|
||||
.ok_or(FormatError::InvalidDatasetStorage(OVERFLOWED))?;
|
||||
match layout {
|
||||
DataLayout::Contiguous {
|
||||
address: Some(address),
|
||||
..
|
||||
} if address
|
||||
.checked_add(data_size)
|
||||
.is_none_or(|end| end > file_len) =>
|
||||
{
|
||||
Err(FormatError::InvalidDatasetStorage(
|
||||
"invalid dataset size, likely file corruption",
|
||||
))
|
||||
}
|
||||
DataLayout::Compact { data } if data.len() as u64 != data_size => {
|
||||
Err(FormatError::InvalidDatasetStorage(
|
||||
"bad value from dataset header - size of compact dataset's data buffer \
|
||||
doesn't match size of dataset data",
|
||||
))
|
||||
}
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many bytes to read from a contiguous dataset's storage of
|
||||
/// `storage_size` bytes (the layout message's size) holding `needed` bytes
|
||||
/// of elements. libhdf5 reads the elements' bytes from the start of the
|
||||
/// storage and ignores storage past them (`H5D__contig_check` checks only
|
||||
/// that the elements fit in the file), so a larger storage reads; one too
|
||||
/// small to hold the elements is an error.
|
||||
pub fn contiguous_read_len(storage_size: u64, needed: usize) -> Result<usize, FormatError> {
|
||||
if storage_size < needed as u64 {
|
||||
return Err(FormatError::DataSizeMismatch {
|
||||
expected: needed,
|
||||
actual: usize::try_from(storage_size).unwrap_or(usize::MAX),
|
||||
});
|
||||
}
|
||||
Ok(needed)
|
||||
}
|
||||
|
||||
/// Zero-copy read of contiguous raw data, returning a borrowed slice.
|
||||
///
|
||||
/// For contiguous layouts, returns a direct `&[u8]` slice into `file_data`.
|
||||
@@ -55,13 +129,7 @@ pub fn read_raw_data_zerocopy<'a>(
|
||||
DataLayout::Contiguous { address, size } => {
|
||||
let addr = address.ok_or(FormatError::NoDataAllocated)?;
|
||||
let addr = addr as usize;
|
||||
let sz = *size as usize;
|
||||
if sz != expected_size {
|
||||
return Err(FormatError::DataSizeMismatch {
|
||||
expected: expected_size,
|
||||
actual: sz,
|
||||
});
|
||||
}
|
||||
let sz = contiguous_read_len(*size, expected_size)?;
|
||||
ensure_len(file_data, addr, sz)?;
|
||||
Ok(Some(&file_data[addr..addr + sz]))
|
||||
}
|
||||
@@ -172,13 +240,7 @@ fn read_raw_data_full_impl(
|
||||
DataLayout::Contiguous { address, size } => {
|
||||
let addr = address.ok_or(FormatError::NoDataAllocated)?;
|
||||
let addr = addr as usize;
|
||||
let sz = *size as usize;
|
||||
if sz != expected_size {
|
||||
return Err(FormatError::DataSizeMismatch {
|
||||
expected: expected_size,
|
||||
actual: sz,
|
||||
});
|
||||
}
|
||||
let sz = contiguous_read_len(*size, expected_size)?;
|
||||
ensure_len(file_data, addr, sz)?;
|
||||
let mut out = crate::bulk_alloc::vec_for_bulk(sz);
|
||||
out.extend_from_slice(&file_data[addr..addr + sz]);
|
||||
@@ -2746,6 +2808,71 @@ mod tests {
|
||||
assert_eq!(result.unwrap(), &[1.5f32, 2.5, 3.5]);
|
||||
}
|
||||
|
||||
/// libhdf5 reads a contiguous dataset's elements from the start of its
|
||||
/// storage and ignores storage past them (cve-2024-32623's scalar
|
||||
/// `/Dset1` has 240 bytes of storage for one 4-byte element). Storage too
|
||||
/// small for the elements is still an error.
|
||||
#[test]
|
||||
fn contiguous_storage_larger_than_the_elements_reads() {
|
||||
let dt = make_f64_le_type();
|
||||
let ds = make_simple_dataspace(&[2]);
|
||||
let mut file_data = vec![0u8; 64];
|
||||
file_data[..8].copy_from_slice(&1.5f64.to_le_bytes());
|
||||
file_data[8..16].copy_from_slice(&2.5f64.to_le_bytes());
|
||||
file_data[16..24].copy_from_slice(&9.0f64.to_le_bytes());
|
||||
let layout = DataLayout::Contiguous {
|
||||
address: Some(0),
|
||||
size: 40,
|
||||
};
|
||||
let raw = read_raw_data(&file_data, &layout, &ds, &dt).unwrap();
|
||||
assert_eq!(raw, file_data[..16]);
|
||||
let zc = read_raw_data_zerocopy(&file_data, &layout, &ds, &dt).unwrap();
|
||||
assert_eq!(zc, Some(&file_data[..16]));
|
||||
let small = DataLayout::Contiguous {
|
||||
address: Some(0),
|
||||
size: 8,
|
||||
};
|
||||
assert!(matches!(
|
||||
read_raw_data(&file_data, &small, &ds, &dt),
|
||||
Err(FormatError::DataSizeMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
/// `H5D__contig_check` / `H5D__compact_init`, run when a dataset opens.
|
||||
#[test]
|
||||
fn dataset_storage_checks_at_open() {
|
||||
let dt = make_f64_le_type();
|
||||
let contiguous = |address| DataLayout::Contiguous { address, size: 0 };
|
||||
// cve-2024-32624 `/Dset_OBJREF`: 2^62 + 2 elements of 8 bytes.
|
||||
let huge = make_simple_dataspace(&[(1 << 62) + 2]);
|
||||
assert_eq!(
|
||||
check_dataset_storage(&contiguous(None), &huge, &dt, 1 << 20),
|
||||
Err(FormatError::InvalidDatasetStorage(
|
||||
"size of dataset's storage overflowed"
|
||||
))
|
||||
);
|
||||
let ds = make_simple_dataspace(&[4]);
|
||||
assert!(check_dataset_storage(&contiguous(Some(100)), &ds, &dt, 132).is_ok());
|
||||
assert!(matches!(
|
||||
check_dataset_storage(&contiguous(Some(100)), &ds, &dt, 131),
|
||||
Err(FormatError::InvalidDatasetStorage(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
check_dataset_storage(&contiguous(Some(u64::MAX - 8)), &ds, &dt, u64::MAX),
|
||||
Err(FormatError::InvalidDatasetStorage(_))
|
||||
));
|
||||
// Not allocated, and (unlike libhdf5) empty at a defined address.
|
||||
assert!(check_dataset_storage(&contiguous(None), &ds, &dt, 0).is_ok());
|
||||
let empty = make_simple_dataspace(&[0]);
|
||||
assert!(check_dataset_storage(&contiguous(Some(64)), &empty, &dt, 64).is_ok());
|
||||
let compact = |n: usize| DataLayout::Compact { data: vec![0; n] };
|
||||
assert!(check_dataset_storage(&compact(32), &ds, &dt, 0).is_ok());
|
||||
assert!(matches!(
|
||||
check_dataset_storage(&compact(24), &ds, &dt, 0),
|
||||
Err(FormatError::InvalidDatasetStorage(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zerocopy_size_mismatch() {
|
||||
let dt = make_f64_le_type();
|
||||
|
||||
@@ -7,6 +7,9 @@ use alloc::vec::Vec;
|
||||
|
||||
use crate::error::FormatError;
|
||||
|
||||
/// Most dimensions a dataspace can have (`H5S_MAX_RANK`).
|
||||
pub const MAX_RANK: u8 = 32;
|
||||
|
||||
/// Type of dataspace.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum DataspaceType {
|
||||
@@ -67,6 +70,12 @@ impl Dataspace {
|
||||
let version = data[0];
|
||||
let rank = data[1];
|
||||
let flags = data[2];
|
||||
// H5O__sdspace_decode's checks.
|
||||
if rank > MAX_RANK {
|
||||
return Err(FormatError::InvalidDataspace(
|
||||
"simple dataspace dimensionality is too large",
|
||||
));
|
||||
}
|
||||
|
||||
let (space_type, header_size) = match version {
|
||||
1 => {
|
||||
@@ -88,6 +97,11 @@ impl Dataspace {
|
||||
2 => DataspaceType::Null,
|
||||
_ => return Err(FormatError::InvalidDataspaceType(type_byte)),
|
||||
};
|
||||
if st != DataspaceType::Simple && rank > 0 {
|
||||
return Err(FormatError::InvalidDataspace(
|
||||
"invalid rank for scalar or NULL dataspace",
|
||||
));
|
||||
}
|
||||
(st, 4usize)
|
||||
}
|
||||
_ => return Err(FormatError::InvalidDataspaceVersion(version)),
|
||||
@@ -107,8 +121,13 @@ impl Dataspace {
|
||||
// Read max dimensions if flags bit 0 is set
|
||||
let max_dimensions = if flags & 0x01 != 0 {
|
||||
let mut max_dims = Vec::with_capacity(rank as usize);
|
||||
for _ in 0..rank {
|
||||
for &dim in &dimensions {
|
||||
let val = read_length(data, pos, length_size)?;
|
||||
if dim > val {
|
||||
return Err(FormatError::InvalidDataspace(
|
||||
"dataspace dimension size is greater than its maximum size",
|
||||
));
|
||||
}
|
||||
max_dims.push(val);
|
||||
pos += ls;
|
||||
}
|
||||
@@ -176,7 +195,6 @@ impl Dataspace {
|
||||
match self.space_type {
|
||||
DataspaceType::Null => Ok(0),
|
||||
DataspaceType::Scalar => Ok(1),
|
||||
DataspaceType::Simple if self.dimensions.is_empty() => Ok(0),
|
||||
DataspaceType::Simple => self
|
||||
.dimensions
|
||||
.iter()
|
||||
@@ -195,18 +213,14 @@ impl Dataspace {
|
||||
match self.space_type {
|
||||
DataspaceType::Null => 0,
|
||||
DataspaceType::Scalar => 1,
|
||||
DataspaceType::Simple => {
|
||||
if self.dimensions.is_empty() {
|
||||
0
|
||||
} else {
|
||||
// Saturate rather than wrap: a wrapped product could
|
||||
// under-size a buffer. Size-critical callers use
|
||||
// `checked_num_elements`.
|
||||
self.dimensions
|
||||
.iter()
|
||||
.fold(1u64, |acc, &d| acc.saturating_mul(d))
|
||||
}
|
||||
}
|
||||
// A simple dataspace of rank 0 holds one element, as in libhdf5
|
||||
// (the product of no dimensions). Saturate rather than wrap: a
|
||||
// wrapped product could under-size a buffer. Size-critical
|
||||
// callers use `checked_num_elements`.
|
||||
DataspaceType::Simple => self
|
||||
.dimensions
|
||||
.iter()
|
||||
.fold(1u64, |acc, &d| acc.saturating_mul(d)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -352,4 +366,44 @@ mod tests {
|
||||
let ds = Dataspace::parse(&data, 8).unwrap();
|
||||
assert_eq!(ds.max_dimensions, Some(vec![10]));
|
||||
}
|
||||
|
||||
/// A simple dataspace of rank 0 (cve-2020-18494's `/dset1`) holds one
|
||||
/// element in libhdf5, which h5py reads as shape `()`. It was 0.
|
||||
#[test]
|
||||
fn simple_rank_zero_holds_one_element() {
|
||||
let data = build_v2_dataspace(0, 0, 1, &[], None);
|
||||
let ds = Dataspace::parse(&data, 8).unwrap();
|
||||
assert_eq!(ds.space_type, DataspaceType::Simple);
|
||||
assert_eq!(ds.num_elements(), 1);
|
||||
assert_eq!(ds.checked_num_elements().unwrap(), 1);
|
||||
}
|
||||
|
||||
/// `H5O__sdspace_decode`'s checks.
|
||||
#[test]
|
||||
fn refuses_what_libhdf5_refuses() {
|
||||
let too_many = build_v2_dataspace(33, 0, 1, &[1; 33], None);
|
||||
assert!(matches!(
|
||||
Dataspace::parse(&too_many, 8),
|
||||
Err(FormatError::InvalidDataspace(_))
|
||||
));
|
||||
let scalar_with_rank = build_v2_dataspace(1, 0, 0, &[4], None);
|
||||
assert!(matches!(
|
||||
Dataspace::parse(&scalar_with_rank, 8),
|
||||
Err(FormatError::InvalidDataspace(_))
|
||||
));
|
||||
let null_with_rank = build_v2_dataspace(1, 0, 2, &[4], None);
|
||||
assert!(matches!(
|
||||
Dataspace::parse(&null_with_rank, 8),
|
||||
Err(FormatError::InvalidDataspace(_))
|
||||
));
|
||||
let over_max = build_v1_dataspace(2, 0x01, &[5, 20], Some(&[10, 10]));
|
||||
assert!(matches!(
|
||||
Dataspace::parse(&over_max, 8),
|
||||
Err(FormatError::InvalidDataspace(_))
|
||||
));
|
||||
// 32 dimensions, and a size equal to the maximum or unlimited, are fine.
|
||||
assert!(Dataspace::parse(&build_v2_dataspace(32, 0, 1, &[1; 32], None), 8).is_ok());
|
||||
let at_max = build_v1_dataspace(2, 0x01, &[10, 20], Some(&[10, u64::MAX]));
|
||||
assert!(Dataspace::parse(&at_max, 8).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,6 +223,26 @@ pub enum FormatError {
|
||||
/// The file's actual length in bytes.
|
||||
actual_len: u64,
|
||||
},
|
||||
/// A link libhdf5 refuses to list: a symbol-table entry with an empty
|
||||
/// name ("invalid link name"). Listing the group fails, as in libhdf5.
|
||||
InvalidLinkName,
|
||||
/// A dataspace message libhdf5 refuses to decode (the reason is
|
||||
/// libhdf5's own error text): more than 32 dimensions, a rank on a
|
||||
/// scalar or null dataspace, a dimension larger than its maximum.
|
||||
InvalidDataspace(&'static str),
|
||||
/// A dataset whose storage libhdf5 refuses when it opens the dataset
|
||||
/// (the reason is libhdf5's own error text): an element count times
|
||||
/// element size that overflows, contiguous storage past the end of the
|
||||
/// file, compact data of the wrong size.
|
||||
InvalidDatasetStorage(&'static str),
|
||||
/// A superblock extension message libhdf5 refuses to decode when it
|
||||
/// opens the file (the reason is libhdf5's own error text): a File Space
|
||||
/// Info message that runs off its end or has a bad page size, a metadata
|
||||
/// cache image outside the file, …
|
||||
InvalidSuperblockExtension(&'static str),
|
||||
/// A metadata cache image block libhdf5 refuses to load (the reason is
|
||||
/// libhdf5's own error text).
|
||||
InvalidCacheImage(&'static str),
|
||||
}
|
||||
|
||||
impl fmt::Display for FormatError {
|
||||
@@ -494,6 +514,21 @@ impl fmt::Display for FormatError {
|
||||
but the file is {actual_len} bytes"
|
||||
)
|
||||
}
|
||||
FormatError::InvalidLinkName => {
|
||||
write!(f, "invalid link name: a group entry has an empty name")
|
||||
}
|
||||
FormatError::InvalidDataspace(why) => {
|
||||
write!(f, "invalid dataspace: {why}")
|
||||
}
|
||||
FormatError::InvalidDatasetStorage(why) => {
|
||||
write!(f, "invalid dataset storage: {why}")
|
||||
}
|
||||
FormatError::InvalidSuperblockExtension(why) => {
|
||||
write!(f, "invalid superblock extension: {why}")
|
||||
}
|
||||
FormatError::InvalidCacheImage(why) => {
|
||||
write!(f, "invalid metadata cache image: {why}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,7 +360,7 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
BuiltinFilter {
|
||||
id: FILTER_SHUFFLE,
|
||||
name: "shuffle",
|
||||
decode: |d, c| shuffle_decompress(d, c.element_size),
|
||||
decode: |d, c| shuffle_decompress(d, shuffle_type_size(c.client_data(), c.element_size)?),
|
||||
encode: Some(|d, c| shuffle_compress(d, c.element_size)),
|
||||
},
|
||||
BuiltinFilter {
|
||||
@@ -464,23 +464,6 @@ pub(crate) static BUILTIN_FILTERS: &[BuiltinFilter] = &[
|
||||
},
|
||||
];
|
||||
|
||||
/// Decode the HDF5 scale-offset filter (id 6).
|
||||
///
|
||||
/// Supports all three scale-offset variants:
|
||||
/// - `H5Z_SO_FLOAT_DSCALE` (0): `value = minval + code / 10^D`
|
||||
/// - `H5Z_SO_FLOAT_ESCALE` (1): `value = minval + code * 2^E`
|
||||
/// - `H5Z_SO_INT` (2): `value = minval + code`
|
||||
///
|
||||
/// Compressed buffer layout: `minbits` (u32 LE) · `minval_width` (1 byte)
|
||||
/// · `minval` (`minval_width` bytes) · 8 reserved bytes · MSB-first packed
|
||||
/// codes (`nelmts * minbits` bits). The all-ones code is reserved for the
|
||||
/// defined fill value.
|
||||
///
|
||||
/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]`=scale type,
|
||||
/// `[1]`=scale factor (decimal digits D for D-scale, binary exponent E for
|
||||
/// E-scale, interpreted as i32 for negative exponents), `[2]`=element count,
|
||||
/// `[4]`=element size, `[5]`=signed flag, `[6]`=byte order (1 = big-endian),
|
||||
/// `[7]`=fill defined, `[8..]`=fill value bits.
|
||||
/// `f64::powi` equivalent that works under `no_std` (no libm/std available).
|
||||
/// Exponentiation by squaring, matching `powi`'s semantics for negative
|
||||
/// exponents via reciprocal.
|
||||
@@ -502,6 +485,32 @@ fn powi_f64(base: f64, mut exp: i32) -> f64 {
|
||||
if neg { 1.0 / result } else { result }
|
||||
}
|
||||
|
||||
/// Decode the HDF5 scale-offset filter (id 6) as libhdf5 does
|
||||
/// (`H5Z__filter_scaleoffset`, reverse direction).
|
||||
///
|
||||
/// - Integers (`H5Z_SO_INT`): `value = minval + code`.
|
||||
/// - Floats, D-scale (`H5Z_SO_FLOAT_DSCALE`): `value = code / 10^D + min`.
|
||||
/// - Floats, E-scale: refused, as libhdf5 refuses it ("E-scaling method not
|
||||
/// supported"); no library writes it.
|
||||
///
|
||||
/// Compressed buffer layout: `minbits` (u32 LE) · the size of `minval` in
|
||||
/// bytes (1 byte; libhdf5 uses at most 8 of them) · `minval` · packed codes
|
||||
/// at byte 21, whatever the stored size of `minval` (`buf_offset` is fixed)
|
||||
/// · MSB-first, `minbits` bits per element. With a fill value defined, the
|
||||
/// all-ones code of `minbits` bits is the fill value — for `minbits == 0`
|
||||
/// that is every element. `minbits` equal to the element's full width means
|
||||
/// the elements are stored as they are (in little-endian order), and an
|
||||
/// integer scale factor of the full width means the filter left the chunk
|
||||
/// untouched.
|
||||
///
|
||||
/// `cd` is the `H5Zscaleoffset.c` parameter block: `[0]` scale type, `[1]`
|
||||
/// scale factor, `[2]` element count, `[3]` class (0 integer, 1 float),
|
||||
/// `[4]` element size, `[5]` signed, `[6]` byte order (1 = big-endian), `[7]`
|
||||
/// fill defined, `[8..]` fill value bits.
|
||||
///
|
||||
/// Packed data too short for its codes is an error (libhdf5 2.0 read past
|
||||
/// the end of the chunk buffer — `cve-2025-2308` — and later releases
|
||||
/// refuse it: "Buffer too short").
|
||||
fn scaleoffset_decompress(
|
||||
data: &[u8],
|
||||
cd: &[u32],
|
||||
@@ -510,74 +519,101 @@ fn scaleoffset_decompress(
|
||||
const H5Z_SO_FLOAT_DSCALE: u32 = 0;
|
||||
const H5Z_SO_FLOAT_ESCALE: u32 = 1;
|
||||
const H5Z_SO_INT: u32 = 2;
|
||||
/// Where the packed codes start (`buf_offset` in `H5Zscaleoffset.c`).
|
||||
const BUF_OFFSET: usize = 21;
|
||||
let err = |why: &str| FormatError::ChunkedReadError(format!("scale-offset: {why}"));
|
||||
if cd.len() < 8 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: missing filter client data".into(),
|
||||
));
|
||||
return Err(err("missing filter client data"));
|
||||
}
|
||||
let scale_type = cd[0];
|
||||
let is_float = scale_type == H5Z_SO_FLOAT_DSCALE || scale_type == H5Z_SO_FLOAT_ESCALE;
|
||||
if scale_type != H5Z_SO_INT && !is_float {
|
||||
return Err(FormatError::UnsupportedFilter(FILTER_SCALEOFFSET));
|
||||
let is_float = match cd[3] {
|
||||
0 => false,
|
||||
1 => true,
|
||||
_ => return Err(err("cannot use C integer datatype for cast")),
|
||||
};
|
||||
if is_float && scale_type != H5Z_SO_FLOAT_DSCALE && scale_type != H5Z_SO_FLOAT_ESCALE
|
||||
|| !is_float && scale_type != H5Z_SO_INT
|
||||
{
|
||||
return Err(err("invalid scale type"));
|
||||
}
|
||||
if scale_type == H5Z_SO_FLOAT_ESCALE {
|
||||
return Err(err("E-scaling method not supported"));
|
||||
}
|
||||
let nelmts = cd[2] as usize;
|
||||
let elem_size = cd[4] as usize;
|
||||
if elem_size == 0 || elem_size > 8 || (is_float && elem_size != 4 && elem_size != 8) {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: unsupported element size".into(),
|
||||
));
|
||||
let size_ok = if is_float {
|
||||
matches!(elem_size, 4 | 8)
|
||||
} else {
|
||||
matches!(elem_size, 1 | 2 | 4 | 8)
|
||||
};
|
||||
if !size_ok {
|
||||
return Err(err("cannot use C integer datatype for cast"));
|
||||
}
|
||||
let full_bits = elem_size * 8;
|
||||
// An integer's scale factor is the number of bits kept; all of them
|
||||
// means the filter stored the chunk as it was.
|
||||
if !is_float && (cd[1] as i32).max(0) as usize > full_bits {
|
||||
return Err(err("minimum number of bits exceeds maximum"));
|
||||
}
|
||||
if !is_float && cd[1] as i32 == full_bits as i32 {
|
||||
return Ok(data.to_vec());
|
||||
}
|
||||
// The decoded output must match the chunk's uncompressed size; reject an
|
||||
// element count that would over-allocate (e.g. minbits == 0 with a huge
|
||||
// nelmts and no packed payload to bound it).
|
||||
let out_bytes = nelmts
|
||||
.checked_mul(elem_size)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
|
||||
.ok_or_else(|| err("size overflow"))?;
|
||||
if expected_bytes != 0 && out_bytes > expected_bytes {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: element count exceeds chunk size".into(),
|
||||
));
|
||||
return Err(err("element count exceeds chunk size"));
|
||||
}
|
||||
let signed = cd[5] == 1;
|
||||
let big_endian = cd[6] == 1;
|
||||
let fill_defined = cd[7] == 1;
|
||||
|
||||
// --- header: minbits, then minval, then 8 reserved bytes ---
|
||||
// --- header: minbits, then the size of minval and minval ---
|
||||
if data.len() < 5 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: truncated header".into(),
|
||||
));
|
||||
return Err(err("buffer too short"));
|
||||
}
|
||||
let minbits = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
|
||||
let minval_width = data[4] as usize;
|
||||
let minval_end = 5 + minval_width;
|
||||
if data.len() < minval_end {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: truncated minval".into(),
|
||||
));
|
||||
if minbits > full_bits {
|
||||
return Err(err("minimum number of bits exceeds size of type"));
|
||||
}
|
||||
let minval_bytes = &data[5..minval_end];
|
||||
let minval_size = usize::from(data[4]).min(8);
|
||||
let minval_bytes = data
|
||||
.get(5..5 + minval_size)
|
||||
.ok_or_else(|| err("buffer too short"))?;
|
||||
let minval = minval_bytes
|
||||
.iter()
|
||||
.rev()
|
||||
.fold(0u64, |acc, &b| (acc << 8) | u64::from(b));
|
||||
|
||||
// --- unpack the per-element codes (MSB-first), shared by both variants ---
|
||||
if minbits > 64 {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: implausible minbits".into(),
|
||||
));
|
||||
// Full precision: the elements follow as they were, little-endian.
|
||||
if minbits == full_bits {
|
||||
let raw = data
|
||||
.get(BUF_OFFSET..)
|
||||
.and_then(|d| d.get(..out_bytes))
|
||||
.ok_or_else(|| err("buffer too short"))?;
|
||||
let mut out = raw.to_vec();
|
||||
if big_endian {
|
||||
for e in out.chunks_exact_mut(elem_size) {
|
||||
e.reverse();
|
||||
}
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
// --- unpack the per-element codes (MSB-first) ---
|
||||
let codes: Vec<u64> = if minbits == 0 {
|
||||
// No packed payload: every element equals minval.
|
||||
// No packed payload: every code is 0.
|
||||
vec![0u64; nelmts]
|
||||
} else {
|
||||
let packed = data.get(minval_end + 8..).ok_or_else(|| {
|
||||
FormatError::ChunkedReadError("scale-offset: truncated packed data".into())
|
||||
})?;
|
||||
let packed = data.get(BUF_OFFSET..).unwrap_or(&[]);
|
||||
let need_bits = nelmts
|
||||
.checked_mul(minbits)
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("scale-offset: size overflow".into()))?;
|
||||
.ok_or_else(|| err("size overflow"))?;
|
||||
if packed.len() * 8 < need_bits {
|
||||
return Err(FormatError::ChunkedReadError(
|
||||
"scale-offset: packed data too short".into(),
|
||||
));
|
||||
return Err(err("packed data too short"));
|
||||
}
|
||||
let mut out = Vec::with_capacity(nelmts);
|
||||
let mut bitpos = 0usize;
|
||||
@@ -592,35 +628,28 @@ fn scaleoffset_decompress(
|
||||
}
|
||||
out
|
||||
};
|
||||
// The fill code (all ones) only exists when there are bits to pack.
|
||||
let has_fill_code = fill_defined && minbits > 0 && minbits < 64;
|
||||
// Computed for all 1..=64 widths; `1 << 64` would overflow, so saturate.
|
||||
let fill_code: u64 = if minbits == 0 {
|
||||
0
|
||||
} else if minbits >= 64 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << minbits) - 1
|
||||
// With a fill value defined, the all-ones code of `minbits` bits (0
|
||||
// when minbits is 0) stands for it. minbits < 64 here.
|
||||
let fill_code: u64 = (1u64 << minbits) - 1;
|
||||
let fill_bits = || {
|
||||
let lo = u64::from(*cd.get(8).unwrap_or(&0));
|
||||
let hi = u64::from(*cd.get(9).unwrap_or(&0));
|
||||
lo | (hi << 32)
|
||||
};
|
||||
|
||||
if is_float {
|
||||
let is_escale = scale_type == H5Z_SO_FLOAT_ESCALE;
|
||||
let scale_factor = cd[1] as i32;
|
||||
let minval = read_le_float(minval_bytes, elem_size);
|
||||
let minval = bits_to_float(minval, elem_size);
|
||||
let fill_value = if fill_defined {
|
||||
let lo = *cd.get(8).unwrap_or(&0) as u64;
|
||||
let hi = *cd.get(9).unwrap_or(&0) as u64;
|
||||
bits_to_float(lo | (hi << 32), elem_size)
|
||||
bits_to_float(fill_bits(), elem_size)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let values: Vec<f64> = codes
|
||||
.iter()
|
||||
.map(|&code| {
|
||||
if has_fill_code && code == fill_code {
|
||||
if fill_defined && code == fill_code {
|
||||
fill_value
|
||||
} else if is_escale {
|
||||
minval + code as f64 * powi_f64(2.0, scale_factor)
|
||||
} else if elem_size == 4 {
|
||||
// H5Z_scaleoffset_modify_3/4 for `float`: the code is
|
||||
// read as an `int` and everything is single precision,
|
||||
@@ -640,21 +669,19 @@ fn scaleoffset_decompress(
|
||||
.collect();
|
||||
Ok(write_floats(&values, elem_size, big_endian))
|
||||
} else {
|
||||
let minval = read_le_int(minval_bytes, signed);
|
||||
let fill_value: i64 = if fill_defined {
|
||||
let lo = *cd.get(8).unwrap_or(&0) as u64;
|
||||
let hi = *cd.get(9).unwrap_or(&0) as u64;
|
||||
sign_extend(lo | (hi << 32), elem_size, signed)
|
||||
sign_extend(fill_bits(), elem_size, signed)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let values: Vec<i64> = codes
|
||||
.iter()
|
||||
.map(|&code| {
|
||||
if has_fill_code && code == fill_code {
|
||||
if fill_defined && code == fill_code {
|
||||
fill_value
|
||||
} else {
|
||||
minval.wrapping_add(code as i64)
|
||||
// `(type)(buf[i] + minval)`: wraps at the element width.
|
||||
(code.wrapping_add(minval)) as i64
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
@@ -662,21 +689,6 @@ fn scaleoffset_decompress(
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a little-endian float of `size` bytes (4 = f32, otherwise f64) as f64.
|
||||
fn read_le_float(bytes: &[u8], size: usize) -> f64 {
|
||||
if size == 4 {
|
||||
let mut b = [0u8; 4];
|
||||
let n = bytes.len().min(4);
|
||||
b[..n].copy_from_slice(&bytes[..n]);
|
||||
f32::from_le_bytes(b) as f64
|
||||
} else {
|
||||
let mut b = [0u8; 8];
|
||||
let n = bytes.len().min(8);
|
||||
b[..n].copy_from_slice(&bytes[..n]);
|
||||
f64::from_le_bytes(b)
|
||||
}
|
||||
}
|
||||
|
||||
/// Interpret the low bits of `raw` as an IEEE float of `size` bytes.
|
||||
fn bits_to_float(raw: u64, size: usize) -> f64 {
|
||||
if size == 4 {
|
||||
@@ -709,16 +721,6 @@ fn write_floats(values: &[f64], elem_size: usize, big_endian: bool) -> Vec<u8> {
|
||||
out
|
||||
}
|
||||
|
||||
/// Read a little-endian integer of `bytes.len()` bytes, sign-extending when
|
||||
/// `signed`. Used for the scale-offset `minval` field.
|
||||
fn read_le_int(bytes: &[u8], signed: bool) -> i64 {
|
||||
let mut raw: u64 = 0;
|
||||
for (i, &b) in bytes.iter().enumerate().take(8) {
|
||||
raw |= (b as u64) << (i * 8);
|
||||
}
|
||||
sign_extend(raw, bytes.len().min(8), signed)
|
||||
}
|
||||
|
||||
/// Interpret the low `size` bytes of `raw` as a (possibly signed) integer.
|
||||
fn sign_extend(raw: u64, size: usize, signed: bool) -> i64 {
|
||||
if size == 0 || size >= 8 {
|
||||
@@ -1422,6 +1424,23 @@ fn zstd_compress(data: &[u8], level: u32) -> Result<Vec<u8>, FormatError> {
|
||||
/// Unshuffle (decompress direction): reconstruct interleaved element bytes.
|
||||
/// On disk: all byte-0s of each element together, then all byte-1s, etc.
|
||||
/// Output: elements in natural order.
|
||||
/// The element size the shuffle filter works with: its parameter, as
|
||||
/// libhdf5 uses it (`H5Z__filter_shuffle`), not the dataset's element size.
|
||||
/// They are the same in every file a library wrote; a corrupt parameter
|
||||
/// larger than the chunk makes libhdf5 leave the chunk as it is, and so
|
||||
/// does [`shuffle_decompress`] (`cve-2025-44905`'s `Shuffle_float_data_be`).
|
||||
/// A zero parameter is an error ("invalid shuffle parameters"); a pipeline
|
||||
/// without the parameter (never written by libhdf5) uses the element size.
|
||||
fn shuffle_type_size(cd: &[u32], element_size: usize) -> Result<usize, FormatError> {
|
||||
match cd {
|
||||
[] => Ok(element_size),
|
||||
[0] | [_, _, ..] => Err(FormatError::FilterError(
|
||||
"invalid shuffle parameters".into(),
|
||||
)),
|
||||
[size] => Ok(*size as usize),
|
||||
}
|
||||
}
|
||||
|
||||
fn shuffle_decompress(data: &[u8], element_size: usize) -> Result<Vec<u8>, FormatError> {
|
||||
let mut result = Vec::new();
|
||||
shuffle_decompress_into(data, element_size, &mut result);
|
||||
@@ -2765,48 +2784,125 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn as_f64(bytes: &[u8]) -> Vec<f64> {
|
||||
bytes
|
||||
.as_chunks::<8>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|c| f64::from_le_bytes(*c))
|
||||
.collect()
|
||||
/// E-scale: libhdf5 refuses it on read and write ("E-scaling method not
|
||||
/// supported"); it was decoded here, never checked against anything.
|
||||
#[test]
|
||||
fn scaleoffset_float_escale_is_refused_as_in_libhdf5() {
|
||||
let cd = [1u32, 1, 4, 1, 8, 0, 0, 0];
|
||||
let mut raw = vec![2, 0, 0, 0, 8];
|
||||
raw.extend_from_slice(&[0; 16]);
|
||||
raw.push(0x1B);
|
||||
assert!(scaleoffset_decompress(&raw, &cd, 0).is_err());
|
||||
}
|
||||
|
||||
/// A scale type that does not match the class is refused, as libhdf5
|
||||
/// refuses it ("invalid scale type").
|
||||
#[test]
|
||||
fn scaleoffset_float_escale_e1() {
|
||||
// f64 [0.0, 2.0, 4.0, 6.0], E=1 (×2^1=2), fill_defined=0.
|
||||
// cd: scale_type=1, E=1, nelmts=4, elem_size=8.
|
||||
let cd = [1u32, 1, 4, 0, 8, 0, 0, 0];
|
||||
let raw: &[u8] = &[
|
||||
2, 0, 0, 0, // minbits=2
|
||||
8, // minval_width=8
|
||||
0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64
|
||||
0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes
|
||||
0x1B, // packed codes: 00 01 10 11 MSB-first
|
||||
];
|
||||
let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap());
|
||||
assert_eq!(got, vec![0.0, 2.0, 4.0, 6.0]);
|
||||
fn scaleoffset_scale_type_must_match_the_class() {
|
||||
let mut raw = vec![2, 0, 0, 0, 8];
|
||||
raw.extend_from_slice(&[0; 16]);
|
||||
raw.push(0x1B);
|
||||
assert!(scaleoffset_decompress(&raw, &[0, 0, 4, 0, 4, 1, 0, 0], 0).is_err());
|
||||
assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 1, 4, 1, 0, 0], 0).is_err());
|
||||
assert!(scaleoffset_decompress(&raw, &[2, 0, 4, 7, 4, 1, 0, 0], 0).is_err());
|
||||
}
|
||||
|
||||
/// `cve-2025-44905` `/Scale_offset_short_data_be`, chunk (4, 0): the
|
||||
/// stored size of `minval` is 0. libhdf5 reads a `minval` of 0 and the
|
||||
/// packed codes from byte 21 regardless; we read them from byte 13
|
||||
/// (5 + size + 8), so the values differed from h5py's.
|
||||
#[test]
|
||||
fn scaleoffset_float_escale_neg_exp() {
|
||||
// f64 [0.0, 0.5, 1.0, 1.5], E=-1 (×2^-1=0.5), fill_defined=0.
|
||||
// cd[1] = 0xFFFF_FFFF which casts to i32 = -1.
|
||||
let cd = [1u32, 0xFFFF_FFFF, 4, 0, 8, 0, 0, 0];
|
||||
let raw: &[u8] = &[
|
||||
2, 0, 0, 0, // minbits=2
|
||||
8, // minval_width=8
|
||||
0, 0, 0, 0, 0, 0, 0, 0, // minval=0.0f64
|
||||
0, 0, 0, 0, 0, 0, 0, 0, // 8 reserved bytes
|
||||
0x1B, // packed codes: 00 01 10 11 MSB-first
|
||||
];
|
||||
let got = as_f64(&scaleoffset_decompress(raw, &cd, 0).unwrap());
|
||||
let exp = [0.0f64, 0.5, 1.0, 1.5];
|
||||
for (g, e) in got.iter().zip(exp.iter()) {
|
||||
assert!((g - e).abs() < 1e-9, "got {g} expected {e}");
|
||||
}
|
||||
fn scaleoffset_codes_start_at_byte_21_whatever_the_minval_size() {
|
||||
// big-endian i16, 12 elements, fill -2 (cd 65534), minbits 3.
|
||||
let mut cd = vec![2u32, 0, 12, 0, 2, 1, 1, 1, 65534];
|
||||
cd.resize(20, 0);
|
||||
let raw = unhex("0300000000d20e00000000000034000000000000000400000000");
|
||||
let got = scaleoffset_decompress(&raw, &cd, 24).unwrap();
|
||||
// Codes of 3 bits from byte 21 (04 00 00 00 00): 0, 1, 0, ...; h5py
|
||||
// reads the chunk's first row as 0, 1, 0.
|
||||
let want: Vec<i16> = vec![0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
let want: Vec<u8> = want.iter().flat_map(|v| v.to_be_bytes()).collect();
|
||||
assert_eq!(got, want);
|
||||
}
|
||||
|
||||
/// With a fill value defined, libhdf5 compares each code with the
|
||||
/// all-ones code of `minbits` bits — which for `minbits == 0` is 0, so a
|
||||
/// chunk with no packed codes reads as all fill values (the compressor
|
||||
/// writes that for a chunk of nothing but fill values). It read as
|
||||
/// `minval` here.
|
||||
#[test]
|
||||
fn scaleoffset_minbits_zero_with_a_fill_value_is_all_fill() {
|
||||
let mut cd = vec![2u32, 0, 3, 0, 4, 1, 0, 1, (-7i32) as u32];
|
||||
cd.resize(20, 0);
|
||||
let mut raw = vec![0, 0, 0, 0, 8];
|
||||
raw.extend_from_slice(&5i64.to_le_bytes());
|
||||
raw.extend_from_slice(&[0; 8]);
|
||||
assert_eq!(
|
||||
scaleoffset_decompress(&raw, &cd, 12).unwrap(),
|
||||
i32_le(&[-7, -7, -7])
|
||||
);
|
||||
// Without a fill value every element is minval.
|
||||
cd[7] = 0;
|
||||
assert_eq!(
|
||||
scaleoffset_decompress(&raw, &cd, 12).unwrap(),
|
||||
i32_le(&[5, 5, 5])
|
||||
);
|
||||
}
|
||||
|
||||
/// `minbits` of the full width stores the elements as they are
|
||||
/// (little-endian), without `minval`; a full-width integer scale factor
|
||||
/// means the filter left the chunk untouched.
|
||||
#[test]
|
||||
fn scaleoffset_full_width_is_stored_as_is() {
|
||||
let mut cd = vec![2u32, 0, 2, 0, 2, 1, 1, 0];
|
||||
cd.resize(20, 0);
|
||||
let mut raw = vec![16, 0, 0, 0, 8];
|
||||
raw.extend_from_slice(&100i64.to_le_bytes());
|
||||
raw.extend_from_slice(&[0; 8]);
|
||||
raw.extend_from_slice(&[0x34, 0x12, 0xfe, 0xff]);
|
||||
assert_eq!(
|
||||
scaleoffset_decompress(&raw, &cd, 4).unwrap(),
|
||||
[0x12, 0x34, 0xff, 0xfe]
|
||||
);
|
||||
cd[1] = 16;
|
||||
assert_eq!(
|
||||
scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).unwrap(),
|
||||
[1, 2, 3, 4]
|
||||
);
|
||||
cd[1] = 17;
|
||||
assert!(scaleoffset_decompress(&[1, 2, 3, 4], &cd, 4).is_err());
|
||||
// minbits wider than the type.
|
||||
cd[1] = 0;
|
||||
raw[0] = 17;
|
||||
assert!(scaleoffset_decompress(&raw, &cd, 4).is_err());
|
||||
}
|
||||
|
||||
/// The shuffle filter uses its own parameter as the element size, as
|
||||
/// libhdf5 does; a parameter larger than the chunk leaves the chunk as
|
||||
/// it is (`cve-2025-44905` `/Shuffle_float_data_be`, whose parameter is
|
||||
/// 4261347332: h5py and h5dump read the stored bytes unshuffled).
|
||||
#[test]
|
||||
fn shuffle_uses_its_parameter() {
|
||||
let data: Vec<u8> = (0..16).collect();
|
||||
let shuffled = shuffle_compress(&data, 4).unwrap();
|
||||
let pipeline = |cd: Vec<u32>| FilterPipeline {
|
||||
version: 2,
|
||||
filters: vec![one_filter(FILTER_SHUFFLE, cd)],
|
||||
};
|
||||
// The dataset's element size says 2; the parameter says 4.
|
||||
assert_eq!(
|
||||
decompress_chunk(&shuffled, &pipeline(vec![4]), 16, 2).unwrap(),
|
||||
data
|
||||
);
|
||||
assert_eq!(
|
||||
decompress_chunk(&shuffled, &pipeline(vec![4_261_347_332]), 16, 4).unwrap(),
|
||||
shuffled
|
||||
);
|
||||
assert!(decompress_chunk(&shuffled, &pipeline(vec![0]), 16, 4).is_err());
|
||||
assert_eq!(
|
||||
decompress_chunk(&shuffled, &pipeline(vec![]), 16, 4).unwrap(),
|
||||
data
|
||||
);
|
||||
}
|
||||
|
||||
// --- N-Bit (filter id 5) --------------------------------------------------
|
||||
|
||||
@@ -21,12 +21,35 @@ pub struct GroupEntry {
|
||||
pub cache_type: u32,
|
||||
}
|
||||
|
||||
/// Given a SymbolTableMessage, resolve all group children.
|
||||
/// Given a SymbolTableMessage, resolve all group children: the group's
|
||||
/// listing.
|
||||
///
|
||||
/// An entry with an empty name fails the listing with
|
||||
/// [`FormatError::InvalidLinkName`], as it fails libhdf5's link iteration
|
||||
/// (`H5G__ent_to_link`: "invalid link name"). Looking a name up
|
||||
/// ([`resolve_path`], and the path resolution in
|
||||
/// [`crate::group_v2::resolve_path_any`]) still works in such a group, as it
|
||||
/// does in libhdf5.
|
||||
pub fn resolve_v1_group_entries(
|
||||
file_data: &[u8],
|
||||
sym_table_msg: &SymbolTableMessage,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<GroupEntry>, FormatError> {
|
||||
let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size)?;
|
||||
if entries.iter().any(|e| e.name.is_empty()) {
|
||||
return Err(FormatError::InvalidLinkName);
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Every entry of a v1 group, empty names included — for looking a name up,
|
||||
/// which never matches an empty name.
|
||||
pub(crate) fn v1_group_entries(
|
||||
file_data: &[u8],
|
||||
sym_table_msg: &SymbolTableMessage,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<GroupEntry>, FormatError> {
|
||||
// Parse local heap
|
||||
let heap = LocalHeap::parse(
|
||||
@@ -207,8 +230,7 @@ pub fn resolve_path(
|
||||
let mut current_sym_table = root_sym_table.clone();
|
||||
|
||||
for (i, component) in components.iter().enumerate() {
|
||||
let entries =
|
||||
resolve_v1_group_entries(file_data, ¤t_sym_table, offset_size, length_size)?;
|
||||
let entries = v1_group_entries(file_data, ¤t_sym_table, offset_size, length_size)?;
|
||||
|
||||
let found = entries.iter().find(|e| e.name == *component);
|
||||
match found {
|
||||
@@ -425,6 +447,22 @@ mod tests {
|
||||
assert_eq!(entries[1].object_header_address, 0x2000);
|
||||
}
|
||||
|
||||
/// cve-2021-46244 `/BAG_root`: a symbol-table entry with an empty name.
|
||||
/// libhdf5 fails the group's listing ("invalid link name"); a lookup of
|
||||
/// the other names still works.
|
||||
#[test]
|
||||
fn empty_entry_name_fails_the_listing_not_a_lookup() {
|
||||
let (file, msg) = build_synthetic_group(&[("", 0x1000, 0), ("elevation", 0x2000, 0)], 8, 8);
|
||||
assert_eq!(
|
||||
resolve_v1_group_entries(&file, &msg, 8, 8).unwrap_err(),
|
||||
FormatError::InvalidLinkName
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_path(&file, &msg, "elevation", 8, 8).unwrap(),
|
||||
0x2000
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_path_single_level() {
|
||||
let (file, msg) =
|
||||
|
||||
@@ -450,7 +450,9 @@ fn resolve_group_entries(
|
||||
.find(|m| m.msg_type == MessageType::SymbolTable)
|
||||
.ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
|
||||
let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
|
||||
group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size)
|
||||
// A lookup: an entry with an empty name (which fails a listing) is
|
||||
// skipped by the name comparison, as in libhdf5.
|
||||
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size)
|
||||
} else if is_v2_group(object_header) {
|
||||
resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
|
||||
} else {
|
||||
|
||||
@@ -121,6 +121,7 @@ pub mod selection;
|
||||
pub mod shared_message;
|
||||
pub mod signature;
|
||||
pub mod superblock;
|
||||
pub mod superblock_ext;
|
||||
pub mod symbol_table;
|
||||
#[cfg(all(
|
||||
test,
|
||||
|
||||
@@ -75,7 +75,40 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||||
})
|
||||
}
|
||||
|
||||
/// The kind of object an object header describes.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ObjectClass {
|
||||
/// A group: the header has a Symbol Table or a Link Info message.
|
||||
Group,
|
||||
/// A dataset: the header has a Datatype and a Dataspace message.
|
||||
Dataset,
|
||||
/// A committed (named) datatype: a Datatype message, no Dataspace.
|
||||
NamedDatatype,
|
||||
}
|
||||
|
||||
impl ObjectHeader {
|
||||
/// The kind of object this header describes, decided as libhdf5 decides
|
||||
/// it (`H5O__obj_class_real`): group first (a Symbol Table or Link Info
|
||||
/// message), then dataset (a Datatype *and* a Dataspace message — not a
|
||||
/// Data Layout message), then named datatype (a Datatype message).
|
||||
/// `None` when none applies; libhdf5 then cannot open the object
|
||||
/// ("unable to determine object type").
|
||||
///
|
||||
/// A header with a Datatype and a Data Layout message but no Dataspace
|
||||
/// is a named datatype to libhdf5, not a dataset.
|
||||
pub fn object_class(&self) -> Option<ObjectClass> {
|
||||
let has = |t: MessageType| self.messages.iter().any(|m| m.msg_type == t);
|
||||
if has(MessageType::SymbolTable) || has(MessageType::LinkInfo) {
|
||||
Some(ObjectClass::Group)
|
||||
} else if has(MessageType::Datatype) && has(MessageType::Dataspace) {
|
||||
Some(ObjectClass::Dataset)
|
||||
} else if has(MessageType::Datatype) {
|
||||
Some(ObjectClass::NamedDatatype)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse an object header at the given offset in the data buffer.
|
||||
///
|
||||
/// `offset_size` and `length_size` come from the superblock.
|
||||
@@ -662,6 +695,54 @@ fn check_message(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn header_with(types: &[MessageType]) -> ObjectHeader {
|
||||
ObjectHeader {
|
||||
version: 2,
|
||||
messages: types
|
||||
.iter()
|
||||
.map(|&msg_type| HeaderMessage {
|
||||
msg_type,
|
||||
size: 0,
|
||||
flags: 0,
|
||||
creation_order: None,
|
||||
data: Vec::new(),
|
||||
})
|
||||
.collect(),
|
||||
reference_count: None,
|
||||
flags: 0,
|
||||
access_time: None,
|
||||
modification_time: None,
|
||||
change_time: None,
|
||||
birth_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_class_follows_libhdf5() {
|
||||
use MessageType::*;
|
||||
let class = |t: &[MessageType]| header_with(t).object_class();
|
||||
assert_eq!(
|
||||
class(&[Datatype, Dataspace, DataLayout]),
|
||||
Some(ObjectClass::Dataset)
|
||||
);
|
||||
// A Data Layout message does not make a dataset without a dataspace
|
||||
// (cve-2024-33874 `/Dset1`: h5py opens it as a named datatype).
|
||||
assert_eq!(
|
||||
class(&[Datatype, DataLayout]),
|
||||
Some(ObjectClass::NamedDatatype)
|
||||
);
|
||||
assert_eq!(class(&[Datatype]), Some(ObjectClass::NamedDatatype));
|
||||
// Group messages win over dataset messages.
|
||||
assert_eq!(
|
||||
class(&[Datatype, Dataspace, SymbolTable]),
|
||||
Some(ObjectClass::Group)
|
||||
);
|
||||
assert_eq!(class(&[LinkInfo]), Some(ObjectClass::Group));
|
||||
// Link messages alone are not a group; nothing is not an object.
|
||||
assert_eq!(class(&[Link]), None);
|
||||
assert_eq!(class(&[]), None);
|
||||
}
|
||||
|
||||
// Helper: build a v1 object header with given messages
|
||||
fn build_v1_header(
|
||||
messages: &[(u16, &[u8], u8)], // (type, data, flags)
|
||||
|
||||
@@ -18,7 +18,7 @@ use alloc::{format, vec, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::string as alloc_or_std;
|
||||
|
||||
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks};
|
||||
use crate::chunked_read::{alloc_output, checked_byte_len, list_chunks_for_read};
|
||||
use crate::data_layout::DataLayout;
|
||||
use crate::data_read::extract_selection_from_buffer;
|
||||
use crate::dataspace::Dataspace;
|
||||
@@ -294,11 +294,12 @@ pub fn read_selection(
|
||||
btree_address: Some(_),
|
||||
..
|
||||
} => {
|
||||
let (chunks, chunk_dims) = list_chunks(
|
||||
let (chunks, chunk_dims) = list_chunks_for_read(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
pipeline,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
|
||||
@@ -0,0 +1,812 @@
|
||||
//! The superblock extension of a version 2 or 3 superblock, and the
|
||||
//! metadata cache image it can point to.
|
||||
//!
|
||||
//! libhdf5 reads the extension when it opens a file (`H5F__super_read`) and
|
||||
//! decodes the messages that configure the file: v1 B-tree "K" values, File
|
||||
//! Space Info, and the Metadata Cache Image. A message that does not decode
|
||||
//! makes the file fail to open, so [`read_superblock_extension`] decodes and
|
||||
//! checks them the way libhdf5 does.
|
||||
//!
|
||||
//! A metadata cache image (written with `H5Pset_mdc_image_config`) is a
|
||||
//! block holding serialized metadata cache entries — object headers, B-tree
|
||||
//! nodes, heaps — each with its file address. libhdf5 loads it into its
|
||||
//! cache before it reads any other metadata (`H5C__load_cache_image`,
|
||||
//! `H5C__reconstruct_cache_contents`), and the entries take the place of
|
||||
//! the file's bytes at their addresses: the file itself may hold stale or
|
||||
//! no metadata there (in `h5clear_mdc_image.h5` the root group's header is
|
||||
//! only in the image). [`CacheImage::apply`] does the same with bytes: it
|
||||
//! writes every entry at its address, so every parser reads what libhdf5
|
||||
//! reads. It writes into whatever the opener gives it — a private
|
||||
//! copy-on-write mapping of the file, or a buffer the opener owns — so the
|
||||
//! file is never copied whole.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{collections::BTreeSet, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::message_type::MessageType;
|
||||
use crate::object_header::ObjectHeader;
|
||||
use crate::superblock::Superblock;
|
||||
|
||||
/// Message type of the File Space Info message.
|
||||
const MSG_FSINFO: u16 = 0x0017;
|
||||
/// Message type of the Metadata Cache Image message.
|
||||
const MSG_MDCI: u16 = 0x0018;
|
||||
/// Header message flag: the library did not know the message when it wrote
|
||||
/// it back (`H5O_MSG_FLAG_WAS_UNKNOWN`); libhdf5 then ignores its contents.
|
||||
const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20;
|
||||
|
||||
/// `H5F_FILE_SPACE_PAGE_SIZE_MIN` / `_MAX`.
|
||||
const PAGE_SIZE_MIN: u64 = 512;
|
||||
const PAGE_SIZE_MAX: u64 = 1024 * 1024 * 1024;
|
||||
/// libhdf5's default file space page size, used for a version 0 message.
|
||||
const PAGE_SIZE_DEFAULT: u64 = 4096;
|
||||
/// Free-space managers whose addresses a persisting version 1 File Space
|
||||
/// Info message lists (`H5F_MEM_PAGE_SUPER` .. `H5F_MEM_PAGE_NTYPES`), and
|
||||
/// a version 0 one (`H5FD_MEM_SUPER` .. `H5FD_MEM_NTYPES`).
|
||||
const FSM_ADDRS_V1: usize = 12;
|
||||
const FSM_ADDRS_V0: usize = 6;
|
||||
|
||||
/// Metadata cache image block limits (`H5Cimage.c`, `H5ACprivate.h`).
|
||||
const MDCI_SIGNATURE: &[u8; 4] = b"MDCI";
|
||||
const MDCI_HAVE_RESIZE_STATUS: u8 = 0x01;
|
||||
const MDCI_ENTRY_IS_FD_PARENT: u8 = 0x04;
|
||||
const MDCI_ENTRY_IS_FD_CHILD: u8 = 0x08;
|
||||
/// `H5AC_NTYPES`: entry type ids are below this.
|
||||
const MDCI_NTYPES: u8 = 30;
|
||||
/// `H5C_RING_NTYPES`.
|
||||
const MDCI_RING_NTYPES: u8 = 6;
|
||||
/// `H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX`.
|
||||
const MDCI_AGE_MAX: u8 = 100;
|
||||
|
||||
/// A decoded File Space Info message (0x0017), mapped to version 1 as
|
||||
/// libhdf5 maps a version 0 one.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FileSpaceInfo {
|
||||
/// Message version as stored (0 or 1).
|
||||
pub version: u8,
|
||||
/// File space strategy (`H5F_fspace_strategy_t`).
|
||||
pub strategy: u8,
|
||||
/// Whether free space is persisted.
|
||||
pub persist: bool,
|
||||
/// Free-space section threshold.
|
||||
pub threshold: u64,
|
||||
/// File space page size.
|
||||
pub page_size: u64,
|
||||
}
|
||||
|
||||
/// Where a metadata cache image block is (Metadata Cache Image message,
|
||||
/// 0x0018).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CacheImageLocation {
|
||||
/// Address of the image block.
|
||||
pub address: u64,
|
||||
/// Length of the image block in bytes.
|
||||
pub length: u64,
|
||||
}
|
||||
|
||||
/// The messages of a superblock extension that libhdf5 decodes at open.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct SuperblockExtension {
|
||||
/// v1 B-tree "K" values (chunk index, symbol table node, symbol table
|
||||
/// leaf), when the extension overrides the defaults.
|
||||
pub btree_k: Option<(u16, u16, u16)>,
|
||||
/// The File Space Info message.
|
||||
pub file_space_info: Option<FileSpaceInfo>,
|
||||
/// The metadata cache image, when the file has one.
|
||||
pub cache_image: Option<CacheImageLocation>,
|
||||
}
|
||||
|
||||
fn ext_err(why: &'static str) -> FormatError {
|
||||
FormatError::InvalidSuperblockExtension(why)
|
||||
}
|
||||
|
||||
const RAN_OFF: &str = "ran off end of input buffer while decoding";
|
||||
|
||||
/// A little-endian cursor over one message or block, failing with
|
||||
/// `overrun` when it runs off the end.
|
||||
struct Cursor<'a> {
|
||||
data: &'a [u8],
|
||||
pos: usize,
|
||||
overrun: FormatError,
|
||||
}
|
||||
|
||||
impl<'a> Cursor<'a> {
|
||||
fn new(data: &'a [u8], overrun: FormatError) -> Self {
|
||||
Cursor {
|
||||
data,
|
||||
pos: 0,
|
||||
overrun,
|
||||
}
|
||||
}
|
||||
|
||||
fn take(&mut self, n: usize) -> Result<&'a [u8], FormatError> {
|
||||
let end = self
|
||||
.pos
|
||||
.checked_add(n)
|
||||
.filter(|&e| e <= self.data.len())
|
||||
.ok_or_else(|| self.overrun.clone())?;
|
||||
let s = &self.data[self.pos..end];
|
||||
self.pos = end;
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
fn u8(&mut self) -> Result<u8, FormatError> {
|
||||
Ok(self.take(1)?[0])
|
||||
}
|
||||
|
||||
fn uint(&mut self, width: u8) -> Result<u64, FormatError> {
|
||||
let b = self.take(width as usize)?;
|
||||
Ok(b.iter()
|
||||
.rev()
|
||||
.fold(0u64, |acc, &x| (acc << 8) | u64::from(x)))
|
||||
}
|
||||
|
||||
/// An address of `width` bytes; `None` when undefined (all ones).
|
||||
fn addr(&mut self, width: u8) -> Result<Option<u64>, FormatError> {
|
||||
let v = self.uint(width)?;
|
||||
let undef = if width >= 8 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << (8 * u32::from(width))) - 1
|
||||
};
|
||||
Ok((v != undef).then_some(v))
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode and check the superblock extension of `sb`, as libhdf5 does when
|
||||
/// it opens the file. `data` is the file from the superblock on, up to the
|
||||
/// end of file the superblock records (its end is libhdf5's "eoa").
|
||||
///
|
||||
/// Returns `Ok(None)` for a superblock without an extension (versions 0
|
||||
/// and 1 have none). A message libhdf5 fails to decode, or a cache image
|
||||
/// that does not lie inside the file, is an error: libhdf5 refuses to open
|
||||
/// such a file (`cve-2020-10810`: a File Space Info message too short for
|
||||
/// the free-space manager addresses it announces; `cve-2020-10812`: a cache
|
||||
/// image past the end of the file).
|
||||
pub fn read_superblock_extension(
|
||||
data: &[u8],
|
||||
sb: &Superblock,
|
||||
) -> Result<Option<SuperblockExtension>, FormatError> {
|
||||
let os = sb.offset_size;
|
||||
let ls = sb.length_size;
|
||||
let undef = if os >= 8 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << (8 * u32::from(os))) - 1
|
||||
};
|
||||
let Some(addr) = sb.superblock_extension_address.filter(|&a| a != undef) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?;
|
||||
let header = ObjectHeader::parse(data, addr, os, ls)?;
|
||||
let eoa = data.len() as u64;
|
||||
|
||||
let mut ext = SuperblockExtension::default();
|
||||
for msg in &header.messages {
|
||||
match msg.msg_type {
|
||||
MessageType::BTreeKValues => {
|
||||
let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF));
|
||||
if c.u8()? != 0 {
|
||||
return Err(ext_err("bad version number for v1 B-tree 'K' message"));
|
||||
}
|
||||
let chunk = c.uint(2)? as u16;
|
||||
let snode = c.uint(2)? as u16;
|
||||
let leaf = c.uint(2)? as u16;
|
||||
ext.btree_k = Some((chunk, snode, leaf));
|
||||
}
|
||||
MessageType::Unknown(MSG_FSINFO) if msg.flags & MSG_FLAG_WAS_UNKNOWN == 0 => {
|
||||
ext.file_space_info = Some(decode_fsinfo(&msg.data, os, ls)?);
|
||||
}
|
||||
MessageType::Unknown(MSG_MDCI) => {
|
||||
let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF));
|
||||
if c.u8()? != 0 {
|
||||
return Err(ext_err(
|
||||
"bad version number for metadata cache image message",
|
||||
));
|
||||
}
|
||||
let address = c.addr(os)?;
|
||||
let length = c.uint(ls)?;
|
||||
let Some(address) = address else {
|
||||
return Err(ext_err("metadata cache image address is undefined"));
|
||||
};
|
||||
if address.checked_add(length).is_none_or(|end| end > eoa) {
|
||||
return Err(ext_err(
|
||||
"metadata cache image: address plus size exceeds file eoa",
|
||||
));
|
||||
}
|
||||
ext.cache_image = Some(CacheImageLocation { address, length });
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(Some(ext))
|
||||
}
|
||||
|
||||
/// `H5O__fsinfo_decode` plus the checks `H5F__super_read` makes on it.
|
||||
fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result<FileSpaceInfo, FormatError> {
|
||||
let mut c = Cursor::new(data, ext_err(RAN_OFF));
|
||||
let version = c.u8()?;
|
||||
let info = if version == 0 {
|
||||
let old_strategy = c.u8()?;
|
||||
let threshold = c.uint(ls)?;
|
||||
// H5F_file_space_type_t: 1 ALL_PERSIST, 2 ALL, 3 AGGR_VFD, 4 VFD.
|
||||
let (strategy, persist) = match old_strategy {
|
||||
1 => {
|
||||
for _ in 0..FSM_ADDRS_V0 {
|
||||
c.addr(os)?;
|
||||
}
|
||||
(0, true)
|
||||
}
|
||||
2 => (0, false),
|
||||
3 => (2, false),
|
||||
4 => (3, false),
|
||||
_ => return Err(ext_err("invalid file space strategy")),
|
||||
};
|
||||
FileSpaceInfo {
|
||||
version,
|
||||
strategy,
|
||||
persist,
|
||||
threshold,
|
||||
page_size: PAGE_SIZE_DEFAULT,
|
||||
}
|
||||
} else {
|
||||
if version > 1 {
|
||||
return Err(ext_err("File space info message's version out of bounds"));
|
||||
}
|
||||
let strategy = c.u8()?;
|
||||
let persist = c.u8()? != 0;
|
||||
let threshold = c.uint(ls)?;
|
||||
let page_size = c.uint(ls)?;
|
||||
if page_size == 0 || page_size > PAGE_SIZE_MAX {
|
||||
return Err(ext_err("invalid page size in file space info"));
|
||||
}
|
||||
c.uint(2)?; // page end metadata threshold
|
||||
c.addr(os)?; // EOA before the free-space managers
|
||||
if persist {
|
||||
for _ in 0..FSM_ADDRS_V1 {
|
||||
c.addr(os)?;
|
||||
}
|
||||
}
|
||||
FileSpaceInfo {
|
||||
version,
|
||||
strategy,
|
||||
persist,
|
||||
threshold,
|
||||
page_size,
|
||||
}
|
||||
};
|
||||
if info.page_size < PAGE_SIZE_MIN {
|
||||
return Err(ext_err("file space page size too small"));
|
||||
}
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
/// One entry of a metadata cache image: `len` bytes at `image_offset` in
|
||||
/// the image block, belonging at file address `address`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct ImageEntry {
|
||||
address: u64,
|
||||
image_offset: usize,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
/// A decoded metadata cache image: where its block is, and the entries it
|
||||
/// holds. [`CacheImage::apply`] writes the entries over a file's bytes.
|
||||
///
|
||||
/// Only the entry list is kept, never a copy of the file: an opener that
|
||||
/// maps the file applies the image to a private copy-on-write mapping, so
|
||||
/// only the pages the entries land on are copied.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CacheImage {
|
||||
location: CacheImageLocation,
|
||||
entries: Vec<ImageEntry>,
|
||||
}
|
||||
|
||||
/// What an opener must do about a file's metadata cache image.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CacheImageState {
|
||||
/// The file has no image: its bytes are its metadata.
|
||||
Absent,
|
||||
/// The file has an image that loads: apply it with [`CacheImage::apply`].
|
||||
Loaded(CacheImage),
|
||||
/// The file has an image libhdf5 fails to load. libhdf5 still opens the
|
||||
/// file (the image loads at the first metadata read), and that read
|
||||
/// fails with this error.
|
||||
Unloadable(FormatError),
|
||||
}
|
||||
|
||||
impl CacheImage {
|
||||
/// Decode the metadata cache image at `location` in `data` (the file
|
||||
/// from the superblock on, up to its recorded end of file). The image is
|
||||
/// checked as libhdf5 checks it (`H5C__decode_cache_image_header`,
|
||||
/// `H5C__reconstruct_cache_entry`): signature and version, the image
|
||||
/// length it records, entry types, rings and ages in range, entry
|
||||
/// addresses inside the file and not repeated, flush-dependency parents
|
||||
/// already in the cache.
|
||||
///
|
||||
/// One check is stricter than libhdf5's: an entry must end inside the
|
||||
/// file. libhdf5 checks only that it starts there, and serves the rest
|
||||
/// from the image; the images libhdf5 writes never do this (every entry
|
||||
/// lies below the image block, which is written last), and the bytes an
|
||||
/// entry would put past the end of file have nowhere to go in a view of
|
||||
/// the file.
|
||||
///
|
||||
/// libhdf5 does not verify the block's trailing checksum when it loads
|
||||
/// an image, so neither does this.
|
||||
pub fn decode(
|
||||
data: &[u8],
|
||||
location: CacheImageLocation,
|
||||
sb: &Superblock,
|
||||
) -> Result<Self, FormatError> {
|
||||
let (offset_size, length_size) = (sb.offset_size, sb.length_size);
|
||||
let bad = FormatError::InvalidCacheImage;
|
||||
let block = image_block(data, location)?;
|
||||
let eoa = data.len() as u64;
|
||||
let mut c = Cursor::new(block, bad(RAN_OFF));
|
||||
|
||||
// Header: signature, version, flags, image data length, entry count.
|
||||
if c.take(4)? != MDCI_SIGNATURE {
|
||||
return Err(bad("bad metadata cache image header signature"));
|
||||
}
|
||||
if c.u8()? != 0 {
|
||||
return Err(bad("bad metadata cache image version"));
|
||||
}
|
||||
if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 {
|
||||
return Err(bad("MDC resize status not yet supported"));
|
||||
}
|
||||
if c.uint(length_size)? != location.length {
|
||||
return Err(bad("bad metadata cache image data length"));
|
||||
}
|
||||
let n_entries = c.uint(4)?;
|
||||
if n_entries == 0 {
|
||||
return Err(bad("bad metadata cache entry count"));
|
||||
}
|
||||
|
||||
let mut entries = Vec::new();
|
||||
// What is in libhdf5's cache when it loads the image: the superblock
|
||||
// and the superblock extension's object header (read to find the
|
||||
// image). Each entry's flush-dependency parents are looked up in the
|
||||
// cache as the entry is inserted (`H5C__reconstruct_cache_contents`
|
||||
// searches the index inside the loop that inserts the entries, in
|
||||
// HDF5 1.14.6 and 2.0.0 alike), so a parent must be one of those or
|
||||
// an earlier entry.
|
||||
let mut cached = BTreeSet::new();
|
||||
cached.insert(0);
|
||||
if let Some(ext) = sb.superblock_extension_address {
|
||||
cached.insert(ext);
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
for _ in 0..n_entries {
|
||||
let type_id = c.u8()?;
|
||||
if type_id >= MDCI_NTYPES {
|
||||
return Err(bad("type id is out of valid range"));
|
||||
}
|
||||
let flags = c.u8()?;
|
||||
if c.u8()? >= MDCI_RING_NTYPES {
|
||||
return Err(bad("ring is out of valid range"));
|
||||
}
|
||||
if c.u8()? > MDCI_AGE_MAX {
|
||||
return Err(bad("entry age is out of policy range"));
|
||||
}
|
||||
let children = c.uint(2)?;
|
||||
// libhdf5 checks the parent flag against the child count only in
|
||||
// debug builds (release builds refuse any entry with children);
|
||||
// the image format's own rule is checked here.
|
||||
if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) {
|
||||
return Err(bad("flush dependency parent flag and child count disagree"));
|
||||
}
|
||||
c.uint(2)?; // dirty dependency children: reset for a read-only open
|
||||
let parents = c.uint(2)?;
|
||||
if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) {
|
||||
return Err(bad("flush dependency child flag and parent count disagree"));
|
||||
}
|
||||
c.uint(4)?; // LRU rank
|
||||
let address = c
|
||||
.addr(offset_size)?
|
||||
.filter(|&a| a < eoa)
|
||||
.ok_or(bad("invalid entry address range"))?;
|
||||
let size = c.uint(length_size)?;
|
||||
if size == 0 {
|
||||
return Err(bad("invalid entry size"));
|
||||
}
|
||||
for _ in 0..parents {
|
||||
let parent = c
|
||||
.addr(offset_size)?
|
||||
.ok_or(bad("invalid flush dependency parent offset"))?;
|
||||
if !seen.contains(&parent) && !cached.contains(&parent) {
|
||||
return Err(bad("fd parent not in cache"));
|
||||
}
|
||||
}
|
||||
let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?;
|
||||
let image_offset = c.pos;
|
||||
c.take(len)?;
|
||||
if address.checked_add(size).is_none_or(|end| end > eoa) {
|
||||
return Err(bad("entry extends past the end of file"));
|
||||
}
|
||||
if !seen.insert(address) {
|
||||
return Err(bad("duplicate addresses in cache"));
|
||||
}
|
||||
entries.push(ImageEntry {
|
||||
address,
|
||||
image_offset,
|
||||
len,
|
||||
});
|
||||
}
|
||||
Ok(CacheImage { location, entries })
|
||||
}
|
||||
|
||||
/// Where the image block is.
|
||||
pub fn location(&self) -> CacheImageLocation {
|
||||
self.location
|
||||
}
|
||||
|
||||
/// The number of entries in the image.
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Whether the image has no entries (a decoded image always has some).
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
/// The file ranges (address, length) the image's entries replace.
|
||||
pub fn entry_ranges(&self) -> impl Iterator<Item = (u64, usize)> + '_ {
|
||||
self.entries.iter().map(|e| (e.address, e.len))
|
||||
}
|
||||
|
||||
/// The image block in `data`, the bytes [`Self::decode`] read it from.
|
||||
pub fn block<'a>(&self, data: &'a [u8]) -> Result<&'a [u8], FormatError> {
|
||||
image_block(data, self.location)
|
||||
}
|
||||
|
||||
/// Write every entry over `dst`, the file's bytes from the superblock
|
||||
/// on (as long as the `data` the image was decoded from), taking the
|
||||
/// entries from `block` (the image block, see [`Self::block`]). `block`
|
||||
/// must not alias `dst`: an entry may land on the block itself.
|
||||
pub fn apply(&self, block: &[u8], dst: &mut [u8]) -> Result<(), FormatError> {
|
||||
let short = || FormatError::InvalidCacheImage("image applied to the wrong file");
|
||||
for e in &self.entries {
|
||||
let src = block
|
||||
.get(e.image_offset..e.image_offset + e.len)
|
||||
.ok_or_else(short)?;
|
||||
let at = usize::try_from(e.address).map_err(|_| short())?;
|
||||
dst.get_mut(at..at + e.len)
|
||||
.ok_or_else(short)?
|
||||
.copy_from_slice(src);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn image_block(data: &[u8], location: CacheImageLocation) -> Result<&[u8], FormatError> {
|
||||
let bad = FormatError::InvalidCacheImage;
|
||||
let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?;
|
||||
let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?;
|
||||
start
|
||||
.checked_add(len)
|
||||
.and_then(|end| data.get(start..end))
|
||||
.ok_or(bad("image block extends past the end of the file"))
|
||||
}
|
||||
|
||||
/// What an opener must do before reading a file's metadata: check the
|
||||
/// superblock extension ([`read_superblock_extension`]; an error means
|
||||
/// libhdf5 refuses to open the file) and decode any metadata cache image
|
||||
/// ([`CacheImage::decode`]). `data` is the file from the superblock on, up
|
||||
/// to its recorded end of file.
|
||||
pub fn cache_image_state(data: &[u8], sb: &Superblock) -> Result<CacheImageState, FormatError> {
|
||||
match read_superblock_extension(data, sb)? {
|
||||
Some(SuperblockExtension {
|
||||
cache_image: Some(location),
|
||||
..
|
||||
}) => Ok(match CacheImage::decode(data, location, sb) {
|
||||
Ok(image) => CacheImageState::Loaded(image),
|
||||
Err(e) => CacheImageState::Unloadable(e),
|
||||
}),
|
||||
_ => Ok(CacheImageState::Absent),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`cache_image_state`] for a reader that holds the file's bytes in a
|
||||
/// buffer of its own: check the superblock extension and write any cache
|
||||
/// image over `data` in place (only the image block is copied). An image
|
||||
/// libhdf5 cannot load is an error here: such a reader has no way to open
|
||||
/// the file and fail each object instead.
|
||||
pub fn apply_cache_image_in_place(data: &mut [u8], sb: &Superblock) -> Result<(), FormatError> {
|
||||
match cache_image_state(data, sb)? {
|
||||
CacheImageState::Absent => Ok(()),
|
||||
CacheImageState::Unloadable(e) => Err(e),
|
||||
CacheImageState::Loaded(image) => {
|
||||
let block = image.block(data)?.to_vec();
|
||||
image.apply(&block, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The file's bytes with the image at `loc` applied.
|
||||
fn apply_cache_image(
|
||||
data: &[u8],
|
||||
loc: CacheImageLocation,
|
||||
sb: &Superblock,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let image = CacheImage::decode(data, loc, sb)?;
|
||||
let mut out = data.to_vec();
|
||||
image.apply(image.block(data)?, &mut out)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn sb_v2(ext: u64) -> Superblock {
|
||||
Superblock {
|
||||
version: 2,
|
||||
offset_size: 8,
|
||||
length_size: 8,
|
||||
base_address: 0,
|
||||
eof_address: 0,
|
||||
root_group_address: 0,
|
||||
group_leaf_node_k: None,
|
||||
group_internal_node_k: None,
|
||||
indexed_storage_internal_node_k: None,
|
||||
free_space_address: None,
|
||||
driver_info_address: None,
|
||||
consistency_flags: 0,
|
||||
superblock_extension_address: Some(ext),
|
||||
checksum: None,
|
||||
page_size: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A file whose superblock extension (a version 1 object header at 48)
|
||||
/// holds the given messages, padded to `len` bytes.
|
||||
fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
for (t, d) in messages {
|
||||
let padded = d.len().div_ceil(8) * 8;
|
||||
body.extend_from_slice(&t.to_le_bytes());
|
||||
body.extend_from_slice(&(padded as u16).to_le_bytes());
|
||||
body.extend_from_slice(&[0x14, 0, 0, 0]);
|
||||
body.extend_from_slice(d);
|
||||
body.resize(body.len() + padded - d.len(), 0);
|
||||
}
|
||||
let mut f = vec![0u8; 48];
|
||||
f.push(1);
|
||||
f.push(0);
|
||||
f.extend_from_slice(&(messages.len() as u16).to_le_bytes());
|
||||
f.extend_from_slice(&1u32.to_le_bytes());
|
||||
f.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
f.extend_from_slice(&[0; 4]);
|
||||
f.extend_from_slice(&body);
|
||||
f.resize(len, 0);
|
||||
f
|
||||
}
|
||||
|
||||
fn fsinfo_v1(page_size: u64, persist: bool, n_addrs: usize) -> Vec<u8> {
|
||||
let mut m = vec![1, 1, u8::from(persist)];
|
||||
m.extend_from_slice(&1u64.to_le_bytes());
|
||||
m.extend_from_slice(&page_size.to_le_bytes());
|
||||
m.extend_from_slice(&0u16.to_le_bytes());
|
||||
m.extend_from_slice(&u64::MAX.to_le_bytes());
|
||||
for _ in 0..n_addrs {
|
||||
m.extend_from_slice(&u64::MAX.to_le_bytes());
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
fn mdci(address: u64, length: u64) -> Vec<u8> {
|
||||
let mut m = vec![0];
|
||||
m.extend_from_slice(&address.to_le_bytes());
|
||||
m.extend_from_slice(&length.to_le_bytes());
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_extension() {
|
||||
assert_eq!(
|
||||
read_superblock_extension(&[0; 64], &sb_v2(u64::MAX)).unwrap(),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_space_info_as_libhdf5_decodes_it() {
|
||||
// What FileWriter::with_page_size writes.
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256);
|
||||
let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap();
|
||||
assert_eq!(ext.file_space_info.unwrap().page_size, 4096);
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 512);
|
||||
assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok());
|
||||
|
||||
let refused = |m: Vec<u8>| {
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &m)], 512);
|
||||
read_superblock_extension(&f, &sb_v2(48)).unwrap_err()
|
||||
};
|
||||
// Persisting, but too short for the manager addresses.
|
||||
let mut short = fsinfo_v1(4096, true, 12);
|
||||
short.truncate(short.len() - 8);
|
||||
assert_eq!(refused(short), ext_err(RAN_OFF));
|
||||
assert!(matches!(
|
||||
refused(fsinfo_v1(256, false, 0)),
|
||||
FormatError::InvalidSuperblockExtension(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
refused(fsinfo_v1(0, false, 0)),
|
||||
FormatError::InvalidSuperblockExtension(_)
|
||||
));
|
||||
let mut v2 = fsinfo_v1(4096, false, 0);
|
||||
v2[0] = 2;
|
||||
assert!(matches!(
|
||||
refused(v2),
|
||||
FormatError::InvalidSuperblockExtension(_)
|
||||
));
|
||||
// cve-2020-10810: version 0, strategy ALL_PERSIST, and a message of
|
||||
// 32 bytes that cannot hold the six addresses that follow.
|
||||
let mut v0 = vec![0u8, 1];
|
||||
v0.extend_from_slice(&[0, 1, 0, 0, 0, 0, 0, 0]);
|
||||
v0.resize(32, 0xff);
|
||||
assert_eq!(refused(v0), ext_err(RAN_OFF));
|
||||
// A version 0 message without persistence is fine.
|
||||
let mut v0 = vec![0u8, 2];
|
||||
v0.extend_from_slice(&[0; 8]);
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &v0)], 256);
|
||||
assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_image_location_must_be_inside_the_file() {
|
||||
let f = file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192);
|
||||
let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
ext.cache_image,
|
||||
Some(CacheImageLocation {
|
||||
address: 128,
|
||||
length: 64
|
||||
})
|
||||
);
|
||||
// cve-2020-10812: 256 MiB at 0x10100 in a 2565-byte file.
|
||||
let f = file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565);
|
||||
assert!(matches!(
|
||||
read_superblock_extension(&f, &sb_v2(48)),
|
||||
Err(FormatError::InvalidSuperblockExtension(_))
|
||||
));
|
||||
let f = file_with_ext(&[(MSG_MDCI, &mdci(u64::MAX, 8))], 256);
|
||||
assert!(read_superblock_extension(&f, &sb_v2(48)).is_err());
|
||||
}
|
||||
|
||||
/// A cache image block with `entries` of (address, bytes).
|
||||
fn image(entries: &[(u64, &[u8])]) -> Vec<u8> {
|
||||
let with_deps: Vec<_> = entries.iter().map(|&(a, b)| (a, b, 0, None)).collect();
|
||||
image_with_deps(&with_deps)
|
||||
}
|
||||
|
||||
/// A cache image block with `entries` of (address, bytes, flush
|
||||
/// dependency children, flush dependency parent).
|
||||
fn image_with_deps(entries: &[(u64, &[u8], u16, Option<u64>)]) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
b.extend_from_slice(MDCI_SIGNATURE);
|
||||
b.push(0);
|
||||
b.push(0);
|
||||
b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below
|
||||
b.extend_from_slice(&(entries.len() as u32).to_le_bytes());
|
||||
for &(addr, bytes, children, parent) in entries {
|
||||
let mut flags = 0x02; // in LRU
|
||||
if children > 0 {
|
||||
flags |= MDCI_ENTRY_IS_FD_PARENT;
|
||||
}
|
||||
if parent.is_some() {
|
||||
flags |= MDCI_ENTRY_IS_FD_CHILD;
|
||||
}
|
||||
b.extend_from_slice(&[5, flags, 1, 0]); // type, flags, ring, age
|
||||
b.extend_from_slice(&children.to_le_bytes());
|
||||
b.extend_from_slice(&0u16.to_le_bytes()); // dirty children
|
||||
b.extend_from_slice(&u16::from(parent.is_some()).to_le_bytes());
|
||||
b.extend_from_slice(&0i32.to_le_bytes());
|
||||
b.extend_from_slice(&addr.to_le_bytes());
|
||||
b.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
|
||||
if let Some(p) = parent {
|
||||
b.extend_from_slice(&p.to_le_bytes());
|
||||
}
|
||||
b.extend_from_slice(bytes);
|
||||
}
|
||||
b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5)
|
||||
let n = b.len() as u64;
|
||||
b[6..14].copy_from_slice(&n.to_le_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_image_entries_replace_the_file_bytes() {
|
||||
let img = image(&[(16, b"HEADER"), (40, b"NODE")]);
|
||||
let mut f = vec![0u8; 64];
|
||||
let at = f.len() as u64;
|
||||
f.extend_from_slice(&img);
|
||||
let loc = CacheImageLocation {
|
||||
address: at,
|
||||
length: img.len() as u64,
|
||||
};
|
||||
let out = apply_cache_image(&f, loc, &sb_v2(u64::MAX)).unwrap();
|
||||
assert_eq!(out.len(), f.len());
|
||||
assert_eq!(&out[16..22], b"HEADER");
|
||||
assert_eq!(&out[40..44], b"NODE");
|
||||
assert_eq!(&out[..16], &f[..16]);
|
||||
|
||||
let bad = |img: Vec<u8>| {
|
||||
let mut f = vec![0u8; 64];
|
||||
f.extend_from_slice(&img);
|
||||
let loc = CacheImageLocation {
|
||||
address: 64,
|
||||
length: img.len() as u64,
|
||||
};
|
||||
apply_cache_image(&f, loc, &sb_v2(u64::MAX)).unwrap_err()
|
||||
};
|
||||
let mut sig = image(&[(16, b"x")]);
|
||||
sig[0] = b'X';
|
||||
assert!(matches!(bad(sig), FormatError::InvalidCacheImage(_)));
|
||||
assert!(matches!(
|
||||
bad(image(&[(16, b"a"), (16, b"b")])),
|
||||
FormatError::InvalidCacheImage("duplicate addresses in cache")
|
||||
));
|
||||
assert!(matches!(
|
||||
bad(image(&[(1 << 20, b"far")])),
|
||||
FormatError::InvalidCacheImage("invalid entry address range")
|
||||
));
|
||||
let mut len = image(&[(16, b"x")]);
|
||||
len[6] ^= 1;
|
||||
assert!(matches!(bad(len), FormatError::InvalidCacheImage(_)));
|
||||
// An entry that starts inside the file (64 bytes, then a 60-byte
|
||||
// image) but runs past its end.
|
||||
assert!(matches!(
|
||||
bad(image(&[(123, b"8 bytes!")])),
|
||||
FormatError::InvalidCacheImage("entry extends past the end of file")
|
||||
));
|
||||
let mut cut = image(&[(16, b"abcdef")]);
|
||||
let n = cut.len() as u64 - 8;
|
||||
cut.truncate(cut.len() - 8);
|
||||
cut[6..14].copy_from_slice(&n.to_le_bytes());
|
||||
assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_)));
|
||||
}
|
||||
|
||||
/// libhdf5 resolves an entry's flush-dependency parents as it inserts
|
||||
/// the entry (`H5C__reconstruct_cache_contents`): a parent must be an
|
||||
/// earlier entry, or the superblock or its extension's object header,
|
||||
/// which are cached before the image loads. A parent listed after its
|
||||
/// child fails ("fd parent not in cache?!?").
|
||||
#[test]
|
||||
fn flush_dependency_parents_must_already_be_cached() {
|
||||
let load = |img: Vec<u8>| {
|
||||
let mut f = vec![0u8; 64];
|
||||
f.extend_from_slice(&img);
|
||||
let loc = CacheImageLocation {
|
||||
address: 64,
|
||||
length: img.len() as u64,
|
||||
};
|
||||
apply_cache_image(&f, loc, &sb_v2(48))
|
||||
};
|
||||
// Parent first, as libhdf5 writes images.
|
||||
assert!(
|
||||
load(image_with_deps(&[
|
||||
(16, b"P", 1, None),
|
||||
(40, b"C", 0, Some(16))
|
||||
]))
|
||||
.is_ok()
|
||||
);
|
||||
// Child first: libhdf5 does not find the parent.
|
||||
assert_eq!(
|
||||
load(image_with_deps(&[
|
||||
(40, b"C", 0, Some(16)),
|
||||
(16, b"P", 1, None)
|
||||
]))
|
||||
.unwrap_err(),
|
||||
FormatError::InvalidCacheImage("fd parent not in cache")
|
||||
);
|
||||
// The superblock extension's header (at 48 here) is in the cache.
|
||||
assert!(load(image_with_deps(&[(40, b"C", 0, Some(48))])).is_ok());
|
||||
// An entry cannot be its own parent.
|
||||
assert!(load(image_with_deps(&[(40, b"C", 1, Some(40))])).is_err());
|
||||
}
|
||||
}
|
||||
@@ -803,7 +803,11 @@ impl<'a, 'r> Sources<'a, 'r> {
|
||||
let resolver = self.resolver.ok_or_else(|| {
|
||||
vds_err("external-file virtual dataset sources require a file resolver")
|
||||
})?;
|
||||
self.cached_file = Some((String::from(name), resolver(name)?));
|
||||
let mut bytes = resolver(name)?;
|
||||
if let Some(b) = bytes.as_mut() {
|
||||
load_source_file(b)?;
|
||||
}
|
||||
self.cached_file = Some((String::from(name), bytes));
|
||||
}
|
||||
// An external file is handed over whole; its addresses are relative
|
||||
// to its superblock, so skip any user block.
|
||||
@@ -851,6 +855,23 @@ impl<'a, 'r> Sources<'a, 'r> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Check an external source file's superblock extension as libhdf5 does
|
||||
/// when it opens the file, and write any metadata cache image over its
|
||||
/// metadata in place: libhdf5 reads the image's entries instead of the
|
||||
/// file's own, possibly stale, bytes (`crate::superblock_ext`). A source
|
||||
/// file whose image cannot be loaded is an error, as other corrupt source
|
||||
/// files are here.
|
||||
fn load_source_file(whole: &mut [u8]) -> Result<(), FormatError> {
|
||||
let base = crate::signature::find_signature(whole)?;
|
||||
let sb = crate::superblock::Superblock::parse(&whole[base..], 0)?;
|
||||
// The end of file the superblock records; a truncated source file is
|
||||
// read as before, up to its length.
|
||||
let end = sb
|
||||
.data_end(base as u64, whole.len() as u64)
|
||||
.map_or(whole.len(), |e| base + e as usize);
|
||||
crate::superblock_ext::apply_cache_image_in_place(&mut whole[base..end], &sb)
|
||||
}
|
||||
|
||||
/// Whether elements of `dt` contain addresses into their own file:
|
||||
/// variable-length data (global-heap IDs) or references.
|
||||
fn holds_file_addresses(dt: &Datatype) -> bool {
|
||||
|
||||
Reference in New Issue
Block a user