From b3058ca46eb36a4d66090de679dd705e6b61106b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 10:28:28 -0500 Subject: [PATCH] feat: decode the superblock extension at open; read metadata cache images libhdf5 decodes the messages of a v2/v3 superblock's extension when it opens a file (H5F__super_read) and refuses the file when one does not decode. We never looked at them, so we opened cve-2020-10810 (a File Space Info message too short for the free-space manager addresses it announces) and cve-2020-10812 (a metadata cache image past the end of the file), both of which libhdf5 refuses. A file written with a metadata cache image keeps its metadata cache entries in an image block the extension points at; libhdf5 loads them over the file's own bytes before it reads any metadata (H5C__load_cache_image, H5C__reconstruct_cache_contents). In h5clear_mdc_image.h5 the root group's header exists only in the image, so every reader failed with InvalidObjectHeaderVersion(0). The new clawhdf5_format::superblock_ext module: - read_superblock_extension decodes the v1 B-tree K, File Space Info and Metadata Cache Image messages with libhdf5's checks (versions, page size 512 B .. 1 GiB, the addresses a persisting message lists, the image inside the file), with the new FormatError::InvalidSuperblockExtension; - apply_cache_image checks an image block as libhdf5 does (signature, version, recorded length, entry types, rings, ages, addresses inside the file and not repeated, flush-dependency parents) and returns the file's bytes with every entry written at its address (FormatError::InvalidCacheImage); - metadata_view does both. File, MmapFile and LazyFile (and so h5rs) call metadata_view at open and read an image file through the patched copy; the conformance probe does the same. The image's trailing checksum is not verified, as libhdf5 does not verify it. tests/fixtures/h5clear_mdc_image.h5 is libhdf5's own test file. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 13 + crates/clawhdf5-format/src/error.rs | 14 + crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/superblock_ext.rs | 630 ++++++++++++++++++ crates/clawhdf5/src/lazy.rs | 17 +- crates/clawhdf5/src/mmap_file.rs | 15 +- crates/clawhdf5/src/reader.rs | 25 +- .../tests/fixtures/h5clear_mdc_image.h5 | Bin 0 -> 23467 bytes .../tests/header_validation_interop.rs | 91 ++- crates/clawhdf5/tests/metadata_cache_image.rs | 48 ++ 10 files changed, 839 insertions(+), 15 deletions(-) create mode 100644 crates/clawhdf5-format/src/superblock_ext.rs create mode 100644 crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 create mode 100644 crates/clawhdf5/tests/metadata_cache_image.rs diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 844f343..c4cf524 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -712,6 +712,19 @@ fn main() { return; } }; + // libhdf5 decodes the superblock extension at open, and loads a + // metadata cache image over the file's own metadata. + let view = match guarded(|| { + clawhdf5_format::superblock_ext::metadata_view(hdf5, &sb).map_err(e) + }) { + Ok(v) => v, + Err(msg) => { + top.insert("open_error".into(), Value::String(msg)); + println!("{}", Value::Object(top)); + return; + } + }; + let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5); top.insert("superblock_version".into(), json!(sb.version)); let ctx = Ctx { data: hdf5, diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index ddd9ba7..7f9e3d2 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -235,6 +235,14 @@ pub enum FormatError { /// element size that overflows, contiguous storage past the end of the /// file, compact data of the wrong size. InvalidDatasetStorage(&'static str), + /// A superblock extension message libhdf5 refuses to decode when it + /// opens the file (the reason is libhdf5's own error text): a File Space + /// Info message that runs off its end or has a bad page size, a metadata + /// cache image outside the file, … + InvalidSuperblockExtension(&'static str), + /// A metadata cache image block libhdf5 refuses to load (the reason is + /// libhdf5's own error text). + InvalidCacheImage(&'static str), } impl fmt::Display for FormatError { @@ -515,6 +523,12 @@ impl fmt::Display for FormatError { FormatError::InvalidDatasetStorage(why) => { write!(f, "invalid dataset storage: {why}") } + FormatError::InvalidSuperblockExtension(why) => { + write!(f, "invalid superblock extension: {why}") + } + FormatError::InvalidCacheImage(why) => { + write!(f, "invalid metadata cache image: {why}") + } } } } diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..e10711b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -118,6 +118,7 @@ pub mod selection; pub mod shared_message; pub mod signature; pub mod superblock; +pub mod superblock_ext; pub mod symbol_table; #[cfg(all( test, diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs new file mode 100644 index 0000000..d7ec3cc --- /dev/null +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -0,0 +1,630 @@ +//! The superblock extension of a version 2 or 3 superblock, and the +//! metadata cache image it can point to. +//! +//! libhdf5 reads the extension when it opens a file (`H5F__super_read`) and +//! decodes the messages that configure the file: v1 B-tree "K" values, File +//! Space Info, and the Metadata Cache Image. A message that does not decode +//! makes the file fail to open, so [`read_superblock_extension`] decodes and +//! checks them the way libhdf5 does. +//! +//! A metadata cache image (written with `H5Pset_mdc_image_config`) is a +//! block holding serialized metadata cache entries — object headers, B-tree +//! nodes, heaps — each with its file address. libhdf5 loads it into its +//! cache before it reads any other metadata (`H5C__load_cache_image`, +//! `H5C__reconstruct_cache_contents`), and the entries take the place of +//! the file's bytes at their addresses: the file itself may hold stale or +//! no metadata there (in `h5clear_mdc_image.h5` the root group's header is +//! only in the image). [`apply_cache_image`] does the same with bytes: it +//! returns a copy of the file with every entry written at its address, so +//! every parser reads what libhdf5 reads. + +#[cfg(not(feature = "std"))] +use alloc::{collections::BTreeSet, vec::Vec}; +#[cfg(feature = "std")] +use std::collections::BTreeSet; + +use crate::error::FormatError; +use crate::message_type::MessageType; +use crate::object_header::ObjectHeader; +use crate::superblock::Superblock; + +/// Message type of the File Space Info message. +const MSG_FSINFO: u16 = 0x0017; +/// Message type of the Metadata Cache Image message. +const MSG_MDCI: u16 = 0x0018; +/// Header message flag: the library did not know the message when it wrote +/// it back (`H5O_MSG_FLAG_WAS_UNKNOWN`); libhdf5 then ignores its contents. +const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20; + +/// `H5F_FILE_SPACE_PAGE_SIZE_MIN` / `_MAX`. +const PAGE_SIZE_MIN: u64 = 512; +const PAGE_SIZE_MAX: u64 = 1024 * 1024 * 1024; +/// libhdf5's default file space page size, used for a version 0 message. +const PAGE_SIZE_DEFAULT: u64 = 4096; +/// Free-space managers whose addresses a persisting version 1 File Space +/// Info message lists (`H5F_MEM_PAGE_SUPER` .. `H5F_MEM_PAGE_NTYPES`), and +/// a version 0 one (`H5FD_MEM_SUPER` .. `H5FD_MEM_NTYPES`). +const FSM_ADDRS_V1: usize = 12; +const FSM_ADDRS_V0: usize = 6; + +/// Metadata cache image block limits (`H5Cimage.c`, `H5ACprivate.h`). +const MDCI_SIGNATURE: &[u8; 4] = b"MDCI"; +const MDCI_HAVE_RESIZE_STATUS: u8 = 0x01; +const MDCI_ENTRY_IS_FD_PARENT: u8 = 0x04; +const MDCI_ENTRY_IS_FD_CHILD: u8 = 0x08; +/// `H5AC_NTYPES`: entry type ids are below this. +const MDCI_NTYPES: u8 = 30; +/// `H5C_RING_NTYPES`. +const MDCI_RING_NTYPES: u8 = 6; +/// `H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX`. +const MDCI_AGE_MAX: u8 = 100; + +/// A decoded File Space Info message (0x0017), mapped to version 1 as +/// libhdf5 maps a version 0 one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileSpaceInfo { + /// Message version as stored (0 or 1). + pub version: u8, + /// File space strategy (`H5F_fspace_strategy_t`). + pub strategy: u8, + /// Whether free space is persisted. + pub persist: bool, + /// Free-space section threshold. + pub threshold: u64, + /// File space page size. + pub page_size: u64, +} + +/// Where a metadata cache image block is (Metadata Cache Image message, +/// 0x0018). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheImageLocation { + /// Address of the image block. + pub address: u64, + /// Length of the image block in bytes. + pub length: u64, +} + +/// The messages of a superblock extension that libhdf5 decodes at open. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SuperblockExtension { + /// v1 B-tree "K" values (chunk index, symbol table node, symbol table + /// leaf), when the extension overrides the defaults. + pub btree_k: Option<(u16, u16, u16)>, + /// The File Space Info message. + pub file_space_info: Option, + /// The metadata cache image, when the file has one. + pub cache_image: Option, +} + +fn ext_err(why: &'static str) -> FormatError { + FormatError::InvalidSuperblockExtension(why) +} + +const RAN_OFF: &str = "ran off end of input buffer while decoding"; + +/// A little-endian cursor over one message or block, failing with +/// `overrun` when it runs off the end. +struct Cursor<'a> { + data: &'a [u8], + pos: usize, + overrun: FormatError, +} + +impl<'a> Cursor<'a> { + fn new(data: &'a [u8], overrun: FormatError) -> Self { + Cursor { + data, + pos: 0, + overrun, + } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], FormatError> { + let end = self + .pos + .checked_add(n) + .filter(|&e| e <= self.data.len()) + .ok_or_else(|| self.overrun.clone())?; + let s = &self.data[self.pos..end]; + self.pos = end; + Ok(s) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn uint(&mut self, width: u8) -> Result { + let b = self.take(width as usize)?; + Ok(b.iter() + .rev() + .fold(0u64, |acc, &x| (acc << 8) | u64::from(x))) + } + + /// An address of `width` bytes; `None` when undefined (all ones). + fn addr(&mut self, width: u8) -> Result, FormatError> { + let v = self.uint(width)?; + let undef = if width >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(width))) - 1 + }; + Ok((v != undef).then_some(v)) + } +} + +/// Decode and check the superblock extension of `sb`, as libhdf5 does when +/// it opens the file. `data` is the file from the superblock on, up to the +/// end of file the superblock records (its end is libhdf5's "eoa"). +/// +/// Returns `Ok(None)` for a superblock without an extension (versions 0 +/// and 1 have none). A message libhdf5 fails to decode, or a cache image +/// that does not lie inside the file, is an error: libhdf5 refuses to open +/// such a file (`cve-2020-10810`: a File Space Info message too short for +/// the free-space manager addresses it announces; `cve-2020-10812`: a cache +/// image past the end of the file). +pub fn read_superblock_extension( + data: &[u8], + sb: &Superblock, +) -> Result, FormatError> { + let os = sb.offset_size; + let ls = sb.length_size; + let undef = if os >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(os))) - 1 + }; + let Some(addr) = sb.superblock_extension_address.filter(|&a| a != undef) else { + return Ok(None); + }; + let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?; + let header = ObjectHeader::parse(data, addr, os, ls)?; + let eoa = data.len() as u64; + + let mut ext = SuperblockExtension::default(); + for msg in &header.messages { + match msg.msg_type { + MessageType::BTreeKValues => { + let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF)); + if c.u8()? != 0 { + return Err(ext_err("bad version number for v1 B-tree 'K' message")); + } + let chunk = c.uint(2)? as u16; + let snode = c.uint(2)? as u16; + let leaf = c.uint(2)? as u16; + ext.btree_k = Some((chunk, snode, leaf)); + } + MessageType::Unknown(MSG_FSINFO) if msg.flags & MSG_FLAG_WAS_UNKNOWN == 0 => { + ext.file_space_info = Some(decode_fsinfo(&msg.data, os, ls)?); + } + MessageType::Unknown(MSG_MDCI) => { + let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF)); + if c.u8()? != 0 { + return Err(ext_err( + "bad version number for metadata cache image message", + )); + } + let address = c.addr(os)?; + let length = c.uint(ls)?; + let Some(address) = address else { + return Err(ext_err("metadata cache image address is undefined")); + }; + if address.checked_add(length).is_none_or(|end| end > eoa) { + return Err(ext_err( + "metadata cache image: address plus size exceeds file eoa", + )); + } + ext.cache_image = Some(CacheImageLocation { address, length }); + } + _ => {} + } + } + Ok(Some(ext)) +} + +/// `H5O__fsinfo_decode` plus the checks `H5F__super_read` makes on it. +fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result { + let mut c = Cursor::new(data, ext_err(RAN_OFF)); + let version = c.u8()?; + let info = if version == 0 { + let old_strategy = c.u8()?; + let threshold = c.uint(ls)?; + // H5F_file_space_type_t: 1 ALL_PERSIST, 2 ALL, 3 AGGR_VFD, 4 VFD. + let (strategy, persist) = match old_strategy { + 1 => { + for _ in 0..FSM_ADDRS_V0 { + c.addr(os)?; + } + (0, true) + } + 2 => (0, false), + 3 => (2, false), + 4 => (3, false), + _ => return Err(ext_err("invalid file space strategy")), + }; + FileSpaceInfo { + version, + strategy, + persist, + threshold, + page_size: PAGE_SIZE_DEFAULT, + } + } else { + if version > 1 { + return Err(ext_err("File space info message's version out of bounds")); + } + let strategy = c.u8()?; + let persist = c.u8()? != 0; + let threshold = c.uint(ls)?; + let page_size = c.uint(ls)?; + if page_size == 0 || page_size > PAGE_SIZE_MAX { + return Err(ext_err("invalid page size in file space info")); + } + c.uint(2)?; // page end metadata threshold + c.addr(os)?; // EOA before the free-space managers + if persist { + for _ in 0..FSM_ADDRS_V1 { + c.addr(os)?; + } + } + FileSpaceInfo { + version, + strategy, + persist, + threshold, + page_size, + } + }; + if info.page_size < PAGE_SIZE_MIN { + return Err(ext_err("file space page size too small")); + } + Ok(info) +} + +/// One entry of a metadata cache image: `len` bytes at `image_offset` in +/// the image block, belonging at file address `address`. +struct ImageEntry { + address: u64, + image_offset: usize, + len: usize, +} + +/// Decode the metadata cache image at `location` in `data` (the file from +/// the superblock on, up to its recorded end of file) and return a copy of +/// `data` with every cached entry written at its address — the metadata +/// libhdf5 reads for this file. The image is checked as libhdf5 checks it +/// (`H5C__decode_cache_image_header`, `H5C__reconstruct_cache_entry`): +/// signature and version, the image length it records, entry types, rings +/// and ages in range, entry addresses inside the file and not repeated, +/// flush-dependency parents that are earlier entries. +/// +/// libhdf5 does not verify the block's trailing checksum when it loads an +/// image, so neither does this. +pub fn apply_cache_image( + data: &[u8], + location: CacheImageLocation, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let bad = FormatError::InvalidCacheImage; + let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; + let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; + let block = start + .checked_add(len) + .and_then(|end| data.get(start..end)) + .ok_or(bad("image block extends past the end of the file"))?; + let eoa = data.len() as u64; + let mut c = Cursor::new(block, bad(RAN_OFF)); + + // Header: signature, version, flags, image data length, entry count. + if c.take(4)? != MDCI_SIGNATURE { + return Err(bad("bad metadata cache image header signature")); + } + if c.u8()? != 0 { + return Err(bad("bad metadata cache image version")); + } + if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 { + return Err(bad("MDC resize status not yet supported")); + } + if c.uint(length_size)? != location.length { + return Err(bad("bad metadata cache image data length")); + } + let n_entries = c.uint(4)?; + if n_entries == 0 { + return Err(bad("bad metadata cache entry count")); + } + + let mut entries = Vec::new(); + let mut seen = BTreeSet::new(); + for _ in 0..n_entries { + let type_id = c.u8()?; + if type_id >= MDCI_NTYPES { + return Err(bad("type id is out of valid range")); + } + let flags = c.u8()?; + if c.u8()? >= MDCI_RING_NTYPES { + return Err(bad("ring is out of valid range")); + } + if c.u8()? > MDCI_AGE_MAX { + return Err(bad("entry age is out of policy range")); + } + let children = c.uint(2)?; + // libhdf5 checks the parent flag against the child count only in + // debug builds (release builds refuse any entry with children); the + // image format's own rule is checked here. + if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) { + return Err(bad("flush dependency parent flag and child count disagree")); + } + c.uint(2)?; // dirty dependency children: reset for a read-only open + let parents = c.uint(2)?; + if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) { + return Err(bad("flush dependency child flag and parent count disagree")); + } + c.uint(4)?; // LRU rank + let address = c + .addr(offset_size)? + .filter(|&a| a < eoa) + .ok_or(bad("invalid entry address range"))?; + let size = c.uint(length_size)?; + if size == 0 { + return Err(bad("invalid entry size")); + } + for _ in 0..parents { + let parent = c + .addr(offset_size)? + .ok_or(bad("invalid flush dependency parent offset"))?; + if !seen.contains(&parent) { + return Err(bad("flush dependency parent not in the image")); + } + } + let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?; + let image_offset = c.pos; + c.take(len)?; + if !seen.insert(address) { + return Err(bad("duplicate addresses in cache")); + } + entries.push(ImageEntry { + address, + image_offset, + len, + }); + } + + let mut out = data.to_vec(); + for e in &entries { + // address < eoa <= usize::MAX, and the entry's bytes came from the + // block, so neither conversion nor the sum can fail. + let at = e.address as usize; + let end = at + e.len; + if end > out.len() { + out.resize(end, 0); + } + out[at..end].copy_from_slice(&block[e.image_offset..e.image_offset + e.len]); + } + Ok(out) +} + +/// What a reader must do before reading a file's metadata, in one call: +/// check the superblock extension ([`read_superblock_extension`]) and, +/// when the file has a metadata cache image, return the file's bytes with +/// the image applied ([`apply_cache_image`]). `Ok(None)` means read `data` +/// as it is. +pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result>, FormatError> { + match read_superblock_extension(data, sb)? { + Some(SuperblockExtension { + cache_image: Some(location), + .. + }) => apply_cache_image(data, location, sb.offset_size, sb.length_size).map(Some), + _ => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sb_v2(ext: u64) -> Superblock { + Superblock { + version: 2, + offset_size: 8, + length_size: 8, + base_address: 0, + eof_address: 0, + root_group_address: 0, + group_leaf_node_k: None, + group_internal_node_k: None, + indexed_storage_internal_node_k: None, + free_space_address: None, + driver_info_address: None, + consistency_flags: 0, + superblock_extension_address: Some(ext), + checksum: None, + page_size: None, + } + } + + /// A file whose superblock extension (a version 1 object header at 48) + /// holds the given messages, padded to `len` bytes. + fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec { + let mut body = Vec::new(); + for (t, d) in messages { + let padded = d.len().div_ceil(8) * 8; + body.extend_from_slice(&t.to_le_bytes()); + body.extend_from_slice(&(padded as u16).to_le_bytes()); + body.extend_from_slice(&[0x14, 0, 0, 0]); + body.extend_from_slice(d); + body.resize(body.len() + padded - d.len(), 0); + } + let mut f = vec![0u8; 48]; + f.push(1); + f.push(0); + f.extend_from_slice(&(messages.len() as u16).to_le_bytes()); + f.extend_from_slice(&1u32.to_le_bytes()); + f.extend_from_slice(&(body.len() as u32).to_le_bytes()); + f.extend_from_slice(&[0; 4]); + f.extend_from_slice(&body); + f.resize(len, 0); + f + } + + fn fsinfo_v1(page_size: u64, persist: bool, n_addrs: usize) -> Vec { + let mut m = vec![1, 1, u8::from(persist)]; + m.extend_from_slice(&1u64.to_le_bytes()); + m.extend_from_slice(&page_size.to_le_bytes()); + m.extend_from_slice(&0u16.to_le_bytes()); + m.extend_from_slice(&u64::MAX.to_le_bytes()); + for _ in 0..n_addrs { + m.extend_from_slice(&u64::MAX.to_le_bytes()); + } + m + } + + fn mdci(address: u64, length: u64) -> Vec { + let mut m = vec![0]; + m.extend_from_slice(&address.to_le_bytes()); + m.extend_from_slice(&length.to_le_bytes()); + m + } + + #[test] + fn no_extension() { + assert_eq!( + read_superblock_extension(&[0; 64], &sb_v2(u64::MAX)).unwrap(), + None + ); + } + + #[test] + fn file_space_info_as_libhdf5_decodes_it() { + // What FileWriter::with_page_size writes. + let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256); + let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap(); + assert_eq!(ext.file_space_info.unwrap().page_size, 4096); + let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 512); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok()); + + let refused = |m: Vec| { + let f = file_with_ext(&[(MSG_FSINFO, &m)], 512); + read_superblock_extension(&f, &sb_v2(48)).unwrap_err() + }; + // Persisting, but too short for the manager addresses. + let mut short = fsinfo_v1(4096, true, 12); + short.truncate(short.len() - 8); + assert_eq!(refused(short), ext_err(RAN_OFF)); + assert!(matches!( + refused(fsinfo_v1(256, false, 0)), + FormatError::InvalidSuperblockExtension(_) + )); + assert!(matches!( + refused(fsinfo_v1(0, false, 0)), + FormatError::InvalidSuperblockExtension(_) + )); + let mut v2 = fsinfo_v1(4096, false, 0); + v2[0] = 2; + assert!(matches!( + refused(v2), + FormatError::InvalidSuperblockExtension(_) + )); + // cve-2020-10810: version 0, strategy ALL_PERSIST, and a message of + // 32 bytes that cannot hold the six addresses that follow. + let mut v0 = vec![0u8, 1]; + v0.extend_from_slice(&[0, 1, 0, 0, 0, 0, 0, 0]); + v0.resize(32, 0xff); + assert_eq!(refused(v0), ext_err(RAN_OFF)); + // A version 0 message without persistence is fine. + let mut v0 = vec![0u8, 2]; + v0.extend_from_slice(&[0; 8]); + let f = file_with_ext(&[(MSG_FSINFO, &v0)], 256); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok()); + } + + #[test] + fn cache_image_location_must_be_inside_the_file() { + let f = file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192); + let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap(); + assert_eq!( + ext.cache_image, + Some(CacheImageLocation { + address: 128, + length: 64 + }) + ); + // cve-2020-10812: 256 MiB at 0x10100 in a 2565-byte file. + let f = file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565); + assert!(matches!( + read_superblock_extension(&f, &sb_v2(48)), + Err(FormatError::InvalidSuperblockExtension(_)) + )); + let f = file_with_ext(&[(MSG_MDCI, &mdci(u64::MAX, 8))], 256); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_err()); + } + + /// A cache image block with `entries` of (address, bytes). + fn image(entries: &[(u64, &[u8])]) -> Vec { + let mut b = Vec::new(); + b.extend_from_slice(MDCI_SIGNATURE); + b.push(0); + b.push(0); + b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below + b.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + for (addr, bytes) in entries { + b.extend_from_slice(&[5, 0x02, 1, 0]); // type, flags (in LRU), ring, age + b.extend_from_slice(&[0; 6]); // children, dirty children, parents + b.extend_from_slice(&0i32.to_le_bytes()); + b.extend_from_slice(&addr.to_le_bytes()); + b.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + b.extend_from_slice(bytes); + } + b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5) + let n = b.len() as u64; + b[6..14].copy_from_slice(&n.to_le_bytes()); + b + } + + #[test] + fn cache_image_entries_replace_the_file_bytes() { + let img = image(&[(16, b"HEADER"), (40, b"NODE")]); + let mut f = vec![0u8; 64]; + let at = f.len() as u64; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: at, + length: img.len() as u64, + }; + let out = apply_cache_image(&f, loc, 8, 8).unwrap(); + assert_eq!(out.len(), f.len()); + assert_eq!(&out[16..22], b"HEADER"); + assert_eq!(&out[40..44], b"NODE"); + assert_eq!(&out[..16], &f[..16]); + + let bad = |img: Vec| { + let mut f = vec![0u8; 64]; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: 64, + length: img.len() as u64, + }; + apply_cache_image(&f, loc, 8, 8).unwrap_err() + }; + let mut sig = image(&[(16, b"x")]); + sig[0] = b'X'; + assert!(matches!(bad(sig), FormatError::InvalidCacheImage(_))); + assert!(matches!( + bad(image(&[(16, b"a"), (16, b"b")])), + FormatError::InvalidCacheImage("duplicate addresses in cache") + )); + assert!(matches!( + bad(image(&[(1 << 20, b"far")])), + FormatError::InvalidCacheImage("invalid entry address range") + )); + let mut len = image(&[(16, b"x")]); + len[6] ^= 1; + assert!(matches!(bad(len), FormatError::InvalidCacheImage(_))); + let mut cut = image(&[(16, b"abcdef")]); + let n = cut.len() as u64 - 8; + cut.truncate(cut.len() - 8); + cut[6..14].copy_from_slice(&n.to_le_bytes()); + assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_))); + } +} diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 3232ee6..8746f5f 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -46,6 +46,9 @@ pub struct LazyFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, + /// The metadata as libhdf5 reads it when the file holds a metadata + /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. + overlay: Option>, root_header: ObjectHeader, /// Cache of parsed object headers, keyed by address. header_cache: RefCell>, @@ -82,7 +85,13 @@ impl LazyFile { let superblock = Superblock::parse(data, 0)?; // Refuse a truncated file; read nothing past the recorded end of file. let end = base + superblock.data_end(base as u64, whole_len)? as usize; - let data = &reader.as_bytes()[base..end]; + // Decode the superblock extension as libhdf5 does at open, and load + // a metadata cache image over the file's metadata. + let overlay = clawhdf5_format::superblock_ext::metadata_view( + &reader.as_bytes()[base..end], + &superblock, + )?; + let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]); let root_header = ObjectHeader::parse( data, superblock.root_group_address as usize, @@ -94,6 +103,7 @@ impl LazyFile { base, end, superblock, + overlay, root_header, header_cache: RefCell::new(HashMap::new()), }) @@ -111,7 +121,10 @@ impl LazyFile { } fn hdf5_bytes(&self) -> &[u8] { - &self.reader.as_bytes()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.reader.as_bytes()[self.base..self.end], + } } /// Returns a reference to the parsed superblock. diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 2a8400b..05d581a 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -38,6 +38,9 @@ pub struct MmapFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, + /// The metadata as libhdf5 reads it when the file holds a metadata + /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. + overlay: Option>, } impl MmapFile { @@ -50,18 +53,28 @@ impl MmapFile { let superblock = Superblock::parse(data, 0)?; // Refuse a truncated file; read nothing past the recorded end of file. let end = base + superblock.data_end(base as u64, whole_len)? as usize; + // Decode the superblock extension as libhdf5 does at open, and load + // a metadata cache image over the file's metadata. + let overlay = clawhdf5_format::superblock_ext::metadata_view( + &reader.as_bytes()[base..end], + &superblock, + )?; Ok(Self { reader, base, end, superblock, + overlay, }) } /// The file's bytes from the superblock on — the space HDF5 addresses /// index into. fn hdf5_bytes(&self) -> &[u8] { - &self.reader.as_bytes()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.reader.as_bytes()[self.base..self.end], + } } /// Size of the user block before the superblock (0 for most files). diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 40ac08b..c892eea 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -20,6 +20,7 @@ use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; @@ -55,6 +56,11 @@ struct FileData { base: usize, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). end: usize, + /// The file's metadata as libhdf5 reads it when the file holds a + /// metadata cache image: the bytes from the superblock to the end of + /// file with the image's entries written in + /// ([`superblock_ext::metadata_view`]). `None` for every other file. + overlay: Option>, } impl FileData { @@ -68,11 +74,26 @@ impl FileData { let end = superblock.data_end(base as u64, whole.len() as u64)?; // data_end is at most the file length (less the user block). let end = base + end as usize; - Ok((Self { backing, base, end }, superblock)) + // libhdf5 decodes the superblock extension at open (a message it + // cannot decode fails the open) and loads a metadata cache image + // over the file's own metadata. + let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?; + Ok(( + Self { + backing, + base, + end, + overlay, + }, + superblock, + )) } fn as_bytes(&self) -> &[u8] { - &self.backing.whole_file()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.backing.whole_file()[self.base..self.end], + } } fn len(&self) -> usize { diff --git a/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 b/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 new file mode 100644 index 0000000000000000000000000000000000000000..6ed8b702cfbd9ae5f61ea2698fb65d404cddf7ef GIT binary patch literal 23467 zcmeI)f1H%_{{Qi5`(dqqtZZ3HrX;IhR;?^nrfgZMtRySRsFg*s5@(UlU}+_6>9DdG z35yOZ$x572o$MhjhpZHXBq57*Rwwm+Jeu+Oer})J_Yb$*_pi_Ax)`>??pPyJ~Cp%tmof1ht{A2aU9r)u8{BZ~V zcin+A3-ZUL1Q)dbx#&OZoOxiNJP;h0xEFUM2hYHmU5`%q-*tQb_}Bi=z5^}z=cS+} zTESO(8|()^J?sw`_p#qT+tbh%?U9a-aPc_qc|God@cakCb99F1=>pHy4MFsP=j;XBx+k}fV%;14;Af_R7=#m$ zg}$&o2f4^Y2>B?$Nc`T#>+?Q&jR(W)JOo~=_s#1)44&6(^4$I5`3Jyj@Y=kN6XCU- zf}t1&ujdSm##tDPb5WRlT*P_;!YD=wrlJ%x@Ou~U-{Av&v%FeYOPE``^2C8pyVlwl?!C`Se6q7n;Gg=+lXC5s)8hWEM; z`ohn>2 zyxynr44y+h)}aCKqY)bs$7XEBxA?tF9y^{4Uo*b0Ll^>IJEtNazRtbR*JCEU*Ad)| za@>Xrc+b73FJc8=K@6+07XN_P{tlY31I^ftKnun%q@pd-k%8a43}?sFP=M1h0=_Rt zVibJMpNYA+3ze9U1@PWiVIivV5Z=QF@cw>`Pw*MM$6sL^zC$M0*cCzaL>BrY8#%~D z9)9mq$c~O9V{kUc;v9^F<4O_ELlnM77UOZ$;3+JDua#OXh4=XOh6dL_`S<`c03;w;5ai87b1*{FbR|4>t#9W;A>_jUPTP6um-+% z+H>v$;cMbxbirZpz8`^JI2t9Gic-u#8D=7aa#Ub0Tv{Uy9gu-T&<#f*3yw=JZO|4S zkqP(diQdS@Ah_&@_UMGJ@Hp<{c$N#_UoNRghsO%Sb9o&1384Tk`@?e{3eV^DcrK6Q zKE6%{z;P!JLogh^?hE1T#=nF2-TtBQdA=8XUD+4L6imf*_9o2%`k{mmz`*RH6!=V=_GcHSqbfVZYg4tc)-ia@b$AaR_Y-V}$F}Vw#PJoj<7arjmYgpQ zzAu{CZbpE|zNZ~O9h-v4g5!~6k?$klKeqdR8Gw`Edw&$h;X-&1-h$`<5F7Csw%{9h zKCi>|PIx_zDW2PN_&)VL>Un%`dM$@1ANOOO1J64Yr^E5)T!c`N+%94rMhSdBmmz`* z*mf1J$1U(Z{TDohCs2!*u#4;3i&i||A70x*@Om6Sdf+JZh4&&CgW>%c0k6w@;utsy z6LB#dW3GnhzX@K`-MAMIq7qezq6S`19b%|Q0~)akZ{mGy!WMjwX0%A*?~8-r^&N{r z@Vo1ZeVK+C@Y=jzx54`}A6}F9Xfd9~Qh1-duGjH4KEOtNfjG9J30_lx zYe_{qG7&@;y5dOm$MFc^bd16ID8?1={!ND0c?G89PjHO(K63wW@8vr z;nEIWa1?Sd1ZJ%HSpsv1d8|n&g}GLYX(+?Zm#tI>c>FpK^Lug7zF9QRoRGa`=d z*o`z^FZHZ#--s<}!Y;I;uRSu+1NNJ1KjK%U(&qTw4dzo0PJz$&*!E{22*<=+ASc-)Tg zoW0=l9gp%c8nzt^&vzOen=XLIy&AJ{JBm;YuVW@EumGNK2^>?`U=HrXVmt?P;7vG& zeTg5j7c21wY~O+%NMU?(JnM$OI1!$A5~jj`X9S+Zb3cR{EKP1d%=#%fZpPqw*W)vM zi+`a38?hChGr&38!ZEZbvf&tN+ac%$$JG!trtA7{ezUkuMX7GX?98Ol)!&s~FM@LC>7Zoj~K z72d%{e2txG*^2iI>(9}IJ!nf?7i3`&@-YTM`plXf42IV+1`{v^rMN!1Hg9gjJa|n{ zU@2ZfJ^qOp)}axb;rVwXm2+iyKJ-O84nuDo53lDeOvL3V!>tHo8fIb+s_;0LVHFzi z3A|U{n<^~A5-f+;^9H;hpCoyoc3>~kcy1?nU47ua8G>ARO@)|%63jpZbK!L?#(X@C zC3p#I@lV9@9d=>A*1TSmk5hTv89gxoA&kZZTn3j8Fw6TP4`*OJtP{<(*>Fim5ZMT! z5Mh)ef=aj?h#nY-e4K+Ru&%9u^{7in^u!<(z;l}Q=Cj9I1eb%*3nySCth1-%R#f3> znB9-T%v}vL@iR2Rx}-g0nRQ5%$F+!|0dcrbfWCADkqtisnD1t{xoqZo9QWyt0XP+& zZ!OGsv)kXQH=9Hg<%@sfQ zhERwwN)bUNqNqg-4HygSD06Z)s^I5mbM8%;W8cBNydGxdL+}{pmN~cu=1&@9jX8Wb z%-I)Uj`}%vJFKHJa1?y5pL63d*US>@g|6rWGsJwdzX<^zrz41LgiweuO5yW6u(sd4 z845qQn<>}eHkcb8zdwAwS!BQCzIn3{W{sKR`A5Oyn^QjD&(HS%4e!CcF+VPW$G8>d zhINXW@fsTNDI%yu6t#%KzBrl?;Ju#?^X4U(T_3|-v0i9T-{Hu{5cu5BVaBv!+wt6S zGY_K?hI!%me*kMU?^rm_`?>ux+=xm%3bVrVm=lM>W0?z%zwYnYdkuo{8bT<9=P5-5 z9xIAkIA%}A4BU!);rLvKe_#W?f#alo=E({;Ry!^>A*EGfjPAwSvD4>m#P`WQ$4T?! zB#g#I@ObyYG50ydU>t9g~-%9vk48JsS)0SH$4>yam6YEo1KCFdL7C$2|?>a4~%TE%4Zn zAdFIYzDh(_Iw@dmsnHQ3Uf~Chme+F$}g} zj#-$8#dr~KU=zHSzu|3cg2(r~Y4ja}EF6yjJZBKu2%!*Rlp=yk*me=7;}$HyV_1&Y z(TJ}R#Zs)ndThlmv~9!R64@AvCfZuE?u?_6hcht=*PsFqzMCG4=x$Vf>~aOVwmIBlu?*jE}hUDr(i71d$ZU1+!{C!>(Uv0F%(6x zCN78dsad)fE(fDOhG7D(#vD9?I=lmyOk~5%55sG#gy)Q*5%t)JCbVMg&xF|+f|+Js znMLOOS1@<2M|;8iEQGmdrkPhB>qn&WxEpd{#)e@wntNv27WT0|&fu|i?NFG*r7%Cu zMsv^W@>oIS!t;4e5ty;Hs7EEt>^iJN96ON8SZr2hBOmVb608TmKr_tC9&o;DBrbx- zai7hwR!pa@7xG})46_k3B{<%(gJfPzf_A2x|}fCZH5%N)`MJAA_IW?Y{tK>}*uS zyjhEnVWzZZob&m&!G3F#dYDCi-uJV95Prt@_|L-UZ-V{iR1chhLREDfX9`}U%j6@h_NCm24&X{ZYI1gr*`QkCmr&ah6KG)1E!(A}fR-gf9 z*KTxVOq<2pJY0g+*nl5konfvG!YI_zW){WKjC9(}f)LD$ec*8%ug$DK!<={$K5sC_!oCRZ#S*N7`LP`y^9Ia^&x_$h*w>n|)p2_u zqHNcp0b3EEEfbFM1qj2jyNtfMaQ|hf$3`@v75zS^FU*mD!ejl4joBhDJ0YmG&TVQHT8bqf5ke9Cw{?Zb%|{WY!sjnQ4OU_j{shnW2$thbc-)_n z#<32E`@RRSV-GTD>kZF04pVV6I-?)*aXzNwc0}&@p}jKNe` z`$n+>@55TNJ)F1Bhx5(my!ElUYh7#2YCW3C+N>{x$Etu?8-um4*_Xk7Jy<*cJptF@ zURY;0!1~zQ*Sgm1HkZv@^VD3t3UgpxXeNGuRG8i7vYBh1y3b-*e>UPrbcOkDcALv) zuE%j7zgIFhJ21{?v-UGU3C!*&JeS9DpLF)=i+mpYIl<2W=6fBy9?#`*+~;&y5AMf0 z2nbz77eX+PXW%Z>U==oEC#)y^-Y5*`?X3CC%XRn)DU9FFznQDW zh``LOgL!M-wWU7`X4Mq7Z$T8VzGW_+%S)ppdMyX zfHuc}b7(9|;TUg*eT_CeHm5?k0A|sBSO)WEJ^OqQbEG>?#Mv-IZpA}*5uRfZ3UL|A zVP750i*L~;HIXwn^0*4m<86G6R`i)KC*mBKOKT9vZe-GygF=`wX2xP{r|$sPX5KJN z!khOw9i&$9?C;jx;KMjn}4ci?fnhK=|a zI?~rKnOT!r$C7=`tTWg)&qksI<%nV>(&6();1byP2wuge2)Gn1TvCg>~4DboOyx+Z?xk zw&pdT%~Ufyjy*8foMRot<8v?re$ToT&I>oe`CR8}M3*22y&r_<-WusP~{?pWHa>#N}B2D9Gl@>oMr4Cj2!Z);ZT(WS7CG#kx5GtIm* zi>!UEYpq$WN3Au@PqWe7GtPFkin1CCw z5HG>wxKDt`L4*)S1X08ghv)J*K^_mmBv@aXn}3J3qu)n)J)SEB_bJEYuts#Q(s@dA z&AfgaW@#GRSuo42*PN4!!WycP{(r%G@HmWx^LEyA&!GX|BY+@62qS_hVu&Na7$1b^ zIUSe5|I3v1qIqdO*n$3RSf3TLJsoD|ax}uZzOM8+#}|fkd*-0+=G-8dV`iW^E@^MmH#;k=?b`T@+zgK0MhogXyk z9)LN#7Upak+vccsnK?Nh=HPmmf4yK0Wvvu}`DQ(|0nM=Hv41jdhILd7pTb({F#7Xg zJ>>JFus@CfZDvmhVfzt<$KM2>Z$8?85v;Ku#Tr;gdHyb}J^n=ad~?hGEl6eT?hSKq z49u^)umom`HCG$@&5fbBp6&Z#P4*#n!o11B*)UV)Ax3)~)@wn85Jm)1#1MzaJ_^=y zmtqdgsde}cX3;S)8>gWXOYuH-B9p!oF#)p>rp@fC#tN8E?w3KI8B~aAsHM$Z+m3c@ zn_Xw%GMG;_cpd(j%ghTQj0mELAr61U6@}n2Ck3uQltQI2Ds{6QcMVKEbc>d0|9gUkq{h@A`MOcj;^mS&Pi}9F=hY-W(XvL4wdijGimLo76mtrog%gu6s zw`=|Be6&kX6ksYUVdh(JTSo`k$6D6qNQ}UhSb&%C30l&hg;B}-Db*^=%-?N&Lem`p7o4vJoAJ)%i zeF$c%_3IK$r`_LUt%Sd$YD1gf`xfFF_+9MZFq6JTsE2iZ25o+B2*Z5#Snv6mKJ(sl z{u|cTet%qw`(VB7_qEo<&Nn-k>^!mgZCz{4YCUSLX&q_KTQ{4#c?iP{t${VG^{BPx zX7+L3x<8z6E`#;4wXbz;TlTRYZDv2`w)5efbS12zHzI|8>so78KUX;aZO%Jy?fI;; zt)Z=t^I=_UcALv#n5Sl?b*#DAfm|5?=ReJNvpWJa*F1Hfboz2o1ZzM)1DNkon9F9a z$8n$YFbj{u&k248h{5bOmpzxqai7Jo&h)c{pA+IR-_35X$8&ic_xSv()F!fc2hP7Dp@kt=C4vdh$Y8FFpzD!JlBgcQPhnHmo5dqrFNONo| z%qeRpbM#%9lZUb$!o@J>o`pH~3(VQ}tj*E!h`_qc9Q+aH++cW)GhohIpT+PM%*o?m z4&H+Kw7mfH@89S~+Ynfr&4$_X7VO`RAdmA=0<)(UW>_=2nggs)h5dJ8DL%w*bfm(>15j= zWb^o3%)(<>ho8`ez9A^V-55>VRak%*u>rp#NZ+ZLg4)4J?9eACy zWT6n&=26tcy4)-e!3=ac8e?z+7UM1K#NqT8;Hu<)y=fbZS*U@Z4b1#r^cTXpYnMJ4 zhnuhjA0mamekj5%aOsO8l%p0KkxE|bT=WDHs_a}J9b{!IbY{dK8Gq21dYhUZyAlufX)|%$0*=X*WY37v~Z|xg`HLLZg z8Ebx;jpm-2md3XAaTwOM)~x2R8Ebx;jpm-$G!_w9`!>R?Hiykv^V4h$p#H9_H5_EY@G|gi;E+T?C@M3$9+yfG493+ ze2H}SISF2m=khr2Qwj5TD>CR0!FkjMwwn>;aRJP`Dwv06mGhFhNa1lGjKz&`-t!$a zqdWa0a5creA`Da=eWH5=w#IiA53+UH>>=!#1HS+ve^J zL}6w&qbJ!BW?cd6J+m%ATNbR>%3wWajb**~cUTV|%ytNu!g}syY(_gW!DFQG*m}`9 z&zIiec#HsTePO*=4(mZ{F>^4JwK-P; zbF2>Llo@Ca6~Uam5a!@>Fy~s*ZjPM;bLt70qmRR!+y!&cj56n}@ywu|FlWC*CXY{r zIe9aZE+WFE}H1&GjQ#xXb61%2-yD+>hUAG(l;EXSm>Iz2jKI6NcNvb`*b{rdi(^>Z@zl`eE9ro*uMio zGH@i)d7O(dDo~3?_+#-b6rv25(RL43;!Cur?RbQ7JC@>Otf9}0&E)Y=OvSy3VH+~i z$sTm$@#&a>hw%n}Mv%TUa4jBz=U$8k>_!%CV-dj;G@_M1L}%#-=h|n(n)@@fCwr{R z&2qmtcIl4^n1ecOK?Z$8QHqC>`yE63MBIs$_y%3*I~`?sBDr5S?O{|Rh9(5*GxH;; zP40Ir?UUf|?$*G013w#_iwgL=Fz42t=eE|i?lpV;p4c4scfHP4`#aus?6V75w2ebK zoY#)S`QtoTpRQ-0*0kkdGUnqoG{N5?kH(FNv!59qLMfu~oB{f>5k@7PFF2HTKNsAH zXJ8$kPTvrixsSpc+dABNV}Ji-hF=WpNxx?`*Zh9etT&&{)E=}IViuNQ15(*%5SGyI z@3GQo^LJDw@O$62@cY`H?Bn;WG4^RfkT#DMftehKwQ>lhaDJdQZT@D<-(&fCz`A=E z{N8s0?u2vF&NoNlJh5}V*0t8G)}z*%){(`u&qoZ-C3mIG+P4hWtk$E}n$B%IZyks8 z&CVrTA6LV=)|%CN)blkUz_!^Oh8Y?|6N2nh2tNxr{~e^wd28pSt+UNp>tkzQ>sqth zTsCvfQ?t_V3p^7IpT=Ud@^5Jh>qVTsZeg-h#&2Dqq%=I|#^9UNS z7ZLhu5l1@P=CbGVIPMc5hs;htOZYj#&j9AT+3odsE|23r7s0t)zlXGbJPXc|I?rmJ z`MsWVqZeW~`#2w5gxg>Z`#I9-v;Lflh3HS)1(=JMu?>gNUVuNr@Ap1{`CEw?nh>PT z%riG@5l1@tV!gT=J8&dzW~+5-9lk~v_BjLArZ2&HPwPVS(7JIUUWYYbcXGo_-wEeM z&Ep83fmxbHe=eq=725+b3G)!c_vl97C^)bAG(1l!qA)W9w3)f)T_s{@g1P!vIPcnl z?GUW{%+!CvdC}uw*4_ZK@B>)S9Ruq%>p<(VFJQg63Ld)~+tz#6qXz3?y*2>Wll!x6 zy?6<%2mcQ1y(8$ip1TR=Wg#M{MI2_C^`Kd2z1INi(feSY{sQaC^I^UC3e2j*$rYb_ zGR(RAV2=F&bLtG3Lyy6nybI>wHkfn6VU8_=IkgA1^nD6*@?@BU55S!J1?JefFlV!2 zj#k2)Y=Svh2y?F1$F$u9bM_0Eqr+fMJ^^!Ze{%3W`u3tP+vnjnJdZfq(SIT?f&JrA zf#ukYj`@=rZIQOR(O0E2R%a|@zp-?EqG)8t0iWZUo|1^J9}X~oHH=Z;SYgPl;{x48bTX8;M{GJHt>Q2tiG?ZUX>QlDJ=o_8tYJ?IkjJ;Sm!xvu5)>QcsQ z%f;89ODcZ@mxkoJihJl1=HKg*#lO*|DY>pYD6t*MNURI|(LPIh=ft`(xsLkddzO;S z#5&s_4YD*R*Rd{%?FxVV$5NQQL7Dz|k0qX5*K|v4m-^!amfYaJ6@yM+AuEb#rnZ8<^Ox$VsdVk4vmG2PM|=e7^4*9}W-M}{ZX1*av}=>>^(V{#qkUExwPBC*asBe8Byu45w;+ZCe{>%!5A zb>^9gbv(JQDNJmao|RbVj!CTd{YSERSbuh6yV40QmZEbK>o-#`tZMxq@23Rqj|8x1kJ39T}JLlk&d@1j%2mB}ezyC9@WhwmA8PxyF plE!V}(_sgH`O_-@>|tYn`#S`ZuOy%J|FsPM_1UfiH`RUhe*p5$_ErD@ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs index 0bfd30e..74dc419 100644 --- a/crates/clawhdf5/tests/header_validation_interop.rs +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -42,8 +42,9 @@ macro_rules! skip_if_no_python { /// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and /// `d` the output directory) and then, for every `NAME.h5` it wrote, -/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR` -/// otherwise. Returns those lines, sorted. +/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset +/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those +/// lines, sorted. fn h5py_verdicts(dir: &Path, body: &str) -> Vec { let script = format!( r#" @@ -54,7 +55,7 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))): name = os.path.basename(path)[:-3] try: with h5py.File(path, "r") as f: - f["d"][()] + f[globals().get("DSET", "d")][()] print(name, "ok") except Exception: print(name, "ERROR") @@ -78,14 +79,14 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))): lines } -/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of -/// whatever type it has). -fn clawhdf5_reads(path: &Path) -> Result<(), String> { +/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes +/// of whatever type it has). +fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> { let file = File::open(path).map_err(|e| format!("open: {e}"))?; - let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?; + let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?; ds.dtype().map_err(|e| format!("dtype: {e}"))?; ds.shape().map_err(|e| format!("shape: {e}"))?; - file.read_multi(&["d"]) + file.read_multi(&[dset]) .map(|_| ()) .map_err(|e| format!("read: {e}")) } @@ -93,10 +94,15 @@ fn clawhdf5_reads(path: &Path) -> Result<(), String> { /// h5py's verdict for each file must be `expected`, and clawhdf5 must read /// exactly the files h5py reads. fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) { + assert_agrees_with_h5py_on(dir, verdicts, expected, "d"); +} + +/// [`assert_agrees_with_h5py`] reading dataset `dset`. +fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) { assert_eq!(verdicts, expected, "h5py's view changed"); for line in verdicts { let (name, verdict) = line.split_once(' ').unwrap(); - let ours = clawhdf5_reads(&dir.join(format!("{name}.h5"))); + let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset); match verdict { "ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"), _ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"), @@ -244,7 +250,7 @@ for libver in ("earliest", "latest"): ], ); for name in ["earliest_size2", "latest_size8"] { - let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err(); + let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err(); assert!( err.contains("stored datatype size in chunk layout"), "{name}: {err}" @@ -557,3 +563,68 @@ for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)): assert!(mm.dataset("d").is_err(), "{name}: MmapFile"); } } + +/// libhdf5 decodes the superblock extension's messages when it opens a file +/// and refuses the file when one does not decode: cve-2020-10810 (a File +/// Space Info message too short for what it announces), cve-2020-10812 (a +/// metadata cache image past the end of the file). We did not look at those +/// messages and opened such files. +#[test] +fn superblock_extension_messages_libhdf5_refuses_are_refused() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5"); + let body = format!( + r#" +{FIX_OHDR_PY} +def ext_addr(buf): + assert buf[8] in (2, 3) + return int.from_bytes(buf[20:28], "little") +def save(name, buf): + open(os.path.join(d, name + ".h5"), "wb").write(buf) + +# A paged file: its superblock extension holds a File Space Info message +# (version 1, strategy page, not persisting, threshold 1, page size 4096). +DSET = "DSET" +good = os.path.join(d, "paged_good.h5") +with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f: + f.create_dataset("DSET", data=np.arange(10, dtype=" ext +# Page size 256 (under libhdf5's minimum of 512). +bad = bytearray(data); bad[at + 8:at + 16] = struct.pack(" ext +length = at + 5 + 8 +bad = bytearray(data); bad[length:length + 8] = struct.pack(" PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5") +} + +fn expected() -> Vec { + (0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect() +} + +#[test] +fn file_reads_through_the_cache_image() { + let file = File::open(fixture()).unwrap(); + assert_eq!(file.root().datasets().unwrap(), ["DSET"]); + let ds = file.dataset("DSET").unwrap(); + assert_eq!(ds.shape().unwrap(), [50, 100]); + assert_eq!(ds.read_i32().unwrap(), expected()); + + let file = File::from_bytes(std::fs::read(fixture()).unwrap()).unwrap(); + assert_eq!( + file.dataset("DSET").unwrap().read_i32().unwrap(), + expected() + ); +} + +#[test] +fn mmap_and_lazy_files_read_through_the_cache_image() { + let mm = MmapFile::open(fixture()).unwrap(); + assert_eq!(mm.dataset("DSET").unwrap().read_i32().unwrap(), expected()); + let lazy = LazyFile::from_bytes(std::fs::read(fixture()).unwrap()).unwrap(); + assert_eq!( + lazy.dataset("DSET").unwrap().read_i32().unwrap(), + expected() + ); +}