diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 844f343..c4cf524 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -712,6 +712,19 @@ fn main() { return; } }; + // libhdf5 decodes the superblock extension at open, and loads a + // metadata cache image over the file's own metadata. + let view = match guarded(|| { + clawhdf5_format::superblock_ext::metadata_view(hdf5, &sb).map_err(e) + }) { + Ok(v) => v, + Err(msg) => { + top.insert("open_error".into(), Value::String(msg)); + println!("{}", Value::Object(top)); + return; + } + }; + let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5); top.insert("superblock_version".into(), json!(sb.version)); let ctx = Ctx { data: hdf5, diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index ddd9ba7..7f9e3d2 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -235,6 +235,14 @@ pub enum FormatError { /// element size that overflows, contiguous storage past the end of the /// file, compact data of the wrong size. InvalidDatasetStorage(&'static str), + /// A superblock extension message libhdf5 refuses to decode when it + /// opens the file (the reason is libhdf5's own error text): a File Space + /// Info message that runs off its end or has a bad page size, a metadata + /// cache image outside the file, … + InvalidSuperblockExtension(&'static str), + /// A metadata cache image block libhdf5 refuses to load (the reason is + /// libhdf5's own error text). + InvalidCacheImage(&'static str), } impl fmt::Display for FormatError { @@ -515,6 +523,12 @@ impl fmt::Display for FormatError { FormatError::InvalidDatasetStorage(why) => { write!(f, "invalid dataset storage: {why}") } + FormatError::InvalidSuperblockExtension(why) => { + write!(f, "invalid superblock extension: {why}") + } + FormatError::InvalidCacheImage(why) => { + write!(f, "invalid metadata cache image: {why}") + } } } } diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 3e30f9d..e10711b 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -118,6 +118,7 @@ pub mod selection; pub mod shared_message; pub mod signature; pub mod superblock; +pub mod superblock_ext; pub mod symbol_table; #[cfg(all( test, diff --git a/crates/clawhdf5-format/src/superblock_ext.rs b/crates/clawhdf5-format/src/superblock_ext.rs new file mode 100644 index 0000000..d7ec3cc --- /dev/null +++ b/crates/clawhdf5-format/src/superblock_ext.rs @@ -0,0 +1,630 @@ +//! The superblock extension of a version 2 or 3 superblock, and the +//! metadata cache image it can point to. +//! +//! libhdf5 reads the extension when it opens a file (`H5F__super_read`) and +//! decodes the messages that configure the file: v1 B-tree "K" values, File +//! Space Info, and the Metadata Cache Image. A message that does not decode +//! makes the file fail to open, so [`read_superblock_extension`] decodes and +//! checks them the way libhdf5 does. +//! +//! A metadata cache image (written with `H5Pset_mdc_image_config`) is a +//! block holding serialized metadata cache entries — object headers, B-tree +//! nodes, heaps — each with its file address. libhdf5 loads it into its +//! cache before it reads any other metadata (`H5C__load_cache_image`, +//! `H5C__reconstruct_cache_contents`), and the entries take the place of +//! the file's bytes at their addresses: the file itself may hold stale or +//! no metadata there (in `h5clear_mdc_image.h5` the root group's header is +//! only in the image). [`apply_cache_image`] does the same with bytes: it +//! returns a copy of the file with every entry written at its address, so +//! every parser reads what libhdf5 reads. + +#[cfg(not(feature = "std"))] +use alloc::{collections::BTreeSet, vec::Vec}; +#[cfg(feature = "std")] +use std::collections::BTreeSet; + +use crate::error::FormatError; +use crate::message_type::MessageType; +use crate::object_header::ObjectHeader; +use crate::superblock::Superblock; + +/// Message type of the File Space Info message. +const MSG_FSINFO: u16 = 0x0017; +/// Message type of the Metadata Cache Image message. +const MSG_MDCI: u16 = 0x0018; +/// Header message flag: the library did not know the message when it wrote +/// it back (`H5O_MSG_FLAG_WAS_UNKNOWN`); libhdf5 then ignores its contents. +const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20; + +/// `H5F_FILE_SPACE_PAGE_SIZE_MIN` / `_MAX`. +const PAGE_SIZE_MIN: u64 = 512; +const PAGE_SIZE_MAX: u64 = 1024 * 1024 * 1024; +/// libhdf5's default file space page size, used for a version 0 message. +const PAGE_SIZE_DEFAULT: u64 = 4096; +/// Free-space managers whose addresses a persisting version 1 File Space +/// Info message lists (`H5F_MEM_PAGE_SUPER` .. `H5F_MEM_PAGE_NTYPES`), and +/// a version 0 one (`H5FD_MEM_SUPER` .. `H5FD_MEM_NTYPES`). +const FSM_ADDRS_V1: usize = 12; +const FSM_ADDRS_V0: usize = 6; + +/// Metadata cache image block limits (`H5Cimage.c`, `H5ACprivate.h`). +const MDCI_SIGNATURE: &[u8; 4] = b"MDCI"; +const MDCI_HAVE_RESIZE_STATUS: u8 = 0x01; +const MDCI_ENTRY_IS_FD_PARENT: u8 = 0x04; +const MDCI_ENTRY_IS_FD_CHILD: u8 = 0x08; +/// `H5AC_NTYPES`: entry type ids are below this. +const MDCI_NTYPES: u8 = 30; +/// `H5C_RING_NTYPES`. +const MDCI_RING_NTYPES: u8 = 6; +/// `H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX`. +const MDCI_AGE_MAX: u8 = 100; + +/// A decoded File Space Info message (0x0017), mapped to version 1 as +/// libhdf5 maps a version 0 one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileSpaceInfo { + /// Message version as stored (0 or 1). + pub version: u8, + /// File space strategy (`H5F_fspace_strategy_t`). + pub strategy: u8, + /// Whether free space is persisted. + pub persist: bool, + /// Free-space section threshold. + pub threshold: u64, + /// File space page size. + pub page_size: u64, +} + +/// Where a metadata cache image block is (Metadata Cache Image message, +/// 0x0018). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheImageLocation { + /// Address of the image block. + pub address: u64, + /// Length of the image block in bytes. + pub length: u64, +} + +/// The messages of a superblock extension that libhdf5 decodes at open. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SuperblockExtension { + /// v1 B-tree "K" values (chunk index, symbol table node, symbol table + /// leaf), when the extension overrides the defaults. + pub btree_k: Option<(u16, u16, u16)>, + /// The File Space Info message. + pub file_space_info: Option, + /// The metadata cache image, when the file has one. + pub cache_image: Option, +} + +fn ext_err(why: &'static str) -> FormatError { + FormatError::InvalidSuperblockExtension(why) +} + +const RAN_OFF: &str = "ran off end of input buffer while decoding"; + +/// A little-endian cursor over one message or block, failing with +/// `overrun` when it runs off the end. +struct Cursor<'a> { + data: &'a [u8], + pos: usize, + overrun: FormatError, +} + +impl<'a> Cursor<'a> { + fn new(data: &'a [u8], overrun: FormatError) -> Self { + Cursor { + data, + pos: 0, + overrun, + } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], FormatError> { + let end = self + .pos + .checked_add(n) + .filter(|&e| e <= self.data.len()) + .ok_or_else(|| self.overrun.clone())?; + let s = &self.data[self.pos..end]; + self.pos = end; + Ok(s) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn uint(&mut self, width: u8) -> Result { + let b = self.take(width as usize)?; + Ok(b.iter() + .rev() + .fold(0u64, |acc, &x| (acc << 8) | u64::from(x))) + } + + /// An address of `width` bytes; `None` when undefined (all ones). + fn addr(&mut self, width: u8) -> Result, FormatError> { + let v = self.uint(width)?; + let undef = if width >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(width))) - 1 + }; + Ok((v != undef).then_some(v)) + } +} + +/// Decode and check the superblock extension of `sb`, as libhdf5 does when +/// it opens the file. `data` is the file from the superblock on, up to the +/// end of file the superblock records (its end is libhdf5's "eoa"). +/// +/// Returns `Ok(None)` for a superblock without an extension (versions 0 +/// and 1 have none). A message libhdf5 fails to decode, or a cache image +/// that does not lie inside the file, is an error: libhdf5 refuses to open +/// such a file (`cve-2020-10810`: a File Space Info message too short for +/// the free-space manager addresses it announces; `cve-2020-10812`: a cache +/// image past the end of the file). +pub fn read_superblock_extension( + data: &[u8], + sb: &Superblock, +) -> Result, FormatError> { + let os = sb.offset_size; + let ls = sb.length_size; + let undef = if os >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(os))) - 1 + }; + let Some(addr) = sb.superblock_extension_address.filter(|&a| a != undef) else { + return Ok(None); + }; + let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?; + let header = ObjectHeader::parse(data, addr, os, ls)?; + let eoa = data.len() as u64; + + let mut ext = SuperblockExtension::default(); + for msg in &header.messages { + match msg.msg_type { + MessageType::BTreeKValues => { + let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF)); + if c.u8()? != 0 { + return Err(ext_err("bad version number for v1 B-tree 'K' message")); + } + let chunk = c.uint(2)? as u16; + let snode = c.uint(2)? as u16; + let leaf = c.uint(2)? as u16; + ext.btree_k = Some((chunk, snode, leaf)); + } + MessageType::Unknown(MSG_FSINFO) if msg.flags & MSG_FLAG_WAS_UNKNOWN == 0 => { + ext.file_space_info = Some(decode_fsinfo(&msg.data, os, ls)?); + } + MessageType::Unknown(MSG_MDCI) => { + let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF)); + if c.u8()? != 0 { + return Err(ext_err( + "bad version number for metadata cache image message", + )); + } + let address = c.addr(os)?; + let length = c.uint(ls)?; + let Some(address) = address else { + return Err(ext_err("metadata cache image address is undefined")); + }; + if address.checked_add(length).is_none_or(|end| end > eoa) { + return Err(ext_err( + "metadata cache image: address plus size exceeds file eoa", + )); + } + ext.cache_image = Some(CacheImageLocation { address, length }); + } + _ => {} + } + } + Ok(Some(ext)) +} + +/// `H5O__fsinfo_decode` plus the checks `H5F__super_read` makes on it. +fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result { + let mut c = Cursor::new(data, ext_err(RAN_OFF)); + let version = c.u8()?; + let info = if version == 0 { + let old_strategy = c.u8()?; + let threshold = c.uint(ls)?; + // H5F_file_space_type_t: 1 ALL_PERSIST, 2 ALL, 3 AGGR_VFD, 4 VFD. + let (strategy, persist) = match old_strategy { + 1 => { + for _ in 0..FSM_ADDRS_V0 { + c.addr(os)?; + } + (0, true) + } + 2 => (0, false), + 3 => (2, false), + 4 => (3, false), + _ => return Err(ext_err("invalid file space strategy")), + }; + FileSpaceInfo { + version, + strategy, + persist, + threshold, + page_size: PAGE_SIZE_DEFAULT, + } + } else { + if version > 1 { + return Err(ext_err("File space info message's version out of bounds")); + } + let strategy = c.u8()?; + let persist = c.u8()? != 0; + let threshold = c.uint(ls)?; + let page_size = c.uint(ls)?; + if page_size == 0 || page_size > PAGE_SIZE_MAX { + return Err(ext_err("invalid page size in file space info")); + } + c.uint(2)?; // page end metadata threshold + c.addr(os)?; // EOA before the free-space managers + if persist { + for _ in 0..FSM_ADDRS_V1 { + c.addr(os)?; + } + } + FileSpaceInfo { + version, + strategy, + persist, + threshold, + page_size, + } + }; + if info.page_size < PAGE_SIZE_MIN { + return Err(ext_err("file space page size too small")); + } + Ok(info) +} + +/// One entry of a metadata cache image: `len` bytes at `image_offset` in +/// the image block, belonging at file address `address`. +struct ImageEntry { + address: u64, + image_offset: usize, + len: usize, +} + +/// Decode the metadata cache image at `location` in `data` (the file from +/// the superblock on, up to its recorded end of file) and return a copy of +/// `data` with every cached entry written at its address — the metadata +/// libhdf5 reads for this file. The image is checked as libhdf5 checks it +/// (`H5C__decode_cache_image_header`, `H5C__reconstruct_cache_entry`): +/// signature and version, the image length it records, entry types, rings +/// and ages in range, entry addresses inside the file and not repeated, +/// flush-dependency parents that are earlier entries. +/// +/// libhdf5 does not verify the block's trailing checksum when it loads an +/// image, so neither does this. +pub fn apply_cache_image( + data: &[u8], + location: CacheImageLocation, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let bad = FormatError::InvalidCacheImage; + let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?; + let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?; + let block = start + .checked_add(len) + .and_then(|end| data.get(start..end)) + .ok_or(bad("image block extends past the end of the file"))?; + let eoa = data.len() as u64; + let mut c = Cursor::new(block, bad(RAN_OFF)); + + // Header: signature, version, flags, image data length, entry count. + if c.take(4)? != MDCI_SIGNATURE { + return Err(bad("bad metadata cache image header signature")); + } + if c.u8()? != 0 { + return Err(bad("bad metadata cache image version")); + } + if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 { + return Err(bad("MDC resize status not yet supported")); + } + if c.uint(length_size)? != location.length { + return Err(bad("bad metadata cache image data length")); + } + let n_entries = c.uint(4)?; + if n_entries == 0 { + return Err(bad("bad metadata cache entry count")); + } + + let mut entries = Vec::new(); + let mut seen = BTreeSet::new(); + for _ in 0..n_entries { + let type_id = c.u8()?; + if type_id >= MDCI_NTYPES { + return Err(bad("type id is out of valid range")); + } + let flags = c.u8()?; + if c.u8()? >= MDCI_RING_NTYPES { + return Err(bad("ring is out of valid range")); + } + if c.u8()? > MDCI_AGE_MAX { + return Err(bad("entry age is out of policy range")); + } + let children = c.uint(2)?; + // libhdf5 checks the parent flag against the child count only in + // debug builds (release builds refuse any entry with children); the + // image format's own rule is checked here. + if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) { + return Err(bad("flush dependency parent flag and child count disagree")); + } + c.uint(2)?; // dirty dependency children: reset for a read-only open + let parents = c.uint(2)?; + if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) { + return Err(bad("flush dependency child flag and parent count disagree")); + } + c.uint(4)?; // LRU rank + let address = c + .addr(offset_size)? + .filter(|&a| a < eoa) + .ok_or(bad("invalid entry address range"))?; + let size = c.uint(length_size)?; + if size == 0 { + return Err(bad("invalid entry size")); + } + for _ in 0..parents { + let parent = c + .addr(offset_size)? + .ok_or(bad("invalid flush dependency parent offset"))?; + if !seen.contains(&parent) { + return Err(bad("flush dependency parent not in the image")); + } + } + let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?; + let image_offset = c.pos; + c.take(len)?; + if !seen.insert(address) { + return Err(bad("duplicate addresses in cache")); + } + entries.push(ImageEntry { + address, + image_offset, + len, + }); + } + + let mut out = data.to_vec(); + for e in &entries { + // address < eoa <= usize::MAX, and the entry's bytes came from the + // block, so neither conversion nor the sum can fail. + let at = e.address as usize; + let end = at + e.len; + if end > out.len() { + out.resize(end, 0); + } + out[at..end].copy_from_slice(&block[e.image_offset..e.image_offset + e.len]); + } + Ok(out) +} + +/// What a reader must do before reading a file's metadata, in one call: +/// check the superblock extension ([`read_superblock_extension`]) and, +/// when the file has a metadata cache image, return the file's bytes with +/// the image applied ([`apply_cache_image`]). `Ok(None)` means read `data` +/// as it is. +pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result>, FormatError> { + match read_superblock_extension(data, sb)? { + Some(SuperblockExtension { + cache_image: Some(location), + .. + }) => apply_cache_image(data, location, sb.offset_size, sb.length_size).map(Some), + _ => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sb_v2(ext: u64) -> Superblock { + Superblock { + version: 2, + offset_size: 8, + length_size: 8, + base_address: 0, + eof_address: 0, + root_group_address: 0, + group_leaf_node_k: None, + group_internal_node_k: None, + indexed_storage_internal_node_k: None, + free_space_address: None, + driver_info_address: None, + consistency_flags: 0, + superblock_extension_address: Some(ext), + checksum: None, + page_size: None, + } + } + + /// A file whose superblock extension (a version 1 object header at 48) + /// holds the given messages, padded to `len` bytes. + fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec { + let mut body = Vec::new(); + for (t, d) in messages { + let padded = d.len().div_ceil(8) * 8; + body.extend_from_slice(&t.to_le_bytes()); + body.extend_from_slice(&(padded as u16).to_le_bytes()); + body.extend_from_slice(&[0x14, 0, 0, 0]); + body.extend_from_slice(d); + body.resize(body.len() + padded - d.len(), 0); + } + let mut f = vec![0u8; 48]; + f.push(1); + f.push(0); + f.extend_from_slice(&(messages.len() as u16).to_le_bytes()); + f.extend_from_slice(&1u32.to_le_bytes()); + f.extend_from_slice(&(body.len() as u32).to_le_bytes()); + f.extend_from_slice(&[0; 4]); + f.extend_from_slice(&body); + f.resize(len, 0); + f + } + + fn fsinfo_v1(page_size: u64, persist: bool, n_addrs: usize) -> Vec { + let mut m = vec![1, 1, u8::from(persist)]; + m.extend_from_slice(&1u64.to_le_bytes()); + m.extend_from_slice(&page_size.to_le_bytes()); + m.extend_from_slice(&0u16.to_le_bytes()); + m.extend_from_slice(&u64::MAX.to_le_bytes()); + for _ in 0..n_addrs { + m.extend_from_slice(&u64::MAX.to_le_bytes()); + } + m + } + + fn mdci(address: u64, length: u64) -> Vec { + let mut m = vec![0]; + m.extend_from_slice(&address.to_le_bytes()); + m.extend_from_slice(&length.to_le_bytes()); + m + } + + #[test] + fn no_extension() { + assert_eq!( + read_superblock_extension(&[0; 64], &sb_v2(u64::MAX)).unwrap(), + None + ); + } + + #[test] + fn file_space_info_as_libhdf5_decodes_it() { + // What FileWriter::with_page_size writes. + let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256); + let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap(); + assert_eq!(ext.file_space_info.unwrap().page_size, 4096); + let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 512); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok()); + + let refused = |m: Vec| { + let f = file_with_ext(&[(MSG_FSINFO, &m)], 512); + read_superblock_extension(&f, &sb_v2(48)).unwrap_err() + }; + // Persisting, but too short for the manager addresses. + let mut short = fsinfo_v1(4096, true, 12); + short.truncate(short.len() - 8); + assert_eq!(refused(short), ext_err(RAN_OFF)); + assert!(matches!( + refused(fsinfo_v1(256, false, 0)), + FormatError::InvalidSuperblockExtension(_) + )); + assert!(matches!( + refused(fsinfo_v1(0, false, 0)), + FormatError::InvalidSuperblockExtension(_) + )); + let mut v2 = fsinfo_v1(4096, false, 0); + v2[0] = 2; + assert!(matches!( + refused(v2), + FormatError::InvalidSuperblockExtension(_) + )); + // cve-2020-10810: version 0, strategy ALL_PERSIST, and a message of + // 32 bytes that cannot hold the six addresses that follow. + let mut v0 = vec![0u8, 1]; + v0.extend_from_slice(&[0, 1, 0, 0, 0, 0, 0, 0]); + v0.resize(32, 0xff); + assert_eq!(refused(v0), ext_err(RAN_OFF)); + // A version 0 message without persistence is fine. + let mut v0 = vec![0u8, 2]; + v0.extend_from_slice(&[0; 8]); + let f = file_with_ext(&[(MSG_FSINFO, &v0)], 256); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok()); + } + + #[test] + fn cache_image_location_must_be_inside_the_file() { + let f = file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192); + let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap(); + assert_eq!( + ext.cache_image, + Some(CacheImageLocation { + address: 128, + length: 64 + }) + ); + // cve-2020-10812: 256 MiB at 0x10100 in a 2565-byte file. + let f = file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565); + assert!(matches!( + read_superblock_extension(&f, &sb_v2(48)), + Err(FormatError::InvalidSuperblockExtension(_)) + )); + let f = file_with_ext(&[(MSG_MDCI, &mdci(u64::MAX, 8))], 256); + assert!(read_superblock_extension(&f, &sb_v2(48)).is_err()); + } + + /// A cache image block with `entries` of (address, bytes). + fn image(entries: &[(u64, &[u8])]) -> Vec { + let mut b = Vec::new(); + b.extend_from_slice(MDCI_SIGNATURE); + b.push(0); + b.push(0); + b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below + b.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + for (addr, bytes) in entries { + b.extend_from_slice(&[5, 0x02, 1, 0]); // type, flags (in LRU), ring, age + b.extend_from_slice(&[0; 6]); // children, dirty children, parents + b.extend_from_slice(&0i32.to_le_bytes()); + b.extend_from_slice(&addr.to_le_bytes()); + b.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + b.extend_from_slice(bytes); + } + b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5) + let n = b.len() as u64; + b[6..14].copy_from_slice(&n.to_le_bytes()); + b + } + + #[test] + fn cache_image_entries_replace_the_file_bytes() { + let img = image(&[(16, b"HEADER"), (40, b"NODE")]); + let mut f = vec![0u8; 64]; + let at = f.len() as u64; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: at, + length: img.len() as u64, + }; + let out = apply_cache_image(&f, loc, 8, 8).unwrap(); + assert_eq!(out.len(), f.len()); + assert_eq!(&out[16..22], b"HEADER"); + assert_eq!(&out[40..44], b"NODE"); + assert_eq!(&out[..16], &f[..16]); + + let bad = |img: Vec| { + let mut f = vec![0u8; 64]; + f.extend_from_slice(&img); + let loc = CacheImageLocation { + address: 64, + length: img.len() as u64, + }; + apply_cache_image(&f, loc, 8, 8).unwrap_err() + }; + let mut sig = image(&[(16, b"x")]); + sig[0] = b'X'; + assert!(matches!(bad(sig), FormatError::InvalidCacheImage(_))); + assert!(matches!( + bad(image(&[(16, b"a"), (16, b"b")])), + FormatError::InvalidCacheImage("duplicate addresses in cache") + )); + assert!(matches!( + bad(image(&[(1 << 20, b"far")])), + FormatError::InvalidCacheImage("invalid entry address range") + )); + let mut len = image(&[(16, b"x")]); + len[6] ^= 1; + assert!(matches!(bad(len), FormatError::InvalidCacheImage(_))); + let mut cut = image(&[(16, b"abcdef")]); + let n = cut.len() as u64 - 8; + cut.truncate(cut.len() - 8); + cut[6..14].copy_from_slice(&n.to_le_bytes()); + assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_))); + } +} diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 3232ee6..8746f5f 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -46,6 +46,9 @@ pub struct LazyFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, + /// The metadata as libhdf5 reads it when the file holds a metadata + /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. + overlay: Option>, root_header: ObjectHeader, /// Cache of parsed object headers, keyed by address. header_cache: RefCell>, @@ -82,7 +85,13 @@ impl LazyFile { let superblock = Superblock::parse(data, 0)?; // Refuse a truncated file; read nothing past the recorded end of file. let end = base + superblock.data_end(base as u64, whole_len)? as usize; - let data = &reader.as_bytes()[base..end]; + // Decode the superblock extension as libhdf5 does at open, and load + // a metadata cache image over the file's metadata. + let overlay = clawhdf5_format::superblock_ext::metadata_view( + &reader.as_bytes()[base..end], + &superblock, + )?; + let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]); let root_header = ObjectHeader::parse( data, superblock.root_group_address as usize, @@ -94,6 +103,7 @@ impl LazyFile { base, end, superblock, + overlay, root_header, header_cache: RefCell::new(HashMap::new()), }) @@ -111,7 +121,10 @@ impl LazyFile { } fn hdf5_bytes(&self) -> &[u8] { - &self.reader.as_bytes()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.reader.as_bytes()[self.base..self.end], + } } /// Returns a reference to the parsed superblock. diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 2a8400b..05d581a 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -38,6 +38,9 @@ pub struct MmapFile { /// End of the HDF5 data (`Superblock::data_end`, absolute). end: usize, superblock: Superblock, + /// The metadata as libhdf5 reads it when the file holds a metadata + /// cache image (see `superblock_ext::metadata_view`); `None` otherwise. + overlay: Option>, } impl MmapFile { @@ -50,18 +53,28 @@ impl MmapFile { let superblock = Superblock::parse(data, 0)?; // Refuse a truncated file; read nothing past the recorded end of file. let end = base + superblock.data_end(base as u64, whole_len)? as usize; + // Decode the superblock extension as libhdf5 does at open, and load + // a metadata cache image over the file's metadata. + let overlay = clawhdf5_format::superblock_ext::metadata_view( + &reader.as_bytes()[base..end], + &superblock, + )?; Ok(Self { reader, base, end, superblock, + overlay, }) } /// The file's bytes from the superblock on — the space HDF5 addresses /// index into. fn hdf5_bytes(&self) -> &[u8] { - &self.reader.as_bytes()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.reader.as_bytes()[self.base..self.end], + } } /// Size of the user block before the superblock (0 for most files). diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 40ac08b..c892eea 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -20,6 +20,7 @@ use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext; use crate::error::Error; use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; @@ -55,6 +56,11 @@ struct FileData { base: usize, /// End of the HDF5 data in the file (`Superblock::data_end`, absolute). end: usize, + /// The file's metadata as libhdf5 reads it when the file holds a + /// metadata cache image: the bytes from the superblock to the end of + /// file with the image's entries written in + /// ([`superblock_ext::metadata_view`]). `None` for every other file. + overlay: Option>, } impl FileData { @@ -68,11 +74,26 @@ impl FileData { let end = superblock.data_end(base as u64, whole.len() as u64)?; // data_end is at most the file length (less the user block). let end = base + end as usize; - Ok((Self { backing, base, end }, superblock)) + // libhdf5 decodes the superblock extension at open (a message it + // cannot decode fails the open) and loads a metadata cache image + // over the file's own metadata. + let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?; + Ok(( + Self { + backing, + base, + end, + overlay, + }, + superblock, + )) } fn as_bytes(&self) -> &[u8] { - &self.backing.whole_file()[self.base..self.end] + match &self.overlay { + Some(v) => v, + None => &self.backing.whole_file()[self.base..self.end], + } } fn len(&self) -> usize { diff --git a/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 b/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 new file mode 100644 index 0000000..6ed8b70 Binary files /dev/null and b/crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5 differ diff --git a/crates/clawhdf5/tests/header_validation_interop.rs b/crates/clawhdf5/tests/header_validation_interop.rs index 0bfd30e..74dc419 100644 --- a/crates/clawhdf5/tests/header_validation_interop.rs +++ b/crates/clawhdf5/tests/header_validation_interop.rs @@ -42,8 +42,9 @@ macro_rules! skip_if_no_python { /// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and /// `d` the output directory) and then, for every `NAME.h5` it wrote, -/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR` -/// otherwise. Returns those lines, sorted. +/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset +/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those +/// lines, sorted. fn h5py_verdicts(dir: &Path, body: &str) -> Vec { let script = format!( r#" @@ -54,7 +55,7 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))): name = os.path.basename(path)[:-3] try: with h5py.File(path, "r") as f: - f["d"][()] + f[globals().get("DSET", "d")][()] print(name, "ok") except Exception: print(name, "ERROR") @@ -78,14 +79,14 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))): lines } -/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of -/// whatever type it has). -fn clawhdf5_reads(path: &Path) -> Result<(), String> { +/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes +/// of whatever type it has). +fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> { let file = File::open(path).map_err(|e| format!("open: {e}"))?; - let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?; + let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?; ds.dtype().map_err(|e| format!("dtype: {e}"))?; ds.shape().map_err(|e| format!("shape: {e}"))?; - file.read_multi(&["d"]) + file.read_multi(&[dset]) .map(|_| ()) .map_err(|e| format!("read: {e}")) } @@ -93,10 +94,15 @@ fn clawhdf5_reads(path: &Path) -> Result<(), String> { /// h5py's verdict for each file must be `expected`, and clawhdf5 must read /// exactly the files h5py reads. fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) { + assert_agrees_with_h5py_on(dir, verdicts, expected, "d"); +} + +/// [`assert_agrees_with_h5py`] reading dataset `dset`. +fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) { assert_eq!(verdicts, expected, "h5py's view changed"); for line in verdicts { let (name, verdict) = line.split_once(' ').unwrap(); - let ours = clawhdf5_reads(&dir.join(format!("{name}.h5"))); + let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset); match verdict { "ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"), _ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"), @@ -244,7 +250,7 @@ for libver in ("earliest", "latest"): ], ); for name in ["earliest_size2", "latest_size8"] { - let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err(); + let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err(); assert!( err.contains("stored datatype size in chunk layout"), "{name}: {err}" @@ -557,3 +563,68 @@ for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)): assert!(mm.dataset("d").is_err(), "{name}: MmapFile"); } } + +/// libhdf5 decodes the superblock extension's messages when it opens a file +/// and refuses the file when one does not decode: cve-2020-10810 (a File +/// Space Info message too short for what it announces), cve-2020-10812 (a +/// metadata cache image past the end of the file). We did not look at those +/// messages and opened such files. +#[test] +fn superblock_extension_messages_libhdf5_refuses_are_refused() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5"); + let body = format!( + r#" +{FIX_OHDR_PY} +def ext_addr(buf): + assert buf[8] in (2, 3) + return int.from_bytes(buf[20:28], "little") +def save(name, buf): + open(os.path.join(d, name + ".h5"), "wb").write(buf) + +# A paged file: its superblock extension holds a File Space Info message +# (version 1, strategy page, not persisting, threshold 1, page size 4096). +DSET = "DSET" +good = os.path.join(d, "paged_good.h5") +with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f: + f.create_dataset("DSET", data=np.arange(10, dtype=" ext +# Page size 256 (under libhdf5's minimum of 512). +bad = bytearray(data); bad[at + 8:at + 16] = struct.pack(" ext +length = at + 5 + 8 +bad = bytearray(data); bad[length:length + 8] = struct.pack(" PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5") +} + +fn expected() -> Vec { + (0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect() +} + +#[test] +fn file_reads_through_the_cache_image() { + let file = File::open(fixture()).unwrap(); + assert_eq!(file.root().datasets().unwrap(), ["DSET"]); + let ds = file.dataset("DSET").unwrap(); + assert_eq!(ds.shape().unwrap(), [50, 100]); + assert_eq!(ds.read_i32().unwrap(), expected()); + + let file = File::from_bytes(std::fs::read(fixture()).unwrap()).unwrap(); + assert_eq!( + file.dataset("DSET").unwrap().read_i32().unwrap(), + expected() + ); +} + +#[test] +fn mmap_and_lazy_files_read_through_the_cache_image() { + let mm = MmapFile::open(fixture()).unwrap(); + assert_eq!(mm.dataset("DSET").unwrap().read_i32().unwrap(), expected()); + let lazy = LazyFile::from_bytes(std::fs::read(fixture()).unwrap()).unwrap(); + assert_eq!( + lazy.dataset("DSET").unwrap().read_i32().unwrap(), + expected() + ); +}