feat: decode the superblock extension at open; read metadata cache images
libhdf5 decodes the messages of a v2/v3 superblock's extension when it opens a file (H5F__super_read) and refuses the file when one does not decode. We never looked at them, so we opened cve-2020-10810 (a File Space Info message too short for the free-space manager addresses it announces) and cve-2020-10812 (a metadata cache image past the end of the file), both of which libhdf5 refuses. A file written with a metadata cache image keeps its metadata cache entries in an image block the extension points at; libhdf5 loads them over the file's own bytes before it reads any metadata (H5C__load_cache_image, H5C__reconstruct_cache_contents). In h5clear_mdc_image.h5 the root group's header exists only in the image, so every reader failed with InvalidObjectHeaderVersion(0). The new clawhdf5_format::superblock_ext module: - read_superblock_extension decodes the v1 B-tree K, File Space Info and Metadata Cache Image messages with libhdf5's checks (versions, page size 512 B .. 1 GiB, the addresses a persisting message lists, the image inside the file), with the new FormatError::InvalidSuperblockExtension; - apply_cache_image checks an image block as libhdf5 does (signature, version, recorded length, entry types, rings, ages, addresses inside the file and not repeated, flush-dependency parents) and returns the file's bytes with every entry written at its address (FormatError::InvalidCacheImage); - metadata_view does both. File, MmapFile and LazyFile (and so h5rs) call metadata_view at open and read an image file through the patched copy; the conformance probe does the same. The image's trailing checksum is not verified, as libhdf5 does not verify it. tests/fixtures/h5clear_mdc_image.h5 is libhdf5's own test file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -712,6 +712,19 @@ fn main() {
|
||||
return;
|
||||
}
|
||||
};
|
||||
// libhdf5 decodes the superblock extension at open, and loads a
|
||||
// metadata cache image over the file's own metadata.
|
||||
let view = match guarded(|| {
|
||||
clawhdf5_format::superblock_ext::metadata_view(hdf5, &sb).map_err(e)
|
||||
}) {
|
||||
Ok(v) => v,
|
||||
Err(msg) => {
|
||||
top.insert("open_error".into(), Value::String(msg));
|
||||
println!("{}", Value::Object(top));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5);
|
||||
top.insert("superblock_version".into(), json!(sb.version));
|
||||
let ctx = Ctx {
|
||||
data: hdf5,
|
||||
|
||||
@@ -235,6 +235,14 @@ pub enum FormatError {
|
||||
/// element size that overflows, contiguous storage past the end of the
|
||||
/// file, compact data of the wrong size.
|
||||
InvalidDatasetStorage(&'static str),
|
||||
/// A superblock extension message libhdf5 refuses to decode when it
|
||||
/// opens the file (the reason is libhdf5's own error text): a File Space
|
||||
/// Info message that runs off its end or has a bad page size, a metadata
|
||||
/// cache image outside the file, …
|
||||
InvalidSuperblockExtension(&'static str),
|
||||
/// A metadata cache image block libhdf5 refuses to load (the reason is
|
||||
/// libhdf5's own error text).
|
||||
InvalidCacheImage(&'static str),
|
||||
}
|
||||
|
||||
impl fmt::Display for FormatError {
|
||||
@@ -515,6 +523,12 @@ impl fmt::Display for FormatError {
|
||||
FormatError::InvalidDatasetStorage(why) => {
|
||||
write!(f, "invalid dataset storage: {why}")
|
||||
}
|
||||
FormatError::InvalidSuperblockExtension(why) => {
|
||||
write!(f, "invalid superblock extension: {why}")
|
||||
}
|
||||
FormatError::InvalidCacheImage(why) => {
|
||||
write!(f, "invalid metadata cache image: {why}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +118,7 @@ pub mod selection;
|
||||
pub mod shared_message;
|
||||
pub mod signature;
|
||||
pub mod superblock;
|
||||
pub mod superblock_ext;
|
||||
pub mod symbol_table;
|
||||
#[cfg(all(
|
||||
test,
|
||||
|
||||
@@ -0,0 +1,630 @@
|
||||
//! The superblock extension of a version 2 or 3 superblock, and the
|
||||
//! metadata cache image it can point to.
|
||||
//!
|
||||
//! libhdf5 reads the extension when it opens a file (`H5F__super_read`) and
|
||||
//! decodes the messages that configure the file: v1 B-tree "K" values, File
|
||||
//! Space Info, and the Metadata Cache Image. A message that does not decode
|
||||
//! makes the file fail to open, so [`read_superblock_extension`] decodes and
|
||||
//! checks them the way libhdf5 does.
|
||||
//!
|
||||
//! A metadata cache image (written with `H5Pset_mdc_image_config`) is a
|
||||
//! block holding serialized metadata cache entries — object headers, B-tree
|
||||
//! nodes, heaps — each with its file address. libhdf5 loads it into its
|
||||
//! cache before it reads any other metadata (`H5C__load_cache_image`,
|
||||
//! `H5C__reconstruct_cache_contents`), and the entries take the place of
|
||||
//! the file's bytes at their addresses: the file itself may hold stale or
|
||||
//! no metadata there (in `h5clear_mdc_image.h5` the root group's header is
|
||||
//! only in the image). [`apply_cache_image`] does the same with bytes: it
|
||||
//! returns a copy of the file with every entry written at its address, so
|
||||
//! every parser reads what libhdf5 reads.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{collections::BTreeSet, vec::Vec};
|
||||
#[cfg(feature = "std")]
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::error::FormatError;
|
||||
use crate::message_type::MessageType;
|
||||
use crate::object_header::ObjectHeader;
|
||||
use crate::superblock::Superblock;
|
||||
|
||||
/// Message type of the File Space Info message.
|
||||
const MSG_FSINFO: u16 = 0x0017;
|
||||
/// Message type of the Metadata Cache Image message.
|
||||
const MSG_MDCI: u16 = 0x0018;
|
||||
/// Header message flag: the library did not know the message when it wrote
|
||||
/// it back (`H5O_MSG_FLAG_WAS_UNKNOWN`); libhdf5 then ignores its contents.
|
||||
const MSG_FLAG_WAS_UNKNOWN: u8 = 0x20;
|
||||
|
||||
/// `H5F_FILE_SPACE_PAGE_SIZE_MIN` / `_MAX`.
|
||||
const PAGE_SIZE_MIN: u64 = 512;
|
||||
const PAGE_SIZE_MAX: u64 = 1024 * 1024 * 1024;
|
||||
/// libhdf5's default file space page size, used for a version 0 message.
|
||||
const PAGE_SIZE_DEFAULT: u64 = 4096;
|
||||
/// Free-space managers whose addresses a persisting version 1 File Space
|
||||
/// Info message lists (`H5F_MEM_PAGE_SUPER` .. `H5F_MEM_PAGE_NTYPES`), and
|
||||
/// a version 0 one (`H5FD_MEM_SUPER` .. `H5FD_MEM_NTYPES`).
|
||||
const FSM_ADDRS_V1: usize = 12;
|
||||
const FSM_ADDRS_V0: usize = 6;
|
||||
|
||||
/// Metadata cache image block limits (`H5Cimage.c`, `H5ACprivate.h`).
|
||||
const MDCI_SIGNATURE: &[u8; 4] = b"MDCI";
|
||||
const MDCI_HAVE_RESIZE_STATUS: u8 = 0x01;
|
||||
const MDCI_ENTRY_IS_FD_PARENT: u8 = 0x04;
|
||||
const MDCI_ENTRY_IS_FD_CHILD: u8 = 0x08;
|
||||
/// `H5AC_NTYPES`: entry type ids are below this.
|
||||
const MDCI_NTYPES: u8 = 30;
|
||||
/// `H5C_RING_NTYPES`.
|
||||
const MDCI_RING_NTYPES: u8 = 6;
|
||||
/// `H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX`.
|
||||
const MDCI_AGE_MAX: u8 = 100;
|
||||
|
||||
/// A decoded File Space Info message (0x0017), mapped to version 1 as
|
||||
/// libhdf5 maps a version 0 one.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FileSpaceInfo {
|
||||
/// Message version as stored (0 or 1).
|
||||
pub version: u8,
|
||||
/// File space strategy (`H5F_fspace_strategy_t`).
|
||||
pub strategy: u8,
|
||||
/// Whether free space is persisted.
|
||||
pub persist: bool,
|
||||
/// Free-space section threshold.
|
||||
pub threshold: u64,
|
||||
/// File space page size.
|
||||
pub page_size: u64,
|
||||
}
|
||||
|
||||
/// Where a metadata cache image block is (Metadata Cache Image message,
|
||||
/// 0x0018).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CacheImageLocation {
|
||||
/// Address of the image block.
|
||||
pub address: u64,
|
||||
/// Length of the image block in bytes.
|
||||
pub length: u64,
|
||||
}
|
||||
|
||||
/// The messages of a superblock extension that libhdf5 decodes at open.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct SuperblockExtension {
|
||||
/// v1 B-tree "K" values (chunk index, symbol table node, symbol table
|
||||
/// leaf), when the extension overrides the defaults.
|
||||
pub btree_k: Option<(u16, u16, u16)>,
|
||||
/// The File Space Info message.
|
||||
pub file_space_info: Option<FileSpaceInfo>,
|
||||
/// The metadata cache image, when the file has one.
|
||||
pub cache_image: Option<CacheImageLocation>,
|
||||
}
|
||||
|
||||
fn ext_err(why: &'static str) -> FormatError {
|
||||
FormatError::InvalidSuperblockExtension(why)
|
||||
}
|
||||
|
||||
const RAN_OFF: &str = "ran off end of input buffer while decoding";
|
||||
|
||||
/// A little-endian cursor over one message or block, failing with
|
||||
/// `overrun` when it runs off the end.
|
||||
struct Cursor<'a> {
|
||||
data: &'a [u8],
|
||||
pos: usize,
|
||||
overrun: FormatError,
|
||||
}
|
||||
|
||||
impl<'a> Cursor<'a> {
|
||||
fn new(data: &'a [u8], overrun: FormatError) -> Self {
|
||||
Cursor {
|
||||
data,
|
||||
pos: 0,
|
||||
overrun,
|
||||
}
|
||||
}
|
||||
|
||||
fn take(&mut self, n: usize) -> Result<&'a [u8], FormatError> {
|
||||
let end = self
|
||||
.pos
|
||||
.checked_add(n)
|
||||
.filter(|&e| e <= self.data.len())
|
||||
.ok_or_else(|| self.overrun.clone())?;
|
||||
let s = &self.data[self.pos..end];
|
||||
self.pos = end;
|
||||
Ok(s)
|
||||
}
|
||||
|
||||
fn u8(&mut self) -> Result<u8, FormatError> {
|
||||
Ok(self.take(1)?[0])
|
||||
}
|
||||
|
||||
fn uint(&mut self, width: u8) -> Result<u64, FormatError> {
|
||||
let b = self.take(width as usize)?;
|
||||
Ok(b.iter()
|
||||
.rev()
|
||||
.fold(0u64, |acc, &x| (acc << 8) | u64::from(x)))
|
||||
}
|
||||
|
||||
/// An address of `width` bytes; `None` when undefined (all ones).
|
||||
fn addr(&mut self, width: u8) -> Result<Option<u64>, FormatError> {
|
||||
let v = self.uint(width)?;
|
||||
let undef = if width >= 8 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << (8 * u32::from(width))) - 1
|
||||
};
|
||||
Ok((v != undef).then_some(v))
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode and check the superblock extension of `sb`, as libhdf5 does when
|
||||
/// it opens the file. `data` is the file from the superblock on, up to the
|
||||
/// end of file the superblock records (its end is libhdf5's "eoa").
|
||||
///
|
||||
/// Returns `Ok(None)` for a superblock without an extension (versions 0
|
||||
/// and 1 have none). A message libhdf5 fails to decode, or a cache image
|
||||
/// that does not lie inside the file, is an error: libhdf5 refuses to open
|
||||
/// such a file (`cve-2020-10810`: a File Space Info message too short for
|
||||
/// the free-space manager addresses it announces; `cve-2020-10812`: a cache
|
||||
/// image past the end of the file).
|
||||
pub fn read_superblock_extension(
|
||||
data: &[u8],
|
||||
sb: &Superblock,
|
||||
) -> Result<Option<SuperblockExtension>, FormatError> {
|
||||
let os = sb.offset_size;
|
||||
let ls = sb.length_size;
|
||||
let undef = if os >= 8 {
|
||||
u64::MAX
|
||||
} else {
|
||||
(1u64 << (8 * u32::from(os))) - 1
|
||||
};
|
||||
let Some(addr) = sb.superblock_extension_address.filter(|&a| a != undef) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let addr = usize::try_from(addr).map_err(|_| ext_err("address out of range"))?;
|
||||
let header = ObjectHeader::parse(data, addr, os, ls)?;
|
||||
let eoa = data.len() as u64;
|
||||
|
||||
let mut ext = SuperblockExtension::default();
|
||||
for msg in &header.messages {
|
||||
match msg.msg_type {
|
||||
MessageType::BTreeKValues => {
|
||||
let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF));
|
||||
if c.u8()? != 0 {
|
||||
return Err(ext_err("bad version number for v1 B-tree 'K' message"));
|
||||
}
|
||||
let chunk = c.uint(2)? as u16;
|
||||
let snode = c.uint(2)? as u16;
|
||||
let leaf = c.uint(2)? as u16;
|
||||
ext.btree_k = Some((chunk, snode, leaf));
|
||||
}
|
||||
MessageType::Unknown(MSG_FSINFO) if msg.flags & MSG_FLAG_WAS_UNKNOWN == 0 => {
|
||||
ext.file_space_info = Some(decode_fsinfo(&msg.data, os, ls)?);
|
||||
}
|
||||
MessageType::Unknown(MSG_MDCI) => {
|
||||
let mut c = Cursor::new(&msg.data, ext_err(RAN_OFF));
|
||||
if c.u8()? != 0 {
|
||||
return Err(ext_err(
|
||||
"bad version number for metadata cache image message",
|
||||
));
|
||||
}
|
||||
let address = c.addr(os)?;
|
||||
let length = c.uint(ls)?;
|
||||
let Some(address) = address else {
|
||||
return Err(ext_err("metadata cache image address is undefined"));
|
||||
};
|
||||
if address.checked_add(length).is_none_or(|end| end > eoa) {
|
||||
return Err(ext_err(
|
||||
"metadata cache image: address plus size exceeds file eoa",
|
||||
));
|
||||
}
|
||||
ext.cache_image = Some(CacheImageLocation { address, length });
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(Some(ext))
|
||||
}
|
||||
|
||||
/// `H5O__fsinfo_decode` plus the checks `H5F__super_read` makes on it.
|
||||
fn decode_fsinfo(data: &[u8], os: u8, ls: u8) -> Result<FileSpaceInfo, FormatError> {
|
||||
let mut c = Cursor::new(data, ext_err(RAN_OFF));
|
||||
let version = c.u8()?;
|
||||
let info = if version == 0 {
|
||||
let old_strategy = c.u8()?;
|
||||
let threshold = c.uint(ls)?;
|
||||
// H5F_file_space_type_t: 1 ALL_PERSIST, 2 ALL, 3 AGGR_VFD, 4 VFD.
|
||||
let (strategy, persist) = match old_strategy {
|
||||
1 => {
|
||||
for _ in 0..FSM_ADDRS_V0 {
|
||||
c.addr(os)?;
|
||||
}
|
||||
(0, true)
|
||||
}
|
||||
2 => (0, false),
|
||||
3 => (2, false),
|
||||
4 => (3, false),
|
||||
_ => return Err(ext_err("invalid file space strategy")),
|
||||
};
|
||||
FileSpaceInfo {
|
||||
version,
|
||||
strategy,
|
||||
persist,
|
||||
threshold,
|
||||
page_size: PAGE_SIZE_DEFAULT,
|
||||
}
|
||||
} else {
|
||||
if version > 1 {
|
||||
return Err(ext_err("File space info message's version out of bounds"));
|
||||
}
|
||||
let strategy = c.u8()?;
|
||||
let persist = c.u8()? != 0;
|
||||
let threshold = c.uint(ls)?;
|
||||
let page_size = c.uint(ls)?;
|
||||
if page_size == 0 || page_size > PAGE_SIZE_MAX {
|
||||
return Err(ext_err("invalid page size in file space info"));
|
||||
}
|
||||
c.uint(2)?; // page end metadata threshold
|
||||
c.addr(os)?; // EOA before the free-space managers
|
||||
if persist {
|
||||
for _ in 0..FSM_ADDRS_V1 {
|
||||
c.addr(os)?;
|
||||
}
|
||||
}
|
||||
FileSpaceInfo {
|
||||
version,
|
||||
strategy,
|
||||
persist,
|
||||
threshold,
|
||||
page_size,
|
||||
}
|
||||
};
|
||||
if info.page_size < PAGE_SIZE_MIN {
|
||||
return Err(ext_err("file space page size too small"));
|
||||
}
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
/// One entry of a metadata cache image: `len` bytes at `image_offset` in
|
||||
/// the image block, belonging at file address `address`.
|
||||
struct ImageEntry {
|
||||
address: u64,
|
||||
image_offset: usize,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
/// Decode the metadata cache image at `location` in `data` (the file from
|
||||
/// the superblock on, up to its recorded end of file) and return a copy of
|
||||
/// `data` with every cached entry written at its address — the metadata
|
||||
/// libhdf5 reads for this file. The image is checked as libhdf5 checks it
|
||||
/// (`H5C__decode_cache_image_header`, `H5C__reconstruct_cache_entry`):
|
||||
/// signature and version, the image length it records, entry types, rings
|
||||
/// and ages in range, entry addresses inside the file and not repeated,
|
||||
/// flush-dependency parents that are earlier entries.
|
||||
///
|
||||
/// libhdf5 does not verify the block's trailing checksum when it loads an
|
||||
/// image, so neither does this.
|
||||
pub fn apply_cache_image(
|
||||
data: &[u8],
|
||||
location: CacheImageLocation,
|
||||
offset_size: u8,
|
||||
length_size: u8,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let bad = FormatError::InvalidCacheImage;
|
||||
let start = usize::try_from(location.address).map_err(|_| bad("address out of range"))?;
|
||||
let len = usize::try_from(location.length).map_err(|_| bad("length out of range"))?;
|
||||
let block = start
|
||||
.checked_add(len)
|
||||
.and_then(|end| data.get(start..end))
|
||||
.ok_or(bad("image block extends past the end of the file"))?;
|
||||
let eoa = data.len() as u64;
|
||||
let mut c = Cursor::new(block, bad(RAN_OFF));
|
||||
|
||||
// Header: signature, version, flags, image data length, entry count.
|
||||
if c.take(4)? != MDCI_SIGNATURE {
|
||||
return Err(bad("bad metadata cache image header signature"));
|
||||
}
|
||||
if c.u8()? != 0 {
|
||||
return Err(bad("bad metadata cache image version"));
|
||||
}
|
||||
if c.u8()? & MDCI_HAVE_RESIZE_STATUS != 0 {
|
||||
return Err(bad("MDC resize status not yet supported"));
|
||||
}
|
||||
if c.uint(length_size)? != location.length {
|
||||
return Err(bad("bad metadata cache image data length"));
|
||||
}
|
||||
let n_entries = c.uint(4)?;
|
||||
if n_entries == 0 {
|
||||
return Err(bad("bad metadata cache entry count"));
|
||||
}
|
||||
|
||||
let mut entries = Vec::new();
|
||||
let mut seen = BTreeSet::new();
|
||||
for _ in 0..n_entries {
|
||||
let type_id = c.u8()?;
|
||||
if type_id >= MDCI_NTYPES {
|
||||
return Err(bad("type id is out of valid range"));
|
||||
}
|
||||
let flags = c.u8()?;
|
||||
if c.u8()? >= MDCI_RING_NTYPES {
|
||||
return Err(bad("ring is out of valid range"));
|
||||
}
|
||||
if c.u8()? > MDCI_AGE_MAX {
|
||||
return Err(bad("entry age is out of policy range"));
|
||||
}
|
||||
let children = c.uint(2)?;
|
||||
// libhdf5 checks the parent flag against the child count only in
|
||||
// debug builds (release builds refuse any entry with children); the
|
||||
// image format's own rule is checked here.
|
||||
if (flags & MDCI_ENTRY_IS_FD_PARENT != 0) != (children > 0) {
|
||||
return Err(bad("flush dependency parent flag and child count disagree"));
|
||||
}
|
||||
c.uint(2)?; // dirty dependency children: reset for a read-only open
|
||||
let parents = c.uint(2)?;
|
||||
if (flags & MDCI_ENTRY_IS_FD_CHILD != 0) != (parents > 0) {
|
||||
return Err(bad("flush dependency child flag and parent count disagree"));
|
||||
}
|
||||
c.uint(4)?; // LRU rank
|
||||
let address = c
|
||||
.addr(offset_size)?
|
||||
.filter(|&a| a < eoa)
|
||||
.ok_or(bad("invalid entry address range"))?;
|
||||
let size = c.uint(length_size)?;
|
||||
if size == 0 {
|
||||
return Err(bad("invalid entry size"));
|
||||
}
|
||||
for _ in 0..parents {
|
||||
let parent = c
|
||||
.addr(offset_size)?
|
||||
.ok_or(bad("invalid flush dependency parent offset"))?;
|
||||
if !seen.contains(&parent) {
|
||||
return Err(bad("flush dependency parent not in the image"));
|
||||
}
|
||||
}
|
||||
let len = usize::try_from(size).map_err(|_| bad(RAN_OFF))?;
|
||||
let image_offset = c.pos;
|
||||
c.take(len)?;
|
||||
if !seen.insert(address) {
|
||||
return Err(bad("duplicate addresses in cache"));
|
||||
}
|
||||
entries.push(ImageEntry {
|
||||
address,
|
||||
image_offset,
|
||||
len,
|
||||
});
|
||||
}
|
||||
|
||||
let mut out = data.to_vec();
|
||||
for e in &entries {
|
||||
// address < eoa <= usize::MAX, and the entry's bytes came from the
|
||||
// block, so neither conversion nor the sum can fail.
|
||||
let at = e.address as usize;
|
||||
let end = at + e.len;
|
||||
if end > out.len() {
|
||||
out.resize(end, 0);
|
||||
}
|
||||
out[at..end].copy_from_slice(&block[e.image_offset..e.image_offset + e.len]);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// What a reader must do before reading a file's metadata, in one call:
|
||||
/// check the superblock extension ([`read_superblock_extension`]) and,
|
||||
/// when the file has a metadata cache image, return the file's bytes with
|
||||
/// the image applied ([`apply_cache_image`]). `Ok(None)` means read `data`
|
||||
/// as it is.
|
||||
pub fn metadata_view(data: &[u8], sb: &Superblock) -> Result<Option<Vec<u8>>, FormatError> {
|
||||
match read_superblock_extension(data, sb)? {
|
||||
Some(SuperblockExtension {
|
||||
cache_image: Some(location),
|
||||
..
|
||||
}) => apply_cache_image(data, location, sb.offset_size, sb.length_size).map(Some),
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sb_v2(ext: u64) -> Superblock {
|
||||
Superblock {
|
||||
version: 2,
|
||||
offset_size: 8,
|
||||
length_size: 8,
|
||||
base_address: 0,
|
||||
eof_address: 0,
|
||||
root_group_address: 0,
|
||||
group_leaf_node_k: None,
|
||||
group_internal_node_k: None,
|
||||
indexed_storage_internal_node_k: None,
|
||||
free_space_address: None,
|
||||
driver_info_address: None,
|
||||
consistency_flags: 0,
|
||||
superblock_extension_address: Some(ext),
|
||||
checksum: None,
|
||||
page_size: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A file whose superblock extension (a version 1 object header at 48)
|
||||
/// holds the given messages, padded to `len` bytes.
|
||||
fn file_with_ext(messages: &[(u16, &[u8])], len: usize) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
for (t, d) in messages {
|
||||
let padded = d.len().div_ceil(8) * 8;
|
||||
body.extend_from_slice(&t.to_le_bytes());
|
||||
body.extend_from_slice(&(padded as u16).to_le_bytes());
|
||||
body.extend_from_slice(&[0x14, 0, 0, 0]);
|
||||
body.extend_from_slice(d);
|
||||
body.resize(body.len() + padded - d.len(), 0);
|
||||
}
|
||||
let mut f = vec![0u8; 48];
|
||||
f.push(1);
|
||||
f.push(0);
|
||||
f.extend_from_slice(&(messages.len() as u16).to_le_bytes());
|
||||
f.extend_from_slice(&1u32.to_le_bytes());
|
||||
f.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
f.extend_from_slice(&[0; 4]);
|
||||
f.extend_from_slice(&body);
|
||||
f.resize(len, 0);
|
||||
f
|
||||
}
|
||||
|
||||
fn fsinfo_v1(page_size: u64, persist: bool, n_addrs: usize) -> Vec<u8> {
|
||||
let mut m = vec![1, 1, u8::from(persist)];
|
||||
m.extend_from_slice(&1u64.to_le_bytes());
|
||||
m.extend_from_slice(&page_size.to_le_bytes());
|
||||
m.extend_from_slice(&0u16.to_le_bytes());
|
||||
m.extend_from_slice(&u64::MAX.to_le_bytes());
|
||||
for _ in 0..n_addrs {
|
||||
m.extend_from_slice(&u64::MAX.to_le_bytes());
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
fn mdci(address: u64, length: u64) -> Vec<u8> {
|
||||
let mut m = vec![0];
|
||||
m.extend_from_slice(&address.to_le_bytes());
|
||||
m.extend_from_slice(&length.to_le_bytes());
|
||||
m
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_extension() {
|
||||
assert_eq!(
|
||||
read_superblock_extension(&[0; 64], &sb_v2(u64::MAX)).unwrap(),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_space_info_as_libhdf5_decodes_it() {
|
||||
// What FileWriter::with_page_size writes.
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, false, 0))], 256);
|
||||
let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap();
|
||||
assert_eq!(ext.file_space_info.unwrap().page_size, 4096);
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &fsinfo_v1(4096, true, 12))], 512);
|
||||
assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok());
|
||||
|
||||
let refused = |m: Vec<u8>| {
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &m)], 512);
|
||||
read_superblock_extension(&f, &sb_v2(48)).unwrap_err()
|
||||
};
|
||||
// Persisting, but too short for the manager addresses.
|
||||
let mut short = fsinfo_v1(4096, true, 12);
|
||||
short.truncate(short.len() - 8);
|
||||
assert_eq!(refused(short), ext_err(RAN_OFF));
|
||||
assert!(matches!(
|
||||
refused(fsinfo_v1(256, false, 0)),
|
||||
FormatError::InvalidSuperblockExtension(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
refused(fsinfo_v1(0, false, 0)),
|
||||
FormatError::InvalidSuperblockExtension(_)
|
||||
));
|
||||
let mut v2 = fsinfo_v1(4096, false, 0);
|
||||
v2[0] = 2;
|
||||
assert!(matches!(
|
||||
refused(v2),
|
||||
FormatError::InvalidSuperblockExtension(_)
|
||||
));
|
||||
// cve-2020-10810: version 0, strategy ALL_PERSIST, and a message of
|
||||
// 32 bytes that cannot hold the six addresses that follow.
|
||||
let mut v0 = vec![0u8, 1];
|
||||
v0.extend_from_slice(&[0, 1, 0, 0, 0, 0, 0, 0]);
|
||||
v0.resize(32, 0xff);
|
||||
assert_eq!(refused(v0), ext_err(RAN_OFF));
|
||||
// A version 0 message without persistence is fine.
|
||||
let mut v0 = vec![0u8, 2];
|
||||
v0.extend_from_slice(&[0; 8]);
|
||||
let f = file_with_ext(&[(MSG_FSINFO, &v0)], 256);
|
||||
assert!(read_superblock_extension(&f, &sb_v2(48)).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_image_location_must_be_inside_the_file() {
|
||||
let f = file_with_ext(&[(MSG_MDCI, &mdci(128, 64))], 192);
|
||||
let ext = read_superblock_extension(&f, &sb_v2(48)).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
ext.cache_image,
|
||||
Some(CacheImageLocation {
|
||||
address: 128,
|
||||
length: 64
|
||||
})
|
||||
);
|
||||
// cve-2020-10812: 256 MiB at 0x10100 in a 2565-byte file.
|
||||
let f = file_with_ext(&[(MSG_MDCI, &mdci(0x10100, 0x1000_0000))], 2565);
|
||||
assert!(matches!(
|
||||
read_superblock_extension(&f, &sb_v2(48)),
|
||||
Err(FormatError::InvalidSuperblockExtension(_))
|
||||
));
|
||||
let f = file_with_ext(&[(MSG_MDCI, &mdci(u64::MAX, 8))], 256);
|
||||
assert!(read_superblock_extension(&f, &sb_v2(48)).is_err());
|
||||
}
|
||||
|
||||
/// A cache image block with `entries` of (address, bytes).
|
||||
fn image(entries: &[(u64, &[u8])]) -> Vec<u8> {
|
||||
let mut b = Vec::new();
|
||||
b.extend_from_slice(MDCI_SIGNATURE);
|
||||
b.push(0);
|
||||
b.push(0);
|
||||
b.extend_from_slice(&0u64.to_le_bytes()); // length, patched below
|
||||
b.extend_from_slice(&(entries.len() as u32).to_le_bytes());
|
||||
for (addr, bytes) in entries {
|
||||
b.extend_from_slice(&[5, 0x02, 1, 0]); // type, flags (in LRU), ring, age
|
||||
b.extend_from_slice(&[0; 6]); // children, dirty children, parents
|
||||
b.extend_from_slice(&0i32.to_le_bytes());
|
||||
b.extend_from_slice(&addr.to_le_bytes());
|
||||
b.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
|
||||
b.extend_from_slice(bytes);
|
||||
}
|
||||
b.extend_from_slice(&[0; 4]); // checksum (not verified, as in libhdf5)
|
||||
let n = b.len() as u64;
|
||||
b[6..14].copy_from_slice(&n.to_le_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_image_entries_replace_the_file_bytes() {
|
||||
let img = image(&[(16, b"HEADER"), (40, b"NODE")]);
|
||||
let mut f = vec![0u8; 64];
|
||||
let at = f.len() as u64;
|
||||
f.extend_from_slice(&img);
|
||||
let loc = CacheImageLocation {
|
||||
address: at,
|
||||
length: img.len() as u64,
|
||||
};
|
||||
let out = apply_cache_image(&f, loc, 8, 8).unwrap();
|
||||
assert_eq!(out.len(), f.len());
|
||||
assert_eq!(&out[16..22], b"HEADER");
|
||||
assert_eq!(&out[40..44], b"NODE");
|
||||
assert_eq!(&out[..16], &f[..16]);
|
||||
|
||||
let bad = |img: Vec<u8>| {
|
||||
let mut f = vec![0u8; 64];
|
||||
f.extend_from_slice(&img);
|
||||
let loc = CacheImageLocation {
|
||||
address: 64,
|
||||
length: img.len() as u64,
|
||||
};
|
||||
apply_cache_image(&f, loc, 8, 8).unwrap_err()
|
||||
};
|
||||
let mut sig = image(&[(16, b"x")]);
|
||||
sig[0] = b'X';
|
||||
assert!(matches!(bad(sig), FormatError::InvalidCacheImage(_)));
|
||||
assert!(matches!(
|
||||
bad(image(&[(16, b"a"), (16, b"b")])),
|
||||
FormatError::InvalidCacheImage("duplicate addresses in cache")
|
||||
));
|
||||
assert!(matches!(
|
||||
bad(image(&[(1 << 20, b"far")])),
|
||||
FormatError::InvalidCacheImage("invalid entry address range")
|
||||
));
|
||||
let mut len = image(&[(16, b"x")]);
|
||||
len[6] ^= 1;
|
||||
assert!(matches!(bad(len), FormatError::InvalidCacheImage(_)));
|
||||
let mut cut = image(&[(16, b"abcdef")]);
|
||||
let n = cut.len() as u64 - 8;
|
||||
cut.truncate(cut.len() - 8);
|
||||
cut[6..14].copy_from_slice(&n.to_le_bytes());
|
||||
assert!(matches!(bad(cut), FormatError::InvalidCacheImage(_)));
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,9 @@ pub struct LazyFile<R: HDF5Read> {
|
||||
/// End of the HDF5 data (`Superblock::data_end`, absolute).
|
||||
end: usize,
|
||||
superblock: Superblock,
|
||||
/// The metadata as libhdf5 reads it when the file holds a metadata
|
||||
/// cache image (see `superblock_ext::metadata_view`); `None` otherwise.
|
||||
overlay: Option<Vec<u8>>,
|
||||
root_header: ObjectHeader,
|
||||
/// Cache of parsed object headers, keyed by address.
|
||||
header_cache: RefCell<HashMap<u64, ObjectHeader>>,
|
||||
@@ -82,7 +85,13 @@ impl<R: HDF5Read> LazyFile<R> {
|
||||
let superblock = Superblock::parse(data, 0)?;
|
||||
// Refuse a truncated file; read nothing past the recorded end of file.
|
||||
let end = base + superblock.data_end(base as u64, whole_len)? as usize;
|
||||
let data = &reader.as_bytes()[base..end];
|
||||
// Decode the superblock extension as libhdf5 does at open, and load
|
||||
// a metadata cache image over the file's metadata.
|
||||
let overlay = clawhdf5_format::superblock_ext::metadata_view(
|
||||
&reader.as_bytes()[base..end],
|
||||
&superblock,
|
||||
)?;
|
||||
let data = overlay.as_deref().unwrap_or(&reader.as_bytes()[base..end]);
|
||||
let root_header = ObjectHeader::parse(
|
||||
data,
|
||||
superblock.root_group_address as usize,
|
||||
@@ -94,6 +103,7 @@ impl<R: HDF5Read> LazyFile<R> {
|
||||
base,
|
||||
end,
|
||||
superblock,
|
||||
overlay,
|
||||
root_header,
|
||||
header_cache: RefCell::new(HashMap::new()),
|
||||
})
|
||||
@@ -111,7 +121,10 @@ impl<R: HDF5Read> LazyFile<R> {
|
||||
}
|
||||
|
||||
fn hdf5_bytes(&self) -> &[u8] {
|
||||
&self.reader.as_bytes()[self.base..self.end]
|
||||
match &self.overlay {
|
||||
Some(v) => v,
|
||||
None => &self.reader.as_bytes()[self.base..self.end],
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to the parsed superblock.
|
||||
|
||||
@@ -38,6 +38,9 @@ pub struct MmapFile {
|
||||
/// End of the HDF5 data (`Superblock::data_end`, absolute).
|
||||
end: usize,
|
||||
superblock: Superblock,
|
||||
/// The metadata as libhdf5 reads it when the file holds a metadata
|
||||
/// cache image (see `superblock_ext::metadata_view`); `None` otherwise.
|
||||
overlay: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl MmapFile {
|
||||
@@ -50,18 +53,28 @@ impl MmapFile {
|
||||
let superblock = Superblock::parse(data, 0)?;
|
||||
// Refuse a truncated file; read nothing past the recorded end of file.
|
||||
let end = base + superblock.data_end(base as u64, whole_len)? as usize;
|
||||
// Decode the superblock extension as libhdf5 does at open, and load
|
||||
// a metadata cache image over the file's metadata.
|
||||
let overlay = clawhdf5_format::superblock_ext::metadata_view(
|
||||
&reader.as_bytes()[base..end],
|
||||
&superblock,
|
||||
)?;
|
||||
Ok(Self {
|
||||
reader,
|
||||
base,
|
||||
end,
|
||||
superblock,
|
||||
overlay,
|
||||
})
|
||||
}
|
||||
|
||||
/// The file's bytes from the superblock on — the space HDF5 addresses
|
||||
/// index into.
|
||||
fn hdf5_bytes(&self) -> &[u8] {
|
||||
&self.reader.as_bytes()[self.base..self.end]
|
||||
match &self.overlay {
|
||||
Some(v) => v,
|
||||
None => &self.reader.as_bytes()[self.base..self.end],
|
||||
}
|
||||
}
|
||||
|
||||
/// Size of the user block before the superblock (0 for most files).
|
||||
|
||||
@@ -20,6 +20,7 @@ use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::signature;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
use clawhdf5_format::superblock_ext;
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
|
||||
@@ -55,6 +56,11 @@ struct FileData {
|
||||
base: usize,
|
||||
/// End of the HDF5 data in the file (`Superblock::data_end`, absolute).
|
||||
end: usize,
|
||||
/// The file's metadata as libhdf5 reads it when the file holds a
|
||||
/// metadata cache image: the bytes from the superblock to the end of
|
||||
/// file with the image's entries written in
|
||||
/// ([`superblock_ext::metadata_view`]). `None` for every other file.
|
||||
overlay: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl FileData {
|
||||
@@ -68,11 +74,26 @@ impl FileData {
|
||||
let end = superblock.data_end(base as u64, whole.len() as u64)?;
|
||||
// data_end is at most the file length (less the user block).
|
||||
let end = base + end as usize;
|
||||
Ok((Self { backing, base, end }, superblock))
|
||||
// libhdf5 decodes the superblock extension at open (a message it
|
||||
// cannot decode fails the open) and loads a metadata cache image
|
||||
// over the file's own metadata.
|
||||
let overlay = superblock_ext::metadata_view(&whole[base..end], &superblock)?;
|
||||
Ok((
|
||||
Self {
|
||||
backing,
|
||||
base,
|
||||
end,
|
||||
overlay,
|
||||
},
|
||||
superblock,
|
||||
))
|
||||
}
|
||||
|
||||
fn as_bytes(&self) -> &[u8] {
|
||||
&self.backing.whole_file()[self.base..self.end]
|
||||
match &self.overlay {
|
||||
Some(v) => v,
|
||||
None => &self.backing.whole_file()[self.base..self.end],
|
||||
}
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
|
||||
Binary file not shown.
@@ -42,8 +42,9 @@ macro_rules! skip_if_no_python {
|
||||
|
||||
/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
|
||||
/// `d` the output directory) and then, for every `NAME.h5` it wrote,
|
||||
/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR`
|
||||
/// otherwise. Returns those lines, sorted.
|
||||
/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset
|
||||
/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those
|
||||
/// lines, sorted.
|
||||
fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
|
||||
let script = format!(
|
||||
r#"
|
||||
@@ -54,7 +55,7 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
|
||||
name = os.path.basename(path)[:-3]
|
||||
try:
|
||||
with h5py.File(path, "r") as f:
|
||||
f["d"][()]
|
||||
f[globals().get("DSET", "d")][()]
|
||||
print(name, "ok")
|
||||
except Exception:
|
||||
print(name, "ERROR")
|
||||
@@ -78,14 +79,14 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
|
||||
lines
|
||||
}
|
||||
|
||||
/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of
|
||||
/// whatever type it has).
|
||||
fn clawhdf5_reads(path: &Path) -> Result<(), String> {
|
||||
/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes
|
||||
/// of whatever type it has).
|
||||
fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> {
|
||||
let file = File::open(path).map_err(|e| format!("open: {e}"))?;
|
||||
let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?;
|
||||
let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?;
|
||||
ds.dtype().map_err(|e| format!("dtype: {e}"))?;
|
||||
ds.shape().map_err(|e| format!("shape: {e}"))?;
|
||||
file.read_multi(&["d"])
|
||||
file.read_multi(&[dset])
|
||||
.map(|_| ())
|
||||
.map_err(|e| format!("read: {e}"))
|
||||
}
|
||||
@@ -93,10 +94,15 @@ fn clawhdf5_reads(path: &Path) -> Result<(), String> {
|
||||
/// h5py's verdict for each file must be `expected`, and clawhdf5 must read
|
||||
/// exactly the files h5py reads.
|
||||
fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
|
||||
assert_agrees_with_h5py_on(dir, verdicts, expected, "d");
|
||||
}
|
||||
|
||||
/// [`assert_agrees_with_h5py`] reading dataset `dset`.
|
||||
fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) {
|
||||
assert_eq!(verdicts, expected, "h5py's view changed");
|
||||
for line in verdicts {
|
||||
let (name, verdict) = line.split_once(' ').unwrap();
|
||||
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")));
|
||||
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset);
|
||||
match verdict {
|
||||
"ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
|
||||
_ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
|
||||
@@ -244,7 +250,7 @@ for libver in ("earliest", "latest"):
|
||||
],
|
||||
);
|
||||
for name in ["earliest_size2", "latest_size8"] {
|
||||
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err();
|
||||
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
|
||||
assert!(
|
||||
err.contains("stored datatype size in chunk layout"),
|
||||
"{name}: {err}"
|
||||
@@ -557,3 +563,68 @@ for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)):
|
||||
assert!(mm.dataset("d").is_err(), "{name}: MmapFile");
|
||||
}
|
||||
}
|
||||
|
||||
/// libhdf5 decodes the superblock extension's messages when it opens a file
|
||||
/// and refuses the file when one does not decode: cve-2020-10810 (a File
|
||||
/// Space Info message too short for what it announces), cve-2020-10812 (a
|
||||
/// metadata cache image past the end of the file). We did not look at those
|
||||
/// messages and opened such files.
|
||||
#[test]
|
||||
fn superblock_extension_messages_libhdf5_refuses_are_refused() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5");
|
||||
let body = format!(
|
||||
r#"
|
||||
{FIX_OHDR_PY}
|
||||
def ext_addr(buf):
|
||||
assert buf[8] in (2, 3)
|
||||
return int.from_bytes(buf[20:28], "little")
|
||||
def save(name, buf):
|
||||
open(os.path.join(d, name + ".h5"), "wb").write(buf)
|
||||
|
||||
# A paged file: its superblock extension holds a File Space Info message
|
||||
# (version 1, strategy page, not persisting, threshold 1, page size 4096).
|
||||
DSET = "DSET"
|
||||
good = os.path.join(d, "paged_good.h5")
|
||||
with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f:
|
||||
f.create_dataset("DSET", data=np.arange(10, dtype="<i4"))
|
||||
data = bytearray(open(good, "rb").read())
|
||||
ext = ext_addr(data)
|
||||
at = data.find(struct.pack("<QQ", 1, 4096), ext)
|
||||
assert at > ext
|
||||
# Page size 256 (under libhdf5's minimum of 512).
|
||||
bad = bytearray(data); bad[at + 8:at + 16] = struct.pack("<Q", 256); fix_ohdr(bad, ext)
|
||||
save("paged_small_page", bad)
|
||||
# Persisting free space, without the manager addresses that then follow.
|
||||
bad = bytearray(data); bad[at - 1] = 1; fix_ohdr(bad, ext)
|
||||
save("paged_persist_short", bad)
|
||||
|
||||
# h5clear_mdc_image.h5 (from libhdf5's tests): a metadata cache image, the
|
||||
# root group's header only in the image. The Metadata Cache Image message
|
||||
# (type 0x18) records the image's address and length.
|
||||
data = bytearray(open(r"{mdc}", "rb").read())
|
||||
save("mdc_good", data)
|
||||
ext = ext_addr(data)
|
||||
at = data.find(bytes([0x18, 17, 0]), ext)
|
||||
assert at > ext
|
||||
length = at + 5 + 8
|
||||
bad = bytearray(data); bad[length:length + 8] = struct.pack("<Q", 1 << 28); fix_ohdr(bad, ext)
|
||||
save("mdc_past_eof", bad)
|
||||
"#,
|
||||
mdc = mdc.display()
|
||||
);
|
||||
let verdicts = h5py_verdicts(dir.path(), &body);
|
||||
assert_agrees_with_h5py_on(
|
||||
dir.path(),
|
||||
&verdicts,
|
||||
&[
|
||||
"mdc_good ok",
|
||||
"mdc_past_eof ERROR",
|
||||
"paged_good ok",
|
||||
"paged_persist_short ERROR",
|
||||
"paged_small_page ERROR",
|
||||
],
|
||||
"DSET",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
//! Files with a metadata cache image read as libhdf5 reads them.
|
||||
//!
|
||||
//! `fixtures/h5clear_mdc_image.h5` comes from libhdf5's tool tests
|
||||
//! (`tools/test/testfiles`): written with a metadata cache image, so its
|
||||
//! superblock extension points at an image block holding the file's
|
||||
//! metadata cache entries, and the root group's object header exists only
|
||||
//! there — the header's own address in the file is zeros. It holds one
|
||||
//! dataset, `/DSET`, 50 x 100 `int32` with `DSET[i][j] = i * j` (h5py
|
||||
//! 3.16 / HDF5 2.0 reads that). Until 2026-09-26 every reader failed on the
|
||||
//! root group: `InvalidObjectHeaderVersion(0)`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clawhdf5::{File, LazyFile, MmapFile};
|
||||
|
||||
fn fixture() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5")
|
||||
}
|
||||
|
||||
fn expected() -> Vec<i32> {
|
||||
(0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_reads_through_the_cache_image() {
|
||||
let file = File::open(fixture()).unwrap();
|
||||
assert_eq!(file.root().datasets().unwrap(), ["DSET"]);
|
||||
let ds = file.dataset("DSET").unwrap();
|
||||
assert_eq!(ds.shape().unwrap(), [50, 100]);
|
||||
assert_eq!(ds.read_i32().unwrap(), expected());
|
||||
|
||||
let file = File::from_bytes(std::fs::read(fixture()).unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
file.dataset("DSET").unwrap().read_i32().unwrap(),
|
||||
expected()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mmap_and_lazy_files_read_through_the_cache_image() {
|
||||
let mm = MmapFile::open(fixture()).unwrap();
|
||||
assert_eq!(mm.dataset("DSET").unwrap().read_i32().unwrap(), expected());
|
||||
let lazy = LazyFile::from_bytes(std::fs::read(fixture()).unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
lazy.dataset("DSET").unwrap().read_i32().unwrap(),
|
||||
expected()
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user