quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p2b-remaining-conformance' into feat/p2b-scale

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/chunked_read.rs
This commit is contained in:
osobh
2026-09-26 11:57:35 -05:00
parent 4b02e7d068 55e0e7e9cf
commit 9a73299594
36 changed files with 3297 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5/src/cache_image.rs
+85
View File
@@ -0,0 +1,85 @@
//! Metadata cache images, as the file openers apply them.
//!
//! A file written with a metadata cache image keeps metadata cache entries
//! (object headers, B-tree nodes, heaps) in an image block, and libhdf5
//! reads those entries in place of the file's own bytes at their addresses
//! (see `clawhdf5_format::superblock_ext`). The metadata parsers read one
//! contiguous byte slice, so the image has to be laid over the file's bytes
//! — without copying the file:
//!
//! - an opener that holds the file in a buffer it owns writes the entries
//! into that buffer (only the image block is copied, as libhdf5 copies
//! it);
//! - an opener that maps the file ([`File::open`](crate::File::open),
//! [`MmapFile`](crate::MmapFile), [`LazyFile::open_mmap`]
//! (crate::LazyFile::open_mmap)) writes them into a private copy-on-write
//! mapping of the file ([`clawhdf5_io::HDF5Read::private_copy`]): only the
//! pages the entries land on are copied, and the rest of the file stays
//! shared with the page cache;
//! - a file without an image is read from the original bytes, as before.
use clawhdf5_format::error::FormatError;
use clawhdf5_format::superblock::Superblock;
use clawhdf5_format::superblock_ext::{self, CacheImageState};
use clawhdf5_io::PrivateCopy;
use crate::error::Error;
/// A file's metadata, as an opener must read it.
pub(crate) enum ImageView {
/// Read the opener's bytes: the file has no cache image, or it was
/// written into a buffer the opener owns.
Plain,
/// The file with its cache image written in: a private copy of the
/// whole file (the HDF5 data at the same offsets as in the file).
Patched(PrivateCopy),
/// The file has a cache image libhdf5 cannot load.
Unloadable(FormatError),
}
/// Check the superblock extension of the file whose bytes are `whole` (the
/// HDF5 data in `base..end`) and lay any cache image over a private copy
/// of the file made by `copy`. An error means libhdf5 refuses the file.
pub(crate) fn private_view(
whole: &[u8],
base: usize,
end: usize,
sb: &Superblock,
copy: impl FnOnce() -> std::io::Result<PrivateCopy>,
) -> Result<ImageView, Error> {
let data = &whole[base..end];
Ok(match superblock_ext::cache_image_state(data, sb)? {
CacheImageState::Absent => ImageView::Plain,
CacheImageState::Unloadable(e) => ImageView::Unloadable(e),
CacheImageState::Loaded(image) => {
let mut view = copy().map_err(Error::Io)?;
let dst =
view.get_mut(base..end)
.ok_or(Error::Format(FormatError::InvalidCacheImage(
"the file changed while it was opened",
)))?;
image.apply(image.block(data)?, dst)?;
ImageView::Patched(view)
}
})
}
/// [`private_view`] for a file held in `whole`, a buffer the opener owns:
/// the image is written into it in place.
pub(crate) fn in_place(
whole: &mut [u8],
base: usize,
end: usize,
sb: &Superblock,
) -> Result<ImageView, Error> {
let data = &mut whole[base..end];
Ok(match superblock_ext::cache_image_state(data, sb)? {
CacheImageState::Absent => ImageView::Plain,
CacheImageState::Unloadable(e) => ImageView::Unloadable(e),
CacheImageState::Loaded(image) => {
let block = image.block(data)?.to_vec();
image.apply(&block, data)?;
ImageView::Plain
}
})
}
crates/clawhdf5/src/lazy.rs
+54 -6
View File
@@ -46,6 +46,13 @@ pub struct LazyFile<R: HDF5Read> {
/// End of the HDF5 data (`Superblock::data_end`, absolute).
end: usize,
superblock: Superblock,
/// A file that holds a metadata cache image, with the image written in:
/// the reader's [`HDF5Read::private_copy`] of the whole file (a
/// copy-on-write mapping for [`clawhdf5_io::MmapReader`], so only the
/// pages the image's entries land on are copied; see
/// `crate::cache_image`). `None` for a file without an image, read
/// straight from the reader.
patched: Option<clawhdf5_io::PrivateCopy>,
root_header: ObjectHeader,
/// Cache of parsed object headers, keyed by address.
header_cache: RefCell<HashMap<u64, ObjectHeader>>,
@@ -82,7 +89,24 @@ impl<R: HDF5Read> LazyFile<R> {
let superblock = Superblock::parse(data, 0)?;
// Refuse a truncated file; read nothing past the recorded end of file.
let end = base + superblock.data_end(base as u64, whole_len)? as usize;
let data = &reader.as_bytes()[base..end];
// Decode the superblock extension as libhdf5 does at open, and load
// a metadata cache image over the file's metadata.
let view =
crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || {
reader.private_copy()
})?;
let patched = match view {
crate::cache_image::ImageView::Plain => None,
crate::cache_image::ImageView::Patched(p) => Some(p),
// libhdf5 opens such a file and fails its first metadata read;
// a LazyFile reads the root group's header at open, so the open
// is that read.
crate::cache_image::ImageView::Unloadable(e) => return Err(e.into()),
};
let data = match &patched {
Some(p) => &p[base..end],
None => &reader.as_bytes()[base..end],
};
let root_header = ObjectHeader::parse(
data,
superblock.root_group_address as usize,
@@ -94,6 +118,7 @@ impl<R: HDF5Read> LazyFile<R> {
base,
end,
superblock,
patched,
root_header,
header_cache: RefCell::new(HashMap::new()),
})
@@ -111,7 +136,10 @@ impl<R: HDF5Read> LazyFile<R> {
}
fn hdf5_bytes(&self) -> &[u8] {
&self.reader.as_bytes()[self.base..self.end]
match &self.patched {
Some(p) => &p[self.base..self.end],
None => &self.reader.as_bytes()[self.base..self.end],
}
}
/// Returns a reference to the parsed superblock.
@@ -135,10 +163,11 @@ impl<R: HDF5Read> LazyFile<R> {
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(path.to_string()));
}
Ok(LazyDataset {
LazyDataset {
file: self,
header: hdr,
})
}
.check_open()
}
/// Resolve a path and return a `LazyGroup` handle.
@@ -284,10 +313,11 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> {
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(name.to_string()));
}
Ok(LazyDataset {
LazyDataset {
file: self.file,
header: hdr,
})
}
.check_open()
}
/// Get a subgroup within this group by name.
@@ -326,6 +356,24 @@ pub struct LazyDataset<'f, R: HDF5Read> {
}
impl<'f, R: HDF5Read> LazyDataset<'f, R> {
/// libhdf5's storage checks when it opens a dataset
/// ([`data_read::check_dataset_storage`]): a dataset whose element count
/// times element size overflows, or whose contiguous storage runs past
/// the end of the file, fails to open. A datatype, dataspace or layout
/// that does not decode is left for the read to report, as before (the
/// dataset still opens, and its attributes can be read).
fn check_open(self) -> Result<Self, Error> {
let decoded = (|| -> Result<_, Error> {
let data = self.required_payload(MessageType::Dataspace)?;
let ds = Dataspace::parse(&data, self.file.length_size())?;
Ok((self.datatype()?, ds, self.data_layout()?))
})();
if let Ok((dt, ds, dl)) = decoded {
data_read::check_dataset_storage(&dl, &ds, &dt, self.file.hdf5_bytes().len() as u64)?;
}
Ok(self)
}
/// Returns the shape (dimensions) of the dataset.
pub fn shape(&self) -> Result<Vec<u64>, Error> {
let ds = self.dataspace()?;
crates/clawhdf5/src/lib.rs
+1
View File
@@ -24,6 +24,7 @@
//! builder.write("output.h5").unwrap();
//! ```
mod cache_image;
pub mod error;
pub mod lazy;
#[cfg(feature = "mmap")]
crates/clawhdf5/src/mmap_file.rs
+69 -16
View File
@@ -38,6 +38,14 @@ pub struct MmapFile {
/// End of the HDF5 data (`Superblock::data_end`, absolute).
end: usize,
superblock: Superblock,
/// A file that holds a metadata cache image, with the image written in:
/// a private copy-on-write mapping of the whole file, so only the pages
/// the image's entries land on are copied (see `crate::cache_image`).
/// `None` for a file without an image, read straight from the mapping.
patched: Option<clawhdf5_io::PrivateCopy>,
/// The file has a metadata cache image libhdf5 cannot load: every
/// object lookup fails with this error (see `File`).
image_error: Option<FormatError>,
}
impl MmapFile {
@@ -50,18 +58,34 @@ impl MmapFile {
let superblock = Superblock::parse(data, 0)?;
// Refuse a truncated file; read nothing past the recorded end of file.
let end = base + superblock.data_end(base as u64, whole_len)? as usize;
// Decode the superblock extension as libhdf5 does at open, and load
// a metadata cache image over the file's metadata.
let view =
crate::cache_image::private_view(reader.as_bytes(), base, end, &superblock, || {
clawhdf5_io::HDF5Read::private_copy(&reader)
})?;
let (patched, image_error) = match view {
crate::cache_image::ImageView::Plain => (None, None),
crate::cache_image::ImageView::Patched(p) => (Some(p), None),
crate::cache_image::ImageView::Unloadable(e) => (None, Some(e)),
};
Ok(Self {
reader,
base,
end,
superblock,
patched,
image_error,
})
}
/// The file's bytes from the superblock on — the space HDF5 addresses
/// index into.
fn hdf5_bytes(&self) -> &[u8] {
&self.reader.as_bytes()[self.base..self.end]
match &self.patched {
Some(p) => &p[self.base..self.end],
None => &self.reader.as_bytes()[self.base..self.end],
}
}
/// Size of the user block before the superblock (0 for most files).
@@ -79,21 +103,22 @@ impl MmapFile {
/// Resolve a path and return a `MmapDataset` handle.
pub fn dataset(&self, path: &str) -> Result<MmapDataset<'_>, Error> {
let data = self.hdf5_bytes();
let data = self.meta()?;
let addr = group_v2::resolve_path_any(data, &self.superblock, path)?;
let hdr = self.parse_header(addr)?;
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(path.to_string()));
}
Ok(MmapDataset {
MmapDataset {
file: self,
header: hdr,
})
}
.check_open()
}
/// Resolve a path and return a `MmapGroup` handle.
pub fn group(&self, path: &str) -> Result<MmapGroup<'_>, Error> {
let data = self.hdf5_bytes();
let data = self.meta()?;
let addr = group_v2::resolve_path_any(data, &self.superblock, path)?;
Ok(MmapGroup {
file: self,
@@ -108,14 +133,29 @@ impl MmapFile {
self.hdf5_bytes()
}
/// The error of a metadata cache image libhdf5 cannot load, when the
/// file has one (see [`crate::File::cache_image_error`]).
pub fn cache_image_error(&self) -> Option<&FormatError> {
self.image_error.as_ref()
}
/// Returns a reference to the parsed superblock.
pub fn superblock(&self) -> &Superblock {
&self.superblock
}
/// The bytes to read metadata from; fails for a file whose cache image
/// cannot be loaded.
fn meta(&self) -> Result<&[u8], FormatError> {
match &self.image_error {
Some(e) => Err(e.clone()),
None => Ok(self.hdf5_bytes()),
}
}
fn parse_header(&self, address: u64) -> Result<ObjectHeader, FormatError> {
ObjectHeader::parse(
self.hdf5_bytes(),
self.meta()?,
address as usize,
self.superblock.offset_size,
self.superblock.length_size,
@@ -211,10 +251,11 @@ impl<'f> MmapGroup<'f> {
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(name.to_string()));
}
Ok(MmapDataset {
MmapDataset {
file: self.file,
header: hdr,
})
}
.check_open()
}
/// Get a subgroup within this group by name.
@@ -235,7 +276,7 @@ impl<'f> MmapGroup<'f> {
/// [`group_v2::resolve_group_children`]); dangling, external and
/// user-defined links are left out.
fn children(&self) -> Result<Vec<GroupEntry>, Error> {
let data = self.file.hdf5_bytes();
let data = self.file.meta()?;
group_v2::resolve_group_children(data, &self.file.superblock, self.address)
.map_err(Error::Format)
}
@@ -253,6 +294,24 @@ pub struct MmapDataset<'f> {
}
impl<'f> MmapDataset<'f> {
/// libhdf5's storage checks when it opens a dataset
/// ([`data_read::check_dataset_storage`]): a dataset whose element count
/// times element size overflows, or whose contiguous storage runs past
/// the end of the file, fails to open. A datatype, dataspace or layout
/// that does not decode is left for the read to report, as before (the
/// dataset still opens, and its attributes can be read).
fn check_open(self) -> Result<Self, Error> {
let decoded = (|| -> Result<_, Error> {
let data = self.required_payload(MessageType::Dataspace)?;
let ds = Dataspace::parse(&data, self.file.length_size())?;
Ok((self.datatype()?, ds, self.data_layout()?))
})();
if let Ok((dt, ds, dl)) = decoded {
data_read::check_dataset_storage(&dl, &ds, &dt, self.file.hdf5_bytes().len() as u64)?;
}
Ok(self)
}
/// Returns the shape (dimensions) of the dataset.
pub fn shape(&self) -> Result<Vec<u64>, Error> {
let ds = self.dataspace()?;
@@ -405,13 +464,7 @@ impl<'f> MmapDataset<'f> {
match &dl {
DataLayout::Contiguous { address, size } => {
let addr = address.ok_or(Error::Format(FormatError::NoDataAllocated))?;
let sz = *size as usize;
if sz != expected {
return Err(Error::Format(FormatError::DataSizeMismatch {
expected,
actual: sz,
}));
}
let sz = clawhdf5_format::data_read::contiguous_read_len(*size, expected)?;
let data = self.file.hdf5_bytes();
let a = addr as usize;
if a + sz > data.len() {
crates/clawhdf5/src/reader.rs
+141 -16
View File
@@ -21,6 +21,7 @@ use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::signature;
use clawhdf5_format::superblock::Superblock;
use crate::cache_image::{self, ImageView};
use crate::error::Error;
use crate::types::{AttrValue, DType, classify_datatype, read_attrs};
@@ -55,12 +56,24 @@ struct FileData {
base: usize,
/// End of the HDF5 data in the file (`Superblock::data_end`, absolute).
end: usize,
/// A mapped file that holds a metadata cache image, with the image
/// written in: a private copy-on-write mapping of the whole file, so
/// only the pages the image's entries land on are copied (see
/// `crate::cache_image`). `None` for every other file: a file without
/// an image is read straight from the mapping, and an owned buffer has
/// the image written into it in place.
patched: Option<clawhdf5_io::PrivateCopy>,
/// The file has a metadata cache image libhdf5 cannot load. libhdf5
/// opens such a file and fails its first metadata read (the image loads
/// then); every object lookup here fails with this error, and no
/// metadata is read from the file's own, possibly stale, bytes.
image_error: Option<FormatError>,
}
impl FileData {
/// Locate the superblock and parse it. A truncated file is refused, and
/// bytes past the recorded end of file are not read, as in libhdf5.
fn new(backing: Backing) -> Result<(Self, Superblock), Error> {
fn new(mut backing: Backing) -> Result<(Self, Superblock), Error> {
let whole = backing.whole_file();
let (user_block, hdf5) = signature::split_user_block(whole)?;
let base = user_block.len();
@@ -68,16 +81,54 @@ impl FileData {
let end = superblock.data_end(base as u64, whole.len() as u64)?;
// data_end is at most the file length (less the user block).
let end = base + end as usize;
Ok((Self { backing, base, end }, superblock))
// libhdf5 decodes the superblock extension at open (a message it
// cannot decode fails the open) and loads a metadata cache image
// over the file's own metadata.
let view = match &mut backing {
Backing::Owned(v) => cache_image::in_place(v, base, end, &superblock)?,
#[cfg(feature = "mmap")]
Backing::Mmap(r) => {
cache_image::private_view(r.as_bytes(), base, end, &superblock, || {
clawhdf5_io::HDF5Read::private_copy(r)
})?
}
};
let (patched, image_error) = match view {
ImageView::Plain => (None, None),
ImageView::Patched(p) => (Some(p), None),
ImageView::Unloadable(e) => (None, Some(e)),
};
Ok((
Self {
backing,
base,
end,
patched,
image_error,
},
superblock,
))
}
fn as_bytes(&self) -> &[u8] {
&self.backing.whole_file()[self.base..self.end]
match &self.patched {
Some(p) => &p[self.base..self.end],
None => &self.backing.whole_file()[self.base..self.end],
}
}
fn len(&self) -> usize {
self.as_bytes().len()
}
/// The bytes to read metadata from; fails for a file whose cache image
/// cannot be loaded (see [`Self::image_error`]).
fn meta(&self) -> Result<&[u8], FormatError> {
match &self.image_error {
Some(e) => Err(e.clone()),
None => Ok(self.as_bytes()),
}
}
}
// ---------------------------------------------------------------------------
@@ -164,16 +215,17 @@ impl File {
///
/// The path uses `/` separators (e.g., `"group1/values"`).
pub fn dataset(&self, path: &str) -> Result<Dataset<'_>, Error> {
let data = self.data.as_bytes();
let data = self.data.meta()?;
let addr = group_v2::resolve_path_any(data, &self.superblock, path)?;
let hdr = self.parse_header(addr)?;
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(path.to_string()));
}
Ok(Dataset {
Dataset {
file: self,
header: hdr,
})
}
.check_open()
}
/// A `Dataset` handle for the object header at `address` (an address
@@ -186,10 +238,11 @@ impl File {
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(format!("object at address {address}")));
}
Ok(Dataset {
Dataset {
file: self,
header: hdr,
})
}
.check_open()
}
/// Resolve a path and return a `Group` handle.
@@ -197,7 +250,7 @@ impl File {
/// The path uses `/` separators (e.g., `"sensors"`).
/// Use `"/"` or `""` for the root group.
pub fn group(&self, path: &str) -> Result<Group<'_>, Error> {
let data = self.data.as_bytes();
let data = self.data.meta()?;
let addr = group_v2::resolve_path_any(data, &self.superblock, path)?;
Ok(Group {
file: self,
@@ -253,11 +306,23 @@ impl File {
/// Returns the file's bytes from the superblock on (after any user
/// block). Every HDF5 address in the file indexes this slice, so it is
/// what the `clawhdf5_format` parsers expect as `file_data`.
/// what the `clawhdf5_format` parsers expect as `file_data`. For a file
/// with a metadata cache image these are the bytes with the image
/// applied; when the image cannot be loaded they are the file's own
/// bytes, whose metadata may be stale (every object lookup fails then).
pub fn as_bytes(&self) -> &[u8] {
self.data.as_bytes()
}
/// The error of a metadata cache image libhdf5 cannot load, when the
/// file has one. Such a file opens, as in libhdf5, and every object
/// lookup fails with this error; code that parses [`Self::as_bytes`]
/// itself should check it first, since those bytes then hold the
/// file's own, possibly stale, metadata.
pub fn cache_image_error(&self) -> Option<&FormatError> {
self.data.image_error.as_ref()
}
/// Size of the user block before the superblock (0 for most files).
/// Matches h5py's `File.userblock_size`.
pub fn user_block_size(&self) -> u64 {
@@ -302,7 +367,7 @@ impl File {
raw: &[u8],
) -> Result<Vec<Vec<u8>>, Error> {
crate::vlen::decode_string_bytes(
self.as_bytes(),
self.data.meta()?,
datatype,
raw,
self.offset_size(),
@@ -320,7 +385,7 @@ impl File {
raw: &[u8],
) -> Result<Vec<Vec<T>>, Error> {
crate::vlen::decode_vlen(
self.as_bytes(),
self.data.meta()?,
datatype,
raw,
self.offset_size(),
@@ -330,7 +395,7 @@ impl File {
fn parse_header(&self, address: u64) -> Result<ObjectHeader, FormatError> {
ObjectHeader::parse(
self.data.as_bytes(),
self.data.meta()?,
address as usize,
self.superblock.offset_size,
self.superblock.length_size,
@@ -427,10 +492,11 @@ impl<'f> Group<'f> {
if !has_message(&hdr, MessageType::DataLayout) {
return Err(Error::NotADataset(name.to_string()));
}
Ok(Dataset {
Dataset {
file: self.file,
header: hdr,
})
}
.check_open()
}
/// Get a subgroup within this group by name.
@@ -451,7 +517,7 @@ impl<'f> Group<'f> {
/// [`group_v2::resolve_group_children`]); dangling, external and
/// user-defined links are left out.
fn children(&self) -> Result<Vec<GroupEntry>, Error> {
let data = self.file.data.as_bytes();
let data = self.file.data.meta()?;
group_v2::resolve_group_children(data, &self.file.superblock, self.address)
.map_err(Error::Format)
}
@@ -469,6 +535,24 @@ pub struct Dataset<'f> {
}
impl<'f> Dataset<'f> {
/// libhdf5's storage checks when it opens a dataset
/// ([`data_read::check_dataset_storage`]): a dataset whose element count
/// times element size overflows, or whose contiguous storage runs past
/// the end of the file, fails to open. A datatype, dataspace or layout
/// that does not decode is left for the read to report, as before (the
/// dataset still opens, and its attributes can be read).
fn check_open(self) -> Result<Self, Error> {
let decoded = (|| -> Result<_, Error> {
let data = self.required_payload(MessageType::Dataspace)?;
let ds = Dataspace::parse(&data, self.file.length_size())?;
Ok((self.datatype()?, ds, self.data_layout()?))
})();
if let Ok((dt, ds, dl)) = decoded {
data_read::check_dataset_storage(&dl, &ds, &dt, self.file.data.len() as u64)?;
}
Ok(self)
}
/// Returns the shape (dimensions) of the dataset.
pub fn shape(&self) -> Result<Vec<u64>, Error> {
let ds = self.dataspace()?;
@@ -1283,3 +1367,44 @@ mod sibling_file_name_tests {
}
}
}
#[cfg(all(test, feature = "mmap"))]
mod zero_copy_tests {
use super::*;
/// Where `File::open` reads metadata from: `Some(true)` for the file's
/// own mapping, `Some(false)` for a private copy-on-write mapping.
fn reads_from_the_mapping(f: &File) -> Option<bool> {
let Backing::Mmap(r) = &f.data.backing else {
return None;
};
let mapped = r.as_bytes()[f.data.base..].as_ptr();
match &f.data.patched {
None => Some(std::ptr::eq(f.as_bytes().as_ptr(), mapped)),
Some(p) => {
assert!(p.is_mapped(), "the image went into a heap copy of the file");
Some(false)
}
}
}
#[test]
fn a_file_without_a_cache_image_is_read_from_the_mapping() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("plain.h5");
let mut b = crate::FileBuilder::new();
b.create_dataset("d").with_f64_data(&[1.0, 2.0]);
b.write(&path).unwrap();
let f = File::open(&path).unwrap();
assert_eq!(reads_from_the_mapping(&f), Some(true));
assert_eq!(f.dataset("d").unwrap().read_f64().unwrap(), [1.0, 2.0]);
}
#[test]
fn a_cache_image_goes_into_a_copy_on_write_mapping() {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures/h5clear_mdc_image.h5");
let f = File::open(path).unwrap();
assert_eq!(reads_from_the_mapping(&f), Some(false));
}
}
crates/clawhdf5/tests/cache_image_memory.rs
+139
View File
@@ -0,0 +1,139 @@
//! Opening a file with a metadata cache image must not copy the file.
//!
//! The image's entries are laid over the file's bytes in a private
//! copy-on-write mapping (`File::open`, `MmapFile::open`,
//! `LazyFile::open_mmap`), so only the pages they land on are copied. The
//! first implementation copied the whole file onto the heap at open: a
//! 1 GiB file that takes a few KB on disk needed 2 GB of memory, and an
//! 8 GiB one aborted the process. Here libhdf5 itself (the library h5py
//! bundles, through ctypes: `H5Pset_mdc_image_config`) adds an image to a
//! 1 GiB sparse file, and the process's resident memory must stay far below
//! the file's size while each opener lists the file and reads its small
//! dataset.
//!
//! One test in its own binary, so no other test's allocations land in the
//! measurement. Linux only (it reads `VmRSS` from `/proc/self/status`).
//! Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
#![cfg(target_os = "linux")]
use std::path::Path;
use std::process::Command;
use clawhdf5::{File, LazyFile, MmapFile};
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py, numpy"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
fn rss_bytes() -> u64 {
let status = std::fs::read_to_string("/proc/self/status").unwrap();
let line = status.lines().find(|l| l.starts_with("VmRSS:")).unwrap();
let kb: u64 = line.split_whitespace().nth(1).unwrap().parse().unwrap();
kb * 1024
}
/// A 1 GiB sparse file: `/big`, 2^27 `f8` with only its last element
/// written, `/small` = 0..10, with a metadata cache image added by libhdf5.
fn make_file(path: &Path) {
let script = format!(
r#"
import ctypes, glob, os, h5py, numpy as np
path = "{path}"
libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, "h5py.libs", "libhdf5-*.so*"))
assert libs, "no libhdf5 bundled with h5py"
lib = ctypes.CDLL(libs[0])
class Cfg(ctypes.Structure):
_fields_ = [("version", ctypes.c_int), ("generate_image", ctypes.c_bool),
("save_resize_status", ctypes.c_bool), ("entry_ageout", ctypes.c_int)]
with h5py.File(path, "w", libver="latest") as f:
d = f.create_dataset("big", shape=(2**27,), dtype="f8")
d[-1] = 7.5
f.create_dataset("small", data=np.arange(10, dtype="<i4"))
fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
cfg = Cfg(1, True, False, -1)
assert lib.H5Pset_mdc_image_config(ctypes.c_int64(fapl.id), ctypes.byref(cfg)) >= 0
f = h5py.File(h5py.h5f.open(path.encode(), h5py.h5f.ACC_RDWR, fapl=fapl))
f["small"][()]; f["big"].shape
f.close()
assert os.path.getsize(path) >= 2**30
with open(path, "rb") as fh:
fh.seek(-(1 << 20), 2)
assert b"MDCI" in fh.read(), "libhdf5 wrote no cache image"
with h5py.File(path, "r") as f:
assert list(f["small"][()]) == list(range(10))
"#,
path = path.display()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
}
#[test]
fn a_cache_image_does_not_copy_the_file() {
if !python_available() {
assert!(
!std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
return;
}
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("sparse_image.h5");
make_file(&path);
const LIMIT: u64 = 256 << 20;
let small: Vec<i32> = (0..10).collect();
let before = rss_bytes();
{
let f = File::open(&path).unwrap();
let mut names = f.root().datasets().unwrap();
names.sort();
assert_eq!(names, ["big", "small"]);
assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small);
assert_eq!(f.dataset("big").unwrap().shape().unwrap(), [1 << 27]);
let grew = rss_bytes().saturating_sub(before);
assert!(
grew < LIMIT,
"File::open: resident memory grew {grew} bytes"
);
}
let before = rss_bytes();
{
let f = MmapFile::open(&path).unwrap();
assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small);
let grew = rss_bytes().saturating_sub(before);
assert!(
grew < LIMIT,
"MmapFile::open: resident memory grew {grew} bytes"
);
}
let before = rss_bytes();
{
let f = LazyFile::open_mmap(&path).unwrap();
assert_eq!(f.dataset("small").unwrap().read_i32().unwrap(), small);
let grew = rss_bytes().saturating_sub(before);
assert!(
grew < LIMIT,
"LazyFile::open_mmap: resident memory grew {grew} bytes"
);
}
}
crates/clawhdf5/tests/fixtures/h5clear_mdc_image.h5
Vendored
BIN
View File
Binary file not shown.
crates/clawhdf5/tests/header_validation_interop.rs
+218 -10
View File
@@ -42,8 +42,9 @@ macro_rules! skip_if_no_python {
/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
/// `d` the output directory) and then, for every `NAME.h5` it wrote,
/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR`
/// otherwise. Returns those lines, sorted.
/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset
/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those
/// lines, sorted.
fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
let script = format!(
r#"
@@ -54,7 +55,7 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
name = os.path.basename(path)[:-3]
try:
with h5py.File(path, "r") as f:
f["d"][()]
f[globals().get("DSET", "d")][()]
print(name, "ok")
except Exception:
print(name, "ERROR")
@@ -78,14 +79,14 @@ for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
lines
}
/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of
/// whatever type it has).
fn clawhdf5_reads(path: &Path) -> Result<(), String> {
/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes
/// of whatever type it has).
fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> {
let file = File::open(path).map_err(|e| format!("open: {e}"))?;
let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?;
let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?;
ds.dtype().map_err(|e| format!("dtype: {e}"))?;
ds.shape().map_err(|e| format!("shape: {e}"))?;
file.read_multi(&["d"])
file.read_multi(&[dset])
.map(|_| ())
.map_err(|e| format!("read: {e}"))
}
@@ -93,10 +94,15 @@ fn clawhdf5_reads(path: &Path) -> Result<(), String> {
/// h5py's verdict for each file must be `expected`, and clawhdf5 must read
/// exactly the files h5py reads.
fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
assert_agrees_with_h5py_on(dir, verdicts, expected, "d");
}
/// [`assert_agrees_with_h5py`] reading dataset `dset`.
fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) {
assert_eq!(verdicts, expected, "h5py's view changed");
for line in verdicts {
let (name, verdict) = line.split_once(' ').unwrap();
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")));
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset);
match verdict {
"ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
_ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
@@ -244,7 +250,7 @@ for libver in ("earliest", "latest"):
],
);
for name in ["earliest_size2", "latest_size8"] {
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err();
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
assert!(
err.contains("stored datatype size in chunk layout"),
"{name}: {err}"
@@ -508,3 +514,205 @@ for libver in ("earliest", "latest"):
);
}
}
/// cve-2024-32624 `/Dset_OBJREF`: a dataspace whose element count times the
/// element size overflows 64 bits. libhdf5 refuses to open the dataset
/// ("size of dataset's storage overflowed"); `File::dataset` used to open it
/// and report its shape, and only reading failed. The same for storage that
/// runs past the end of the file ("invalid dataset size, likely file
/// corruption").
#[test]
fn dataset_storage_libhdf5_refuses_at_open_is_refused_at_open() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
// A contiguous int64 dataset of 3 elements, libver earliest (a version 1
// dataspace holding the size and the maximum size). "overflow" makes
// both 2^62 + 2 (x 8 bytes overflows); "pasteof" makes both 1000 (the
// storage runs past the end of the file).
let verdicts = h5py_verdicts(
dir.path(),
r#"
good = os.path.join(d, "good.h5")
with h5py.File(good, "w", libver="earliest") as f:
f.create_dataset("d", data=np.array([7, 8, 9], dtype="<i8"))
data = bytearray(open(good, "rb").read())
three = struct.pack("<QQ", 3, 3)
at = data.find(three)
assert at > 0 and data.find(three, at + 1) < 0
for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)):
bad = bytearray(data)
bad[at:at + 16] = struct.pack("<QQ", n, n)
open(os.path.join(d, f"{name}.h5"), "wb").write(bad)
"#,
);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&["good ok", "overflow ERROR", "pasteof ERROR"],
);
for name in ["overflow", "pasteof"] {
let path = dir.path().join(format!("{name}.h5"));
let file = File::open(&path).unwrap();
let err = file.dataset("d").unwrap_err();
assert!(
err.to_string().contains("invalid dataset storage"),
"{name}: {err}"
);
assert!(file.root().dataset("d").is_err(), "{name}");
let mm = clawhdf5::MmapFile::open(&path).unwrap();
assert!(mm.dataset("d").is_err(), "{name}: MmapFile");
}
}
/// libhdf5 decodes the superblock extension's messages when it opens a file
/// and refuses the file when one does not decode: cve-2020-10810 (a File
/// Space Info message too short for what it announces), cve-2020-10812 (a
/// metadata cache image past the end of the file). We did not look at those
/// messages and opened such files.
#[test]
fn superblock_extension_messages_libhdf5_refuses_are_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5");
let body = format!(
r#"
{FIX_OHDR_PY}
def ext_addr(buf):
assert buf[8] in (2, 3)
return int.from_bytes(buf[20:28], "little")
def save(name, buf):
open(os.path.join(d, name + ".h5"), "wb").write(buf)
# A paged file: its superblock extension holds a File Space Info message
# (version 1, strategy page, not persisting, threshold 1, page size 4096).
DSET = "DSET"
good = os.path.join(d, "paged_good.h5")
with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f:
f.create_dataset("DSET", data=np.arange(10, dtype="<i4"))
data = bytearray(open(good, "rb").read())
ext = ext_addr(data)
at = data.find(struct.pack("<QQ", 1, 4096), ext)
assert at > ext
# Page size 256 (under libhdf5's minimum of 512).
bad = bytearray(data); bad[at + 8:at + 16] = struct.pack("<Q", 256); fix_ohdr(bad, ext)
save("paged_small_page", bad)
# Persisting free space, without the manager addresses that then follow.
bad = bytearray(data); bad[at - 1] = 1; fix_ohdr(bad, ext)
save("paged_persist_short", bad)
# h5clear_mdc_image.h5 (from libhdf5's tests): a metadata cache image, the
# root group's header only in the image. The Metadata Cache Image message
# (type 0x18) records the image's address and length.
data = bytearray(open(r"{mdc}", "rb").read())
save("mdc_good", data)
ext = ext_addr(data)
at = data.find(bytes([0x18, 17, 0]), ext)
assert at > ext
length = at + 5 + 8
bad = bytearray(data); bad[length:length + 8] = struct.pack("<Q", 1 << 28); fix_ohdr(bad, ext)
save("mdc_past_eof", bad)
"#,
mdc = mdc.display()
);
let verdicts = h5py_verdicts(dir.path(), &body);
assert_agrees_with_h5py_on(
dir.path(),
&verdicts,
&[
"mdc_good ok",
"mdc_past_eof ERROR",
"paged_good ok",
"paged_persist_short ERROR",
"paged_small_page ERROR",
],
"DSET",
);
}
/// An unfiltered chunk the index records at less than the chunk's size
/// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever
/// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk
/// size returned from index for unfiltered chunk"); we used to read the
/// rest as zeros, and now refuse it.
#[test]
fn unfiltered_chunk_of_the_wrong_size_is_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
good = os.path.join(d, "good.h5")
with h5py.File(good, "w", libver="earliest") as f:
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,), fillvalue=-1)
data = bytearray(open(good, "rb").read())
tree = data.find(b"TREE")
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
tree = data.find(b"TREE", tree + 1)
assert tree > 0 and data[tree + 5] == 0
key = tree + 8 + 16 # the first chunk's key: size, filter mask, offsets
second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
"#,
);
assert_eq!(
clawhdf5_reads(&dir.path().join("good.h5"), "d"),
Ok(()),
"good"
);
let err = clawhdf5_reads(&dir.path().join("short_chunk.h5"), "d").unwrap_err();
assert!(err.starts_with("read:"), "short_chunk: {err}");
}
/// A v1 B-tree chunk key whose element-size coordinate is not 0. libhdf5
/// looks each chunk up with that coordinate set to 0 (`H5B_find`,
/// `H5D__btree_cmp3`, `H5D__btree_found`): in a 1-D dataset it still finds
/// the chunk, in a 2-D one it does not and reads fill values
/// (`cve-2025-44905` `/Shuffle_float_data_le`). Both must read exactly what
/// h5py reads: the 1-D file was refused, and before that the 2-D chunk was
/// read as if its key were well formed.
#[test]
fn chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
def corrupt(name, shape, chunks, which, element_offset):
path = os.path.join(d, name + ".h5")
with h5py.File(path, "w", libver="earliest") as f:
f.create_dataset("d", data=np.arange(np.prod(shape), dtype="<i4").reshape(shape),
chunks=chunks, fillvalue=-1)
data = bytearray(open(path, "rb").read())
tree = data.find(b"TREE")
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
tree = data.find(b"TREE", tree + 1)
assert tree > 0 and data[tree + 5] == 0
ndims = len(shape) + 1
key_size = 8 + 8 * ndims
key = tree + 8 + 16 + which * (key_size + 8)
last = key + 8 + 8 * (ndims - 1)
assert struct.unpack_from("<Q", data, last) == (0,)
struct.pack_into("<Q", data, last, element_offset)
open(path, "wb").write(data)
with h5py.File(path, "r") as f:
open(path + ".txt", "w").write(" ".join(map(str, f["d"][()].ravel())))
corrupt("one_d", (100,), (37,), 1, 4096)
corrupt("two_d", (4, 6), (2, 3), 1, 4)
corrupt("two_d_first", (4, 6), (2, 3), 0, 8)
"#,
);
for name in ["one_d", "two_d", "two_d_first"] {
let path = dir.path().join(format!("{name}.h5"));
let expected: Vec<i32> = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt")))
.unwrap()
.split_whitespace()
.map(|v| v.parse().unwrap())
.collect();
let file = File::open(&path).unwrap();
let got = file.dataset("d").unwrap().read_i32();
assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}");
}
}
crates/clawhdf5/tests/metadata_cache_image.rs
+94
View File
@@ -0,0 +1,94 @@
//! Files with a metadata cache image read as libhdf5 reads them.
//!
//! `fixtures/h5clear_mdc_image.h5` comes from libhdf5's tool tests
//! (`tools/test/testfiles`): written with a metadata cache image, so its
//! superblock extension points at an image block holding the file's
//! metadata cache entries, and the root group's object header exists only
//! there — the header's own address in the file is zeros. It holds one
//! dataset, `/DSET`, 50 x 100 `int32` with `DSET[i][j] = i * j` (h5py
//! 3.16 / HDF5 2.0 reads that). Until 2026-09-26 every reader failed on the
//! root group: `InvalidObjectHeaderVersion(0)`.
use std::path::PathBuf;
use clawhdf5::{File, LazyFile, MmapFile};
fn fixture() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5")
}
fn expected() -> Vec<i32> {
(0..50).flat_map(|i| (0..100).map(move |j| i * j)).collect()
}
#[test]
fn file_reads_through_the_cache_image() {
let file = File::open(fixture()).unwrap();
assert_eq!(file.root().datasets().unwrap(), ["DSET"]);
let ds = file.dataset("DSET").unwrap();
assert_eq!(ds.shape().unwrap(), [50, 100]);
assert_eq!(ds.read_i32().unwrap(), expected());
let file = File::from_bytes(std::fs::read(fixture()).unwrap()).unwrap();
assert_eq!(
file.dataset("DSET").unwrap().read_i32().unwrap(),
expected()
);
}
#[test]
fn mmap_and_lazy_files_read_through_the_cache_image() {
let mm = MmapFile::open(fixture()).unwrap();
assert_eq!(mm.dataset("DSET").unwrap().read_i32().unwrap(), expected());
let lazy = LazyFile::from_bytes(std::fs::read(fixture()).unwrap()).unwrap();
assert_eq!(
lazy.dataset("DSET").unwrap().read_i32().unwrap(),
expected()
);
}
/// A file whose cache image libhdf5 cannot load: libhdf5 opens it, and the
/// first metadata read fails with the image's error (h5py:
/// `Unable to get group info (Bad metadata cache image header signature)`);
/// later reads get the file's own bytes, which here are stale (the root
/// group's header is zeros: "bad object header version number"). The
/// openers agree on the open and the failure: `File` and `MmapFile` open
/// and fail every object lookup with the image's error (never reading the
/// stale bytes), `LazyFile` reads the root group's header at open and so
/// fails there. `File::open` used to refuse the file, while the
/// conformance probe reported it as h5py does.
#[test]
fn an_image_libhdf5_cannot_load_fails_every_object() {
let mut bytes = std::fs::read(fixture()).unwrap();
let at = bytes.windows(4).position(|w| w == b"MDCI").unwrap();
bytes[at] = b'X';
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("bad_image.h5");
std::fs::write(&path, &bytes).unwrap();
let is_image_error = |e: clawhdf5::Error| {
matches!(
e,
clawhdf5::Error::Format(clawhdf5_format::error::FormatError::InvalidCacheImage(
"bad metadata cache image header signature"
))
)
};
for file in [
File::open(&path).unwrap(),
File::from_bytes(bytes.clone()).unwrap(),
] {
assert!(is_image_error(file.root().datasets().unwrap_err()));
assert!(is_image_error(file.root().attrs().unwrap_err()));
assert!(is_image_error(file.dataset("DSET").unwrap_err()));
assert!(is_image_error(file.group("/").map(|_| ()).unwrap_err()));
assert!(is_image_error(file.dataset_at(96).unwrap_err()));
}
let mm = MmapFile::open(&path).unwrap();
assert!(is_image_error(mm.root().datasets().unwrap_err()));
assert!(is_image_error(mm.dataset("DSET").unwrap_err()));
assert!(is_image_error(mm.group("/").map(|_| ()).unwrap_err()));
assert!(is_image_error(
LazyFile::from_bytes(bytes).map(|_| ()).unwrap_err()
));
}
crates/clawhdf5/tests/scaleoffset_interop.rs
+151
View File
@@ -0,0 +1,151 @@
//! Scale-offset datasets written by h5py (libhdf5) read exactly as h5py
//! reads them.
//!
//! h5py writes the whole matrix the filter has: every integer type (`i1` ..
//! `u8`) and `f4`/`f8`, both byte orders, with and without a fill value
//! (the type's minimum, maximum, or another value), with random, constant,
//! all-fill and extreme data, and every interesting `scaleoffset` setting
//! (0 = let libhdf5 choose the bits, a few bits, full width less one, full
//! width; decimal scale factors 0..7 for floats) — about 1480 datasets. For
//! each one h5py's decoded values are stored uncompressed next to it, and
//! the raw bytes clawhdf5 decodes must equal them.
//!
//! Until 2026-09-26 clawhdf5 silently returned wrong values for 332 of
//! these, in every release that decoded scale-offset: ordinary `u8`, `u4`,
//! `i8` and `f4` data with a wide range (where libhdf5 stores the elements
//! as they are, at full width), chunks whose `minval` field was recorded at
//! another size than 8 bytes, and chunks with `minbits` 0 and a fill value.
//!
//! Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
use std::process::Command;
use clawhdf5::File;
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py, numpy"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
const GENERATE: &str = r#"
import sys, h5py, numpy as np
rng = np.random.default_rng(7)
out, expect = sys.argv[1], sys.argv[2]
made = []
with h5py.File(out, "w") as f:
def make(name, a, so, fv, desc):
try:
d = f.create_dataset(name, data=a, chunks=(16,), scaleoffset=so, fillvalue=fv)
except Exception:
return # a combination libhdf5 refuses to write
d.attrs["desc"] = desc
made.append(name)
i = 0
for dt in ["i1", "u1", "i2", "u2", "i4", "u4", "i8", "u8"]:
for bo in "<>":
t = np.dtype(bo + dt)
info = np.iinfo(t)
for so in [0, 1, 3, 8 * t.itemsize - 1, 8 * t.itemsize]:
for fill in [None, "min", "max", "mid"]:
for pat in ["rand", "const", "allfill", "extreme"]:
n = 64
fv = {None: None, "min": info.min, "max": info.max, "mid": t.type(7)}[fill]
if pat == "rand":
lo = max(info.min, -50) if so else info.min
hi = min(info.max, 50) if so else info.max
a = rng.integers(lo, hi, size=n, endpoint=True,
dtype=np.int64 if t.kind == "i" else np.uint64).astype(t)
elif pat == "const":
a = np.full(n, 3, t)
elif pat == "extreme":
a = np.array([info.min, info.max] * (n // 2), t)
else:
if fv is None:
continue
a = np.full(n, fv, t)
make(f"d{i}", a, so, fv, f"{bo}{dt} so={so} fill={fill} pat={pat}")
i += 1
for dt in ["f4", "f8"]:
for bo in "<>":
t = np.dtype(bo + dt)
for so in [0, 1, 2, 4, 7]:
for fill in [None, -1.5, 0.0]:
for pat in ["rand", "const", "allfill", "neg", "big"]:
n = 50
if pat == "rand":
a = rng.normal(size=n).astype(t) * 10
elif pat == "const":
a = np.full(n, 2.25, t)
elif pat == "neg":
a = -np.abs(rng.normal(size=n)).astype(t) * 1000
elif pat == "big":
a = rng.normal(size=n).astype(t) * 1e6
else:
if fill is None:
continue
a = np.full(n, fill, t)
make(f"d{i}", a, so, fill, f"{bo}{dt} so={so} fill={fill} pat={pat}")
i += 1
# What libhdf5 decodes, stored uncompressed in the same datatype.
with h5py.File(out, "r") as f, h5py.File(expect, "w") as e:
for name in made:
e.create_dataset(name, data=f[name][()])
print(len(made))
"#;
#[test]
fn every_scale_offset_dataset_reads_as_h5py_reads_it() {
if !python_available() {
assert!(
!std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
return;
}
let dir = tempfile::tempdir().unwrap();
let written = dir.path().join("scaleoffset.h5");
let expected = dir.path().join("expected.h5");
let out = Command::new(python())
.args(["-c", GENERATE])
.arg(&written)
.arg(&expected)
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
let count: usize = String::from_utf8_lossy(&out.stdout).trim().parse().unwrap();
assert!(count > 1400, "only {count} datasets written");
let file = File::open(&written).unwrap();
let reference = File::open(&expected).unwrap();
let mut names = reference.root().datasets().unwrap();
names.sort();
assert_eq!(names.len(), count);
let mut wrong = Vec::new();
for name in &names {
let want = reference.read_multi(&[name]).unwrap().remove(0);
let got = file.read_multi(&[name]).map(|mut v| v.remove(0));
if got.as_ref().ok() != Some(&want) {
let desc = file.dataset(name).unwrap().attrs().unwrap().remove("desc");
wrong.push(format!("{name} {desc:?}: {:?}", got.map(|g| g.len())));
}
}
assert!(
wrong.is_empty(),
"{} of {count} scale-offset datasets differ from h5py:\n{}",
wrong.len(),
wrong.join("\n")
);
}
crates/clawhdf5/tests/vds_interop.rs
+23
View File
@@ -486,3 +486,26 @@ fn vds_libhdf5_test_files() {
vec![5, 10, 10]
);
}
/// A source file with a metadata cache image (libhdf5's
/// `h5clear_mdc_image.h5`, whose root group's header exists only in the
/// image) is read through the image, as libhdf5 opens it. Source files were
/// read from their own bytes, and this one failed on the root group.
#[test]
fn vds_source_file_with_a_metadata_cache_image() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5");
std::fs::copy(&fixture, dir.path().join("src.h5")).unwrap();
generate(
dir.path(),
r#"
with h5py.File("vds.h5", "w", libver="latest") as f:
lay = h5py.VirtualLayout(shape=(3, 100), dtype="i4")
lay[0:3, :] = h5py.VirtualSource("src.h5", "DSET", shape=(50, 100))[5:8, :]
f.create_virtual_dataset("v", lay)
expect("vds.h5", "v", "image_source")
"#,
);
assert_matches_libhdf5(dir.path(), "vds.h5", "v", "image_source");
}