osobhandClaude Opus 5.5
76c97f6c94
On a backend without the file in memory, a structure read whose length comes from untrusted header fields was clamped only by the end of the file, so a crafted size made one read (and copy) of up to the rest of the file. Each such read now covers what the parser actually uses: - local heap names: read in growing pieces (64 bytes first, then 4x) up to the end of the data segment, instead of the rest of the segment per name (quadratic for a big symbol-table group); - fractal heap indirect blocks: the doubling-table geometry locates the entry covering the object, and the first read ends at that entry; only if it is unallocated does the walk read the rest of the block (it visits every entry then). One walk implementation serves both; - paged fixed/extensible array data blocks over 1 MiB: the prefix and page bitmap, then each page in use on its own (smaller blocks are still one read); - blocks under one checksum (non-paged array data blocks, extensible array index and super blocks): the bounds check that comes first (the checksum's; the page bitmap's for a super block) is made against the file length before reading (Window::check_extent), so a block claimed past the end of the file costs no read. With the checksum feature off the parser has no such first check and the old read stands. Other windows were already bounded (the superblock and object header prefixes, the fractal heap header by a u16, SOHM tables by u8/u16 counts) or are exact reads checked against the file length first. In memory nothing changes: the pieces are borrowed slices. Tests: CountingStorage over a crafted heap (16 MiB file, width and rows 0xFFFF: under 1 KiB read, 16.7 MB before), a heap segment claiming 64 MiB (one 64-byte read per short name), long names at every piece boundary, a fixed array block claimed past the end of a 16 MiB file (under 64 bytes read), and in the equivalence harness an h5py file with a 2.4 MB fixed array block and a >1 MiB extensible array block, whole and cut at 97 points: every chunk index agrees with the slice read and the largest takes 205 KB (2.4 MB and 1.2 MB when read whole). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
702 lines
27 KiB
Rust
702 lines
27 KiB
Rust
//! Equivalence harness for the range-read migration
|
|
//! (`docs/design/range-reads.md`, milestone M1).
|
|
//!
|
|
//! Every metadata parser converted to [`Storage`] must give exactly what its
|
|
//! `&[u8]` form gives. This walks real files — the fixtures, files h5py
|
|
//! writes to exercise the less common structures, and optionally the
|
|
//! conformance corpus — and, for every object, runs each converted parser
|
|
//! twice: over the file as a slice, and over a [`CountingStorage`] that
|
|
//! serves the same bytes through `read_at` only (`as_contiguous()` is
|
|
//! `None`, so no parser can fall back to the whole slice). The results must
|
|
//! be identical, value for value and error for error.
|
|
//!
|
|
//! The one allowed difference is [`FormatError::ContiguousStorageRequired`]
|
|
//! from the storage path, and only from the structures still indexed by a v2
|
|
//! B-tree (dense attributes, a SOHM B-tree index, huge fractal-heap objects;
|
|
//! see `CONTIGUOUS_REQUIRED`), which fail cleanly instead of reading the
|
|
//! whole file. Those are counted; the error from any other site or check
|
|
//! fails the harness.
|
|
//!
|
|
//! Milestones M2/M3 extend `check_object` with the raw-data and group
|
|
//! parsers as they are converted.
|
|
//!
|
|
//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/
|
|
//! `.nc`/`.h5ad` file under those directories (the conformance corpus is
|
|
//! `conformance/.cache/corpus`); `CLAWHDF5_STORAGE_REPORT=1` prints the
|
|
//! per-file read counts.
|
|
//! - The h5py-written files honour `CLAWHDF5_PYTHON` and
|
|
//! `CLAWHDF5_REQUIRE_INTEROP` like the facade's interop tests.
|
|
|
|
use std::collections::{HashSet, VecDeque};
|
|
use std::fmt::Debug;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
use clawhdf5_format::attribute::{
|
|
extract_attributes_full, extract_attributes_full_in, extract_attributes_tolerant,
|
|
extract_attributes_tolerant_in,
|
|
};
|
|
use clawhdf5_format::attribute_info::AttributeInfoMessage;
|
|
use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in};
|
|
use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::dataspace::Dataspace;
|
|
use clawhdf5_format::datatype::Datatype;
|
|
use clawhdf5_format::error::FormatError;
|
|
use clawhdf5_format::extensible_array::{
|
|
ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in,
|
|
};
|
|
use clawhdf5_format::fill_value::{dataset_fill_value_from_storage, dataset_fill_value_in};
|
|
use clawhdf5_format::fixed_array::{
|
|
FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in,
|
|
};
|
|
use clawhdf5_format::fractal_heap::FractalHeapHeader;
|
|
use clawhdf5_format::link_info::LinkInfoMessage;
|
|
use clawhdf5_format::local_heap::LocalHeap;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
use clawhdf5_format::shared_message::{
|
|
self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in,
|
|
parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in,
|
|
};
|
|
use clawhdf5_format::signature::split_user_block;
|
|
use clawhdf5_format::storage::{CountingStorage, Storage};
|
|
use clawhdf5_format::superblock::Superblock;
|
|
use clawhdf5_format::superblock_ext::{
|
|
cache_image_state, cache_image_state_in, read_superblock_extension,
|
|
read_superblock_extension_in,
|
|
};
|
|
use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode};
|
|
|
|
/// Objects visited per file, heap objects read per heap: enough to cover
|
|
/// every structure kind while keeping a 35 000-group file fast.
|
|
const MAX_OBJECTS: usize = 1500;
|
|
const MAX_HEAP_IDS: usize = 200;
|
|
|
|
/// The structures that still need the whole file in memory, because they
|
|
/// are found through a version-2 B-tree (not converted yet), and the checks
|
|
/// that can reach each of them. Anything else answering
|
|
/// [`FormatError::ContiguousStorageRequired`] is a converted parser falling
|
|
/// back to the whole file, and fails the harness.
|
|
const CONTIGUOUS_REQUIRED: &[(&str, &[&str])] = &[
|
|
(
|
|
"dense attribute storage (a v2 B-tree)",
|
|
&["attributes", "attributes (tolerant)"],
|
|
),
|
|
(
|
|
"a shared-message B-tree index",
|
|
&[
|
|
"SOHM B-tree",
|
|
"shared message",
|
|
"fill value",
|
|
"attributes",
|
|
"attributes (tolerant)",
|
|
],
|
|
),
|
|
(
|
|
"a huge fractal-heap object's B-tree",
|
|
&["heap object", "attributes", "attributes (tolerant)"],
|
|
),
|
|
];
|
|
|
|
fn may_require_contiguous(check: &str, site: &str) -> bool {
|
|
CONTIGUOUS_REQUIRED
|
|
.iter()
|
|
.any(|(s, checks)| *s == site && checks.contains(&check))
|
|
}
|
|
|
|
#[derive(Default, Debug)]
|
|
struct Tally {
|
|
files: usize,
|
|
objects: usize,
|
|
checks: usize,
|
|
contiguous_required: usize,
|
|
reads: u64,
|
|
bytes: u64,
|
|
/// Chunk indexes (fixed and extensible arrays) read, and the most bytes
|
|
/// one of them took through the storage.
|
|
chunk_indexes: usize,
|
|
max_chunk_index_bytes: u64,
|
|
}
|
|
|
|
struct Walk<'a> {
|
|
slice: &'a [u8],
|
|
storage: &'a CountingStorage,
|
|
name: String,
|
|
tally: &'a mut Tally,
|
|
}
|
|
|
|
impl Walk<'_> {
|
|
/// The storage result must equal the slice result, or be the clean
|
|
/// "needs the whole file" error.
|
|
fn same<T: Debug>(
|
|
&mut self,
|
|
what: &str,
|
|
want: &Result<T, FormatError>,
|
|
got: &Result<T, FormatError>,
|
|
) {
|
|
self.tally.checks += 1;
|
|
if let Err(FormatError::ContiguousStorageRequired(site)) = got {
|
|
assert!(
|
|
may_require_contiguous(what, site),
|
|
"{}: {what} fell back to the whole file ({site}), which only the \
|
|
v2-B-tree-indexed structures may do",
|
|
self.name
|
|
);
|
|
self.tally.contiguous_required += 1;
|
|
return;
|
|
}
|
|
let (w, g) = (format!("{want:?}"), format!("{got:?}"));
|
|
assert!(
|
|
w == g,
|
|
"{}: {what} differs\n slice: {}\n storage: {}",
|
|
self.name,
|
|
&w[..w.len().min(600)],
|
|
&g[..g.len().min(600)]
|
|
);
|
|
}
|
|
|
|
fn index_read(&mut self, bytes_before: u64) {
|
|
self.tally.chunk_indexes += 1;
|
|
let bytes = self.storage.bytes_read() - bytes_before;
|
|
self.tally.max_chunk_index_bytes = self.tally.max_chunk_index_bytes.max(bytes);
|
|
}
|
|
|
|
fn st(&self) -> &dyn Storage {
|
|
self.storage
|
|
}
|
|
|
|
fn run(&mut self) {
|
|
let slice = self.slice;
|
|
let sb = Superblock::parse(slice, 0);
|
|
self.same("superblock", &sb, &Superblock::parse_in(self.st(), 0));
|
|
let Ok(sb) = sb else { return };
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
|
|
let want = read_superblock_extension(slice, &sb);
|
|
self.same(
|
|
"superblock extension",
|
|
&want,
|
|
&read_superblock_extension_in(self.st(), &sb),
|
|
);
|
|
let want = cache_image_state(slice, &sb);
|
|
self.same("cache image", &want, &cache_image_state_in(self.st(), &sb));
|
|
let table = load_sohm_table(slice, os, ls);
|
|
self.same("SOHM table", &table, &load_sohm_table_in(self.st(), os, ls));
|
|
if let Ok(Some(table)) = &table {
|
|
for idx in &table.indexes {
|
|
if idx.index_type == 0 {
|
|
let want =
|
|
parse_sohm_list(slice, idx.index_addr as usize, idx.num_messages, os);
|
|
let got = parse_sohm_list_in(self.st(), idx.index_addr, idx.num_messages, os);
|
|
self.same("SOHM list", &want, &got);
|
|
} else {
|
|
let want = parse_sohm_btree_entries(slice, idx.index_addr as usize, os, ls);
|
|
let got = parse_sohm_btree_entries_in(self.st(), idx.index_addr, os, ls);
|
|
self.same("SOHM B-tree", &want, &got);
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut seen = HashSet::new();
|
|
let mut queue = VecDeque::from([sb.root_group_address]);
|
|
while let Some(addr) = queue.pop_front() {
|
|
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
|
|
continue;
|
|
}
|
|
self.tally.objects += 1;
|
|
self.check_object(&sb, addr);
|
|
// Traversal only (group lookups are milestone M0/M3 work).
|
|
if let Ok(children) =
|
|
clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr)
|
|
{
|
|
queue.extend(children.iter().map(|c| c.object_header_address));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn check_object(&mut self, sb: &Superblock, addr: u64) {
|
|
let slice = self.slice;
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
let header = ObjectHeader::parse(slice, addr as usize, os, ls);
|
|
self.same(
|
|
"object header",
|
|
&header,
|
|
&ObjectHeader::parse_in(self.st(), addr, os, ls),
|
|
);
|
|
let Ok(header) = header else { return };
|
|
|
|
let want = extract_attributes_full(slice, &header, os, ls);
|
|
self.same(
|
|
"attributes",
|
|
&want,
|
|
&extract_attributes_full_in(self.st(), &header, os, ls),
|
|
);
|
|
let want = extract_attributes_tolerant(slice, &header, os, ls);
|
|
let got = extract_attributes_tolerant_in(self.st(), &header, os, ls);
|
|
self.same("attributes (tolerant)", &want, &got);
|
|
let want = dataset_fill_value_in(slice, &header.messages, os, ls);
|
|
self.same(
|
|
"fill value",
|
|
&want,
|
|
&dataset_fill_value_from_storage(self.st(), &header.messages, os, ls),
|
|
);
|
|
|
|
for msg in &header.messages {
|
|
if shared_message::is_shared(msg.flags) {
|
|
let want = message_data_with_sohm(slice, msg, os, ls);
|
|
let got = message_data_with_sohm_in(self.st(), msg, os, ls);
|
|
self.same("shared message", &want, &got);
|
|
}
|
|
match msg.msg_type {
|
|
MessageType::SymbolTable => {
|
|
if let Ok(stm) = SymbolTableMessage::parse(&msg.data, os) {
|
|
self.check_v1_group(&stm, os, ls);
|
|
}
|
|
}
|
|
MessageType::LinkInfo => {
|
|
if let Ok(li) = LinkInfoMessage::parse(&msg.data, os) {
|
|
self.check_heap(
|
|
li.fractal_heap_address,
|
|
li.btree_name_index_address,
|
|
4,
|
|
os,
|
|
ls,
|
|
);
|
|
}
|
|
}
|
|
MessageType::AttributeInfo => {
|
|
if let Ok(ai) = AttributeInfoMessage::parse(&msg.data, os) {
|
|
self.check_heap(
|
|
ai.fractal_heap_address,
|
|
ai.btree_name_index_address,
|
|
0,
|
|
os,
|
|
ls,
|
|
);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
self.check_layout(&header, os, ls);
|
|
}
|
|
|
|
/// A symbol-table group: its local heap, B-tree, nodes and names.
|
|
fn check_v1_group(&mut self, stm: &SymbolTableMessage, os: u8, ls: u8) {
|
|
let slice = self.slice;
|
|
let heap = LocalHeap::parse(slice, stm.local_heap_address as usize, os, ls);
|
|
self.same(
|
|
"local heap",
|
|
&heap,
|
|
&LocalHeap::parse_in(self.st(), stm.local_heap_address, os, ls),
|
|
);
|
|
let nodes = collect_symbol_table_nodes(slice, stm.btree_address, os, ls);
|
|
let got = collect_symbol_table_nodes_in(self.st(), stm.btree_address, os, ls);
|
|
self.same("group B-tree", &nodes, &got);
|
|
let Ok(heap) = heap else { return };
|
|
let want = heap.validate_free_list(slice, ls);
|
|
self.same(
|
|
"local heap free list",
|
|
&want,
|
|
&heap.validate_free_list_in(self.st(), ls),
|
|
);
|
|
let Ok(nodes) = nodes else { return };
|
|
for &node in nodes.iter().take(MAX_HEAP_IDS) {
|
|
let snod = SymbolTableNode::parse(slice, node as usize, os);
|
|
self.same(
|
|
"symbol table node",
|
|
&snod,
|
|
&SymbolTableNode::parse_in(self.st(), node, os),
|
|
);
|
|
let Ok(snod) = snod else { continue };
|
|
for e in &snod.entries {
|
|
let want = heap.read_string(slice, e.link_name_offset);
|
|
self.same(
|
|
"link name",
|
|
&want,
|
|
&heap.read_string_in(self.st(), e.link_name_offset),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A dense group's or dense attributes' fractal heap: the header, and
|
|
/// the objects its name index points at. `id_at` is where the heap ID
|
|
/// starts in a name-index record (after the hash for links).
|
|
fn check_heap(&mut self, heap: Option<u64>, index: Option<u64>, id_at: usize, os: u8, ls: u8) {
|
|
let slice = self.slice;
|
|
let Some(heap_addr) = heap else { return };
|
|
let fh = FractalHeapHeader::parse(slice, heap_addr as usize, os, ls);
|
|
self.same(
|
|
"fractal heap",
|
|
&fh,
|
|
&FractalHeapHeader::parse_in(self.st(), heap_addr, os, ls),
|
|
);
|
|
let (Ok(fh), Some(index)) = (fh, index) else {
|
|
return;
|
|
};
|
|
let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else {
|
|
return;
|
|
};
|
|
let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else {
|
|
return;
|
|
};
|
|
let id_len = fh.heap_id_length as usize;
|
|
for rec in records.iter().take(MAX_HEAP_IDS) {
|
|
let Some(id) = rec.data.get(id_at..id_at + id_len) else {
|
|
continue;
|
|
};
|
|
let want = fh.read_managed_object(slice, id, os);
|
|
self.same(
|
|
"heap object",
|
|
&want,
|
|
&fh.read_managed_object_in(self.st(), id, os),
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A dataset's layout: VDS mappings, and fixed/extensible array chunk
|
|
/// indexes.
|
|
fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) {
|
|
let slice = self.slice;
|
|
let find = |t: MessageType| {
|
|
header
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == t)
|
|
.and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok())
|
|
};
|
|
let Some(layout) = find(MessageType::DataLayout) else {
|
|
return;
|
|
};
|
|
let Ok(layout) = DataLayout::parse(&layout, os, ls) else {
|
|
return;
|
|
};
|
|
match &layout {
|
|
DataLayout::Virtual { .. } => {
|
|
let (mut want, mut got) = (layout.clone(), layout.clone());
|
|
let w = want.resolve_vds_mappings(slice, ls).map(|()| want);
|
|
let g = got.resolve_vds_mappings_in(self.st(), ls).map(|()| got);
|
|
self.same("VDS mappings", &w, &g);
|
|
}
|
|
DataLayout::Chunked {
|
|
chunk_dimensions,
|
|
btree_address: Some(addr),
|
|
version: 4,
|
|
chunk_index_type: Some(kind @ (3 | 4)),
|
|
..
|
|
} => {
|
|
let (Some(ds), Some(dt)) =
|
|
(find(MessageType::Dataspace), find(MessageType::Datatype))
|
|
else {
|
|
return;
|
|
};
|
|
let (Ok(ds), Ok((dt, _))) = (Dataspace::parse(&ds, ls), Datatype::parse(&dt))
|
|
else {
|
|
return;
|
|
};
|
|
let rank = ds.dimensions.len();
|
|
if chunk_dimensions.len() < rank {
|
|
return;
|
|
}
|
|
let dims = &chunk_dimensions[..rank];
|
|
let max = ds.max_dimensions.as_deref();
|
|
let es = dt.type_size();
|
|
if *kind == 3 {
|
|
let h = FixedArrayHeader::parse(slice, *addr as usize, os, ls);
|
|
self.same(
|
|
"fixed array header",
|
|
&h,
|
|
&FixedArrayHeader::parse_in(self.st(), *addr, os, ls),
|
|
);
|
|
let Ok(h) = h else { return };
|
|
let want =
|
|
read_fixed_array_chunks(slice, &h, &ds.dimensions, max, dims, es, os, ls);
|
|
let before = self.storage.bytes_read();
|
|
let got = read_fixed_array_chunks_in(
|
|
self.st(),
|
|
&h,
|
|
&ds.dimensions,
|
|
max,
|
|
dims,
|
|
es,
|
|
os,
|
|
ls,
|
|
);
|
|
self.index_read(before);
|
|
self.same("fixed array chunks", &want, &got);
|
|
} else {
|
|
let h = ExtensibleArrayHeader::parse(slice, *addr as usize, os, ls);
|
|
let got = ExtensibleArrayHeader::parse_in(self.st(), *addr, os, ls);
|
|
self.same("extensible array header", &h, &got);
|
|
let Ok(h) = h else { return };
|
|
let want = read_extensible_array_chunks(
|
|
slice,
|
|
&h,
|
|
&ds.dimensions,
|
|
max,
|
|
dims,
|
|
es,
|
|
os,
|
|
ls,
|
|
);
|
|
let before = self.storage.bytes_read();
|
|
let got = read_extensible_array_chunks_in(
|
|
self.st(),
|
|
&h,
|
|
&ds.dimensions,
|
|
max,
|
|
dims,
|
|
es,
|
|
os,
|
|
ls,
|
|
);
|
|
self.index_read(before);
|
|
self.same("extensible array chunks", &want, &got);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn check_file(path: &Path, tally: &mut Tally) {
|
|
let Ok(bytes) = std::fs::read(path) else {
|
|
return;
|
|
};
|
|
check_bytes(&path.display().to_string(), &bytes, tally);
|
|
}
|
|
|
|
fn check_bytes(name: &str, bytes: &[u8], tally: &mut Tally) {
|
|
let Ok((_, hdf5)) = split_user_block(bytes) else {
|
|
return;
|
|
};
|
|
let storage = CountingStorage::new(hdf5.to_vec());
|
|
let before = (tally.checks, tally.objects);
|
|
let mut walk = Walk {
|
|
slice: hdf5,
|
|
storage: &storage,
|
|
name: name.to_string(),
|
|
tally,
|
|
};
|
|
walk.run();
|
|
tally.files += 1;
|
|
tally.reads += storage.reads();
|
|
tally.bytes += storage.bytes_read();
|
|
if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") {
|
|
eprintln!(
|
|
"{:>6} objects {:>7} checks {:>8} reads {:>12} bytes {}",
|
|
tally.objects - before.1,
|
|
tally.checks - before.0,
|
|
storage.reads(),
|
|
storage.bytes_read(),
|
|
name
|
|
);
|
|
}
|
|
}
|
|
|
|
fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
|
return;
|
|
};
|
|
for e in entries.flatten() {
|
|
let p = e.path();
|
|
if p.is_dir() {
|
|
hdf5_files(&p, out);
|
|
} else if p
|
|
.extension()
|
|
.and_then(|x| x.to_str())
|
|
.is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf"))
|
|
{
|
|
out.push(p);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn fixtures_parse_identically_through_storage() {
|
|
let mut files = Vec::new();
|
|
hdf5_files(
|
|
&Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"),
|
|
&mut files,
|
|
);
|
|
files.sort();
|
|
assert!(files.len() >= 40, "{} fixtures", files.len());
|
|
let mut tally = Tally::default();
|
|
for f in &files {
|
|
check_file(f, &mut tally);
|
|
}
|
|
eprintln!("fixtures: {tally:?}");
|
|
assert!(tally.objects >= 150 && tally.checks >= 1000, "{tally:?}");
|
|
// Reads really went through read_at.
|
|
assert!(tally.reads > tally.objects as u64);
|
|
}
|
|
|
|
#[test]
|
|
fn corpus_parses_identically_through_storage() {
|
|
let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else {
|
|
eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus");
|
|
return;
|
|
};
|
|
let mut files = Vec::new();
|
|
for d in std::env::split_paths(&dirs) {
|
|
hdf5_files(&d, &mut files);
|
|
}
|
|
files.sort();
|
|
let mut tally = Tally::default();
|
|
for f in &files {
|
|
check_file(f, &mut tally);
|
|
}
|
|
eprintln!("corpus: {tally:?}");
|
|
assert!(tally.files > 0);
|
|
}
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
/// Files h5py writes to cover what the fixtures do not: extensible arrays
|
|
/// deep enough for super blocks and paged data blocks, paged fixed arrays,
|
|
/// big symbol-table and dense groups, dense and shared attributes, a SOHM
|
|
/// list and a SOHM B-tree, committed datatypes, and a user block.
|
|
const GENERATE: &str = r#"
|
|
import ctypes, glob, os, sys, h5py, numpy as np
|
|
out = sys.argv[1]
|
|
def p(n): return os.path.join(out, n)
|
|
|
|
with h5py.File(p('ea.h5'), 'w', libver='latest') as f:
|
|
# One unlimited dimension: extensible array. 3000 chunks reach super
|
|
# blocks; 2-element chunks of int8 keep data small.
|
|
d = f.create_dataset('ea', shape=(6000,), maxshape=(None,), chunks=(2,), dtype='i1')
|
|
d[:] = np.arange(6000) % 100
|
|
d2 = f.create_dataset('ea_deflate', shape=(4000, 3), maxshape=(None, 3), chunks=(2, 3),
|
|
dtype='f4', compression='gzip')
|
|
d2[:] = np.random.default_rng(1).random((4000, 3))
|
|
d3 = f.create_dataset('ea_sparse', shape=(100000,), maxshape=(None,), chunks=(4,), dtype='i2')
|
|
d3[0:8] = 1; d3[50000:50004] = 2; d3[99996:] = 3
|
|
fa = f.create_dataset('fa_paged', shape=(5000,), chunks=(1,), dtype='u1')
|
|
fa[::3] = 7
|
|
fa2 = f.create_dataset('fa', shape=(40, 40), chunks=(8, 8), dtype='f8', compression='gzip')
|
|
fa2[:] = 1.5
|
|
for i in range(30):
|
|
f.attrs[f'a{i}'] = np.arange(i + 1)
|
|
g = f.create_group('dense')
|
|
for i in range(300):
|
|
g.create_group(f'child{i:04d}').attrs['i'] = i
|
|
f['committed'] = np.dtype([('x', 'i4'), ('y', 'f8')])
|
|
f.create_dataset('uses_committed', shape=(3,), dtype=f['committed'])
|
|
f['uses_committed'].attrs.create('ta', data=np.zeros(2, dtype=f['committed'].dtype), dtype=f['committed'])
|
|
f.attrs['vl'] = ['alpha', 'beta', 'gamma']
|
|
|
|
with h5py.File(p('v1_groups.h5'), 'w', libver='earliest', userblock_size=512) as f:
|
|
for i in range(400):
|
|
g = f.create_group(f'g{i:04d}')
|
|
g.attrs['n'] = i
|
|
f.create_dataset('x', data=np.arange(10))
|
|
|
|
# Paged chunk indexes whose data blocks are bigger than the storage reads
|
|
# in one piece (1 MiB), with two chunks written: a fixed array of 300 000
|
|
# chunks (a 2.4 MB data block) and an extensible array grown to 1.2e9 (its
|
|
# last data block holds 131 072 chunks: over 1 MiB).
|
|
with h5py.File(p('big_paged.h5'), 'w', libver='latest') as f:
|
|
d = f.create_dataset('fa', shape=(300000,), chunks=(1,), dtype='u1')
|
|
d[5] = 1; d[250000] = 2
|
|
e = f.create_dataset('ea', shape=(1,), maxshape=(None,), chunks=(1,), dtype='u1')
|
|
e.resize((1200000000,)); e[10] = 1; e[1100000000] = 3
|
|
|
|
libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))
|
|
if libs:
|
|
lib = ctypes.CDLL(libs[0])
|
|
lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]
|
|
lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]
|
|
lib.H5Pset_shared_mesg_phase_change.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint]
|
|
for name, list_max in [('sohm_list.h5', 50), ('sohm_btree.h5', 0)]:
|
|
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
|
|
assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0
|
|
# datatype, dataspace, fill value, filter pipeline, attribute
|
|
assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, 0x02 | 0x04 | 0x08 | 0x10 | 0x20, 1) >= 0
|
|
assert lib.H5Pset_shared_mesg_phase_change(fcpl.id, list_max, 0) >= 0
|
|
fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
|
|
fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
|
|
fid = h5py.h5f.create(p(name).encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)
|
|
with h5py.File(fid) as f:
|
|
for i in range(20):
|
|
d = f.create_dataset(f'd{i}', shape=(10, i + 1), dtype='f4', fillvalue=-1.0,
|
|
chunks=(5, 1), compression='gzip')
|
|
d.attrs['units'] = 'metres per second, a long enough string to share'
|
|
d.attrs['scale'] = np.arange(20, dtype='f8')
|
|
print('ok')
|
|
"#;
|
|
|
|
#[test]
|
|
fn h5py_files_parse_identically_through_storage() {
|
|
let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("storage_equivalence");
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let out = Command::new(python())
|
|
.args(["-c", GENERATE, dir.to_str().unwrap()])
|
|
.output();
|
|
match out {
|
|
Ok(o) if o.status.success() => {}
|
|
Ok(o) if interop_required() => panic!(
|
|
"h5py generation failed:\n{}\n{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
),
|
|
Err(e) if interop_required() => panic!("python not available: {e}"),
|
|
_ => {
|
|
eprintln!("python3 with h5py unavailable; skipping");
|
|
return;
|
|
}
|
|
}
|
|
let mut files = Vec::new();
|
|
hdf5_files(&dir, &mut files);
|
|
files.sort();
|
|
let names: Vec<_> = files
|
|
.iter()
|
|
.map(|f| f.file_name().unwrap().to_string_lossy().into_owned())
|
|
.collect();
|
|
for want in ["ea.h5", "v1_groups.h5", "big_paged.h5"] {
|
|
assert!(names.iter().any(|n| n == want), "{names:?}");
|
|
}
|
|
if interop_required() {
|
|
assert!(names.iter().any(|n| n == "sohm_btree.h5"), "{names:?}");
|
|
}
|
|
let mut tally = Tally::default();
|
|
for f in &files {
|
|
if f.ends_with("big_paged.h5") {
|
|
// Only the pages in use are read, not the whole data blocks:
|
|
// read in one piece, the fixed array took 2.4 MB and the
|
|
// extensible array 1.2 MB (its super block's page bitmap and
|
|
// block addresses are most of what remains).
|
|
let mut big = Tally::default();
|
|
check_file(f, &mut big);
|
|
eprintln!("big paged blocks: {big:?}");
|
|
assert_eq!(big.chunk_indexes, 2, "{big:?}");
|
|
assert!(big.max_chunk_index_bytes < 256 << 10, "{big:?}");
|
|
// Truncated anywhere, the page-by-page reads still agree with
|
|
// the slice reads (errors included).
|
|
let bytes = std::fs::read(f).unwrap();
|
|
for cut in (0..bytes.len()).step_by(bytes.len() / 97) {
|
|
check_bytes(
|
|
&format!("big_paged.h5 cut at {cut}"),
|
|
&bytes[..cut],
|
|
&mut big,
|
|
);
|
|
}
|
|
eprintln!("big paged blocks, truncated: {big:?}");
|
|
assert!(big.chunk_indexes > 100, "{big:?}");
|
|
}
|
|
check_file(f, &mut tally);
|
|
}
|
|
eprintln!("h5py files: {tally:?}");
|
|
assert!(tally.objects >= 700, "{tally:?}");
|
|
// Dense attributes and the SOHM B-tree are the known clean errors.
|
|
assert!(tally.contiguous_required > 0, "{tally:?}");
|
|
}
|