osobhandClaude Opus 5.5
e5359354b7
The harness counted ContiguousStorageRequired from any parser as an
allowed difference, so a converted module that wrongly fell back to the
whole file would still pass. It now accepts the error only from the
three sites that are not converted yet (dense attribute storage, a SOHM
B-tree index, huge fractal-heap objects, all found through a v2 B-tree)
and only in the checks that can reach them; anything else fails with
the check and the site named.
Checked by making LocalHeap::parse_in return the error first: the
fixture and h5py runs fail ("local heap fell back to the whole file").
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
651 lines
25 KiB
Rust
651 lines
25 KiB
Rust
//! Equivalence harness for the range-read migration
|
|
//! (`docs/design/range-reads.md`, milestone M1).
|
|
//!
|
|
//! Every metadata parser converted to [`Storage`] must give exactly what its
|
|
//! `&[u8]` form gives. This walks real files — the fixtures, files h5py
|
|
//! writes to exercise the less common structures, and optionally the
|
|
//! conformance corpus — and, for every object, runs each converted parser
|
|
//! twice: over the file as a slice, and over a [`CountingStorage`] that
|
|
//! serves the same bytes through `read_at` only (`as_contiguous()` is
|
|
//! `None`, so no parser can fall back to the whole slice). The results must
|
|
//! be identical, value for value and error for error.
|
|
//!
|
|
//! The one allowed difference is [`FormatError::ContiguousStorageRequired`]
|
|
//! from the storage path, and only from the structures still indexed by a v2
|
|
//! B-tree (dense attributes, a SOHM B-tree index, huge fractal-heap objects;
|
|
//! see `CONTIGUOUS_REQUIRED`), which fail cleanly instead of reading the
|
|
//! whole file. Those are counted; the error from any other site or check
|
|
//! fails the harness.
|
|
//!
|
|
//! Milestones M2/M3 extend `check_object` with the raw-data and group
|
|
//! parsers as they are converted.
|
|
//!
|
|
//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/
|
|
//! `.nc`/`.h5ad` file under those directories (the conformance corpus is
|
|
//! `conformance/.cache/corpus`); `CLAWHDF5_STORAGE_REPORT=1` prints the
|
|
//! per-file read counts.
|
|
//! - The h5py-written files honour `CLAWHDF5_PYTHON` and
|
|
//! `CLAWHDF5_REQUIRE_INTEROP` like the facade's interop tests.
|
|
|
|
use std::collections::{HashSet, VecDeque};
|
|
use std::fmt::Debug;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
use clawhdf5_format::attribute::{
|
|
extract_attributes_full, extract_attributes_full_in, extract_attributes_tolerant,
|
|
extract_attributes_tolerant_in,
|
|
};
|
|
use clawhdf5_format::attribute_info::AttributeInfoMessage;
|
|
use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in};
|
|
use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::dataspace::Dataspace;
|
|
use clawhdf5_format::datatype::Datatype;
|
|
use clawhdf5_format::error::FormatError;
|
|
use clawhdf5_format::extensible_array::{
|
|
ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in,
|
|
};
|
|
use clawhdf5_format::fill_value::{dataset_fill_value_from_storage, dataset_fill_value_in};
|
|
use clawhdf5_format::fixed_array::{
|
|
FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in,
|
|
};
|
|
use clawhdf5_format::fractal_heap::FractalHeapHeader;
|
|
use clawhdf5_format::link_info::LinkInfoMessage;
|
|
use clawhdf5_format::local_heap::LocalHeap;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
use clawhdf5_format::shared_message::{
|
|
self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in,
|
|
parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in,
|
|
};
|
|
use clawhdf5_format::signature::split_user_block;
|
|
use clawhdf5_format::storage::{CountingStorage, Storage};
|
|
use clawhdf5_format::superblock::Superblock;
|
|
use clawhdf5_format::superblock_ext::{
|
|
cache_image_state, cache_image_state_in, read_superblock_extension,
|
|
read_superblock_extension_in,
|
|
};
|
|
use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode};
|
|
|
|
/// Objects visited per file, heap objects read per heap: enough to cover
|
|
/// every structure kind while keeping a 35 000-group file fast.
|
|
const MAX_OBJECTS: usize = 1500;
|
|
const MAX_HEAP_IDS: usize = 200;
|
|
|
|
/// The structures that still need the whole file in memory, because they
|
|
/// are found through a version-2 B-tree (not converted yet), and the checks
|
|
/// that can reach each of them. Anything else answering
|
|
/// [`FormatError::ContiguousStorageRequired`] is a converted parser falling
|
|
/// back to the whole file, and fails the harness.
|
|
const CONTIGUOUS_REQUIRED: &[(&str, &[&str])] = &[
|
|
(
|
|
"dense attribute storage (a v2 B-tree)",
|
|
&["attributes", "attributes (tolerant)"],
|
|
),
|
|
(
|
|
"a shared-message B-tree index",
|
|
&[
|
|
"SOHM B-tree",
|
|
"shared message",
|
|
"fill value",
|
|
"attributes",
|
|
"attributes (tolerant)",
|
|
],
|
|
),
|
|
(
|
|
"a huge fractal-heap object's B-tree",
|
|
&["heap object", "attributes", "attributes (tolerant)"],
|
|
),
|
|
];
|
|
|
|
fn may_require_contiguous(check: &str, site: &str) -> bool {
|
|
CONTIGUOUS_REQUIRED
|
|
.iter()
|
|
.any(|(s, checks)| *s == site && checks.contains(&check))
|
|
}
|
|
|
|
#[derive(Default, Debug)]
|
|
struct Tally {
|
|
files: usize,
|
|
objects: usize,
|
|
checks: usize,
|
|
contiguous_required: usize,
|
|
reads: u64,
|
|
bytes: u64,
|
|
}
|
|
|
|
struct Walk<'a> {
|
|
slice: &'a [u8],
|
|
storage: &'a CountingStorage,
|
|
name: String,
|
|
tally: &'a mut Tally,
|
|
}
|
|
|
|
impl Walk<'_> {
|
|
/// The storage result must equal the slice result, or be the clean
|
|
/// "needs the whole file" error.
|
|
fn same<T: Debug>(
|
|
&mut self,
|
|
what: &str,
|
|
want: &Result<T, FormatError>,
|
|
got: &Result<T, FormatError>,
|
|
) {
|
|
self.tally.checks += 1;
|
|
if let Err(FormatError::ContiguousStorageRequired(site)) = got {
|
|
assert!(
|
|
may_require_contiguous(what, site),
|
|
"{}: {what} fell back to the whole file ({site}), which only the \
|
|
v2-B-tree-indexed structures may do",
|
|
self.name
|
|
);
|
|
self.tally.contiguous_required += 1;
|
|
return;
|
|
}
|
|
let (w, g) = (format!("{want:?}"), format!("{got:?}"));
|
|
assert!(
|
|
w == g,
|
|
"{}: {what} differs\n slice: {}\n storage: {}",
|
|
self.name,
|
|
&w[..w.len().min(600)],
|
|
&g[..g.len().min(600)]
|
|
);
|
|
}
|
|
|
|
fn st(&self) -> &dyn Storage {
|
|
self.storage
|
|
}
|
|
|
|
fn run(&mut self) {
|
|
let slice = self.slice;
|
|
let sb = Superblock::parse(slice, 0);
|
|
self.same("superblock", &sb, &Superblock::parse_in(self.st(), 0));
|
|
let Ok(sb) = sb else { return };
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
|
|
let want = read_superblock_extension(slice, &sb);
|
|
self.same(
|
|
"superblock extension",
|
|
&want,
|
|
&read_superblock_extension_in(self.st(), &sb),
|
|
);
|
|
let want = cache_image_state(slice, &sb);
|
|
self.same("cache image", &want, &cache_image_state_in(self.st(), &sb));
|
|
let table = load_sohm_table(slice, os, ls);
|
|
self.same("SOHM table", &table, &load_sohm_table_in(self.st(), os, ls));
|
|
if let Ok(Some(table)) = &table {
|
|
for idx in &table.indexes {
|
|
if idx.index_type == 0 {
|
|
let want =
|
|
parse_sohm_list(slice, idx.index_addr as usize, idx.num_messages, os);
|
|
let got = parse_sohm_list_in(self.st(), idx.index_addr, idx.num_messages, os);
|
|
self.same("SOHM list", &want, &got);
|
|
} else {
|
|
let want = parse_sohm_btree_entries(slice, idx.index_addr as usize, os, ls);
|
|
let got = parse_sohm_btree_entries_in(self.st(), idx.index_addr, os, ls);
|
|
self.same("SOHM B-tree", &want, &got);
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut seen = HashSet::new();
|
|
let mut queue = VecDeque::from([sb.root_group_address]);
|
|
while let Some(addr) = queue.pop_front() {
|
|
if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
|
|
continue;
|
|
}
|
|
self.tally.objects += 1;
|
|
self.check_object(&sb, addr);
|
|
// Traversal only (group lookups are milestone M0/M3 work).
|
|
if let Ok(children) =
|
|
clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr)
|
|
{
|
|
queue.extend(children.iter().map(|c| c.object_header_address));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn check_object(&mut self, sb: &Superblock, addr: u64) {
|
|
let slice = self.slice;
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
let header = ObjectHeader::parse(slice, addr as usize, os, ls);
|
|
self.same(
|
|
"object header",
|
|
&header,
|
|
&ObjectHeader::parse_in(self.st(), addr, os, ls),
|
|
);
|
|
let Ok(header) = header else { return };
|
|
|
|
let want = extract_attributes_full(slice, &header, os, ls);
|
|
self.same(
|
|
"attributes",
|
|
&want,
|
|
&extract_attributes_full_in(self.st(), &header, os, ls),
|
|
);
|
|
let want = extract_attributes_tolerant(slice, &header, os, ls);
|
|
let got = extract_attributes_tolerant_in(self.st(), &header, os, ls);
|
|
self.same("attributes (tolerant)", &want, &got);
|
|
let want = dataset_fill_value_in(slice, &header.messages, os, ls);
|
|
self.same(
|
|
"fill value",
|
|
&want,
|
|
&dataset_fill_value_from_storage(self.st(), &header.messages, os, ls),
|
|
);
|
|
|
|
for msg in &header.messages {
|
|
if shared_message::is_shared(msg.flags) {
|
|
let want = message_data_with_sohm(slice, msg, os, ls);
|
|
let got = message_data_with_sohm_in(self.st(), msg, os, ls);
|
|
self.same("shared message", &want, &got);
|
|
}
|
|
match msg.msg_type {
|
|
MessageType::SymbolTable => {
|
|
if let Ok(stm) = SymbolTableMessage::parse(&msg.data, os) {
|
|
self.check_v1_group(&stm, os, ls);
|
|
}
|
|
}
|
|
MessageType::LinkInfo => {
|
|
if let Ok(li) = LinkInfoMessage::parse(&msg.data, os) {
|
|
self.check_heap(
|
|
li.fractal_heap_address,
|
|
li.btree_name_index_address,
|
|
4,
|
|
os,
|
|
ls,
|
|
);
|
|
}
|
|
}
|
|
MessageType::AttributeInfo => {
|
|
if let Ok(ai) = AttributeInfoMessage::parse(&msg.data, os) {
|
|
self.check_heap(
|
|
ai.fractal_heap_address,
|
|
ai.btree_name_index_address,
|
|
0,
|
|
os,
|
|
ls,
|
|
);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
self.check_layout(&header, os, ls);
|
|
}
|
|
|
|
/// A symbol-table group: its local heap, B-tree, nodes and names.
|
|
fn check_v1_group(&mut self, stm: &SymbolTableMessage, os: u8, ls: u8) {
|
|
let slice = self.slice;
|
|
let heap = LocalHeap::parse(slice, stm.local_heap_address as usize, os, ls);
|
|
self.same(
|
|
"local heap",
|
|
&heap,
|
|
&LocalHeap::parse_in(self.st(), stm.local_heap_address, os, ls),
|
|
);
|
|
let nodes = collect_symbol_table_nodes(slice, stm.btree_address, os, ls);
|
|
let got = collect_symbol_table_nodes_in(self.st(), stm.btree_address, os, ls);
|
|
self.same("group B-tree", &nodes, &got);
|
|
let Ok(heap) = heap else { return };
|
|
let want = heap.validate_free_list(slice, ls);
|
|
self.same(
|
|
"local heap free list",
|
|
&want,
|
|
&heap.validate_free_list_in(self.st(), ls),
|
|
);
|
|
let Ok(nodes) = nodes else { return };
|
|
for &node in nodes.iter().take(MAX_HEAP_IDS) {
|
|
let snod = SymbolTableNode::parse(slice, node as usize, os);
|
|
self.same(
|
|
"symbol table node",
|
|
&snod,
|
|
&SymbolTableNode::parse_in(self.st(), node, os),
|
|
);
|
|
let Ok(snod) = snod else { continue };
|
|
for e in &snod.entries {
|
|
let want = heap.read_string(slice, e.link_name_offset);
|
|
self.same(
|
|
"link name",
|
|
&want,
|
|
&heap.read_string_in(self.st(), e.link_name_offset),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A dense group's or dense attributes' fractal heap: the header, and
|
|
/// the objects its name index points at. `id_at` is where the heap ID
|
|
/// starts in a name-index record (after the hash for links).
|
|
fn check_heap(&mut self, heap: Option<u64>, index: Option<u64>, id_at: usize, os: u8, ls: u8) {
|
|
let slice = self.slice;
|
|
let Some(heap_addr) = heap else { return };
|
|
let fh = FractalHeapHeader::parse(slice, heap_addr as usize, os, ls);
|
|
self.same(
|
|
"fractal heap",
|
|
&fh,
|
|
&FractalHeapHeader::parse_in(self.st(), heap_addr, os, ls),
|
|
);
|
|
let (Ok(fh), Some(index)) = (fh, index) else {
|
|
return;
|
|
};
|
|
let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else {
|
|
return;
|
|
};
|
|
let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else {
|
|
return;
|
|
};
|
|
let id_len = fh.heap_id_length as usize;
|
|
for rec in records.iter().take(MAX_HEAP_IDS) {
|
|
let Some(id) = rec.data.get(id_at..id_at + id_len) else {
|
|
continue;
|
|
};
|
|
let want = fh.read_managed_object(slice, id, os);
|
|
self.same(
|
|
"heap object",
|
|
&want,
|
|
&fh.read_managed_object_in(self.st(), id, os),
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A dataset's layout: VDS mappings, and fixed/extensible array chunk
|
|
/// indexes.
|
|
fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) {
|
|
let slice = self.slice;
|
|
let find = |t: MessageType| {
|
|
header
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == t)
|
|
.and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok())
|
|
};
|
|
let Some(layout) = find(MessageType::DataLayout) else {
|
|
return;
|
|
};
|
|
let Ok(layout) = DataLayout::parse(&layout, os, ls) else {
|
|
return;
|
|
};
|
|
match &layout {
|
|
DataLayout::Virtual { .. } => {
|
|
let (mut want, mut got) = (layout.clone(), layout.clone());
|
|
let w = want.resolve_vds_mappings(slice, ls).map(|()| want);
|
|
let g = got.resolve_vds_mappings_in(self.st(), ls).map(|()| got);
|
|
self.same("VDS mappings", &w, &g);
|
|
}
|
|
DataLayout::Chunked {
|
|
chunk_dimensions,
|
|
btree_address: Some(addr),
|
|
version: 4,
|
|
chunk_index_type: Some(kind @ (3 | 4)),
|
|
..
|
|
} => {
|
|
let (Some(ds), Some(dt)) =
|
|
(find(MessageType::Dataspace), find(MessageType::Datatype))
|
|
else {
|
|
return;
|
|
};
|
|
let (Ok(ds), Ok((dt, _))) = (Dataspace::parse(&ds, ls), Datatype::parse(&dt))
|
|
else {
|
|
return;
|
|
};
|
|
let rank = ds.dimensions.len();
|
|
if chunk_dimensions.len() < rank {
|
|
return;
|
|
}
|
|
let dims = &chunk_dimensions[..rank];
|
|
let max = ds.max_dimensions.as_deref();
|
|
let es = dt.type_size();
|
|
if *kind == 3 {
|
|
let h = FixedArrayHeader::parse(slice, *addr as usize, os, ls);
|
|
self.same(
|
|
"fixed array header",
|
|
&h,
|
|
&FixedArrayHeader::parse_in(self.st(), *addr, os, ls),
|
|
);
|
|
let Ok(h) = h else { return };
|
|
let want =
|
|
read_fixed_array_chunks(slice, &h, &ds.dimensions, max, dims, es, os, ls);
|
|
let got = read_fixed_array_chunks_in(
|
|
self.st(),
|
|
&h,
|
|
&ds.dimensions,
|
|
max,
|
|
dims,
|
|
es,
|
|
os,
|
|
ls,
|
|
);
|
|
self.same("fixed array chunks", &want, &got);
|
|
} else {
|
|
let h = ExtensibleArrayHeader::parse(slice, *addr as usize, os, ls);
|
|
let got = ExtensibleArrayHeader::parse_in(self.st(), *addr, os, ls);
|
|
self.same("extensible array header", &h, &got);
|
|
let Ok(h) = h else { return };
|
|
let want = read_extensible_array_chunks(
|
|
slice,
|
|
&h,
|
|
&ds.dimensions,
|
|
max,
|
|
dims,
|
|
es,
|
|
os,
|
|
ls,
|
|
);
|
|
let got = read_extensible_array_chunks_in(
|
|
self.st(),
|
|
&h,
|
|
&ds.dimensions,
|
|
max,
|
|
dims,
|
|
es,
|
|
os,
|
|
ls,
|
|
);
|
|
self.same("extensible array chunks", &want, &got);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn check_file(path: &Path, tally: &mut Tally) {
|
|
let Ok(bytes) = std::fs::read(path) else {
|
|
return;
|
|
};
|
|
let Ok((_, hdf5)) = split_user_block(&bytes) else {
|
|
return;
|
|
};
|
|
let storage = CountingStorage::new(hdf5.to_vec());
|
|
let before = (tally.checks, tally.objects);
|
|
let mut walk = Walk {
|
|
slice: hdf5,
|
|
storage: &storage,
|
|
name: path.display().to_string(),
|
|
tally,
|
|
};
|
|
walk.run();
|
|
tally.files += 1;
|
|
tally.reads += storage.reads();
|
|
tally.bytes += storage.bytes_read();
|
|
if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") {
|
|
eprintln!(
|
|
"{:>6} objects {:>7} checks {:>8} reads {:>12} bytes {}",
|
|
tally.objects - before.1,
|
|
tally.checks - before.0,
|
|
storage.reads(),
|
|
storage.bytes_read(),
|
|
path.display()
|
|
);
|
|
}
|
|
}
|
|
|
|
fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
|
return;
|
|
};
|
|
for e in entries.flatten() {
|
|
let p = e.path();
|
|
if p.is_dir() {
|
|
hdf5_files(&p, out);
|
|
} else if p
|
|
.extension()
|
|
.and_then(|x| x.to_str())
|
|
.is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf"))
|
|
{
|
|
out.push(p);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn fixtures_parse_identically_through_storage() {
|
|
let mut files = Vec::new();
|
|
hdf5_files(
|
|
&Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"),
|
|
&mut files,
|
|
);
|
|
files.sort();
|
|
assert!(files.len() >= 40, "{} fixtures", files.len());
|
|
let mut tally = Tally::default();
|
|
for f in &files {
|
|
check_file(f, &mut tally);
|
|
}
|
|
eprintln!("fixtures: {tally:?}");
|
|
assert!(tally.objects >= 150 && tally.checks >= 1000, "{tally:?}");
|
|
// Reads really went through read_at.
|
|
assert!(tally.reads > tally.objects as u64);
|
|
}
|
|
|
|
#[test]
|
|
fn corpus_parses_identically_through_storage() {
|
|
let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else {
|
|
eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus");
|
|
return;
|
|
};
|
|
let mut files = Vec::new();
|
|
for d in std::env::split_paths(&dirs) {
|
|
hdf5_files(&d, &mut files);
|
|
}
|
|
files.sort();
|
|
let mut tally = Tally::default();
|
|
for f in &files {
|
|
check_file(f, &mut tally);
|
|
}
|
|
eprintln!("corpus: {tally:?}");
|
|
assert!(tally.files > 0);
|
|
}
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
/// Files h5py writes to cover what the fixtures do not: extensible arrays
|
|
/// deep enough for super blocks and paged data blocks, paged fixed arrays,
|
|
/// big symbol-table and dense groups, dense and shared attributes, a SOHM
|
|
/// list and a SOHM B-tree, committed datatypes, and a user block.
|
|
const GENERATE: &str = r#"
|
|
import ctypes, glob, os, sys, h5py, numpy as np
|
|
out = sys.argv[1]
|
|
def p(n): return os.path.join(out, n)
|
|
|
|
with h5py.File(p('ea.h5'), 'w', libver='latest') as f:
|
|
# One unlimited dimension: extensible array. 3000 chunks reach super
|
|
# blocks; 2-element chunks of int8 keep data small.
|
|
d = f.create_dataset('ea', shape=(6000,), maxshape=(None,), chunks=(2,), dtype='i1')
|
|
d[:] = np.arange(6000) % 100
|
|
d2 = f.create_dataset('ea_deflate', shape=(4000, 3), maxshape=(None, 3), chunks=(2, 3),
|
|
dtype='f4', compression='gzip')
|
|
d2[:] = np.random.default_rng(1).random((4000, 3))
|
|
d3 = f.create_dataset('ea_sparse', shape=(100000,), maxshape=(None,), chunks=(4,), dtype='i2')
|
|
d3[0:8] = 1; d3[50000:50004] = 2; d3[99996:] = 3
|
|
fa = f.create_dataset('fa_paged', shape=(5000,), chunks=(1,), dtype='u1')
|
|
fa[::3] = 7
|
|
fa2 = f.create_dataset('fa', shape=(40, 40), chunks=(8, 8), dtype='f8', compression='gzip')
|
|
fa2[:] = 1.5
|
|
for i in range(30):
|
|
f.attrs[f'a{i}'] = np.arange(i + 1)
|
|
g = f.create_group('dense')
|
|
for i in range(300):
|
|
g.create_group(f'child{i:04d}').attrs['i'] = i
|
|
f['committed'] = np.dtype([('x', 'i4'), ('y', 'f8')])
|
|
f.create_dataset('uses_committed', shape=(3,), dtype=f['committed'])
|
|
f['uses_committed'].attrs.create('ta', data=np.zeros(2, dtype=f['committed'].dtype), dtype=f['committed'])
|
|
f.attrs['vl'] = ['alpha', 'beta', 'gamma']
|
|
|
|
with h5py.File(p('v1_groups.h5'), 'w', libver='earliest', userblock_size=512) as f:
|
|
for i in range(400):
|
|
g = f.create_group(f'g{i:04d}')
|
|
g.attrs['n'] = i
|
|
f.create_dataset('x', data=np.arange(10))
|
|
|
|
libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))
|
|
if libs:
|
|
lib = ctypes.CDLL(libs[0])
|
|
lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]
|
|
lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]
|
|
lib.H5Pset_shared_mesg_phase_change.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint]
|
|
for name, list_max in [('sohm_list.h5', 50), ('sohm_btree.h5', 0)]:
|
|
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
|
|
assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0
|
|
# datatype, dataspace, fill value, filter pipeline, attribute
|
|
assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, 0x02 | 0x04 | 0x08 | 0x10 | 0x20, 1) >= 0
|
|
assert lib.H5Pset_shared_mesg_phase_change(fcpl.id, list_max, 0) >= 0
|
|
fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
|
|
fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
|
|
fid = h5py.h5f.create(p(name).encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)
|
|
with h5py.File(fid) as f:
|
|
for i in range(20):
|
|
d = f.create_dataset(f'd{i}', shape=(10, i + 1), dtype='f4', fillvalue=-1.0,
|
|
chunks=(5, 1), compression='gzip')
|
|
d.attrs['units'] = 'metres per second, a long enough string to share'
|
|
d.attrs['scale'] = np.arange(20, dtype='f8')
|
|
print('ok')
|
|
"#;
|
|
|
|
#[test]
|
|
fn h5py_files_parse_identically_through_storage() {
|
|
let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("storage_equivalence");
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let out = Command::new(python())
|
|
.args(["-c", GENERATE, dir.to_str().unwrap()])
|
|
.output();
|
|
match out {
|
|
Ok(o) if o.status.success() => {}
|
|
Ok(o) if interop_required() => panic!(
|
|
"h5py generation failed:\n{}\n{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
),
|
|
Err(e) if interop_required() => panic!("python not available: {e}"),
|
|
_ => {
|
|
eprintln!("python3 with h5py unavailable; skipping");
|
|
return;
|
|
}
|
|
}
|
|
let mut files = Vec::new();
|
|
hdf5_files(&dir, &mut files);
|
|
files.sort();
|
|
let names: Vec<_> = files
|
|
.iter()
|
|
.map(|f| f.file_name().unwrap().to_string_lossy().into_owned())
|
|
.collect();
|
|
for want in ["ea.h5", "v1_groups.h5"] {
|
|
assert!(names.iter().any(|n| n == want), "{names:?}");
|
|
}
|
|
if interop_required() {
|
|
assert!(names.iter().any(|n| n == "sohm_btree.h5"), "{names:?}");
|
|
}
|
|
let mut tally = Tally::default();
|
|
for f in &files {
|
|
check_file(f, &mut tally);
|
|
}
|
|
eprintln!("h5py files: {tally:?}");
|
|
assert!(tally.objects >= 700, "{tally:?}");
|
|
// Dense attributes and the SOHM B-tree are the known clean errors.
|
|
assert!(tally.contiguous_required > 0, "{tally:?}");
|
|
}
|