osobhandClaude Opus 5.5
773f427f16
FileEditor::set_attr now handles every attribute storage libhdf5 uses for version-2 object headers: - objects that track (and index) attribute creation order: compact attributes carry their creation index in the message header, the Attribute Info message its maximum; - the move to dense storage when an object reaches its compact limit (or an attribute is too large for a header message), as H5O__attr_create does it: a new fractal heap, name index (v2 B-tree type 8) and, when creation order is indexed, creation-order index (type 9); the compact attributes moved over in header message order, their messages freed; - objects already in dense storage (h5py- or clawhdf5-written): new attributes inserted (H5A__dense_insert), an attribute replaced by one of the same encoded size rewritten in its heap object (H5A__dense_write), otherwise removed from both indexes and the heap and inserted anew. edit/fheap.rs follows H5HF: managed objects go to the best-fitting free section of the heap's free-space manager (FSHD/FSSE, kept as libhdf5 keeps it — sorted sections, counts, section info reallocated when its size changes, the manager deleted when empty); otherwise to a new direct block: the root direct block of an empty heap, else the block at the allocation iterator in the root indirect block (created from the root direct block, doubled as needed), with libhdf5's managed-space, allocated space, free space and iterator bookkeeping. Objects above the managed limit are huge objects in their own space, indexed by the huge-object B-tree (type 1). Removed objects return their space merged with adjacent free space. Refused before anything is written: heaps with I/O filters, child indirect blocks, an object larger than the next heap block (libhdf5 skips blocks and records them as free space), free sections other than those inside direct blocks, removing a direct block's last object (libhdf5 frees the block), directly addressed huge objects. Attributes are encoded as libhdf5 does when h5py opens a file r+ (low bound "earliest"): message version 1 (3 for non-ASCII names), simple dataspaces with their maximum dimensions. Header chunks are now visited in libhdf5's order (FIFO), which is also the order attributes move to dense storage in. Tests (edit_coverage_interop): 40 attributes on each of a plain group, a group tracking and indexing creation order, and a dataset (earliest, v110, latest), some above the 4 KiB managed limit, then same-size rewrites: the heap statistics, free-space sections and both index B-trees node for node equal libhdf5's doing the same through h5py; then replacements of other sizes, h5py adds/deletes/rewrites; h5py, h5dump, h5rs check and our reader agree throughout, h5py's attribute count included. clawhdf5-written dense storage (tracked and untracked) is extended the same way; refusals leave the file byte for byte as it was. edit_interop's attribute test now expects dense storage and tracked creation order to work. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1326 lines
47 KiB
Rust
1326 lines
47 KiB
Rust
//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
|
|
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
|
|
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
|
|
//! (and moved there), and space reused within an editing session. Files are
|
|
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
|
|
//! clawhdf5; after each edit h5py must read exactly what a model says,
|
|
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
|
|
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
|
|
//! same operations can be done by libhdf5, the structures it builds (B-tree
|
|
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
|
|
//!
|
|
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
|
|
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::path::Path;
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5::{Error, File, FileEditor, Selection};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn available(cmd: &str, args: &[&str]) -> bool {
|
|
Command::new(cmd)
|
|
.args(args)
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Whether the tools are here; false (skip) or a panic when interop is
|
|
/// required.
|
|
fn tools_ok() -> bool {
|
|
let ok =
|
|
available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
|
|
if !ok {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
|
|
);
|
|
eprintln!("SKIP: h5py/numpy or h5dump not available");
|
|
}
|
|
ok
|
|
}
|
|
|
|
fn py(script: &str) -> String {
|
|
let o = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("run python");
|
|
assert!(
|
|
o.status.success(),
|
|
"python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
String::from_utf8_lossy(&o.stdout).trim().to_string()
|
|
}
|
|
|
|
fn text(o: &Output) -> String {
|
|
format!(
|
|
"{}{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
)
|
|
}
|
|
|
|
/// The file is valid for libhdf5's tools and for `h5rs check --data`.
|
|
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
|
|
fn check_tools(path: &Path, h5dump: bool) {
|
|
let p = path.to_str().unwrap();
|
|
let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(["check", "--data", "-q", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
|
|
if h5dump {
|
|
let o = Command::new("h5dump")
|
|
.args(["-o", "/dev/null", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(
|
|
o.status.success() && o.stderr.is_empty(),
|
|
"h5dump {p}:\n{}",
|
|
text(&o)
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A dataset's expected contents.
|
|
#[derive(Clone, Debug)]
|
|
struct Model {
|
|
shape: Vec<u64>,
|
|
/// Row-major values.
|
|
data: Vec<i32>,
|
|
}
|
|
|
|
impl Model {
|
|
fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
|
|
let n: u64 = shape.iter().product();
|
|
Self {
|
|
shape: shape.to_vec(),
|
|
data: (0..n).map(f).collect(),
|
|
}
|
|
}
|
|
|
|
fn index(&self, c: &[u64]) -> usize {
|
|
c.iter()
|
|
.zip(&self.shape)
|
|
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
|
}
|
|
|
|
/// Change the extent to `shape`: elements inside both keep their
|
|
/// values, new ones are `fill`.
|
|
fn resize(&mut self, shape: &[u64], fill: i32) {
|
|
let old = self.clone();
|
|
*self = Self::new(shape, |_| fill);
|
|
let n: u64 = old.shape.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; old.shape.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = r % old.shape[d];
|
|
r /= old.shape[d];
|
|
}
|
|
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
|
let i = self.index(&c);
|
|
self.data[i] = old.data[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Apply a hyperslab write of `vals` (row-major over the block).
|
|
fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
|
|
let n: u64 = count.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; count.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = start[d] + r % count[d];
|
|
r /= count[d];
|
|
}
|
|
let i = self.index(&c);
|
|
self.data[i] = vals[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
fn block(start: &[u64], count: &[u64]) -> Selection {
|
|
Selection::Hyperslab {
|
|
start: start.to_vec(),
|
|
stride: vec![1; start.len()],
|
|
count: count.to_vec(),
|
|
block: vec![1; start.len()],
|
|
}
|
|
}
|
|
|
|
/// h5py and our reader both read `m` from dataset `name`.
|
|
fn verify(path: &Path, name: &str, m: &Model) {
|
|
let f = File::open(path).unwrap();
|
|
let ds = f.dataset(name).unwrap();
|
|
assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
|
|
let got = ds.read_i32().unwrap();
|
|
if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
|
|
panic!(
|
|
"our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
|
|
path.display(),
|
|
m.shape,
|
|
got[i],
|
|
m.data[i]
|
|
);
|
|
}
|
|
let exp = path.with_extension("expect");
|
|
let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
|
|
std::fs::write(&exp, bytes).unwrap();
|
|
let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
a = f[{name:?}][()]\n\
|
|
e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
|
|
assert a.shape == e.shape, (a.shape, e.shape)\n\
|
|
bad = np.argwhere(a != e)\n\
|
|
assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
|
|
p = path.to_str().unwrap(),
|
|
e = exp.to_str().unwrap(),
|
|
s = shape.join(",")
|
|
));
|
|
}
|
|
|
|
/// A deterministic pseudo-random sequence (SplitMix64).
|
|
struct Rng(u64);
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
|
let mut z = self.0;
|
|
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
|
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
|
z ^ (z >> 31)
|
|
}
|
|
fn below(&mut self, n: u64) -> u64 {
|
|
self.next() % n
|
|
}
|
|
}
|
|
|
|
fn tmpdir() -> tempfile::TempDir {
|
|
let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
|
|
tempfile::TempDir::new_in(base).unwrap()
|
|
}
|
|
|
|
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
|
match r {
|
|
Err(Error::Unsupported(_)) => {}
|
|
other => panic!("expected Error::Unsupported, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn le(b: &[u8]) -> u64 {
|
|
b.iter()
|
|
.take(8)
|
|
.enumerate()
|
|
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
|
}
|
|
|
|
fn enc_size(v: u64) -> usize {
|
|
((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
|
|
}
|
|
|
|
/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
|
|
/// (8-byte addresses and lengths): its depth, record count, and every
|
|
/// node's (depth, records) breadth-first.
|
|
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let h = &b[hdr..];
|
|
assert_eq!(&h[0..4], b"BTHD");
|
|
let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
|
|
let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
|
|
let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
|
|
let root = le(&h[16..24]);
|
|
let root_n = le(&h[24..26]);
|
|
let total = le(&h[26..34]);
|
|
// Node geometry, as H5B2__hdr_init computes it.
|
|
let leaf_max = (node_size - 10) / rs;
|
|
let nrec_w = enc_size(leaf_max);
|
|
let mut cum = vec![(leaf_max, 0usize)];
|
|
for d in 1..=u64::from(depth) {
|
|
let (below, below_w) = cum[d as usize - 1];
|
|
let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
|
|
let max = (node_size - 10 - ptr) / (rs + ptr);
|
|
let c = (max + 1) * below + max;
|
|
cum.push((c, enc_size(c)));
|
|
}
|
|
let mut out = Vec::new();
|
|
let mut level = vec![(root, root_n)];
|
|
let mut dep = depth;
|
|
loop {
|
|
let mut next = Vec::new();
|
|
for &(addr, n) in &level {
|
|
out.push((dep, n));
|
|
if dep == 0 {
|
|
assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
|
|
continue;
|
|
}
|
|
let node = &b[addr as usize..];
|
|
assert_eq!(&node[0..4], b"BTIN");
|
|
let mut q = 6 + (n * rs) as usize;
|
|
let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
|
|
for _ in 0..=n {
|
|
let a = le(&node[q..q + 8]);
|
|
let c = le(&node[q + 8..q + 8 + nrec_w]);
|
|
q += 8 + nrec_w + all_w;
|
|
next.push((a, c));
|
|
}
|
|
}
|
|
if dep == 0 {
|
|
break;
|
|
}
|
|
dep -= 1;
|
|
level = next;
|
|
}
|
|
(depth, total, out)
|
|
}
|
|
|
|
/// The shape of the file's only chunk B-tree (record type 10 or 11).
|
|
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let b = std::fs::read(path).unwrap();
|
|
let hdrs: Vec<usize> = b
|
|
.windows(6)
|
|
.enumerate()
|
|
.filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
|
|
.map(|(i, _)| i)
|
|
.collect();
|
|
assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
|
|
bt2_shape_at(&b, hdrs[0])
|
|
}
|
|
|
|
/// Grow a dataset with two unlimited dimensions in both directions many
|
|
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
|
|
/// chunk enters the index as it is written, in the same order as the
|
|
/// editor's) and with the editor: the same values, and the same B-tree —
|
|
/// depth, record count, and every node's record count — through leaf and
|
|
/// internal splits, redistributions and a depth increase.
|
|
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
|
|
// Extents: both dimensions grow; new columns fall between existing
|
|
// chunks in the index's key order (row first).
|
|
let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
|
|
let val = |r: u64, c: u64| (r * 1000 + c) as i32;
|
|
let create = |p: &Path, write: bool| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
|
|
dtype='<i4'{extra})\n\
|
|
\x20 if {write}:\n\
|
|
\x20 r0, c0 = 0, 0\n\
|
|
\x20 for r, c in {steps:?}:\n\
|
|
\x20 d.resize((r, c))\n\
|
|
\x20 g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
|
|
\x20 if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
|
|
\x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\
|
|
\x20 r0, c0 = r, c\n",
|
|
p = p.to_str().unwrap(),
|
|
write = if write { "True" } else { "False" },
|
|
))
|
|
};
|
|
create(&a, true);
|
|
create(&b, false);
|
|
let mut m = Model::new(&[0, 0], |_| 0);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
let (mut r0, mut c0) = (0u64, 0u64);
|
|
for &(r, c) in &steps {
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], 0);
|
|
if c > c0 && r0 > 0 {
|
|
let vals: Vec<i32> = (0..r0)
|
|
.flat_map(|i| (c0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[0, c0], &[r0, c - c0], &vals);
|
|
}
|
|
if r > r0 {
|
|
let vals: Vec<i32> = (r0..r)
|
|
.flat_map(|i| (0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, 0], &[r - r0, c], &vals);
|
|
}
|
|
(r0, c0) = (r, c);
|
|
}
|
|
drop(ed);
|
|
verify(&b, "x", &m);
|
|
verify(&a, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
|
|
assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
|
|
assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
|
|
// libhdf5 goes on growing what the editor built.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 r, c = d.shape\n\
|
|
\x20 d.resize((r + 3, c + 2))\n\
|
|
\x20 d[:, c:] = -1\n\
|
|
\x20 d[r:, :] = -2\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let (r, c) = (m.shape[0], m.shape[1]);
|
|
m.resize(&[r + 3, c + 2], 0);
|
|
m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
|
|
m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Dataset `name`'s layout: its chunk index type (0 for a version-1
|
|
/// B-tree), index address and chunk rank (element-size dimension
|
|
/// included).
|
|
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let m = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap();
|
|
match DataLayout::parse(&m.data, os, ls).unwrap() {
|
|
DataLayout::Chunked {
|
|
chunk_dimensions,
|
|
btree_address,
|
|
chunk_index_type,
|
|
..
|
|
} => (
|
|
chunk_index_type.unwrap_or(0),
|
|
btree_address,
|
|
chunk_dimensions.len(),
|
|
),
|
|
other => panic!("{other:?}"),
|
|
}
|
|
}
|
|
|
|
/// A version-1 chunk B-tree's shape: every node's (level, entries),
|
|
/// breadth-first from the root.
|
|
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
|
|
let ks = 8 + 8 * ndims;
|
|
let mut out = Vec::new();
|
|
let mut level = vec![root];
|
|
while !level.is_empty() {
|
|
let mut next = Vec::new();
|
|
for a in level {
|
|
let n = &b[a as usize..];
|
|
assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
|
|
let lv = n[5];
|
|
let e = le(&n[6..8]);
|
|
out.push((lv, e));
|
|
if lv > 0 {
|
|
for i in 0..e as usize {
|
|
next.push(le(
|
|
&n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
|
|
));
|
|
}
|
|
}
|
|
}
|
|
level = next;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The shape of dataset `name`'s chunk index, for comparing with the one
|
|
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
|
|
/// nothing to compare (Fixed Array, implicit).
|
|
fn index_shape(path: &Path, name: &str) -> String {
|
|
let b = std::fs::read(path).unwrap();
|
|
match chunk_index(path, name) {
|
|
(_, None, _) => "none".into(),
|
|
(0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
|
|
(5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
|
|
(4, Some(a), _) => {
|
|
let s: Vec<u64> = (0..6)
|
|
.map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
|
|
.collect();
|
|
format!("ea {s:?}")
|
|
}
|
|
(t, Some(_), _) => format!("type {t}"),
|
|
}
|
|
}
|
|
|
|
/// One step of a resize workload.
|
|
#[derive(Debug, Clone)]
|
|
enum Op {
|
|
Resize(Vec<u64>),
|
|
/// A block write; values from `vals`.
|
|
Write(Vec<u64>, Vec<u64>, u64),
|
|
}
|
|
|
|
fn op_vals(seed: u64, n: u64) -> Vec<i32> {
|
|
(0..n)
|
|
.map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
|
|
.collect()
|
|
}
|
|
|
|
/// Random resizes (shrinking and growing any dimension within `max`)
|
|
/// and block writes.
|
|
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
|
|
let mut shape = start.to_vec();
|
|
let mut ops = Vec::new();
|
|
for k in 0..n {
|
|
if k % 3 == 0 {
|
|
shape = (0..shape.len())
|
|
.map(|d| {
|
|
let cap = max[d].min(shape[d] * 2 + 6);
|
|
rng.below(cap + 1)
|
|
})
|
|
.collect();
|
|
ops.push(Op::Resize(shape.clone()));
|
|
} else if shape.iter().all(|&s| s > 0) {
|
|
let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
|
|
let cnt: Vec<u64> = (0..shape.len())
|
|
.map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
|
|
.collect();
|
|
ops.push(Op::Write(st, cnt, rng.next() % 1000));
|
|
}
|
|
}
|
|
ops
|
|
}
|
|
|
|
/// The python statements applying `ops` to dataset `d`.
|
|
fn py_ops(ops: &[Op]) -> String {
|
|
let mut s = String::new();
|
|
for op in ops {
|
|
match op {
|
|
Op::Resize(shape) => {
|
|
s += &format!("\x20 d.resize({shape:?})\n")
|
|
.replace('[', "(")
|
|
.replace(']', ",)");
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let n: u64 = cnt.iter().product();
|
|
let sl: Vec<String> = st
|
|
.iter()
|
|
.zip(cnt)
|
|
.map(|(a, c)| format!("{a}:{}", a + c))
|
|
.collect();
|
|
s += &format!(
|
|
"\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
|
|
.astype('<i4').reshape({cnt:?})\n",
|
|
sl.join(", ")
|
|
)
|
|
.replace("reshape([", "reshape((")
|
|
.replace("])\n", ",))\n");
|
|
}
|
|
}
|
|
}
|
|
s
|
|
}
|
|
|
|
/// The same resize workload — shrinking and growing along every
|
|
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
|
|
/// editor on dataset `x` created by `create` (python, `f` open): the same
|
|
/// values as a model where elements that come back after a shrink read as
|
|
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
|
|
/// check` accept, and h5py can go on.
|
|
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
|
|
let mk = |p: &Path| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
{create}",
|
|
p = p.to_str().unwrap()
|
|
))
|
|
};
|
|
mk(&a);
|
|
mk(&b);
|
|
let (start, max) = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
|
|
p = b.to_str().unwrap()
|
|
))
|
|
.split_once("] [")
|
|
.map(|(x, y)| {
|
|
let p = |s: &str| -> Vec<u64> {
|
|
s.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
};
|
|
(p(x), p(y))
|
|
})
|
|
.unwrap();
|
|
let initial = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let mut m = Model {
|
|
shape: start.clone(),
|
|
data: initial
|
|
.split_whitespace()
|
|
.map(|v| v.parse().unwrap())
|
|
.collect(),
|
|
};
|
|
let mut rng = Rng(seed);
|
|
let ops = random_resize_ops(&mut rng, &start, &max, 60);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f['x']\n{o}",
|
|
p = a.to_str().unwrap(),
|
|
o = py_ops(&ops)
|
|
));
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, op) in ops.iter().enumerate() {
|
|
match op {
|
|
Op::Resize(s) => {
|
|
ed.resize("x", s)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.resize(s, fill);
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let vals = op_vals(*seed, cnt.iter().product());
|
|
ed.write_values("x", &block(st, cnt), &vals)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.write_block(st, cnt, &vals);
|
|
}
|
|
}
|
|
}
|
|
drop(ed);
|
|
if tag == "implicit" {
|
|
// An implicit index cannot drop chunks: libhdf5 leaves the data of
|
|
// chunks wholly outside a shrunk extent in place and does not
|
|
// refill those that come back unless they lie past the extent it
|
|
// grows from, so they can show old values. The editor must match
|
|
// libhdf5, not the model.
|
|
let got = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = a.to_str().unwrap()
|
|
));
|
|
m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
|
|
}
|
|
verify(&a, "x", &m);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
assert_eq!(
|
|
index_shape(&b, "x"),
|
|
index_shape(&a, "x"),
|
|
"{tag}: chunk index differs from libhdf5's"
|
|
);
|
|
// libhdf5 goes on: grow back to the start shape and write everything.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 d.resize({s:?})\n\
|
|
\x20 d[...] = 3\n",
|
|
p = b.to_str().unwrap(),
|
|
s = start
|
|
));
|
|
let n: u64 = start.iter().product();
|
|
verify(
|
|
&b,
|
|
"x",
|
|
&Model {
|
|
shape: start.clone(),
|
|
data: vec![3; n as usize],
|
|
},
|
|
);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
|
|
/// dimension, mixed with growth and writes, on every chunk index: the
|
|
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
|
|
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
|
|
/// and implicit (early allocation), unfiltered and deflated.
|
|
#[test]
|
|
fn shrink_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let kinds: &[(&str, &str, i32)] = &[
|
|
(
|
|
"ea",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
|
|
maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
|
|
-1,
|
|
),
|
|
(
|
|
"bt2",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
|
|
maxshape=(None, None), chunks=(3, 2){z})\n",
|
|
0,
|
|
),
|
|
(
|
|
"fa",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
|
|
maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
|
|
5,
|
|
),
|
|
(
|
|
"1d",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
|
|
maxshape=(None,), chunks=(4,){z})\n",
|
|
0,
|
|
),
|
|
];
|
|
let mut seed = 100;
|
|
for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
for (kind, create, fill) in kinds {
|
|
for (zi, z) in ["", ", compression='gzip', shuffle=True"]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
seed += 1;
|
|
let tag = format!("{kind}_{li}_{zi}");
|
|
shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
|
|
}
|
|
}
|
|
}
|
|
// Implicit index: early allocation, no filters, fixed maximum shape.
|
|
shrink_workload(
|
|
"implicit",
|
|
"'latest'",
|
|
"\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
7,
|
|
false,
|
|
99,
|
|
);
|
|
}
|
|
|
|
/// An implicit chunk index (early allocation, no filters, fixed maximum
|
|
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
|
|
/// by its position in the maximum chunk grid. The reader once used the
|
|
/// current grid, and returned other chunks' values for every row after the
|
|
/// first.
|
|
#[test]
|
|
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("implicit_max.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
|
|
verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
|
|
// The editor places new values by the same grid.
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
|
|
.unwrap();
|
|
drop(ed);
|
|
let mut m = Model::new(&[8, 9], |i| i as i32);
|
|
m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
#[test]
|
|
fn btree2_chunk_index_growth_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
|
|
bt2_growth(lv, *dump, "", &format!("plain{i}"));
|
|
bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
|
|
}
|
|
}
|
|
|
|
/// Random writes into a dataset with two unlimited dimensions (chunks
|
|
/// created in any order) and growth, with a small node size so the tree
|
|
/// gets deep, against a model; then h5py continues.
|
|
#[test]
|
|
fn btree2_random_chunk_order() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("bt2_random.h5");
|
|
// A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
|
|
// hundred chunks).
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
|
|
dtype='<i4', fillvalue=-7)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = Model::new(&[40, 40], |_| -7);
|
|
let mut rng = Rng(42);
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for step in 0..400 {
|
|
if step % 50 == 49 {
|
|
let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], -7);
|
|
}
|
|
let r0 = rng.below(m.shape[0]);
|
|
let c0 = rng.below(m.shape[1]);
|
|
let cnt = [
|
|
1 + rng.below((m.shape[0] - r0).min(3)),
|
|
1 + rng.below((m.shape[1] - c0).min(3)),
|
|
];
|
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1])
|
|
.map(|_| (rng.next() % 1_000_000) as i32)
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, c0], &cnt, &vals);
|
|
if step % 100 == 99 {
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
ed = FileEditor::open(&path).unwrap();
|
|
}
|
|
}
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
let shape = chunk_bt2_shape(&path);
|
|
assert!(shape.0 >= 1, "{shape:?}");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'][...] = 5\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let n = m.data.len();
|
|
m.data = vec![5; n];
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
// ---- dense attributes ----
|
|
|
|
/// An attribute value both sides can write with the same encoded size: a
|
|
/// 1-D int64 array or a fixed-length string.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
enum AV {
|
|
Ints(Vec<i64>),
|
|
Str(String),
|
|
}
|
|
|
|
impl AV {
|
|
fn value(&self) -> clawhdf5::AttrValue {
|
|
match self {
|
|
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
|
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
|
}
|
|
}
|
|
|
|
fn py(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
|
AV::Str(s) => format!("np.bytes_({s:?})"),
|
|
}
|
|
}
|
|
|
|
fn py_expect(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("{v:?}"),
|
|
AV::Str(s) => format!("{s:?}"),
|
|
}
|
|
}
|
|
|
|
fn make(i: u64, salt: u64) -> Self {
|
|
match i % 11 {
|
|
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
|
1 | 4 | 8 => AV::Str(
|
|
(0..10 + (i * 13 + salt) % 300)
|
|
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
|
.collect(),
|
|
),
|
|
_ => AV::Ints(
|
|
(0..1 + (i + salt) % 9)
|
|
.map(|k| (k * 31 + salt) as i64)
|
|
.collect(),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An object's dense attribute storage as libhdf5 would compare it: the
|
|
/// heap's statistics and root shape, its free sections (heap offsets and
|
|
/// sizes), and the shapes of the name and creation-order index B-trees.
|
|
fn dense_info(path: &Path, obj: &str) -> String {
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
assert_eq!((os, ls), (8, 8));
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let Some(m) = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
|
else {
|
|
return "no attribute info".into();
|
|
};
|
|
let d = &m.data;
|
|
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
|
let heap = le(&d[p..p + 8]);
|
|
p += 8;
|
|
let name = le(&d[p..p + 8]);
|
|
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
|
if heap == u64::MAX {
|
|
return "compact".into();
|
|
}
|
|
let b = std::fs::read(path).unwrap();
|
|
let h = &b[heap as usize..];
|
|
assert_eq!(&h[0..4], b"FRHP");
|
|
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
|
// size, huge objs (8-byte fields from offset 14).
|
|
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
|
let fs = stat(3);
|
|
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
|
// index, start rows, root address.
|
|
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
|
let mut out = format!(
|
|
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
|
stat(0),
|
|
stat(2),
|
|
stat(4),
|
|
stat(5),
|
|
stat(6),
|
|
stat(7),
|
|
stat(8),
|
|
stat(9),
|
|
le(&h[rows_at..rows_at + 2])
|
|
);
|
|
if fs != u64::MAX {
|
|
let s = &b[fs as usize..];
|
|
assert_eq!(&s[0..4], b"FSHD");
|
|
let tot = le(&s[6..14]);
|
|
let n = le(&s[14..22]);
|
|
// counts (4 x 8), 4 x u16, max section size, section info address.
|
|
let at = 6 + 32 + 8 + 8;
|
|
let sect_addr = le(&s[at..at + 8]);
|
|
let sect_size = le(&s[at + 8..at + 16]);
|
|
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
|
// Sections after the prefix (signature, version, header address);
|
|
// trailing zero padding and the checksum are left out.
|
|
let body = &ss[13..ss.len() - 4];
|
|
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
|
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
|
} else {
|
|
out += " no-fs";
|
|
}
|
|
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
|
if let Some(o) = order {
|
|
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
|
}
|
|
out
|
|
}
|
|
|
|
fn hex(b: &[u8]) -> String {
|
|
b.iter().map(|x| format!("{x:02x}")).collect()
|
|
}
|
|
|
|
/// h5py and our reader both see `want` on each object (other attributes
|
|
/// may exist; these must have these values), and h5py's attribute count
|
|
/// agrees with libhdf5's object info.
|
|
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = if o == "d" {
|
|
f.dataset(o).unwrap().attrs().unwrap()
|
|
} else {
|
|
f.group(o).unwrap().attrs().unwrap()
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
|
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
|
fn py_r_plus(path: &Path, lines: &[String]) {
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
|
p = path.to_str().unwrap(),
|
|
l = lines.concat()
|
|
);
|
|
let sp = path.with_extension("ops.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
|
}
|
|
|
|
type AttrOp = (String, String, AV);
|
|
|
|
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
|
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
|
want.push((o.into(), n.into(), v));
|
|
}
|
|
|
|
/// The same attribute workload by libhdf5 and by the editor on objects
|
|
/// with compact attributes that move to dense storage (a plain group, a
|
|
/// group tracking and indexing creation order, a dataset): 40 new
|
|
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
|
/// same-size rewrites — the heaps, their free space and both index B-trees
|
|
/// must come out as libhdf5 makes them; then replacements of another size,
|
|
/// then h5py adds, deletes and rewrites attributes.
|
|
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
|
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
|
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
|
\x20 for o in objs:\n\
|
|
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
let objs = ["g", "t", "d"];
|
|
let mut want: Vec<AttrOp> = Vec::new();
|
|
for o in objs {
|
|
for i in 0..3i64 {
|
|
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
|
}
|
|
}
|
|
// Phase 1: new attributes.
|
|
let mut ops: Vec<AttrOp> = Vec::new();
|
|
for i in 0..40u64 {
|
|
for (k, o) in objs.iter().enumerate() {
|
|
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
|
}
|
|
}
|
|
let lines: Vec<String> = ops
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops {
|
|
ed.set_attr(o, n, &v.value())
|
|
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
check_attr_values(&a, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
|
);
|
|
}
|
|
// Phase 2: same-size rewrites (H5Awrite in place).
|
|
let ops2: Vec<AttrOp> = ops
|
|
.iter()
|
|
.step_by(4)
|
|
.map(|(o, n, v)| {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
|
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
|
};
|
|
(o.clone(), n.clone(), nv)
|
|
})
|
|
.collect();
|
|
let lines: Vec<String> = ops2
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops2 {
|
|
ed.set_attr(o, n, &v.value()).unwrap();
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
|
);
|
|
}
|
|
// Phase 3: replacements of another size (values only: h5py replaces
|
|
// through a temporary attribute and a rename).
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
|
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
|
};
|
|
let before = std::fs::read(&b).unwrap();
|
|
match ed.set_attr(o, n, &nv.value()) {
|
|
Ok(()) => set_want(&mut want, o, n, nv),
|
|
Err(Error::Unsupported(msg)) => {
|
|
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
|
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
|
}
|
|
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
// libhdf5 goes on: adds, deletes and rewrites.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['g', 't', 'd']:\n\
|
|
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
|
\x20 del f[o].attrs['n3']\n\
|
|
\x20 del f[o].attrs['c1']\n\
|
|
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attributes_match_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
dense_workload(lv, *dump, &format!("{i}"));
|
|
}
|
|
}
|
|
|
|
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
|
/// layout, no free-space manager), with and without tracked creation
|
|
/// order: attributes added (moving a dataset's attributes to dense storage
|
|
/// too), rewritten and replaced, then h5py goes on.
|
|
#[test]
|
|
fn dense_attributes_on_clawhdf5_files() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for track in [false, true] {
|
|
let dir = tmpdir();
|
|
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
|
let mut b = clawhdf5::FileBuilder::new();
|
|
b.track_order(track);
|
|
for i in 0..12i64 {
|
|
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
|
}
|
|
b.create_dataset("d")
|
|
.with_i32_data(&[1, 2, 3])
|
|
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
|
b.write(&path).unwrap();
|
|
let mut want: Vec<AttrOp> = (0..12i64)
|
|
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
|
.collect();
|
|
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for i in 0..30u64 {
|
|
for (k, o) in ["/", "d"].iter().enumerate() {
|
|
// Small attributes: a larger one needs a heap block bigger than
|
|
// the next (see dense_attribute_refusals_change_nothing).
|
|
let v = match AV::make(i, k as u64 + 7) {
|
|
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
|
|
v => v,
|
|
};
|
|
ed.set_attr(o, &format!("n{i}"), &v.value())
|
|
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
|
set_want(&mut want, o, &format!("n{i}"), v);
|
|
}
|
|
}
|
|
// Rewrites in place, then replacements of another size.
|
|
for i in (0..12i64).step_by(3) {
|
|
let v = AV::Ints(vec![-i; 3]);
|
|
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
|
set_want(&mut want, "/", &format!("r{i}"), v);
|
|
}
|
|
for i in (1..30u64).step_by(7) {
|
|
let v = AV::Str(format!("replaced {i}"));
|
|
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
|
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
|
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
|
Err(e) => panic!("{e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&path, true);
|
|
let want_py: Vec<AttrOp> = want
|
|
.iter()
|
|
.map(|(o, n, v)| {
|
|
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
|
(o, n.clone(), v.clone())
|
|
})
|
|
.collect();
|
|
check_root_and_attrs(&path, &want_py);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['/', 'd']:\n\
|
|
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
|
\x20 del f[o].attrs['n2']\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n2");
|
|
check_tools(&path, true);
|
|
check_root_and_attrs(&path, &want);
|
|
}
|
|
}
|
|
|
|
/// `check_attr_values`, with the root group as "/".
|
|
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = match o.as_str() {
|
|
"/" => f.root().attrs().unwrap(),
|
|
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
|
_ => f.group(o).unwrap().attrs().unwrap(),
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n)\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// What the editor refuses in dense storage — an object larger than the
|
|
/// next heap block (libhdf5 would skip blocks and record their space as
|
|
/// free, which this editor does not do) — is `Error::Unsupported`, and the
|
|
/// file is left byte for byte as it was.
|
|
#[test]
|
|
fn dense_attribute_refusals_change_nothing() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("dense_refuse.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let before = std::fs::read(&path).unwrap();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
|
|
drop(ed);
|
|
assert!(
|
|
std::fs::read(&path).unwrap() == before,
|
|
"a refused edit wrote"
|
|
);
|
|
// What libhdf5 does with it instead works on the untouched file.
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
|
|
\x20 assert len(f['g'].attrs) == 13\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
check_tools(&path, true);
|
|
}
|