osobhandClaude Opus 5.5
dc9cfba6bb
A FileEditor now keeps the space its edits free — a filtered chunk that moved, chunks a shrink removed, B-tree nodes merged away, a heap's replaced root indirect block or free-space section info, huge objects replaced — and later edits allocate from it (best fit, lowest address among equals, zeroed) before growing the file. An edit never reuses what it frees itself: until it is committed the file still refers to that space. Reused blocks are written in the commit's first phase with the space past the old end of file (nothing on disk refers to them yet), before any existing byte changes, so the crash-safety ordering holds. Space still free when the editor is dropped is leaked, as libhdf5 leaks it without a persistent free-space manager (files that have one, or use paged aggregation, are still refused at open). FileEditor::reusable_bytes reports what is left to reuse. Tests: FreeList merging and best fit, the edit-local rule and the commit split (image unit tests); a shrink followed by regrowth writing the same data reuses every removed chunk and leaves the file size unchanged, while one editor per edit grows the file, h5py/h5dump/h5rs check read both and h5py continues (freed_space_is_reused_within_a_session). measure_append_waste (edit_interop, ignored), same workload, one editor, before -> after (bytes; libhdf5 in brackets), on tank 2026-09-26: gzip chunks 1024, 1000 appends of 100: 307210 -> 306780 (306058); gzip chunks 4096, 2000 appends of 10: 119684 -> 79829 (50292); unfiltered unchanged (no space is freed). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1399 lines
50 KiB
Rust
1399 lines
50 KiB
Rust
//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
|
|
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
|
|
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
|
|
//! (and moved there), and space reused within an editing session. Files are
|
|
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
|
|
//! clawhdf5; after each edit h5py must read exactly what a model says,
|
|
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
|
|
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
|
|
//! same operations can be done by libhdf5, the structures it builds (B-tree
|
|
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
|
|
//!
|
|
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
|
|
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::path::Path;
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5::{Error, File, FileEditor, Selection};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn available(cmd: &str, args: &[&str]) -> bool {
|
|
Command::new(cmd)
|
|
.args(args)
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Whether the tools are here; false (skip) or a panic when interop is
|
|
/// required.
|
|
fn tools_ok() -> bool {
|
|
let ok =
|
|
available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
|
|
if !ok {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
|
|
);
|
|
eprintln!("SKIP: h5py/numpy or h5dump not available");
|
|
}
|
|
ok
|
|
}
|
|
|
|
fn py(script: &str) -> String {
|
|
let o = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("run python");
|
|
assert!(
|
|
o.status.success(),
|
|
"python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
String::from_utf8_lossy(&o.stdout).trim().to_string()
|
|
}
|
|
|
|
fn text(o: &Output) -> String {
|
|
format!(
|
|
"{}{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
)
|
|
}
|
|
|
|
/// The file is valid for libhdf5's tools and for `h5rs check --data`.
|
|
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
|
|
fn check_tools(path: &Path, h5dump: bool) {
|
|
let p = path.to_str().unwrap();
|
|
let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(["check", "--data", "-q", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
|
|
if h5dump {
|
|
let o = Command::new("h5dump")
|
|
.args(["-o", "/dev/null", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(
|
|
o.status.success() && o.stderr.is_empty(),
|
|
"h5dump {p}:\n{}",
|
|
text(&o)
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A dataset's expected contents.
|
|
#[derive(Clone, Debug)]
|
|
struct Model {
|
|
shape: Vec<u64>,
|
|
/// Row-major values.
|
|
data: Vec<i32>,
|
|
}
|
|
|
|
impl Model {
|
|
fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
|
|
let n: u64 = shape.iter().product();
|
|
Self {
|
|
shape: shape.to_vec(),
|
|
data: (0..n).map(f).collect(),
|
|
}
|
|
}
|
|
|
|
fn index(&self, c: &[u64]) -> usize {
|
|
c.iter()
|
|
.zip(&self.shape)
|
|
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
|
}
|
|
|
|
/// Change the extent to `shape`: elements inside both keep their
|
|
/// values, new ones are `fill`.
|
|
fn resize(&mut self, shape: &[u64], fill: i32) {
|
|
let old = self.clone();
|
|
*self = Self::new(shape, |_| fill);
|
|
let n: u64 = old.shape.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; old.shape.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = r % old.shape[d];
|
|
r /= old.shape[d];
|
|
}
|
|
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
|
let i = self.index(&c);
|
|
self.data[i] = old.data[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Apply a hyperslab write of `vals` (row-major over the block).
|
|
fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
|
|
let n: u64 = count.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; count.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = start[d] + r % count[d];
|
|
r /= count[d];
|
|
}
|
|
let i = self.index(&c);
|
|
self.data[i] = vals[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
fn block(start: &[u64], count: &[u64]) -> Selection {
|
|
Selection::Hyperslab {
|
|
start: start.to_vec(),
|
|
stride: vec![1; start.len()],
|
|
count: count.to_vec(),
|
|
block: vec![1; start.len()],
|
|
}
|
|
}
|
|
|
|
/// h5py and our reader both read `m` from dataset `name`.
|
|
fn verify(path: &Path, name: &str, m: &Model) {
|
|
let f = File::open(path).unwrap();
|
|
let ds = f.dataset(name).unwrap();
|
|
assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
|
|
let got = ds.read_i32().unwrap();
|
|
if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
|
|
panic!(
|
|
"our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
|
|
path.display(),
|
|
m.shape,
|
|
got[i],
|
|
m.data[i]
|
|
);
|
|
}
|
|
let exp = path.with_extension("expect");
|
|
let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
|
|
std::fs::write(&exp, bytes).unwrap();
|
|
let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
a = f[{name:?}][()]\n\
|
|
e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
|
|
assert a.shape == e.shape, (a.shape, e.shape)\n\
|
|
bad = np.argwhere(a != e)\n\
|
|
assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
|
|
p = path.to_str().unwrap(),
|
|
e = exp.to_str().unwrap(),
|
|
s = shape.join(",")
|
|
));
|
|
}
|
|
|
|
/// A deterministic pseudo-random sequence (SplitMix64).
|
|
struct Rng(u64);
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
|
let mut z = self.0;
|
|
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
|
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
|
z ^ (z >> 31)
|
|
}
|
|
fn below(&mut self, n: u64) -> u64 {
|
|
self.next() % n
|
|
}
|
|
}
|
|
|
|
fn tmpdir() -> tempfile::TempDir {
|
|
let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
|
|
tempfile::TempDir::new_in(base).unwrap()
|
|
}
|
|
|
|
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
|
match r {
|
|
Err(Error::Unsupported(_)) => {}
|
|
other => panic!("expected Error::Unsupported, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn le(b: &[u8]) -> u64 {
|
|
b.iter()
|
|
.take(8)
|
|
.enumerate()
|
|
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
|
}
|
|
|
|
fn enc_size(v: u64) -> usize {
|
|
((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
|
|
}
|
|
|
|
/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
|
|
/// (8-byte addresses and lengths): its depth, record count, and every
|
|
/// node's (depth, records) breadth-first.
|
|
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let h = &b[hdr..];
|
|
assert_eq!(&h[0..4], b"BTHD");
|
|
let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
|
|
let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
|
|
let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
|
|
let root = le(&h[16..24]);
|
|
let root_n = le(&h[24..26]);
|
|
let total = le(&h[26..34]);
|
|
// Node geometry, as H5B2__hdr_init computes it.
|
|
let leaf_max = (node_size - 10) / rs;
|
|
let nrec_w = enc_size(leaf_max);
|
|
let mut cum = vec![(leaf_max, 0usize)];
|
|
for d in 1..=u64::from(depth) {
|
|
let (below, below_w) = cum[d as usize - 1];
|
|
let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
|
|
let max = (node_size - 10 - ptr) / (rs + ptr);
|
|
let c = (max + 1) * below + max;
|
|
cum.push((c, enc_size(c)));
|
|
}
|
|
let mut out = Vec::new();
|
|
let mut level = vec![(root, root_n)];
|
|
let mut dep = depth;
|
|
loop {
|
|
let mut next = Vec::new();
|
|
for &(addr, n) in &level {
|
|
out.push((dep, n));
|
|
if dep == 0 {
|
|
assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
|
|
continue;
|
|
}
|
|
let node = &b[addr as usize..];
|
|
assert_eq!(&node[0..4], b"BTIN");
|
|
let mut q = 6 + (n * rs) as usize;
|
|
let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
|
|
for _ in 0..=n {
|
|
let a = le(&node[q..q + 8]);
|
|
let c = le(&node[q + 8..q + 8 + nrec_w]);
|
|
q += 8 + nrec_w + all_w;
|
|
next.push((a, c));
|
|
}
|
|
}
|
|
if dep == 0 {
|
|
break;
|
|
}
|
|
dep -= 1;
|
|
level = next;
|
|
}
|
|
(depth, total, out)
|
|
}
|
|
|
|
/// The shape of the file's only chunk B-tree (record type 10 or 11).
|
|
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let b = std::fs::read(path).unwrap();
|
|
let hdrs: Vec<usize> = b
|
|
.windows(6)
|
|
.enumerate()
|
|
.filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
|
|
.map(|(i, _)| i)
|
|
.collect();
|
|
assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
|
|
bt2_shape_at(&b, hdrs[0])
|
|
}
|
|
|
|
/// Grow a dataset with two unlimited dimensions in both directions many
|
|
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
|
|
/// chunk enters the index as it is written, in the same order as the
|
|
/// editor's) and with the editor: the same values, and the same B-tree —
|
|
/// depth, record count, and every node's record count — through leaf and
|
|
/// internal splits, redistributions and a depth increase.
|
|
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
|
|
// Extents: both dimensions grow; new columns fall between existing
|
|
// chunks in the index's key order (row first).
|
|
let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
|
|
let val = |r: u64, c: u64| (r * 1000 + c) as i32;
|
|
let create = |p: &Path, write: bool| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
|
|
dtype='<i4'{extra})\n\
|
|
\x20 if {write}:\n\
|
|
\x20 r0, c0 = 0, 0\n\
|
|
\x20 for r, c in {steps:?}:\n\
|
|
\x20 d.resize((r, c))\n\
|
|
\x20 g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
|
|
\x20 if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
|
|
\x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\
|
|
\x20 r0, c0 = r, c\n",
|
|
p = p.to_str().unwrap(),
|
|
write = if write { "True" } else { "False" },
|
|
))
|
|
};
|
|
create(&a, true);
|
|
create(&b, false);
|
|
let mut m = Model::new(&[0, 0], |_| 0);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
let (mut r0, mut c0) = (0u64, 0u64);
|
|
for &(r, c) in &steps {
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], 0);
|
|
if c > c0 && r0 > 0 {
|
|
let vals: Vec<i32> = (0..r0)
|
|
.flat_map(|i| (c0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[0, c0], &[r0, c - c0], &vals);
|
|
}
|
|
if r > r0 {
|
|
let vals: Vec<i32> = (r0..r)
|
|
.flat_map(|i| (0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, 0], &[r - r0, c], &vals);
|
|
}
|
|
(r0, c0) = (r, c);
|
|
}
|
|
drop(ed);
|
|
verify(&b, "x", &m);
|
|
verify(&a, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
|
|
assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
|
|
assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
|
|
// libhdf5 goes on growing what the editor built.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 r, c = d.shape\n\
|
|
\x20 d.resize((r + 3, c + 2))\n\
|
|
\x20 d[:, c:] = -1\n\
|
|
\x20 d[r:, :] = -2\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let (r, c) = (m.shape[0], m.shape[1]);
|
|
m.resize(&[r + 3, c + 2], 0);
|
|
m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
|
|
m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Dataset `name`'s layout: its chunk index type (0 for a version-1
|
|
/// B-tree), index address and chunk rank (element-size dimension
|
|
/// included).
|
|
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let m = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap();
|
|
match DataLayout::parse(&m.data, os, ls).unwrap() {
|
|
DataLayout::Chunked {
|
|
chunk_dimensions,
|
|
btree_address,
|
|
chunk_index_type,
|
|
..
|
|
} => (
|
|
chunk_index_type.unwrap_or(0),
|
|
btree_address,
|
|
chunk_dimensions.len(),
|
|
),
|
|
other => panic!("{other:?}"),
|
|
}
|
|
}
|
|
|
|
/// A version-1 chunk B-tree's shape: every node's (level, entries),
|
|
/// breadth-first from the root.
|
|
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
|
|
let ks = 8 + 8 * ndims;
|
|
let mut out = Vec::new();
|
|
let mut level = vec![root];
|
|
while !level.is_empty() {
|
|
let mut next = Vec::new();
|
|
for a in level {
|
|
let n = &b[a as usize..];
|
|
assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
|
|
let lv = n[5];
|
|
let e = le(&n[6..8]);
|
|
out.push((lv, e));
|
|
if lv > 0 {
|
|
for i in 0..e as usize {
|
|
next.push(le(
|
|
&n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
|
|
));
|
|
}
|
|
}
|
|
}
|
|
level = next;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The shape of dataset `name`'s chunk index, for comparing with the one
|
|
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
|
|
/// nothing to compare (Fixed Array, implicit).
|
|
fn index_shape(path: &Path, name: &str) -> String {
|
|
let b = std::fs::read(path).unwrap();
|
|
match chunk_index(path, name) {
|
|
(_, None, _) => "none".into(),
|
|
(0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
|
|
(5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
|
|
(4, Some(a), _) => {
|
|
let s: Vec<u64> = (0..6)
|
|
.map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
|
|
.collect();
|
|
format!("ea {s:?}")
|
|
}
|
|
(t, Some(_), _) => format!("type {t}"),
|
|
}
|
|
}
|
|
|
|
/// One step of a resize workload.
|
|
#[derive(Debug, Clone)]
|
|
enum Op {
|
|
Resize(Vec<u64>),
|
|
/// A block write; values from `vals`.
|
|
Write(Vec<u64>, Vec<u64>, u64),
|
|
}
|
|
|
|
fn op_vals(seed: u64, n: u64) -> Vec<i32> {
|
|
(0..n)
|
|
.map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
|
|
.collect()
|
|
}
|
|
|
|
/// Random resizes (shrinking and growing any dimension within `max`)
|
|
/// and block writes.
|
|
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
|
|
let mut shape = start.to_vec();
|
|
let mut ops = Vec::new();
|
|
for k in 0..n {
|
|
if k % 3 == 0 {
|
|
shape = (0..shape.len())
|
|
.map(|d| {
|
|
let cap = max[d].min(shape[d] * 2 + 6);
|
|
rng.below(cap + 1)
|
|
})
|
|
.collect();
|
|
ops.push(Op::Resize(shape.clone()));
|
|
} else if shape.iter().all(|&s| s > 0) {
|
|
let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
|
|
let cnt: Vec<u64> = (0..shape.len())
|
|
.map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
|
|
.collect();
|
|
ops.push(Op::Write(st, cnt, rng.next() % 1000));
|
|
}
|
|
}
|
|
ops
|
|
}
|
|
|
|
/// The python statements applying `ops` to dataset `d`.
|
|
fn py_ops(ops: &[Op]) -> String {
|
|
let mut s = String::new();
|
|
for op in ops {
|
|
match op {
|
|
Op::Resize(shape) => {
|
|
s += &format!("\x20 d.resize({shape:?})\n")
|
|
.replace('[', "(")
|
|
.replace(']', ",)");
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let n: u64 = cnt.iter().product();
|
|
let sl: Vec<String> = st
|
|
.iter()
|
|
.zip(cnt)
|
|
.map(|(a, c)| format!("{a}:{}", a + c))
|
|
.collect();
|
|
s += &format!(
|
|
"\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
|
|
.astype('<i4').reshape({cnt:?})\n",
|
|
sl.join(", ")
|
|
)
|
|
.replace("reshape([", "reshape((")
|
|
.replace("])\n", ",))\n");
|
|
}
|
|
}
|
|
}
|
|
s
|
|
}
|
|
|
|
/// The same resize workload — shrinking and growing along every
|
|
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
|
|
/// editor on dataset `x` created by `create` (python, `f` open): the same
|
|
/// values as a model where elements that come back after a shrink read as
|
|
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
|
|
/// check` accept, and h5py can go on.
|
|
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
|
|
let mk = |p: &Path| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
{create}",
|
|
p = p.to_str().unwrap()
|
|
))
|
|
};
|
|
mk(&a);
|
|
mk(&b);
|
|
let (start, max) = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
|
|
p = b.to_str().unwrap()
|
|
))
|
|
.split_once("] [")
|
|
.map(|(x, y)| {
|
|
let p = |s: &str| -> Vec<u64> {
|
|
s.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
};
|
|
(p(x), p(y))
|
|
})
|
|
.unwrap();
|
|
let initial = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let mut m = Model {
|
|
shape: start.clone(),
|
|
data: initial
|
|
.split_whitespace()
|
|
.map(|v| v.parse().unwrap())
|
|
.collect(),
|
|
};
|
|
let mut rng = Rng(seed);
|
|
let ops = random_resize_ops(&mut rng, &start, &max, 60);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f['x']\n{o}",
|
|
p = a.to_str().unwrap(),
|
|
o = py_ops(&ops)
|
|
));
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, op) in ops.iter().enumerate() {
|
|
match op {
|
|
Op::Resize(s) => {
|
|
ed.resize("x", s)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.resize(s, fill);
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let vals = op_vals(*seed, cnt.iter().product());
|
|
ed.write_values("x", &block(st, cnt), &vals)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.write_block(st, cnt, &vals);
|
|
}
|
|
}
|
|
}
|
|
drop(ed);
|
|
if tag == "implicit" {
|
|
// An implicit index cannot drop chunks: libhdf5 leaves the data of
|
|
// chunks wholly outside a shrunk extent in place and does not
|
|
// refill those that come back unless they lie past the extent it
|
|
// grows from, so they can show old values. The editor must match
|
|
// libhdf5, not the model.
|
|
let got = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = a.to_str().unwrap()
|
|
));
|
|
m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
|
|
}
|
|
verify(&a, "x", &m);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
assert_eq!(
|
|
index_shape(&b, "x"),
|
|
index_shape(&a, "x"),
|
|
"{tag}: chunk index differs from libhdf5's"
|
|
);
|
|
// libhdf5 goes on: grow back to the start shape and write everything.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 d.resize({s:?})\n\
|
|
\x20 d[...] = 3\n",
|
|
p = b.to_str().unwrap(),
|
|
s = start
|
|
));
|
|
let n: u64 = start.iter().product();
|
|
verify(
|
|
&b,
|
|
"x",
|
|
&Model {
|
|
shape: start.clone(),
|
|
data: vec![3; n as usize],
|
|
},
|
|
);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
|
|
/// dimension, mixed with growth and writes, on every chunk index: the
|
|
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
|
|
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
|
|
/// and implicit (early allocation), unfiltered and deflated.
|
|
#[test]
|
|
fn shrink_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let kinds: &[(&str, &str, i32)] = &[
|
|
(
|
|
"ea",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
|
|
maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
|
|
-1,
|
|
),
|
|
(
|
|
"bt2",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
|
|
maxshape=(None, None), chunks=(3, 2){z})\n",
|
|
0,
|
|
),
|
|
(
|
|
"fa",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
|
|
maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
|
|
5,
|
|
),
|
|
(
|
|
"1d",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
|
|
maxshape=(None,), chunks=(4,){z})\n",
|
|
0,
|
|
),
|
|
];
|
|
let mut seed = 100;
|
|
for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
for (kind, create, fill) in kinds {
|
|
for (zi, z) in ["", ", compression='gzip', shuffle=True"]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
seed += 1;
|
|
let tag = format!("{kind}_{li}_{zi}");
|
|
shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
|
|
}
|
|
}
|
|
}
|
|
// Implicit index: early allocation, no filters, fixed maximum shape.
|
|
shrink_workload(
|
|
"implicit",
|
|
"'latest'",
|
|
"\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
7,
|
|
false,
|
|
99,
|
|
);
|
|
}
|
|
|
|
/// An implicit chunk index (early allocation, no filters, fixed maximum
|
|
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
|
|
/// by its position in the maximum chunk grid. The reader once used the
|
|
/// current grid, and returned other chunks' values for every row after the
|
|
/// first.
|
|
#[test]
|
|
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("implicit_max.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
|
|
verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
|
|
// The editor places new values by the same grid.
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
|
|
.unwrap();
|
|
drop(ed);
|
|
let mut m = Model::new(&[8, 9], |i| i as i32);
|
|
m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
#[test]
|
|
fn btree2_chunk_index_growth_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
|
|
bt2_growth(lv, *dump, "", &format!("plain{i}"));
|
|
bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
|
|
}
|
|
}
|
|
|
|
/// Random writes into a dataset with two unlimited dimensions (chunks
|
|
/// created in any order) and growth, with a small node size so the tree
|
|
/// gets deep, against a model; then h5py continues.
|
|
#[test]
|
|
fn btree2_random_chunk_order() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("bt2_random.h5");
|
|
// A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
|
|
// hundred chunks).
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
|
|
dtype='<i4', fillvalue=-7)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = Model::new(&[40, 40], |_| -7);
|
|
let mut rng = Rng(42);
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for step in 0..400 {
|
|
if step % 50 == 49 {
|
|
let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], -7);
|
|
}
|
|
let r0 = rng.below(m.shape[0]);
|
|
let c0 = rng.below(m.shape[1]);
|
|
let cnt = [
|
|
1 + rng.below((m.shape[0] - r0).min(3)),
|
|
1 + rng.below((m.shape[1] - c0).min(3)),
|
|
];
|
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1])
|
|
.map(|_| (rng.next() % 1_000_000) as i32)
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, c0], &cnt, &vals);
|
|
if step % 100 == 99 {
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
ed = FileEditor::open(&path).unwrap();
|
|
}
|
|
}
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
let shape = chunk_bt2_shape(&path);
|
|
assert!(shape.0 >= 1, "{shape:?}");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'][...] = 5\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let n = m.data.len();
|
|
m.data = vec![5; n];
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
// ---- dense attributes ----
|
|
|
|
/// An attribute value both sides can write with the same encoded size: a
|
|
/// 1-D int64 array or a fixed-length string.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
enum AV {
|
|
Ints(Vec<i64>),
|
|
Str(String),
|
|
}
|
|
|
|
impl AV {
|
|
fn value(&self) -> clawhdf5::AttrValue {
|
|
match self {
|
|
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
|
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
|
}
|
|
}
|
|
|
|
fn py(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
|
AV::Str(s) => format!("np.bytes_({s:?})"),
|
|
}
|
|
}
|
|
|
|
fn py_expect(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("{v:?}"),
|
|
AV::Str(s) => format!("{s:?}"),
|
|
}
|
|
}
|
|
|
|
fn make(i: u64, salt: u64) -> Self {
|
|
match i % 11 {
|
|
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
|
1 | 4 | 8 => AV::Str(
|
|
(0..10 + (i * 13 + salt) % 300)
|
|
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
|
.collect(),
|
|
),
|
|
_ => AV::Ints(
|
|
(0..1 + (i + salt) % 9)
|
|
.map(|k| (k * 31 + salt) as i64)
|
|
.collect(),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An object's dense attribute storage as libhdf5 would compare it: the
|
|
/// heap's statistics and root shape, its free sections (heap offsets and
|
|
/// sizes), and the shapes of the name and creation-order index B-trees.
|
|
fn dense_info(path: &Path, obj: &str) -> String {
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
assert_eq!((os, ls), (8, 8));
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let Some(m) = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
|
else {
|
|
return "no attribute info".into();
|
|
};
|
|
let d = &m.data;
|
|
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
|
let heap = le(&d[p..p + 8]);
|
|
p += 8;
|
|
let name = le(&d[p..p + 8]);
|
|
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
|
if heap == u64::MAX {
|
|
return "compact".into();
|
|
}
|
|
let b = std::fs::read(path).unwrap();
|
|
let h = &b[heap as usize..];
|
|
assert_eq!(&h[0..4], b"FRHP");
|
|
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
|
// size, huge objs (8-byte fields from offset 14).
|
|
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
|
let fs = stat(3);
|
|
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
|
// index, start rows, root address.
|
|
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
|
let mut out = format!(
|
|
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
|
stat(0),
|
|
stat(2),
|
|
stat(4),
|
|
stat(5),
|
|
stat(6),
|
|
stat(7),
|
|
stat(8),
|
|
stat(9),
|
|
le(&h[rows_at..rows_at + 2])
|
|
);
|
|
if fs != u64::MAX {
|
|
let s = &b[fs as usize..];
|
|
assert_eq!(&s[0..4], b"FSHD");
|
|
let tot = le(&s[6..14]);
|
|
let n = le(&s[14..22]);
|
|
// counts (4 x 8), 4 x u16, max section size, section info address.
|
|
let at = 6 + 32 + 8 + 8;
|
|
let sect_addr = le(&s[at..at + 8]);
|
|
let sect_size = le(&s[at + 8..at + 16]);
|
|
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
|
// Sections after the prefix (signature, version, header address);
|
|
// trailing zero padding and the checksum are left out.
|
|
let body = &ss[13..ss.len() - 4];
|
|
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
|
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
|
} else {
|
|
out += " no-fs";
|
|
}
|
|
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
|
if let Some(o) = order {
|
|
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
|
}
|
|
out
|
|
}
|
|
|
|
fn hex(b: &[u8]) -> String {
|
|
b.iter().map(|x| format!("{x:02x}")).collect()
|
|
}
|
|
|
|
/// h5py and our reader both see `want` on each object (other attributes
|
|
/// may exist; these must have these values), and h5py's attribute count
|
|
/// agrees with libhdf5's object info.
|
|
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = if o == "d" {
|
|
f.dataset(o).unwrap().attrs().unwrap()
|
|
} else {
|
|
f.group(o).unwrap().attrs().unwrap()
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
|
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
|
fn py_r_plus(path: &Path, lines: &[String]) {
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
|
p = path.to_str().unwrap(),
|
|
l = lines.concat()
|
|
);
|
|
let sp = path.with_extension("ops.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
|
}
|
|
|
|
type AttrOp = (String, String, AV);
|
|
|
|
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
|
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
|
want.push((o.into(), n.into(), v));
|
|
}
|
|
|
|
/// The same attribute workload by libhdf5 and by the editor on objects
|
|
/// with compact attributes that move to dense storage (a plain group, a
|
|
/// group tracking and indexing creation order, a dataset): 40 new
|
|
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
|
/// same-size rewrites — the heaps, their free space and both index B-trees
|
|
/// must come out as libhdf5 makes them; then replacements of another size,
|
|
/// then h5py adds, deletes and rewrites attributes.
|
|
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
|
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
|
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
|
\x20 for o in objs:\n\
|
|
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
let objs = ["g", "t", "d"];
|
|
let mut want: Vec<AttrOp> = Vec::new();
|
|
for o in objs {
|
|
for i in 0..3i64 {
|
|
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
|
}
|
|
}
|
|
// Phase 1: new attributes.
|
|
let mut ops: Vec<AttrOp> = Vec::new();
|
|
for i in 0..40u64 {
|
|
for (k, o) in objs.iter().enumerate() {
|
|
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
|
}
|
|
}
|
|
let lines: Vec<String> = ops
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops {
|
|
ed.set_attr(o, n, &v.value())
|
|
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
check_attr_values(&a, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
|
);
|
|
}
|
|
// Phase 2: same-size rewrites (H5Awrite in place).
|
|
let ops2: Vec<AttrOp> = ops
|
|
.iter()
|
|
.step_by(4)
|
|
.map(|(o, n, v)| {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
|
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
|
};
|
|
(o.clone(), n.clone(), nv)
|
|
})
|
|
.collect();
|
|
let lines: Vec<String> = ops2
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops2 {
|
|
ed.set_attr(o, n, &v.value()).unwrap();
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
|
);
|
|
}
|
|
// Phase 3: replacements of another size (values only: h5py replaces
|
|
// through a temporary attribute and a rename).
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
|
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
|
};
|
|
let before = std::fs::read(&b).unwrap();
|
|
match ed.set_attr(o, n, &nv.value()) {
|
|
Ok(()) => set_want(&mut want, o, n, nv),
|
|
Err(Error::Unsupported(msg)) => {
|
|
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
|
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
|
}
|
|
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
// libhdf5 goes on: adds, deletes and rewrites.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['g', 't', 'd']:\n\
|
|
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
|
\x20 del f[o].attrs['n3']\n\
|
|
\x20 del f[o].attrs['c1']\n\
|
|
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attributes_match_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
dense_workload(lv, *dump, &format!("{i}"));
|
|
}
|
|
}
|
|
|
|
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
|
/// layout, no free-space manager), with and without tracked creation
|
|
/// order: attributes added (moving a dataset's attributes to dense storage
|
|
/// too), rewritten and replaced, then h5py goes on.
|
|
#[test]
|
|
fn dense_attributes_on_clawhdf5_files() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for track in [false, true] {
|
|
let dir = tmpdir();
|
|
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
|
let mut b = clawhdf5::FileBuilder::new();
|
|
b.track_order(track);
|
|
for i in 0..12i64 {
|
|
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
|
}
|
|
b.create_dataset("d")
|
|
.with_i32_data(&[1, 2, 3])
|
|
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
|
b.write(&path).unwrap();
|
|
let mut want: Vec<AttrOp> = (0..12i64)
|
|
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
|
.collect();
|
|
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for i in 0..30u64 {
|
|
for (k, o) in ["/", "d"].iter().enumerate() {
|
|
// Small attributes: a larger one needs a heap block bigger than
|
|
// the next (see dense_attribute_refusals_change_nothing).
|
|
let v = match AV::make(i, k as u64 + 7) {
|
|
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
|
|
v => v,
|
|
};
|
|
ed.set_attr(o, &format!("n{i}"), &v.value())
|
|
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
|
set_want(&mut want, o, &format!("n{i}"), v);
|
|
}
|
|
}
|
|
// Rewrites in place, then replacements of another size.
|
|
for i in (0..12i64).step_by(3) {
|
|
let v = AV::Ints(vec![-i; 3]);
|
|
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
|
set_want(&mut want, "/", &format!("r{i}"), v);
|
|
}
|
|
for i in (1..30u64).step_by(7) {
|
|
let v = AV::Str(format!("replaced {i}"));
|
|
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
|
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
|
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
|
Err(e) => panic!("{e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&path, true);
|
|
let want_py: Vec<AttrOp> = want
|
|
.iter()
|
|
.map(|(o, n, v)| {
|
|
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
|
(o, n.clone(), v.clone())
|
|
})
|
|
.collect();
|
|
check_root_and_attrs(&path, &want_py);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['/', 'd']:\n\
|
|
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
|
\x20 del f[o].attrs['n2']\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n2");
|
|
check_tools(&path, true);
|
|
check_root_and_attrs(&path, &want);
|
|
}
|
|
}
|
|
|
|
/// `check_attr_values`, with the root group as "/".
|
|
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = match o.as_str() {
|
|
"/" => f.root().attrs().unwrap(),
|
|
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
|
_ => f.group(o).unwrap().attrs().unwrap(),
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n)\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// What the editor refuses in dense storage — an object larger than the
|
|
/// next heap block (libhdf5 would skip blocks and record their space as
|
|
/// free, which this editor does not do) — is `Error::Unsupported`, and the
|
|
/// file is left byte for byte as it was.
|
|
#[test]
|
|
fn dense_attribute_refusals_change_nothing() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("dense_refuse.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let before = std::fs::read(&path).unwrap();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
|
|
drop(ed);
|
|
assert!(
|
|
std::fs::read(&path).unwrap() == before,
|
|
"a refused edit wrote"
|
|
);
|
|
// What libhdf5 does with it instead works on the untouched file.
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
|
|
\x20 assert len(f['g'].attrs) == 13\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
check_tools(&path, true);
|
|
}
|
|
|
|
/// Space one edit frees is reused by later edits of the same editor: the
|
|
/// chunks a shrink removes are where the chunks of the following growth
|
|
/// go, so the file does not grow; with a new editor per edit (nothing to
|
|
/// reuse) it does. h5py, h5dump and `h5rs check` read the result, and
|
|
/// h5py goes on.
|
|
#[test]
|
|
fn freed_space_is_reused_within_a_session() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let mut sizes = Vec::new();
|
|
for session in [true, false] {
|
|
let path = dir.path().join(format!("reuse_{session}.h5"));
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 f.create_dataset('x', data=np.zeros(2000, dtype='<i4'), maxshape=(None,), \
|
|
chunks=(100,), compression='gzip')\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut rng = Rng(5);
|
|
let vals: Vec<i32> = (0..2000).map(|_| rng.next() as i32).collect();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &Selection::All, &vals).unwrap();
|
|
drop(ed);
|
|
let len0 = std::fs::metadata(&path).unwrap().len();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.resize("x", &[1000]).unwrap();
|
|
if session {
|
|
assert!(ed.reusable_bytes() > 0);
|
|
} else {
|
|
ed = {
|
|
drop(ed);
|
|
FileEditor::open(&path).unwrap()
|
|
};
|
|
}
|
|
ed.resize("x", &[2000]).unwrap();
|
|
ed.write_values("x", &block(&[1000], &[1000]), &vals[1000..])
|
|
.unwrap();
|
|
if session {
|
|
assert_eq!(ed.reusable_bytes(), 0, "every freed chunk is reused");
|
|
}
|
|
drop(ed);
|
|
let len1 = std::fs::metadata(&path).unwrap().len();
|
|
sizes.push((len0, len1));
|
|
let m = Model {
|
|
shape: vec![2000],
|
|
data: vals.clone(),
|
|
};
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, true);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'].resize((2100,))\n\
|
|
\x20 f['x'][2000:] = 9\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = m;
|
|
m.resize(&[2100], 0);
|
|
m.write_block(&[2000], &[100], &[9; 100]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, true);
|
|
}
|
|
let (reuse, fresh) = (sizes[0], sizes[1]);
|
|
assert_eq!(
|
|
reuse.1, reuse.0,
|
|
"a session reusing freed chunks does not grow the file"
|
|
);
|
|
assert!(fresh.1 > fresh.0, "without reuse the file grows");
|
|
}
|