osobhandClaude Opus 5.5
8236b0e30a
An attribute needing a heap block larger than the next one was refused
("skipping blocks too small for an object", "a first object too large for
the starting block"); once an object's move to dense storage was refused
it refused every new attribute, so 24% of set_attr calls in the review's
random workload failed.
Following H5HF__hdr_update_iter, H5HF__man_iblock_root_create/_double and
H5HF__hdr_skip_blocks, the smaller blocks are now skipped: the iterator
moves past them and they become an indirect free section with a first
row section (serialized, class 1, as H5HF__sect_indirect_serialize writes
it) and ghost normal rows, added as returned space so it merges with a
range skipped just before it (H5HF__sect_indirect_merge_row). Later
objects that best-fit a row section get a block created there
(H5HF__man_iblock_alloc_row / H5HF__sect_indirect_reduce_row: from the
start or end of the range, or from its middle, which splits it, with
libhdf5's span bookkeeping). Heaps with such sections, as libhdf5 writes
them, are now read too (they were refused at open).
dense_skipped_blocks_match_libhdf5 drives every path (merge, split, end,
last entry, row wrap) on earliest/v110/latest files against libhdf5
doing the same edits one session each; heaps, free sections and index
B-trees are equal after every phase. The refusal test now checks the
skip against libhdf5 and keeps a real refusal (last object in a block);
clawhdf5-written heaps get 1-4 KiB attributes too. The three tests fail
on the previous fheap.rs. Random workload refusals: 24% -> 2.2%, all the
documented last-object-in-a-block case.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1789 lines
66 KiB
Rust
1789 lines
66 KiB
Rust
//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
|
|
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
|
|
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
|
|
//! (and moved there), and space reused within an editing session. Files are
|
|
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
|
|
//! clawhdf5; after each edit h5py must read exactly what a model says,
|
|
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
|
|
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
|
|
//! same operations can be done by libhdf5, the structures it builds (B-tree
|
|
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
|
|
//!
|
|
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
|
|
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::path::Path;
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5::{Error, File, FileEditor, Selection};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn available(cmd: &str, args: &[&str]) -> bool {
|
|
Command::new(cmd)
|
|
.args(args)
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Whether the tools are here; false (skip) or a panic when interop is
|
|
/// required.
|
|
fn tools_ok() -> bool {
|
|
let ok =
|
|
available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
|
|
if !ok {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
|
|
);
|
|
eprintln!("SKIP: h5py/numpy or h5dump not available");
|
|
}
|
|
ok
|
|
}
|
|
|
|
fn py(script: &str) -> String {
|
|
let o = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("run python");
|
|
assert!(
|
|
o.status.success(),
|
|
"python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
String::from_utf8_lossy(&o.stdout).trim().to_string()
|
|
}
|
|
|
|
fn text(o: &Output) -> String {
|
|
format!(
|
|
"{}{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
)
|
|
}
|
|
|
|
/// The file is valid for libhdf5's tools and for `h5rs check --data`.
|
|
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
|
|
fn check_tools(path: &Path, h5dump: bool) {
|
|
let p = path.to_str().unwrap();
|
|
let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(["check", "--data", "-q", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
|
|
if h5dump {
|
|
let o = Command::new("h5dump")
|
|
.args(["-o", "/dev/null", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(
|
|
o.status.success() && o.stderr.is_empty(),
|
|
"h5dump {p}:\n{}",
|
|
text(&o)
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A dataset's expected contents.
|
|
#[derive(Clone, Debug)]
|
|
struct Model {
|
|
shape: Vec<u64>,
|
|
/// Row-major values.
|
|
data: Vec<i32>,
|
|
}
|
|
|
|
impl Model {
|
|
fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
|
|
let n: u64 = shape.iter().product();
|
|
Self {
|
|
shape: shape.to_vec(),
|
|
data: (0..n).map(f).collect(),
|
|
}
|
|
}
|
|
|
|
fn index(&self, c: &[u64]) -> usize {
|
|
c.iter()
|
|
.zip(&self.shape)
|
|
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
|
}
|
|
|
|
/// Change the extent to `shape`: elements inside both keep their
|
|
/// values, new ones are `fill`.
|
|
fn resize(&mut self, shape: &[u64], fill: i32) {
|
|
let old = self.clone();
|
|
*self = Self::new(shape, |_| fill);
|
|
let n: u64 = old.shape.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; old.shape.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = r % old.shape[d];
|
|
r /= old.shape[d];
|
|
}
|
|
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
|
let i = self.index(&c);
|
|
self.data[i] = old.data[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Apply a hyperslab write of `vals` (row-major over the block).
|
|
fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
|
|
let n: u64 = count.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; count.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = start[d] + r % count[d];
|
|
r /= count[d];
|
|
}
|
|
let i = self.index(&c);
|
|
self.data[i] = vals[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
fn block(start: &[u64], count: &[u64]) -> Selection {
|
|
Selection::Hyperslab {
|
|
start: start.to_vec(),
|
|
stride: vec![1; start.len()],
|
|
count: count.to_vec(),
|
|
block: vec![1; start.len()],
|
|
}
|
|
}
|
|
|
|
/// h5py and our reader both read `m` from dataset `name`.
|
|
fn verify(path: &Path, name: &str, m: &Model) {
|
|
let f = File::open(path).unwrap();
|
|
let ds = f.dataset(name).unwrap();
|
|
assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
|
|
let got = ds.read_i32().unwrap();
|
|
if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
|
|
panic!(
|
|
"our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
|
|
path.display(),
|
|
m.shape,
|
|
got[i],
|
|
m.data[i]
|
|
);
|
|
}
|
|
let exp = path.with_extension("expect");
|
|
let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
|
|
std::fs::write(&exp, bytes).unwrap();
|
|
let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
a = f[{name:?}][()]\n\
|
|
e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
|
|
assert a.shape == e.shape, (a.shape, e.shape)\n\
|
|
bad = np.argwhere(a != e)\n\
|
|
assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
|
|
p = path.to_str().unwrap(),
|
|
e = exp.to_str().unwrap(),
|
|
s = shape.join(",")
|
|
));
|
|
}
|
|
|
|
/// A deterministic pseudo-random sequence (SplitMix64).
|
|
struct Rng(u64);
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
|
let mut z = self.0;
|
|
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
|
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
|
z ^ (z >> 31)
|
|
}
|
|
fn below(&mut self, n: u64) -> u64 {
|
|
self.next() % n
|
|
}
|
|
}
|
|
|
|
fn tmpdir() -> tempfile::TempDir {
|
|
let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
|
|
tempfile::TempDir::new_in(base).unwrap()
|
|
}
|
|
|
|
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
|
match r {
|
|
Err(Error::Unsupported(_)) => {}
|
|
other => panic!("expected Error::Unsupported, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn le(b: &[u8]) -> u64 {
|
|
b.iter()
|
|
.take(8)
|
|
.enumerate()
|
|
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
|
}
|
|
|
|
fn enc_size(v: u64) -> usize {
|
|
((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
|
|
}
|
|
|
|
/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
|
|
/// (8-byte addresses and lengths): its depth, record count, and every
|
|
/// node's (depth, records) breadth-first.
|
|
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let h = &b[hdr..];
|
|
assert_eq!(&h[0..4], b"BTHD");
|
|
let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
|
|
let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
|
|
let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
|
|
let root = le(&h[16..24]);
|
|
let root_n = le(&h[24..26]);
|
|
let total = le(&h[26..34]);
|
|
// Node geometry, as H5B2__hdr_init computes it.
|
|
let leaf_max = (node_size - 10) / rs;
|
|
let nrec_w = enc_size(leaf_max);
|
|
let mut cum = vec![(leaf_max, 0usize)];
|
|
for d in 1..=u64::from(depth) {
|
|
let (below, below_w) = cum[d as usize - 1];
|
|
let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
|
|
let max = (node_size - 10 - ptr) / (rs + ptr);
|
|
let c = (max + 1) * below + max;
|
|
cum.push((c, enc_size(c)));
|
|
}
|
|
let mut out = Vec::new();
|
|
let mut level = vec![(root, root_n)];
|
|
let mut dep = depth;
|
|
loop {
|
|
let mut next = Vec::new();
|
|
for &(addr, n) in &level {
|
|
out.push((dep, n));
|
|
if dep == 0 {
|
|
assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
|
|
continue;
|
|
}
|
|
let node = &b[addr as usize..];
|
|
assert_eq!(&node[0..4], b"BTIN");
|
|
let mut q = 6 + (n * rs) as usize;
|
|
let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
|
|
for _ in 0..=n {
|
|
let a = le(&node[q..q + 8]);
|
|
let c = le(&node[q + 8..q + 8 + nrec_w]);
|
|
q += 8 + nrec_w + all_w;
|
|
next.push((a, c));
|
|
}
|
|
}
|
|
if dep == 0 {
|
|
break;
|
|
}
|
|
dep -= 1;
|
|
level = next;
|
|
}
|
|
(depth, total, out)
|
|
}
|
|
|
|
/// The shape of the file's only chunk B-tree (record type 10 or 11).
|
|
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let b = std::fs::read(path).unwrap();
|
|
let hdrs: Vec<usize> = b
|
|
.windows(6)
|
|
.enumerate()
|
|
.filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
|
|
.map(|(i, _)| i)
|
|
.collect();
|
|
assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
|
|
bt2_shape_at(&b, hdrs[0])
|
|
}
|
|
|
|
/// Grow a dataset with two unlimited dimensions in both directions many
|
|
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
|
|
/// chunk enters the index as it is written, in the same order as the
|
|
/// editor's) and with the editor: the same values, and the same B-tree —
|
|
/// depth, record count, and every node's record count — through leaf and
|
|
/// internal splits, redistributions and a depth increase.
|
|
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
|
|
// Extents: both dimensions grow; new columns fall between existing
|
|
// chunks in the index's key order (row first).
|
|
let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
|
|
let val = |r: u64, c: u64| (r * 1000 + c) as i32;
|
|
let create = |p: &Path, write: bool| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
|
|
dtype='<i4'{extra})\n\
|
|
\x20 if {write}:\n\
|
|
\x20 r0, c0 = 0, 0\n\
|
|
\x20 for r, c in {steps:?}:\n\
|
|
\x20 d.resize((r, c))\n\
|
|
\x20 g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
|
|
\x20 if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
|
|
\x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\
|
|
\x20 r0, c0 = r, c\n",
|
|
p = p.to_str().unwrap(),
|
|
write = if write { "True" } else { "False" },
|
|
))
|
|
};
|
|
create(&a, true);
|
|
create(&b, false);
|
|
let mut m = Model::new(&[0, 0], |_| 0);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
let (mut r0, mut c0) = (0u64, 0u64);
|
|
for &(r, c) in &steps {
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], 0);
|
|
if c > c0 && r0 > 0 {
|
|
let vals: Vec<i32> = (0..r0)
|
|
.flat_map(|i| (c0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[0, c0], &[r0, c - c0], &vals);
|
|
}
|
|
if r > r0 {
|
|
let vals: Vec<i32> = (r0..r)
|
|
.flat_map(|i| (0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, 0], &[r - r0, c], &vals);
|
|
}
|
|
(r0, c0) = (r, c);
|
|
}
|
|
drop(ed);
|
|
verify(&b, "x", &m);
|
|
verify(&a, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
|
|
assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
|
|
assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
|
|
// libhdf5 goes on growing what the editor built.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 r, c = d.shape\n\
|
|
\x20 d.resize((r + 3, c + 2))\n\
|
|
\x20 d[:, c:] = -1\n\
|
|
\x20 d[r:, :] = -2\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let (r, c) = (m.shape[0], m.shape[1]);
|
|
m.resize(&[r + 3, c + 2], 0);
|
|
m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
|
|
m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Dataset `name`'s layout: its chunk index type (0 for a version-1
|
|
/// B-tree), index address and chunk rank (element-size dimension
|
|
/// included).
|
|
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let m = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap();
|
|
match DataLayout::parse(&m.data, os, ls).unwrap() {
|
|
DataLayout::Chunked {
|
|
chunk_dimensions,
|
|
btree_address,
|
|
chunk_index_type,
|
|
..
|
|
} => (
|
|
chunk_index_type.unwrap_or(0),
|
|
btree_address,
|
|
chunk_dimensions.len(),
|
|
),
|
|
other => panic!("{other:?}"),
|
|
}
|
|
}
|
|
|
|
/// A version-1 chunk B-tree's shape: every node's (level, entries),
|
|
/// breadth-first from the root.
|
|
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
|
|
let ks = 8 + 8 * ndims;
|
|
let mut out = Vec::new();
|
|
let mut level = vec![root];
|
|
while !level.is_empty() {
|
|
let mut next = Vec::new();
|
|
for a in level {
|
|
let n = &b[a as usize..];
|
|
assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
|
|
let lv = n[5];
|
|
let e = le(&n[6..8]);
|
|
out.push((lv, e));
|
|
if lv > 0 {
|
|
for i in 0..e as usize {
|
|
next.push(le(
|
|
&n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
|
|
));
|
|
}
|
|
}
|
|
}
|
|
level = next;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The shape of dataset `name`'s chunk index, for comparing with the one
|
|
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
|
|
/// nothing to compare (Fixed Array, implicit).
|
|
fn index_shape(path: &Path, name: &str) -> String {
|
|
let b = std::fs::read(path).unwrap();
|
|
match chunk_index(path, name) {
|
|
(_, None, _) => "none".into(),
|
|
(0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
|
|
(5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
|
|
(4, Some(a), _) => {
|
|
let s: Vec<u64> = (0..6)
|
|
.map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
|
|
.collect();
|
|
format!("ea {s:?}")
|
|
}
|
|
(t, Some(_), _) => format!("type {t}"),
|
|
}
|
|
}
|
|
|
|
/// One step of a resize workload.
|
|
#[derive(Debug, Clone)]
|
|
enum Op {
|
|
Resize(Vec<u64>),
|
|
/// A block write; values from `vals`.
|
|
Write(Vec<u64>, Vec<u64>, u64),
|
|
}
|
|
|
|
fn op_vals(seed: u64, n: u64) -> Vec<i32> {
|
|
(0..n)
|
|
.map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
|
|
.collect()
|
|
}
|
|
|
|
/// Random resizes (shrinking and growing any dimension within `max`)
|
|
/// and block writes.
|
|
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
|
|
let mut shape = start.to_vec();
|
|
let mut ops = Vec::new();
|
|
for k in 0..n {
|
|
if k % 3 == 0 {
|
|
shape = (0..shape.len())
|
|
.map(|d| {
|
|
let cap = max[d].min(shape[d] * 2 + 6);
|
|
rng.below(cap + 1)
|
|
})
|
|
.collect();
|
|
ops.push(Op::Resize(shape.clone()));
|
|
} else if shape.iter().all(|&s| s > 0) {
|
|
let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
|
|
let cnt: Vec<u64> = (0..shape.len())
|
|
.map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
|
|
.collect();
|
|
ops.push(Op::Write(st, cnt, rng.next() % 1000));
|
|
}
|
|
}
|
|
ops
|
|
}
|
|
|
|
/// The python statements applying `ops` to dataset `d`.
|
|
fn py_ops(ops: &[Op]) -> String {
|
|
let mut s = String::new();
|
|
for op in ops {
|
|
match op {
|
|
Op::Resize(shape) => {
|
|
s += &format!("\x20 d.resize({shape:?})\n")
|
|
.replace('[', "(")
|
|
.replace(']', ",)");
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let n: u64 = cnt.iter().product();
|
|
let sl: Vec<String> = st
|
|
.iter()
|
|
.zip(cnt)
|
|
.map(|(a, c)| format!("{a}:{}", a + c))
|
|
.collect();
|
|
s += &format!(
|
|
"\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
|
|
.astype('<i4').reshape({cnt:?})\n",
|
|
sl.join(", ")
|
|
)
|
|
.replace("reshape([", "reshape((")
|
|
.replace("])\n", ",))\n");
|
|
}
|
|
}
|
|
}
|
|
s
|
|
}
|
|
|
|
/// The same resize workload — shrinking and growing along every
|
|
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
|
|
/// editor on dataset `x` created by `create` (python, `f` open): the same
|
|
/// values as a model where elements that come back after a shrink read as
|
|
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
|
|
/// check` accept, and h5py can go on.
|
|
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
|
|
let mk = |p: &Path| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
{create}",
|
|
p = p.to_str().unwrap()
|
|
))
|
|
};
|
|
mk(&a);
|
|
mk(&b);
|
|
let (start, max) = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
|
|
p = b.to_str().unwrap()
|
|
))
|
|
.split_once("] [")
|
|
.map(|(x, y)| {
|
|
let p = |s: &str| -> Vec<u64> {
|
|
s.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
};
|
|
(p(x), p(y))
|
|
})
|
|
.unwrap();
|
|
let initial = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let mut m = Model {
|
|
shape: start.clone(),
|
|
data: initial
|
|
.split_whitespace()
|
|
.map(|v| v.parse().unwrap())
|
|
.collect(),
|
|
};
|
|
let mut rng = Rng(seed);
|
|
let ops = random_resize_ops(&mut rng, &start, &max, 60);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f['x']\n{o}",
|
|
p = a.to_str().unwrap(),
|
|
o = py_ops(&ops)
|
|
));
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, op) in ops.iter().enumerate() {
|
|
match op {
|
|
Op::Resize(s) => {
|
|
ed.resize("x", s)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.resize(s, fill);
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let vals = op_vals(*seed, cnt.iter().product());
|
|
ed.write_values("x", &block(st, cnt), &vals)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.write_block(st, cnt, &vals);
|
|
}
|
|
}
|
|
}
|
|
drop(ed);
|
|
if tag == "implicit" {
|
|
// An implicit index cannot drop chunks: libhdf5 leaves the data of
|
|
// chunks wholly outside a shrunk extent in place and does not
|
|
// refill those that come back unless they lie past the extent it
|
|
// grows from, so they can show old values. The editor must match
|
|
// libhdf5, not the model.
|
|
let got = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = a.to_str().unwrap()
|
|
));
|
|
m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
|
|
}
|
|
verify(&a, "x", &m);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
assert_eq!(
|
|
index_shape(&b, "x"),
|
|
index_shape(&a, "x"),
|
|
"{tag}: chunk index differs from libhdf5's"
|
|
);
|
|
// libhdf5 goes on: grow back to the start shape and write everything.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 d.resize({s:?})\n\
|
|
\x20 d[...] = 3\n",
|
|
p = b.to_str().unwrap(),
|
|
s = start
|
|
));
|
|
let n: u64 = start.iter().product();
|
|
verify(
|
|
&b,
|
|
"x",
|
|
&Model {
|
|
shape: start.clone(),
|
|
data: vec![3; n as usize],
|
|
},
|
|
);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
|
|
/// dimension, mixed with growth and writes, on every chunk index: the
|
|
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
|
|
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
|
|
/// and implicit (early allocation), unfiltered and deflated.
|
|
#[test]
|
|
fn shrink_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let kinds: &[(&str, &str, i32)] = &[
|
|
(
|
|
"ea",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
|
|
maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
|
|
-1,
|
|
),
|
|
(
|
|
"bt2",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
|
|
maxshape=(None, None), chunks=(3, 2){z})\n",
|
|
0,
|
|
),
|
|
(
|
|
"fa",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
|
|
maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
|
|
5,
|
|
),
|
|
(
|
|
"1d",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
|
|
maxshape=(None,), chunks=(4,){z})\n",
|
|
0,
|
|
),
|
|
];
|
|
let mut seed = 100;
|
|
for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
for (kind, create, fill) in kinds {
|
|
for (zi, z) in ["", ", compression='gzip', shuffle=True"]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
seed += 1;
|
|
let tag = format!("{kind}_{li}_{zi}");
|
|
shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
|
|
}
|
|
}
|
|
}
|
|
// Implicit index: early allocation, no filters, fixed maximum shape.
|
|
shrink_workload(
|
|
"implicit",
|
|
"'latest'",
|
|
"\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
7,
|
|
false,
|
|
99,
|
|
);
|
|
// Early allocation with unlimited dimensions (Extensible Array,
|
|
// version-2 B-tree; version-1 B-tree under `earliest`), filtered and
|
|
// not: growth allocates and fills every new chunk, as libhdf5 does.
|
|
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
for (j, (max, z)) in [
|
|
("(None, 9)", ""),
|
|
("(None, None)", ""),
|
|
("(None, 9)", ", compression='gzip'"),
|
|
("(None, None)", ", compression='gzip'"),
|
|
]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
shrink_workload(
|
|
&format!("early_{i}_{j}"),
|
|
lv,
|
|
&format!(
|
|
"\x20 f.create_dataset('x', data=np.arange(72, dtype='<i4').reshape(8, 9), \
|
|
maxshape={max}, chunks=(3, 4), fillvalue=-4{z})\n\
|
|
\x20 dcpl = f['x'].id.get_create_plist()\n\
|
|
\x20 del f['x']\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, \
|
|
h5s.create_simple((8, 9), tuple(h5s.UNLIMITED if m is None else m for m in {max})), \
|
|
dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n"
|
|
),
|
|
-4,
|
|
*dump,
|
|
200 + (i * 4 + j) as u64,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An implicit chunk index (early allocation, no filters, fixed maximum
|
|
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
|
|
/// by its position in the maximum chunk grid. The reader once used the
|
|
/// current grid, and returned other chunks' values for every row after the
|
|
/// first.
|
|
#[test]
|
|
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("implicit_max.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
|
|
verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
|
|
// The editor places new values by the same grid.
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
|
|
.unwrap();
|
|
drop(ed);
|
|
let mut m = Model::new(&[8, 9], |i| i as i32);
|
|
m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
#[test]
|
|
fn btree2_chunk_index_growth_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
|
|
bt2_growth(lv, *dump, "", &format!("plain{i}"));
|
|
bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
|
|
}
|
|
}
|
|
|
|
/// Random writes into a dataset with two unlimited dimensions (chunks
|
|
/// created in any order) and growth, with a small node size so the tree
|
|
/// gets deep, against a model; then h5py continues.
|
|
#[test]
|
|
fn btree2_random_chunk_order() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("bt2_random.h5");
|
|
// A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
|
|
// hundred chunks).
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
|
|
dtype='<i4', fillvalue=-7)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = Model::new(&[40, 40], |_| -7);
|
|
let mut rng = Rng(42);
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for step in 0..400 {
|
|
if step % 50 == 49 {
|
|
let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], -7);
|
|
}
|
|
let r0 = rng.below(m.shape[0]);
|
|
let c0 = rng.below(m.shape[1]);
|
|
let cnt = [
|
|
1 + rng.below((m.shape[0] - r0).min(3)),
|
|
1 + rng.below((m.shape[1] - c0).min(3)),
|
|
];
|
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1])
|
|
.map(|_| (rng.next() % 1_000_000) as i32)
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, c0], &cnt, &vals);
|
|
if step % 100 == 99 {
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
ed = FileEditor::open(&path).unwrap();
|
|
}
|
|
}
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
let shape = chunk_bt2_shape(&path);
|
|
assert!(shape.0 >= 1, "{shape:?}");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'][...] = 5\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let n = m.data.len();
|
|
m.data = vec![5; n];
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
// ---- dense attributes ----
|
|
|
|
/// An attribute value both sides can write with the same encoded size: a
|
|
/// 1-D int64 array or a fixed-length string.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
enum AV {
|
|
Ints(Vec<i64>),
|
|
Str(String),
|
|
}
|
|
|
|
impl AV {
|
|
fn value(&self) -> clawhdf5::AttrValue {
|
|
match self {
|
|
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
|
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
|
}
|
|
}
|
|
|
|
fn py(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
|
AV::Str(s) => format!("np.bytes_({s:?})"),
|
|
}
|
|
}
|
|
|
|
fn py_expect(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("{v:?}"),
|
|
AV::Str(s) => format!("{s:?}"),
|
|
}
|
|
}
|
|
|
|
fn make(i: u64, salt: u64) -> Self {
|
|
match i % 11 {
|
|
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
|
1 | 4 | 8 => AV::Str(
|
|
(0..10 + (i * 13 + salt) % 300)
|
|
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
|
.collect(),
|
|
),
|
|
_ => AV::Ints(
|
|
(0..1 + (i + salt) % 9)
|
|
.map(|k| (k * 31 + salt) as i64)
|
|
.collect(),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An object's dense attribute storage as libhdf5 would compare it: the
|
|
/// heap's statistics and root shape, its free sections (heap offsets and
|
|
/// sizes), and the shapes of the name and creation-order index B-trees.
|
|
fn dense_info(path: &Path, obj: &str) -> String {
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
assert_eq!((os, ls), (8, 8));
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let Some(m) = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
|
else {
|
|
return "no attribute info".into();
|
|
};
|
|
let d = &m.data;
|
|
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
|
let heap = le(&d[p..p + 8]);
|
|
p += 8;
|
|
let name = le(&d[p..p + 8]);
|
|
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
|
if heap == u64::MAX {
|
|
return "compact".into();
|
|
}
|
|
let b = std::fs::read(path).unwrap();
|
|
let h = &b[heap as usize..];
|
|
assert_eq!(&h[0..4], b"FRHP");
|
|
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
|
// size, huge objs (8-byte fields from offset 14).
|
|
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
|
let fs = stat(3);
|
|
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
|
// index, start rows, root address.
|
|
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
|
let mut out = format!(
|
|
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
|
stat(0),
|
|
stat(2),
|
|
stat(4),
|
|
stat(5),
|
|
stat(6),
|
|
stat(7),
|
|
stat(8),
|
|
stat(9),
|
|
le(&h[rows_at..rows_at + 2])
|
|
);
|
|
if fs != u64::MAX {
|
|
let s = &b[fs as usize..];
|
|
assert_eq!(&s[0..4], b"FSHD");
|
|
let tot = le(&s[6..14]);
|
|
let n = le(&s[14..22]);
|
|
// counts (4 x 8), 4 x u16, max section size, section info address.
|
|
let at = 6 + 32 + 8 + 8;
|
|
let sect_addr = le(&s[at..at + 8]);
|
|
let sect_size = le(&s[at + 8..at + 16]);
|
|
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
|
// Sections after the prefix (signature, version, header address);
|
|
// trailing zero padding and the checksum are left out.
|
|
let body = &ss[13..ss.len() - 4];
|
|
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
|
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
|
} else {
|
|
out += " no-fs";
|
|
}
|
|
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
|
if let Some(o) = order {
|
|
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
|
}
|
|
out
|
|
}
|
|
|
|
fn hex(b: &[u8]) -> String {
|
|
b.iter().map(|x| format!("{x:02x}")).collect()
|
|
}
|
|
|
|
/// h5py and our reader both see `want` on each object (other attributes
|
|
/// may exist; these must have these values), and h5py's attribute count
|
|
/// agrees with libhdf5's object info.
|
|
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = if o == "d" {
|
|
f.dataset(o).unwrap().attrs().unwrap()
|
|
} else {
|
|
f.group(o).unwrap().attrs().unwrap()
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
|
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
|
fn py_r_plus(path: &Path, lines: &[String]) {
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
|
p = path.to_str().unwrap(),
|
|
l = lines.concat()
|
|
);
|
|
let sp = path.with_extension("ops.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
|
}
|
|
|
|
type AttrOp = (String, String, AV);
|
|
|
|
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
|
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
|
want.push((o.into(), n.into(), v));
|
|
}
|
|
|
|
/// The same attribute workload by libhdf5 and by the editor on objects
|
|
/// with compact attributes that move to dense storage (a plain group, a
|
|
/// group tracking and indexing creation order, a dataset): 40 new
|
|
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
|
/// same-size rewrites — the heaps, their free space and both index B-trees
|
|
/// must come out as libhdf5 makes them; then replacements of another size,
|
|
/// then h5py adds, deletes and rewrites attributes.
|
|
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
|
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
|
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
|
\x20 for o in objs:\n\
|
|
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
let objs = ["g", "t", "d"];
|
|
let mut want: Vec<AttrOp> = Vec::new();
|
|
for o in objs {
|
|
for i in 0..3i64 {
|
|
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
|
}
|
|
}
|
|
// Phase 1: new attributes.
|
|
let mut ops: Vec<AttrOp> = Vec::new();
|
|
for i in 0..40u64 {
|
|
for (k, o) in objs.iter().enumerate() {
|
|
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
|
}
|
|
}
|
|
let lines: Vec<String> = ops
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops {
|
|
ed.set_attr(o, n, &v.value())
|
|
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
check_attr_values(&a, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
|
);
|
|
}
|
|
// Phase 2: same-size rewrites (H5Awrite in place).
|
|
let ops2: Vec<AttrOp> = ops
|
|
.iter()
|
|
.step_by(4)
|
|
.map(|(o, n, v)| {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
|
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
|
};
|
|
(o.clone(), n.clone(), nv)
|
|
})
|
|
.collect();
|
|
let lines: Vec<String> = ops2
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops2 {
|
|
ed.set_attr(o, n, &v.value()).unwrap();
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
|
);
|
|
}
|
|
// Phase 3: replacements of another size (values only: h5py replaces
|
|
// through a temporary attribute and a rename).
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
|
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
|
};
|
|
let before = std::fs::read(&b).unwrap();
|
|
match ed.set_attr(o, n, &nv.value()) {
|
|
Ok(()) => set_want(&mut want, o, n, nv),
|
|
Err(Error::Unsupported(msg)) => {
|
|
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
|
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
|
}
|
|
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
// libhdf5 goes on: adds, deletes and rewrites.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['g', 't', 'd']:\n\
|
|
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
|
\x20 del f[o].attrs['n3']\n\
|
|
\x20 del f[o].attrs['c1']\n\
|
|
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attributes_match_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
dense_workload(lv, *dump, &format!("{i}"));
|
|
}
|
|
}
|
|
|
|
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
|
/// layout, no free-space manager), with and without tracked creation
|
|
/// order: attributes added (moving a dataset's attributes to dense storage
|
|
/// too), rewritten and replaced, then h5py goes on.
|
|
#[test]
|
|
fn dense_attributes_on_clawhdf5_files() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for track in [false, true] {
|
|
let dir = tmpdir();
|
|
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
|
let mut b = clawhdf5::FileBuilder::new();
|
|
b.track_order(track);
|
|
for i in 0..12i64 {
|
|
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
|
}
|
|
b.create_dataset("d")
|
|
.with_i32_data(&[1, 2, 3])
|
|
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
|
b.write(&path).unwrap();
|
|
let mut want: Vec<AttrOp> = (0..12i64)
|
|
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
|
.collect();
|
|
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for i in 0..30u64 {
|
|
for (k, o) in ["/", "d"].iter().enumerate() {
|
|
// Strings up to 300 bytes and of 5000 (huge objects), and
|
|
// every tenth one of 1-4 KiB (a heap block larger than the
|
|
// next: blocks skipped).
|
|
let v = match AV::make(i, k as u64 + 7) {
|
|
AV::Str(s) if i % 10 == 3 => AV::Str(s.repeat(3000 / s.len().max(1) + 1)),
|
|
v => v,
|
|
};
|
|
ed.set_attr(o, &format!("n{i}"), &v.value())
|
|
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
|
set_want(&mut want, o, &format!("n{i}"), v);
|
|
}
|
|
}
|
|
// Rewrites in place, then replacements of another size.
|
|
for i in (0..12i64).step_by(3) {
|
|
let v = AV::Ints(vec![-i; 3]);
|
|
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
|
set_want(&mut want, "/", &format!("r{i}"), v);
|
|
}
|
|
for i in (1..30u64).step_by(7) {
|
|
let v = AV::Str(format!("replaced {i}"));
|
|
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
|
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
|
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
|
Err(e) => panic!("{e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&path, true);
|
|
let want_py: Vec<AttrOp> = want
|
|
.iter()
|
|
.map(|(o, n, v)| {
|
|
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
|
(o, n.clone(), v.clone())
|
|
})
|
|
.collect();
|
|
check_root_and_attrs(&path, &want_py);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['/', 'd']:\n\
|
|
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
|
\x20 del f[o].attrs['n2']\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n2");
|
|
check_tools(&path, true);
|
|
check_root_and_attrs(&path, &want);
|
|
}
|
|
}
|
|
|
|
/// `check_attr_values`, with the root group as "/".
|
|
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = match o.as_str() {
|
|
"/" => f.root().attrs().unwrap(),
|
|
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
|
_ => f.group(o).unwrap().attrs().unwrap(),
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n)\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// An attribute that needs a heap block larger than the next one: libhdf5
|
|
/// skips the smaller blocks (`H5HF__hdr_skip_blocks`) and records them as
|
|
/// an indirect free section, and so does the editor — the heap and its
|
|
/// free space come out as libhdf5's. Replacing that attribute, the only
|
|
/// object in its block, with one of another size is still refused (libhdf5
|
|
/// frees the block), and the refusal leaves the file byte-identical.
|
|
#[test]
|
|
fn dense_attribute_refusals_change_nothing() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("dense_refuse.h5");
|
|
let twin = dir.path().join("dense_refuse_h5py.h5");
|
|
for p in [&path, &twin] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
let big = AV::Str("x".repeat(2000));
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.set_attr("g", "big", &big.value()).unwrap();
|
|
drop(ed);
|
|
py_r_plus(
|
|
&twin,
|
|
&[format!("\x20 f['g'].attrs.create('big', {})\n", big.py())],
|
|
);
|
|
assert_eq!(dense_info(&path, "g"), dense_info(&twin, "g"));
|
|
check_tools(&path, true);
|
|
let before = std::fs::read(&path).unwrap();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("y".repeat(1500))));
|
|
drop(ed);
|
|
assert!(
|
|
std::fs::read(&path).unwrap() == before,
|
|
"a refused edit wrote"
|
|
);
|
|
// What libhdf5 does with it instead works on the untouched file.
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['g'].attrs['big'] = 'y' * 1500\n\
|
|
\x20 assert len(f['g'].attrs) == 13\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
check_tools(&path, true);
|
|
}
|
|
|
|
/// Attributes of every size up to the heap's 4 KiB managed limit, in an
|
|
/// order that makes libhdf5 skip heap blocks in every way it does: a first
|
|
/// attribute too large for the heap's starting block (at the move to dense
|
|
/// storage), a block larger than the rest of the current row, a root
|
|
/// indirect block doubled past rows, two skipped ranges merged; later small
|
|
/// attributes go into the skipped blocks (from either end of a skipped
|
|
/// range, and from its middle, which splits it). libhdf5 does the same
|
|
/// operations, one file session each (as the editor re-reads the heap for
|
|
/// each edit); after every phase the heaps, their free sections (single,
|
|
/// row and indirect) and index B-trees must be libhdf5's.
|
|
#[test]
|
|
fn dense_skipped_blocks_match_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (libver, h5dump) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join("skip_h5py.h5");
|
|
let b = dir.path().join("skip_edit.h5");
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
|
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
|
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
|
\x20 for k, o in enumerate(objs):\n\
|
|
\x20 o.attrs.create('c0', np.bytes_('s' * (1200 + 700 * k)))\n\
|
|
\x20 for i in range(1, 8): o.attrs.create(f'c{{i}}', np.array([i], dtype='<i8'))\n\
|
|
\x20 h = f.create_group('h', track_order=True)\n\
|
|
\x20 for i in range(8): h.attrs.create(f'c{{i}}', np.array([i], dtype='<i8'))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
let objs = ["g", "t", "d"];
|
|
let mut want: Vec<AttrOp> = Vec::new();
|
|
for (k, o) in objs.iter().enumerate() {
|
|
want.push((
|
|
o.to_string(),
|
|
"c0".into(),
|
|
AV::Str("s".repeat(1200 + 700 * k)),
|
|
));
|
|
for i in 1..8i64 {
|
|
want.push((o.to_string(), format!("c{i}"), AV::Ints(vec![i])));
|
|
}
|
|
}
|
|
for i in 0..8i64 {
|
|
want.push(("h".into(), format!("c{i}"), AV::Ints(vec![i])));
|
|
}
|
|
// "h" starts with small attributes, so its heap has a root direct
|
|
// block and then a one-row root indirect block when the first
|
|
// large attribute comes: the rest of that row and the rows the
|
|
// doubling adds are skipped as two ranges, which merge; the next
|
|
// attributes take blocks from the start, the end and the middle
|
|
// (splitting it) of the merged range, and use up whole rows.
|
|
let h_phases: [&[usize]; 4] = [
|
|
&[0; 20],
|
|
&[4060, 1500, 0, 3000, 0, 1500, 0, 600],
|
|
&[1000, 0, 700, 3000, 0, 0, 1500, 0, 900, 0, 0, 0, 1800],
|
|
&[0, 800, 0, 0, 0, 2000, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 0, 0],
|
|
];
|
|
// Sizes (string lengths, or 0 for a small int array) in phases.
|
|
let phases: [&[usize]; 4] = [
|
|
&[0, 3000, 0, 700, 3900, 0, 150],
|
|
&[2500, 90, 0, 3500, 1800, 60, 3990, 0],
|
|
&[0, 0, 40, 300, 0, 900, 20, 0, 0, 500, 0, 30, 1000, 0, 0, 200],
|
|
&[0, 250, 0, 40, 600, 0, 0, 1100, 80, 0, 0, 350, 0, 0, 0, 10],
|
|
];
|
|
let mut n = 0;
|
|
let value = |len: usize, n: usize, k: usize| {
|
|
if len == 0 {
|
|
AV::Ints((0..1 + (n + k) as i64 % 7).collect())
|
|
} else {
|
|
AV::Str(
|
|
(0..len + 11 * k)
|
|
.map(|j| (b'a' + ((j + n) % 26) as u8) as char)
|
|
.collect(),
|
|
)
|
|
}
|
|
};
|
|
for (ph, sizes) in phases.iter().enumerate() {
|
|
let mut ops: Vec<AttrOp> = Vec::new();
|
|
for &len in *sizes {
|
|
for (k, o) in objs.iter().enumerate() {
|
|
ops.push((o.to_string(), format!("n{n}"), value(len, n, k)));
|
|
}
|
|
n += 1;
|
|
}
|
|
for &len in h_phases[ph] {
|
|
ops.push(("h".into(), format!("n{n}"), value(len, n, 0)));
|
|
n += 1;
|
|
}
|
|
// libhdf5: one session per operation.
|
|
let lines: Vec<String> = ops
|
|
.iter()
|
|
.map(|(o, nm, v)| {
|
|
format!(
|
|
"with h5py.File({p:?}, 'r+') as f: f[{o:?}].attrs.create({nm:?}, {})\n",
|
|
v.py(),
|
|
p = a.to_str().unwrap()
|
|
)
|
|
})
|
|
.collect();
|
|
let sp = a.with_extension(format!("ops{ph}.py"));
|
|
std::fs::write(&sp, format!("import h5py, numpy as np\n{}", lines.concat())).unwrap();
|
|
let out = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(
|
|
out.status.success(),
|
|
"h5py workload failed:\n{}",
|
|
text(&out)
|
|
);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, nm, v) in &ops {
|
|
ed.set_attr(o, nm, &v.value())
|
|
.unwrap_or_else(|e| panic!("{libver} phase {ph}: set {o}/{nm}: {e}"));
|
|
set_want(&mut want, o, nm, v.clone());
|
|
}
|
|
drop(ed);
|
|
for o in objs.iter().chain(&["h"]) {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{libver}: dense storage of {o} differs from libhdf5's after phase {ph}"
|
|
);
|
|
}
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
// libhdf5 goes on with the editor's file.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['g', 't', 'd', 'h']:\n\
|
|
\x20 for i in range(6): f[o].attrs[f'late{{i}}'] = 'z' * (i * 700 + 5)\n\
|
|
\x20 del f[o].attrs['c1']\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, nm, _)| nm != "c1");
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
}
|
|
|
|
/// Space one edit frees is reused by later edits of the same editor: the
|
|
/// chunks a shrink removes are where the chunks of the following growth
|
|
/// go, so the file does not grow; with a new editor per edit (nothing to
|
|
/// reuse) it does. h5py, h5dump and `h5rs check` read the result, and
|
|
/// h5py goes on.
|
|
#[test]
|
|
fn freed_space_is_reused_within_a_session() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let mut sizes = Vec::new();
|
|
for session in [true, false] {
|
|
let path = dir.path().join(format!("reuse_{session}.h5"));
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 f.create_dataset('x', data=np.zeros(2000, dtype='<i4'), maxshape=(None,), \
|
|
chunks=(100,), compression='gzip')\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut rng = Rng(5);
|
|
let vals: Vec<i32> = (0..2000).map(|_| rng.next() as i32).collect();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &Selection::All, &vals).unwrap();
|
|
drop(ed);
|
|
let len0 = std::fs::metadata(&path).unwrap().len();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.resize("x", &[1000]).unwrap();
|
|
if session {
|
|
assert!(ed.reusable_bytes() > 0);
|
|
} else {
|
|
ed = {
|
|
drop(ed);
|
|
FileEditor::open(&path).unwrap()
|
|
};
|
|
}
|
|
ed.resize("x", &[2000]).unwrap();
|
|
ed.write_values("x", &block(&[1000], &[1000]), &vals[1000..])
|
|
.unwrap();
|
|
if session {
|
|
assert_eq!(ed.reusable_bytes(), 0, "every freed chunk is reused");
|
|
}
|
|
drop(ed);
|
|
let len1 = std::fs::metadata(&path).unwrap().len();
|
|
sizes.push((len0, len1));
|
|
let m = Model {
|
|
shape: vec![2000],
|
|
data: vals.clone(),
|
|
};
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, true);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'].resize((2100,))\n\
|
|
\x20 f['x'][2000:] = 9\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = m;
|
|
m.resize(&[2100], 0);
|
|
m.write_block(&[2000], &[100], &[9; 100]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, true);
|
|
}
|
|
let (reuse, fresh) = (sizes[0], sizes[1]);
|
|
assert_eq!(
|
|
reuse.1, reuse.0,
|
|
"a session reusing freed chunks does not grow the file"
|
|
);
|
|
assert!(fresh.1 > fresh.0, "without reuse the file grows");
|
|
}
|
|
|
|
/// Replacing the last huge attribute (larger than the heap's managed
|
|
/// limit) of an object with a small one deletes the heap's huge-object
|
|
/// B-tree, as libhdf5 does when it closes the heap (`H5HF__huge_term`). A
|
|
/// heap left with an empty huge-object B-tree made read-only libhdf5 fail
|
|
/// to list the attributes ("no write intent on file").
|
|
#[test]
|
|
fn last_huge_attribute_replaced() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let a = dir.path().join("huge_h5py.h5");
|
|
let b = dir.path().join("huge_edit.h5");
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i in range(10): g.attrs.create(f'k{{i}}', np.array([i], dtype='<i8'))\n\
|
|
\x20 g.attrs.create('big', np.bytes_('h' * 6000))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['g'].attrs.create('big', np.array([1, 2], dtype='<i8'))\n",
|
|
p = a.to_str().unwrap()
|
|
));
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
ed.set_attr("g", "big", &clawhdf5::AttrValue::I64Array(vec![1, 2]))
|
|
.unwrap();
|
|
drop(ed);
|
|
check_tools(&b, true);
|
|
let mut want: Vec<AttrOp> = (0..10i64)
|
|
.map(|i| ("g".to_string(), format!("k{i}"), AV::Ints(vec![i])))
|
|
.collect();
|
|
want.push(("g".into(), "big".into(), AV::Ints(vec![1, 2])));
|
|
check_attr_values(&b, &want);
|
|
let info = |p: &Path| {
|
|
let s = dense_info(p, "g");
|
|
s[..s.find(" fs ").or(s.find(" no-fs")).unwrap()].to_string()
|
|
};
|
|
assert_eq!(info(&b), info(&a), "heap after the replacement");
|
|
// A huge attribute again starts the huge-object B-tree over.
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
ed.set_attr("g", "big2", &clawhdf5::AttrValue::String("x".repeat(7000)))
|
|
.unwrap();
|
|
drop(ed);
|
|
want.push(("g".into(), "big2".into(), AV::Str("x".repeat(7000))));
|
|
check_tools(&b, true);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
|
|
/// Datasets clawhdf5 writes — a version-2 B-tree index (two unlimited
|
|
/// dimensions, its own node size and a single leaf sized to its records),
|
|
/// an Extensible Array, a Fixed Array (fixed maximum shape), deflated and
|
|
/// not — resized up and down and written at random, against a model; h5py,
|
|
/// h5dump and `h5rs check` read the result and h5py goes on.
|
|
#[test]
|
|
fn resize_clawhdf5_written_datasets() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("ours_resize.h5");
|
|
let mut b = clawhdf5::FileBuilder::new();
|
|
let grid: Vec<i32> = (0..48).collect();
|
|
let specs: [(&str, [u64; 2], [u64; 2], bool); 4] = [
|
|
("bt2", [6, 8], [u64::MAX, u64::MAX], false),
|
|
("bt2_gz", [6, 8], [u64::MAX, u64::MAX], true),
|
|
("ea", [6, 8], [u64::MAX, 8], true),
|
|
("fa", [6, 8], [20, 12], false),
|
|
];
|
|
for (name, shape, max, gz) in specs {
|
|
let d = b
|
|
.create_dataset(name)
|
|
.with_i32_data(&grid)
|
|
.with_shape(&shape)
|
|
.with_maxshape(&max)
|
|
.with_chunks(&[2, 3]);
|
|
if gz {
|
|
d.with_deflate(4);
|
|
}
|
|
}
|
|
b.write(&path).unwrap();
|
|
assert_eq!(chunk_index(&path, "bt2").0, 5, "version-2 B-tree index");
|
|
assert_eq!(chunk_index(&path, "ea").0, 4, "Extensible Array index");
|
|
let mut models: Vec<Model> = specs
|
|
.iter()
|
|
.map(|_| Model::new(&[6, 8], |i| i as i32))
|
|
.collect();
|
|
let mut rng = Rng(77);
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for step in 0..120 {
|
|
let k = rng.below(4) as usize;
|
|
let (name, _, max, _) = specs[k];
|
|
let m = &mut models[k];
|
|
if step % 4 == 0 {
|
|
let s: Vec<u64> = (0..2)
|
|
.map(|d| rng.below((m.shape[d] * 2 + 4).min(max[d]) + 1))
|
|
.collect();
|
|
ed.resize(name, &s).unwrap();
|
|
m.resize(&s, 0);
|
|
} else if m.shape.iter().all(|&s| s > 0) {
|
|
let st: Vec<u64> = m.shape.iter().map(|&s| rng.below(s)).collect();
|
|
let cnt: Vec<u64> = (0..2)
|
|
.map(|d| 1 + rng.below((m.shape[d] - st[d]).min(5)))
|
|
.collect();
|
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1]).map(|_| rng.next() as i32).collect();
|
|
ed.write_values(name, &block(&st, &cnt), &vals).unwrap();
|
|
m.write_block(&st, &cnt, &vals);
|
|
}
|
|
}
|
|
drop(ed);
|
|
for ((name, ..), m) in specs.iter().zip(&models) {
|
|
verify(&path, name, m);
|
|
}
|
|
check_tools(&path, true);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for n in ['bt2', 'bt2_gz', 'ea', 'fa']:\n\
|
|
\x20 d = f[n]\n\
|
|
\x20 d.resize((6, 8))\n\
|
|
\x20 d[...] = np.arange(48, dtype='<i4').reshape(6, 8) * 2\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
for (name, ..) in specs {
|
|
verify(&path, name, &Model::new(&[6, 8], |i| i as i32 * 2));
|
|
}
|
|
check_tools(&path, true);
|
|
}
|
|
|
|
/// Shrinking a huge, sparse dataset costs time and memory in the chunks
|
|
/// that exist, not in the chunk coordinates cut off. The editor once stored
|
|
/// every coordinate of the cut-off region (about 62 bytes each), so this
|
|
/// 2 x 10^12-coordinate shrink ran out of memory.
|
|
#[test]
|
|
fn shrinking_a_huge_sparse_dataset_is_bounded() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
const N: u64 = 1_000_000_000_000;
|
|
for libver in ["earliest", "v110"] {
|
|
let path = dir.path().join(format!("sparse_{libver}.h5"));
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'w', libver=({libver:?}, 'latest')) as f:\n\
|
|
\x20 d = f.create_dataset('b', shape=(4, {N}), maxshape=(None, None), chunks=(1, 1), dtype='<i4')\n\
|
|
\x20 d[0, 0:5] = [1, 2, 3, 4, 5]\n\
|
|
\x20 d[0, 900000000000] = 6\n\
|
|
\x20 d[1, {N} - 1] = 7\n\
|
|
\x20 d[2, 7] = 8\n\
|
|
\x20 d[3, 500000000000] = 9\n\
|
|
\x20 assert d.id.get_num_chunks() == 9\n",
|
|
p = path.to_str().unwrap(),
|
|
));
|
|
let start = std::time::Instant::now();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.resize("b", &[2, 600_000_000_000]).unwrap();
|
|
drop(ed);
|
|
let took = start.elapsed();
|
|
assert!(
|
|
took < std::time::Duration::from_secs(30),
|
|
"{libver}: shrink took {took:?}"
|
|
);
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'r') as f:\n\
|
|
\x20 d = f['b']\n\
|
|
\x20 assert d.shape == (2, 600000000000), d.shape\n\
|
|
\x20 assert d.id.get_num_chunks() == 5, d.id.get_num_chunks()\n\
|
|
\x20 assert list(d[0, 0:6]) == [1, 2, 3, 4, 5, 0], d[0, 0:6]\n\
|
|
\x20 assert d[1, 599999999999] == 0\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['b']\n\
|
|
\x20 d.resize((4, {N}))\n\
|
|
\x20 assert d[0, 900000000000] == 0 and d[1, {N} - 1] == 0\n\
|
|
\x20 assert d[2, 7] == 0 and d[3, 500000000000] == 0\n",
|
|
p = path.to_str().unwrap(),
|
|
));
|
|
let f = File::open(&path).unwrap();
|
|
let ds = f.dataset("b").unwrap();
|
|
let got = ds.read_selection(&block(&[0, 0], &[2, 6])).unwrap();
|
|
let got: Vec<i32> = got
|
|
.as_chunks::<4>()
|
|
.0
|
|
.iter()
|
|
.map(|c| i32::from_le_bytes(*c))
|
|
.collect();
|
|
assert_eq!(got, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0], "{libver}");
|
|
}
|
|
}
|