osobhandClaude Opus 5.5
930921e8cb
prune_plan stored one Vec<u64> for every chunk coordinate of the region a shrink cuts off, existing or not, so a sparse dataset exhausted memory (about 62 bytes per coordinate; (4, 2e7) with chunks (1, 1) took 2.5 GB, larger extents never finished). It now places each existing chunk in H5D__chunk_prune_by_extent's walk (its pass, then its coordinates) and sorts, which gives the same chunks, order and actions in memory and time proportional to the chunks that exist. A unit test checks the plan against the full walk (kept as the test's reference) for 3000 random extents and chunk subsets. The interop test shrinks a (4, 10^12) dataset with chunks (1, 1) and 9 chunks (v1 and v2 B-tree): 0.56 s and 43 MB peak; the old code aborted on allocation under an 8 GB limit. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1631 lines
59 KiB
Rust
1631 lines
59 KiB
Rust
//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
|
|
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
|
|
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
|
|
//! (and moved there), and space reused within an editing session. Files are
|
|
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
|
|
//! clawhdf5; after each edit h5py must read exactly what a model says,
|
|
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
|
|
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
|
|
//! same operations can be done by libhdf5, the structures it builds (B-tree
|
|
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
|
|
//!
|
|
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
|
|
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::path::Path;
|
|
use std::process::{Command, Output};
|
|
|
|
use clawhdf5::{Error, File, FileEditor, Selection};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn available(cmd: &str, args: &[&str]) -> bool {
|
|
Command::new(cmd)
|
|
.args(args)
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Whether the tools are here; false (skip) or a panic when interop is
|
|
/// required.
|
|
fn tools_ok() -> bool {
|
|
let ok =
|
|
available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
|
|
if !ok {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
|
|
);
|
|
eprintln!("SKIP: h5py/numpy or h5dump not available");
|
|
}
|
|
ok
|
|
}
|
|
|
|
fn py(script: &str) -> String {
|
|
let o = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("run python");
|
|
assert!(
|
|
o.status.success(),
|
|
"python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
String::from_utf8_lossy(&o.stdout).trim().to_string()
|
|
}
|
|
|
|
fn text(o: &Output) -> String {
|
|
format!(
|
|
"{}{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
)
|
|
}
|
|
|
|
/// The file is valid for libhdf5's tools and for `h5rs check --data`.
|
|
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
|
|
fn check_tools(path: &Path, h5dump: bool) {
|
|
let p = path.to_str().unwrap();
|
|
let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
|
.args(["check", "--data", "-q", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
|
|
if h5dump {
|
|
let o = Command::new("h5dump")
|
|
.args(["-o", "/dev/null", p])
|
|
.output()
|
|
.unwrap();
|
|
assert!(
|
|
o.status.success() && o.stderr.is_empty(),
|
|
"h5dump {p}:\n{}",
|
|
text(&o)
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A dataset's expected contents.
|
|
#[derive(Clone, Debug)]
|
|
struct Model {
|
|
shape: Vec<u64>,
|
|
/// Row-major values.
|
|
data: Vec<i32>,
|
|
}
|
|
|
|
impl Model {
|
|
fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
|
|
let n: u64 = shape.iter().product();
|
|
Self {
|
|
shape: shape.to_vec(),
|
|
data: (0..n).map(f).collect(),
|
|
}
|
|
}
|
|
|
|
fn index(&self, c: &[u64]) -> usize {
|
|
c.iter()
|
|
.zip(&self.shape)
|
|
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
|
}
|
|
|
|
/// Change the extent to `shape`: elements inside both keep their
|
|
/// values, new ones are `fill`.
|
|
fn resize(&mut self, shape: &[u64], fill: i32) {
|
|
let old = self.clone();
|
|
*self = Self::new(shape, |_| fill);
|
|
let n: u64 = old.shape.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; old.shape.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = r % old.shape[d];
|
|
r /= old.shape[d];
|
|
}
|
|
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
|
let i = self.index(&c);
|
|
self.data[i] = old.data[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Apply a hyperslab write of `vals` (row-major over the block).
|
|
fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
|
|
let n: u64 = count.iter().product();
|
|
for flat in 0..n {
|
|
let mut c = vec![0u64; count.len()];
|
|
let mut r = flat;
|
|
for d in (0..c.len()).rev() {
|
|
c[d] = start[d] + r % count[d];
|
|
r /= count[d];
|
|
}
|
|
let i = self.index(&c);
|
|
self.data[i] = vals[flat as usize];
|
|
}
|
|
}
|
|
}
|
|
|
|
fn block(start: &[u64], count: &[u64]) -> Selection {
|
|
Selection::Hyperslab {
|
|
start: start.to_vec(),
|
|
stride: vec![1; start.len()],
|
|
count: count.to_vec(),
|
|
block: vec![1; start.len()],
|
|
}
|
|
}
|
|
|
|
/// h5py and our reader both read `m` from dataset `name`.
|
|
fn verify(path: &Path, name: &str, m: &Model) {
|
|
let f = File::open(path).unwrap();
|
|
let ds = f.dataset(name).unwrap();
|
|
assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
|
|
let got = ds.read_i32().unwrap();
|
|
if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
|
|
panic!(
|
|
"our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
|
|
path.display(),
|
|
m.shape,
|
|
got[i],
|
|
m.data[i]
|
|
);
|
|
}
|
|
let exp = path.with_extension("expect");
|
|
let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
|
|
std::fs::write(&exp, bytes).unwrap();
|
|
let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
a = f[{name:?}][()]\n\
|
|
e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
|
|
assert a.shape == e.shape, (a.shape, e.shape)\n\
|
|
bad = np.argwhere(a != e)\n\
|
|
assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
|
|
p = path.to_str().unwrap(),
|
|
e = exp.to_str().unwrap(),
|
|
s = shape.join(",")
|
|
));
|
|
}
|
|
|
|
/// A deterministic pseudo-random sequence (SplitMix64).
|
|
struct Rng(u64);
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
|
let mut z = self.0;
|
|
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
|
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
|
z ^ (z >> 31)
|
|
}
|
|
fn below(&mut self, n: u64) -> u64 {
|
|
self.next() % n
|
|
}
|
|
}
|
|
|
|
fn tmpdir() -> tempfile::TempDir {
|
|
let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
|
|
tempfile::TempDir::new_in(base).unwrap()
|
|
}
|
|
|
|
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
|
match r {
|
|
Err(Error::Unsupported(_)) => {}
|
|
other => panic!("expected Error::Unsupported, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn le(b: &[u8]) -> u64 {
|
|
b.iter()
|
|
.take(8)
|
|
.enumerate()
|
|
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
|
}
|
|
|
|
fn enc_size(v: u64) -> usize {
|
|
((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
|
|
}
|
|
|
|
/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
|
|
/// (8-byte addresses and lengths): its depth, record count, and every
|
|
/// node's (depth, records) breadth-first.
|
|
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let h = &b[hdr..];
|
|
assert_eq!(&h[0..4], b"BTHD");
|
|
let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
|
|
let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
|
|
let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
|
|
let root = le(&h[16..24]);
|
|
let root_n = le(&h[24..26]);
|
|
let total = le(&h[26..34]);
|
|
// Node geometry, as H5B2__hdr_init computes it.
|
|
let leaf_max = (node_size - 10) / rs;
|
|
let nrec_w = enc_size(leaf_max);
|
|
let mut cum = vec![(leaf_max, 0usize)];
|
|
for d in 1..=u64::from(depth) {
|
|
let (below, below_w) = cum[d as usize - 1];
|
|
let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
|
|
let max = (node_size - 10 - ptr) / (rs + ptr);
|
|
let c = (max + 1) * below + max;
|
|
cum.push((c, enc_size(c)));
|
|
}
|
|
let mut out = Vec::new();
|
|
let mut level = vec![(root, root_n)];
|
|
let mut dep = depth;
|
|
loop {
|
|
let mut next = Vec::new();
|
|
for &(addr, n) in &level {
|
|
out.push((dep, n));
|
|
if dep == 0 {
|
|
assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
|
|
continue;
|
|
}
|
|
let node = &b[addr as usize..];
|
|
assert_eq!(&node[0..4], b"BTIN");
|
|
let mut q = 6 + (n * rs) as usize;
|
|
let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
|
|
for _ in 0..=n {
|
|
let a = le(&node[q..q + 8]);
|
|
let c = le(&node[q + 8..q + 8 + nrec_w]);
|
|
q += 8 + nrec_w + all_w;
|
|
next.push((a, c));
|
|
}
|
|
}
|
|
if dep == 0 {
|
|
break;
|
|
}
|
|
dep -= 1;
|
|
level = next;
|
|
}
|
|
(depth, total, out)
|
|
}
|
|
|
|
/// The shape of the file's only chunk B-tree (record type 10 or 11).
|
|
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
|
|
let b = std::fs::read(path).unwrap();
|
|
let hdrs: Vec<usize> = b
|
|
.windows(6)
|
|
.enumerate()
|
|
.filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
|
|
.map(|(i, _)| i)
|
|
.collect();
|
|
assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
|
|
bt2_shape_at(&b, hdrs[0])
|
|
}
|
|
|
|
/// Grow a dataset with two unlimited dimensions in both directions many
|
|
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
|
|
/// chunk enters the index as it is written, in the same order as the
|
|
/// editor's) and with the editor: the same values, and the same B-tree —
|
|
/// depth, record count, and every node's record count — through leaf and
|
|
/// internal splits, redistributions and a depth increase.
|
|
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
|
|
// Extents: both dimensions grow; new columns fall between existing
|
|
// chunks in the index's key order (row first).
|
|
let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
|
|
let val = |r: u64, c: u64| (r * 1000 + c) as i32;
|
|
let create = |p: &Path, write: bool| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
|
|
dtype='<i4'{extra})\n\
|
|
\x20 if {write}:\n\
|
|
\x20 r0, c0 = 0, 0\n\
|
|
\x20 for r, c in {steps:?}:\n\
|
|
\x20 d.resize((r, c))\n\
|
|
\x20 g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
|
|
\x20 if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
|
|
\x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\
|
|
\x20 r0, c0 = r, c\n",
|
|
p = p.to_str().unwrap(),
|
|
write = if write { "True" } else { "False" },
|
|
))
|
|
};
|
|
create(&a, true);
|
|
create(&b, false);
|
|
let mut m = Model::new(&[0, 0], |_| 0);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
let (mut r0, mut c0) = (0u64, 0u64);
|
|
for &(r, c) in &steps {
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], 0);
|
|
if c > c0 && r0 > 0 {
|
|
let vals: Vec<i32> = (0..r0)
|
|
.flat_map(|i| (c0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[0, c0], &[r0, c - c0], &vals);
|
|
}
|
|
if r > r0 {
|
|
let vals: Vec<i32> = (r0..r)
|
|
.flat_map(|i| (0..c).map(move |j| val(i, j)))
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, 0], &[r - r0, c], &vals);
|
|
}
|
|
(r0, c0) = (r, c);
|
|
}
|
|
drop(ed);
|
|
verify(&b, "x", &m);
|
|
verify(&a, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
|
|
assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
|
|
assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
|
|
// libhdf5 goes on growing what the editor built.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 r, c = d.shape\n\
|
|
\x20 d.resize((r + 3, c + 2))\n\
|
|
\x20 d[:, c:] = -1\n\
|
|
\x20 d[r:, :] = -2\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let (r, c) = (m.shape[0], m.shape[1]);
|
|
m.resize(&[r + 3, c + 2], 0);
|
|
m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
|
|
m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Dataset `name`'s layout: its chunk index type (0 for a version-1
|
|
/// B-tree), index address and chunk rank (element-size dimension
|
|
/// included).
|
|
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let m = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap();
|
|
match DataLayout::parse(&m.data, os, ls).unwrap() {
|
|
DataLayout::Chunked {
|
|
chunk_dimensions,
|
|
btree_address,
|
|
chunk_index_type,
|
|
..
|
|
} => (
|
|
chunk_index_type.unwrap_or(0),
|
|
btree_address,
|
|
chunk_dimensions.len(),
|
|
),
|
|
other => panic!("{other:?}"),
|
|
}
|
|
}
|
|
|
|
/// A version-1 chunk B-tree's shape: every node's (level, entries),
|
|
/// breadth-first from the root.
|
|
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
|
|
let ks = 8 + 8 * ndims;
|
|
let mut out = Vec::new();
|
|
let mut level = vec![root];
|
|
while !level.is_empty() {
|
|
let mut next = Vec::new();
|
|
for a in level {
|
|
let n = &b[a as usize..];
|
|
assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
|
|
let lv = n[5];
|
|
let e = le(&n[6..8]);
|
|
out.push((lv, e));
|
|
if lv > 0 {
|
|
for i in 0..e as usize {
|
|
next.push(le(
|
|
&n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
|
|
));
|
|
}
|
|
}
|
|
}
|
|
level = next;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The shape of dataset `name`'s chunk index, for comparing with the one
|
|
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
|
|
/// nothing to compare (Fixed Array, implicit).
|
|
fn index_shape(path: &Path, name: &str) -> String {
|
|
let b = std::fs::read(path).unwrap();
|
|
match chunk_index(path, name) {
|
|
(_, None, _) => "none".into(),
|
|
(0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
|
|
(5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
|
|
(4, Some(a), _) => {
|
|
let s: Vec<u64> = (0..6)
|
|
.map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
|
|
.collect();
|
|
format!("ea {s:?}")
|
|
}
|
|
(t, Some(_), _) => format!("type {t}"),
|
|
}
|
|
}
|
|
|
|
/// One step of a resize workload.
|
|
#[derive(Debug, Clone)]
|
|
enum Op {
|
|
Resize(Vec<u64>),
|
|
/// A block write; values from `vals`.
|
|
Write(Vec<u64>, Vec<u64>, u64),
|
|
}
|
|
|
|
fn op_vals(seed: u64, n: u64) -> Vec<i32> {
|
|
(0..n)
|
|
.map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
|
|
.collect()
|
|
}
|
|
|
|
/// Random resizes (shrinking and growing any dimension within `max`)
|
|
/// and block writes.
|
|
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
|
|
let mut shape = start.to_vec();
|
|
let mut ops = Vec::new();
|
|
for k in 0..n {
|
|
if k % 3 == 0 {
|
|
shape = (0..shape.len())
|
|
.map(|d| {
|
|
let cap = max[d].min(shape[d] * 2 + 6);
|
|
rng.below(cap + 1)
|
|
})
|
|
.collect();
|
|
ops.push(Op::Resize(shape.clone()));
|
|
} else if shape.iter().all(|&s| s > 0) {
|
|
let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
|
|
let cnt: Vec<u64> = (0..shape.len())
|
|
.map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
|
|
.collect();
|
|
ops.push(Op::Write(st, cnt, rng.next() % 1000));
|
|
}
|
|
}
|
|
ops
|
|
}
|
|
|
|
/// The python statements applying `ops` to dataset `d`.
|
|
fn py_ops(ops: &[Op]) -> String {
|
|
let mut s = String::new();
|
|
for op in ops {
|
|
match op {
|
|
Op::Resize(shape) => {
|
|
s += &format!("\x20 d.resize({shape:?})\n")
|
|
.replace('[', "(")
|
|
.replace(']', ",)");
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let n: u64 = cnt.iter().product();
|
|
let sl: Vec<String> = st
|
|
.iter()
|
|
.zip(cnt)
|
|
.map(|(a, c)| format!("{a}:{}", a + c))
|
|
.collect();
|
|
s += &format!(
|
|
"\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
|
|
.astype('<i4').reshape({cnt:?})\n",
|
|
sl.join(", ")
|
|
)
|
|
.replace("reshape([", "reshape((")
|
|
.replace("])\n", ",))\n");
|
|
}
|
|
}
|
|
}
|
|
s
|
|
}
|
|
|
|
/// The same resize workload — shrinking and growing along every
|
|
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
|
|
/// editor on dataset `x` created by `create` (python, `f` open): the same
|
|
/// values as a model where elements that come back after a shrink read as
|
|
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
|
|
/// check` accept, and h5py can go on.
|
|
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
|
|
let mk = |p: &Path| {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
|
{create}",
|
|
p = p.to_str().unwrap()
|
|
))
|
|
};
|
|
mk(&a);
|
|
mk(&b);
|
|
let (start, max) = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
|
|
p = b.to_str().unwrap()
|
|
))
|
|
.split_once("] [")
|
|
.map(|(x, y)| {
|
|
let p = |s: &str| -> Vec<u64> {
|
|
s.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
};
|
|
(p(x), p(y))
|
|
})
|
|
.unwrap();
|
|
let initial = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
let mut m = Model {
|
|
shape: start.clone(),
|
|
data: initial
|
|
.split_whitespace()
|
|
.map(|v| v.parse().unwrap())
|
|
.collect(),
|
|
};
|
|
let mut rng = Rng(seed);
|
|
let ops = random_resize_ops(&mut rng, &start, &max, 60);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
|
|
\x20 d = f['x']\n{o}",
|
|
p = a.to_str().unwrap(),
|
|
o = py_ops(&ops)
|
|
));
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, op) in ops.iter().enumerate() {
|
|
match op {
|
|
Op::Resize(s) => {
|
|
ed.resize("x", s)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.resize(s, fill);
|
|
}
|
|
Op::Write(st, cnt, seed) => {
|
|
let vals = op_vals(*seed, cnt.iter().product());
|
|
ed.write_values("x", &block(st, cnt), &vals)
|
|
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
|
m.write_block(st, cnt, &vals);
|
|
}
|
|
}
|
|
}
|
|
drop(ed);
|
|
if tag == "implicit" {
|
|
// An implicit index cannot drop chunks: libhdf5 leaves the data of
|
|
// chunks wholly outside a shrunk extent in place and does not
|
|
// refill those that come back unless they lie past the extent it
|
|
// grows from, so they can show old values. The editor must match
|
|
// libhdf5, not the model.
|
|
let got = py(&format!(
|
|
"import h5py\n\
|
|
d = h5py.File({p:?}, 'r')['x']\n\
|
|
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
|
p = a.to_str().unwrap()
|
|
));
|
|
m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
|
|
}
|
|
verify(&a, "x", &m);
|
|
verify(&b, "x", &m);
|
|
check_tools(&b, h5dump);
|
|
assert_eq!(
|
|
index_shape(&b, "x"),
|
|
index_shape(&a, "x"),
|
|
"{tag}: chunk index differs from libhdf5's"
|
|
);
|
|
// libhdf5 goes on: grow back to the start shape and write everything.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['x']\n\
|
|
\x20 d.resize({s:?})\n\
|
|
\x20 d[...] = 3\n",
|
|
p = b.to_str().unwrap(),
|
|
s = start
|
|
));
|
|
let n: u64 = start.iter().product();
|
|
verify(
|
|
&b,
|
|
"x",
|
|
&Model {
|
|
shape: start.clone(),
|
|
data: vec![3; n as usize],
|
|
},
|
|
);
|
|
check_tools(&b, h5dump);
|
|
}
|
|
|
|
/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
|
|
/// dimension, mixed with growth and writes, on every chunk index: the
|
|
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
|
|
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
|
|
/// and implicit (early allocation), unfiltered and deflated.
|
|
#[test]
|
|
fn shrink_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let kinds: &[(&str, &str, i32)] = &[
|
|
(
|
|
"ea",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
|
|
maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
|
|
-1,
|
|
),
|
|
(
|
|
"bt2",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
|
|
maxshape=(None, None), chunks=(3, 2){z})\n",
|
|
0,
|
|
),
|
|
(
|
|
"fa",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
|
|
maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
|
|
5,
|
|
),
|
|
(
|
|
"1d",
|
|
"\x20 d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
|
|
maxshape=(None,), chunks=(4,){z})\n",
|
|
0,
|
|
),
|
|
];
|
|
let mut seed = 100;
|
|
for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
for (kind, create, fill) in kinds {
|
|
for (zi, z) in ["", ", compression='gzip', shuffle=True"]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
seed += 1;
|
|
let tag = format!("{kind}_{li}_{zi}");
|
|
shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
|
|
}
|
|
}
|
|
}
|
|
// Implicit index: early allocation, no filters, fixed maximum shape.
|
|
shrink_workload(
|
|
"implicit",
|
|
"'latest'",
|
|
"\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
7,
|
|
false,
|
|
99,
|
|
);
|
|
// Early allocation with unlimited dimensions (Extensible Array,
|
|
// version-2 B-tree; version-1 B-tree under `earliest`), filtered and
|
|
// not: growth allocates and fills every new chunk, as libhdf5 does.
|
|
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
for (j, (max, z)) in [
|
|
("(None, 9)", ""),
|
|
("(None, None)", ""),
|
|
("(None, 9)", ", compression='gzip'"),
|
|
("(None, None)", ", compression='gzip'"),
|
|
]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
shrink_workload(
|
|
&format!("early_{i}_{j}"),
|
|
lv,
|
|
&format!(
|
|
"\x20 f.create_dataset('x', data=np.arange(72, dtype='<i4').reshape(8, 9), \
|
|
maxshape={max}, chunks=(3, 4), fillvalue=-4{z})\n\
|
|
\x20 dcpl = f['x'].id.get_create_plist()\n\
|
|
\x20 del f['x']\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, \
|
|
h5s.create_simple((8, 9), tuple(h5s.UNLIMITED if m is None else m for m in {max})), \
|
|
dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n"
|
|
),
|
|
-4,
|
|
*dump,
|
|
200 + (i * 4 + j) as u64,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An implicit chunk index (early allocation, no filters, fixed maximum
|
|
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
|
|
/// by its position in the maximum chunk grid. The reader once used the
|
|
/// current grid, and returned other chunks' values for every row after the
|
|
/// first.
|
|
#[test]
|
|
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("implicit_max.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
|
\x20 dcpl.set_chunk((3, 4))\n\
|
|
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
|
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
|
|
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
|
|
verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
|
|
// The editor places new values by the same grid.
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
|
|
.unwrap();
|
|
drop(ed);
|
|
let mut m = Model::new(&[8, 9], |i| i as i32);
|
|
m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
#[test]
|
|
fn btree2_chunk_index_growth_matches_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
|
|
bt2_growth(lv, *dump, "", &format!("plain{i}"));
|
|
bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
|
|
}
|
|
}
|
|
|
|
/// Random writes into a dataset with two unlimited dimensions (chunks
|
|
/// created in any order) and growth, with a small node size so the tree
|
|
/// gets deep, against a model; then h5py continues.
|
|
#[test]
|
|
fn btree2_random_chunk_order() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("bt2_random.h5");
|
|
// A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
|
|
// hundred chunks).
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
from h5py import h5p, h5d, h5s, h5t\n\
|
|
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
|
\x20 f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
|
|
dtype='<i4', fillvalue=-7)\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = Model::new(&[40, 40], |_| -7);
|
|
let mut rng = Rng(42);
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for step in 0..400 {
|
|
if step % 50 == 49 {
|
|
let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
|
|
ed.resize("x", &[r, c]).unwrap();
|
|
m.resize(&[r, c], -7);
|
|
}
|
|
let r0 = rng.below(m.shape[0]);
|
|
let c0 = rng.below(m.shape[1]);
|
|
let cnt = [
|
|
1 + rng.below((m.shape[0] - r0).min(3)),
|
|
1 + rng.below((m.shape[1] - c0).min(3)),
|
|
];
|
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1])
|
|
.map(|_| (rng.next() % 1_000_000) as i32)
|
|
.collect();
|
|
ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
|
|
.unwrap();
|
|
m.write_block(&[r0, c0], &cnt, &vals);
|
|
if step % 100 == 99 {
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
ed = FileEditor::open(&path).unwrap();
|
|
}
|
|
}
|
|
drop(ed);
|
|
verify(&path, "x", &m);
|
|
let shape = chunk_bt2_shape(&path);
|
|
assert!(shape.0 >= 1, "{shape:?}");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'][...] = 5\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let n = m.data.len();
|
|
m.data = vec![5; n];
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, false);
|
|
}
|
|
|
|
// ---- dense attributes ----
|
|
|
|
/// An attribute value both sides can write with the same encoded size: a
|
|
/// 1-D int64 array or a fixed-length string.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
enum AV {
|
|
Ints(Vec<i64>),
|
|
Str(String),
|
|
}
|
|
|
|
impl AV {
|
|
fn value(&self) -> clawhdf5::AttrValue {
|
|
match self {
|
|
AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
|
|
AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
|
|
}
|
|
}
|
|
|
|
fn py(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
|
|
AV::Str(s) => format!("np.bytes_({s:?})"),
|
|
}
|
|
}
|
|
|
|
fn py_expect(&self) -> String {
|
|
match self {
|
|
AV::Ints(v) => format!("{v:?}"),
|
|
AV::Str(s) => format!("{s:?}"),
|
|
}
|
|
}
|
|
|
|
fn make(i: u64, salt: u64) -> Self {
|
|
match i % 11 {
|
|
5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
|
|
1 | 4 | 8 => AV::Str(
|
|
(0..10 + (i * 13 + salt) % 300)
|
|
.map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
|
|
.collect(),
|
|
),
|
|
_ => AV::Ints(
|
|
(0..1 + (i + salt) % 9)
|
|
.map(|k| (k * 31 + salt) as i64)
|
|
.collect(),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// An object's dense attribute storage as libhdf5 would compare it: the
|
|
/// heap's statistics and root shape, its free sections (heap offsets and
|
|
/// sizes), and the shapes of the name and creation-order index B-trees.
|
|
fn dense_info(path: &Path, obj: &str) -> String {
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
let f = File::open(path).unwrap();
|
|
let sb = f.superblock();
|
|
let (os, ls) = (sb.offset_size, sb.length_size);
|
|
assert_eq!((os, ls), (8, 8));
|
|
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
|
|
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
|
let Some(m) = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::AttributeInfo)
|
|
else {
|
|
return "no attribute info".into();
|
|
};
|
|
let d = &m.data;
|
|
let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
|
|
let heap = le(&d[p..p + 8]);
|
|
p += 8;
|
|
let name = le(&d[p..p + 8]);
|
|
let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
|
|
if heap == u64::MAX {
|
|
return "compact".into();
|
|
}
|
|
let b = std::fs::read(path).unwrap();
|
|
let h = &b[heap as usize..];
|
|
assert_eq!(&h[0..4], b"FRHP");
|
|
// next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
|
|
// size, huge objs (8-byte fields from offset 14).
|
|
let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
|
|
let fs = stat(3);
|
|
// Root rows: after 12 8-byte fields, width, start, max direct, max
|
|
// index, start rows, root address.
|
|
let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
|
|
let mut out = format!(
|
|
"heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
|
|
stat(0),
|
|
stat(2),
|
|
stat(4),
|
|
stat(5),
|
|
stat(6),
|
|
stat(7),
|
|
stat(8),
|
|
stat(9),
|
|
le(&h[rows_at..rows_at + 2])
|
|
);
|
|
if fs != u64::MAX {
|
|
let s = &b[fs as usize..];
|
|
assert_eq!(&s[0..4], b"FSHD");
|
|
let tot = le(&s[6..14]);
|
|
let n = le(&s[14..22]);
|
|
// counts (4 x 8), 4 x u16, max section size, section info address.
|
|
let at = 6 + 32 + 8 + 8;
|
|
let sect_addr = le(&s[at..at + 8]);
|
|
let sect_size = le(&s[at + 8..at + 16]);
|
|
let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
|
|
// Sections after the prefix (signature, version, header address);
|
|
// trailing zero padding and the checksum are left out.
|
|
let body = &ss[13..ss.len() - 4];
|
|
let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
|
|
out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
|
|
} else {
|
|
out += " no-fs";
|
|
}
|
|
out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
|
|
if let Some(o) = order {
|
|
out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
|
|
}
|
|
out
|
|
}
|
|
|
|
fn hex(b: &[u8]) -> String {
|
|
b.iter().map(|x| format!("{x:02x}")).collect()
|
|
}
|
|
|
|
/// h5py and our reader both see `want` on each object (other attributes
|
|
/// may exist; these must have these values), and h5py's attribute count
|
|
/// agrees with libhdf5's object info.
|
|
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = if o == "d" {
|
|
f.dataset(o).unwrap().attrs().unwrap()
|
|
} else {
|
|
f.group(o).unwrap().attrs().unwrap()
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
|
|
\x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
|
|
fn py_r_plus(path: &Path, lines: &[String]) {
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n{l}",
|
|
p = path.to_str().unwrap(),
|
|
l = lines.concat()
|
|
);
|
|
let sp = path.with_extension("ops.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
|
|
}
|
|
|
|
type AttrOp = (String, String, AV);
|
|
|
|
fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
|
|
want.retain(|(wo, wn, _)| !(wo == o && wn == n));
|
|
want.push((o.into(), n.into(), v));
|
|
}
|
|
|
|
/// The same attribute workload by libhdf5 and by the editor on objects
|
|
/// with compact attributes that move to dense storage (a plain group, a
|
|
/// group tracking and indexing creation order, a dataset): 40 new
|
|
/// attributes each (some larger than the heap's 4 KiB managed limit), then
|
|
/// same-size rewrites — the heaps, their free space and both index B-trees
|
|
/// must come out as libhdf5 makes them; then replacements of another size,
|
|
/// then h5py adds, deletes and rewrites attributes.
|
|
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
|
|
let dir = tmpdir();
|
|
let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
|
|
let b = dir.path().join(format!("dense_{tag}_edit.h5"));
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
|
\x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \
|
|
f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
|
|
\x20 for o in objs:\n\
|
|
\x20 for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
let objs = ["g", "t", "d"];
|
|
let mut want: Vec<AttrOp> = Vec::new();
|
|
for o in objs {
|
|
for i in 0..3i64 {
|
|
want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
|
|
}
|
|
}
|
|
// Phase 1: new attributes.
|
|
let mut ops: Vec<AttrOp> = Vec::new();
|
|
for i in 0..40u64 {
|
|
for (k, o) in objs.iter().enumerate() {
|
|
ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
|
|
}
|
|
}
|
|
let lines: Vec<String> = ops
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops {
|
|
ed.set_attr(o, n, &v.value())
|
|
.unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
check_attr_values(&a, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after insertions"
|
|
);
|
|
}
|
|
// Phase 2: same-size rewrites (H5Awrite in place).
|
|
let ops2: Vec<AttrOp> = ops
|
|
.iter()
|
|
.step_by(4)
|
|
.map(|(o, n, v)| {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
|
|
AV::Str(s) => AV::Str(s.chars().rev().collect()),
|
|
};
|
|
(o.clone(), n.clone(), nv)
|
|
})
|
|
.collect();
|
|
let lines: Vec<String> = ops2
|
|
.iter()
|
|
.map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
|
|
.collect();
|
|
py_r_plus(&a, &lines);
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (o, n, v) in &ops2 {
|
|
ed.set_attr(o, n, &v.value()).unwrap();
|
|
set_want(&mut want, o, n, v.clone());
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
for o in objs {
|
|
assert_eq!(
|
|
dense_info(&b, o),
|
|
dense_info(&a, o),
|
|
"{tag}: dense storage of {o} differs from libhdf5's after rewrites"
|
|
);
|
|
}
|
|
// Phase 3: replacements of another size (values only: h5py replaces
|
|
// through a temporary attribute and a rename).
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
|
|
let nv = match v {
|
|
AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
|
|
AV::Str(s) => AV::Str(format!("{s}-{k}")),
|
|
};
|
|
let before = std::fs::read(&b).unwrap();
|
|
match ed.set_attr(o, n, &nv.value()) {
|
|
Ok(()) => set_want(&mut want, o, n, nv),
|
|
Err(Error::Unsupported(msg)) => {
|
|
assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
|
|
assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
|
|
}
|
|
Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
// libhdf5 goes on: adds, deletes and rewrites.
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['g', 't', 'd']:\n\
|
|
\x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
|
|
\x20 del f[o].attrs['n3']\n\
|
|
\x20 del f[o].attrs['c1']\n\
|
|
\x20 f[o].attrs['n7'] = 'rewritten by h5py'\n",
|
|
p = b.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
|
|
check_tools(&b, h5dump);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
|
|
#[test]
|
|
fn dense_attributes_match_libhdf5() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
|
.iter()
|
|
.enumerate()
|
|
{
|
|
dense_workload(lv, *dump, &format!("{i}"));
|
|
}
|
|
}
|
|
|
|
/// Dense attributes in files clawhdf5 writes (its own heap and index
|
|
/// layout, no free-space manager), with and without tracked creation
|
|
/// order: attributes added (moving a dataset's attributes to dense storage
|
|
/// too), rewritten and replaced, then h5py goes on.
|
|
#[test]
|
|
fn dense_attributes_on_clawhdf5_files() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
for track in [false, true] {
|
|
let dir = tmpdir();
|
|
let path = dir.path().join(format!("ours_dense_{track}.h5"));
|
|
let mut b = clawhdf5::FileBuilder::new();
|
|
b.track_order(track);
|
|
for i in 0..12i64 {
|
|
b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
|
|
}
|
|
b.create_dataset("d")
|
|
.with_i32_data(&[1, 2, 3])
|
|
.set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
|
|
b.write(&path).unwrap();
|
|
let mut want: Vec<AttrOp> = (0..12i64)
|
|
.map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
|
|
.collect();
|
|
want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for i in 0..30u64 {
|
|
for (k, o) in ["/", "d"].iter().enumerate() {
|
|
// Small attributes: a larger one needs a heap block bigger than
|
|
// the next (see dense_attribute_refusals_change_nothing).
|
|
let v = match AV::make(i, k as u64 + 7) {
|
|
AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
|
|
v => v,
|
|
};
|
|
ed.set_attr(o, &format!("n{i}"), &v.value())
|
|
.unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
|
|
set_want(&mut want, o, &format!("n{i}"), v);
|
|
}
|
|
}
|
|
// Rewrites in place, then replacements of another size.
|
|
for i in (0..12i64).step_by(3) {
|
|
let v = AV::Ints(vec![-i; 3]);
|
|
ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
|
|
set_want(&mut want, "/", &format!("r{i}"), v);
|
|
}
|
|
for i in (1..30u64).step_by(7) {
|
|
let v = AV::Str(format!("replaced {i}"));
|
|
match ed.set_attr("d", &format!("n{i}"), &v.value()) {
|
|
Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
|
|
Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
|
|
Err(e) => panic!("{e}"),
|
|
}
|
|
}
|
|
drop(ed);
|
|
check_tools(&path, true);
|
|
let want_py: Vec<AttrOp> = want
|
|
.iter()
|
|
.map(|(o, n, v)| {
|
|
let o = if o == "/" { "/".to_string() } else { o.clone() };
|
|
(o, n.clone(), v.clone())
|
|
})
|
|
.collect();
|
|
check_root_and_attrs(&path, &want_py);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for o in ['/', 'd']:\n\
|
|
\x20 f[o].attrs['from_h5py'] = np.arange(3)\n\
|
|
\x20 del f[o].attrs['n2']\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
want.retain(|(_, n, _)| n != "n2");
|
|
check_tools(&path, true);
|
|
check_root_and_attrs(&path, &want);
|
|
}
|
|
}
|
|
|
|
/// `check_attr_values`, with the root group as "/".
|
|
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
|
|
let f = File::open(path).unwrap();
|
|
for (o, n, v) in want {
|
|
let attrs = match o.as_str() {
|
|
"/" => f.root().attrs().unwrap(),
|
|
"d" => f.dataset(o).unwrap().attrs().unwrap(),
|
|
_ => f.group(o).unwrap().attrs().unwrap(),
|
|
};
|
|
let got = attrs
|
|
.get(n.as_str())
|
|
.unwrap_or_else(|| panic!("{o}/{n} missing"));
|
|
match (got, v) {
|
|
(clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
|
|
(clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
|
|
other => panic!("{o}/{n}: {other:?}"),
|
|
}
|
|
}
|
|
let exp: Vec<String> = want
|
|
.iter()
|
|
.map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
|
|
.collect();
|
|
let script = format!(
|
|
"import h5py, numpy as np\n\
|
|
f = h5py.File({p:?}, 'r')\n\
|
|
want = [{w}]\n\
|
|
for o, n, v in want:\n\
|
|
\x20 a = f[o].attrs[n]\n\
|
|
\x20 a = a.decode() if isinstance(a, bytes) else a\n\
|
|
\x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\
|
|
\x20 a = [a] if isinstance(a, int) else a\n\
|
|
\x20 assert a == v, (o, n)\n\
|
|
for o in set(x[0] for x in want):\n\
|
|
\x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
|
|
p = path.to_str().unwrap(),
|
|
w = exp.join(", ")
|
|
);
|
|
let sp = path.with_extension("check.py");
|
|
std::fs::write(&sp, script).unwrap();
|
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
|
}
|
|
|
|
/// What the editor refuses in dense storage — an object larger than the
|
|
/// next heap block (libhdf5 would skip blocks and record their space as
|
|
/// free, which this editor does not do) — is `Error::Unsupported`, and the
|
|
/// file is left byte for byte as it was.
|
|
#[test]
|
|
fn dense_attribute_refusals_change_nothing() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("dense_refuse.h5");
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let before = std::fs::read(&path).unwrap();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
|
|
drop(ed);
|
|
assert!(
|
|
std::fs::read(&path).unwrap() == before,
|
|
"a refused edit wrote"
|
|
);
|
|
// What libhdf5 does with it instead works on the untouched file.
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['g'].attrs['big'] = 'x' * 2000\n\
|
|
\x20 assert len(f['g'].attrs) == 13\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
check_tools(&path, true);
|
|
}
|
|
|
|
/// Space one edit frees is reused by later edits of the same editor: the
|
|
/// chunks a shrink removes are where the chunks of the following growth
|
|
/// go, so the file does not grow; with a new editor per edit (nothing to
|
|
/// reuse) it does. h5py, h5dump and `h5rs check` read the result, and
|
|
/// h5py goes on.
|
|
#[test]
|
|
fn freed_space_is_reused_within_a_session() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let mut sizes = Vec::new();
|
|
for session in [true, false] {
|
|
let path = dir.path().join(format!("reuse_{session}.h5"));
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 f.create_dataset('x', data=np.zeros(2000, dtype='<i4'), maxshape=(None,), \
|
|
chunks=(100,), compression='gzip')\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut rng = Rng(5);
|
|
let vals: Vec<i32> = (0..2000).map(|_| rng.next() as i32).collect();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.write_values("x", &Selection::All, &vals).unwrap();
|
|
drop(ed);
|
|
let len0 = std::fs::metadata(&path).unwrap().len();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.resize("x", &[1000]).unwrap();
|
|
if session {
|
|
assert!(ed.reusable_bytes() > 0);
|
|
} else {
|
|
ed = {
|
|
drop(ed);
|
|
FileEditor::open(&path).unwrap()
|
|
};
|
|
}
|
|
ed.resize("x", &[2000]).unwrap();
|
|
ed.write_values("x", &block(&[1000], &[1000]), &vals[1000..])
|
|
.unwrap();
|
|
if session {
|
|
assert_eq!(ed.reusable_bytes(), 0, "every freed chunk is reused");
|
|
}
|
|
drop(ed);
|
|
let len1 = std::fs::metadata(&path).unwrap().len();
|
|
sizes.push((len0, len1));
|
|
let m = Model {
|
|
shape: vec![2000],
|
|
data: vals.clone(),
|
|
};
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, true);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['x'].resize((2100,))\n\
|
|
\x20 f['x'][2000:] = 9\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
let mut m = m;
|
|
m.resize(&[2100], 0);
|
|
m.write_block(&[2000], &[100], &[9; 100]);
|
|
verify(&path, "x", &m);
|
|
check_tools(&path, true);
|
|
}
|
|
let (reuse, fresh) = (sizes[0], sizes[1]);
|
|
assert_eq!(
|
|
reuse.1, reuse.0,
|
|
"a session reusing freed chunks does not grow the file"
|
|
);
|
|
assert!(fresh.1 > fresh.0, "without reuse the file grows");
|
|
}
|
|
|
|
/// Replacing the last huge attribute (larger than the heap's managed
|
|
/// limit) of an object with a small one deletes the heap's huge-object
|
|
/// B-tree, as libhdf5 does when it closes the heap (`H5HF__huge_term`). A
|
|
/// heap left with an empty huge-object B-tree made read-only libhdf5 fail
|
|
/// to list the attributes ("no write intent on file").
|
|
#[test]
|
|
fn last_huge_attribute_replaced() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let a = dir.path().join("huge_h5py.h5");
|
|
let b = dir.path().join("huge_edit.h5");
|
|
for p in [&a, &b] {
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'w', libver='v110') as f:\n\
|
|
\x20 g = f.create_group('g')\n\
|
|
\x20 for i in range(10): g.attrs.create(f'k{{i}}', np.array([i], dtype='<i8'))\n\
|
|
\x20 g.attrs.create('big', np.bytes_('h' * 6000))\n",
|
|
p = p.to_str().unwrap()
|
|
));
|
|
}
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 f['g'].attrs.create('big', np.array([1, 2], dtype='<i8'))\n",
|
|
p = a.to_str().unwrap()
|
|
));
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
ed.set_attr("g", "big", &clawhdf5::AttrValue::I64Array(vec![1, 2]))
|
|
.unwrap();
|
|
drop(ed);
|
|
check_tools(&b, true);
|
|
let mut want: Vec<AttrOp> = (0..10i64)
|
|
.map(|i| ("g".to_string(), format!("k{i}"), AV::Ints(vec![i])))
|
|
.collect();
|
|
want.push(("g".into(), "big".into(), AV::Ints(vec![1, 2])));
|
|
check_attr_values(&b, &want);
|
|
let info = |p: &Path| {
|
|
let s = dense_info(p, "g");
|
|
s[..s.find(" fs ").or(s.find(" no-fs")).unwrap()].to_string()
|
|
};
|
|
assert_eq!(info(&b), info(&a), "heap after the replacement");
|
|
// A huge attribute again starts the huge-object B-tree over.
|
|
let mut ed = FileEditor::open(&b).unwrap();
|
|
ed.set_attr("g", "big2", &clawhdf5::AttrValue::String("x".repeat(7000)))
|
|
.unwrap();
|
|
drop(ed);
|
|
want.push(("g".into(), "big2".into(), AV::Str("x".repeat(7000))));
|
|
check_tools(&b, true);
|
|
check_attr_values(&b, &want);
|
|
}
|
|
|
|
/// Datasets clawhdf5 writes — a version-2 B-tree index (two unlimited
|
|
/// dimensions, its own node size and a single leaf sized to its records),
|
|
/// an Extensible Array, a Fixed Array (fixed maximum shape), deflated and
|
|
/// not — resized up and down and written at random, against a model; h5py,
|
|
/// h5dump and `h5rs check` read the result and h5py goes on.
|
|
#[test]
|
|
fn resize_clawhdf5_written_datasets() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
let path = dir.path().join("ours_resize.h5");
|
|
let mut b = clawhdf5::FileBuilder::new();
|
|
let grid: Vec<i32> = (0..48).collect();
|
|
let specs: [(&str, [u64; 2], [u64; 2], bool); 4] = [
|
|
("bt2", [6, 8], [u64::MAX, u64::MAX], false),
|
|
("bt2_gz", [6, 8], [u64::MAX, u64::MAX], true),
|
|
("ea", [6, 8], [u64::MAX, 8], true),
|
|
("fa", [6, 8], [20, 12], false),
|
|
];
|
|
for (name, shape, max, gz) in specs {
|
|
let d = b
|
|
.create_dataset(name)
|
|
.with_i32_data(&grid)
|
|
.with_shape(&shape)
|
|
.with_maxshape(&max)
|
|
.with_chunks(&[2, 3]);
|
|
if gz {
|
|
d.with_deflate(4);
|
|
}
|
|
}
|
|
b.write(&path).unwrap();
|
|
assert_eq!(chunk_index(&path, "bt2").0, 5, "version-2 B-tree index");
|
|
assert_eq!(chunk_index(&path, "ea").0, 4, "Extensible Array index");
|
|
let mut models: Vec<Model> = specs
|
|
.iter()
|
|
.map(|_| Model::new(&[6, 8], |i| i as i32))
|
|
.collect();
|
|
let mut rng = Rng(77);
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
for step in 0..120 {
|
|
let k = rng.below(4) as usize;
|
|
let (name, _, max, _) = specs[k];
|
|
let m = &mut models[k];
|
|
if step % 4 == 0 {
|
|
let s: Vec<u64> = (0..2)
|
|
.map(|d| rng.below((m.shape[d] * 2 + 4).min(max[d]) + 1))
|
|
.collect();
|
|
ed.resize(name, &s).unwrap();
|
|
m.resize(&s, 0);
|
|
} else if m.shape.iter().all(|&s| s > 0) {
|
|
let st: Vec<u64> = m.shape.iter().map(|&s| rng.below(s)).collect();
|
|
let cnt: Vec<u64> = (0..2)
|
|
.map(|d| 1 + rng.below((m.shape[d] - st[d]).min(5)))
|
|
.collect();
|
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1]).map(|_| rng.next() as i32).collect();
|
|
ed.write_values(name, &block(&st, &cnt), &vals).unwrap();
|
|
m.write_block(&st, &cnt, &vals);
|
|
}
|
|
}
|
|
drop(ed);
|
|
for ((name, ..), m) in specs.iter().zip(&models) {
|
|
verify(&path, name, m);
|
|
}
|
|
check_tools(&path, true);
|
|
py(&format!(
|
|
"import h5py, numpy as np\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 for n in ['bt2', 'bt2_gz', 'ea', 'fa']:\n\
|
|
\x20 d = f[n]\n\
|
|
\x20 d.resize((6, 8))\n\
|
|
\x20 d[...] = np.arange(48, dtype='<i4').reshape(6, 8) * 2\n",
|
|
p = path.to_str().unwrap()
|
|
));
|
|
for (name, ..) in specs {
|
|
verify(&path, name, &Model::new(&[6, 8], |i| i as i32 * 2));
|
|
}
|
|
check_tools(&path, true);
|
|
}
|
|
|
|
/// Shrinking a huge, sparse dataset costs time and memory in the chunks
|
|
/// that exist, not in the chunk coordinates cut off. The editor once stored
|
|
/// every coordinate of the cut-off region (about 62 bytes each), so this
|
|
/// 2 x 10^12-coordinate shrink ran out of memory.
|
|
#[test]
|
|
fn shrinking_a_huge_sparse_dataset_is_bounded() {
|
|
if !tools_ok() {
|
|
return;
|
|
}
|
|
let dir = tmpdir();
|
|
const N: u64 = 1_000_000_000_000;
|
|
for libver in ["earliest", "v110"] {
|
|
let path = dir.path().join(format!("sparse_{libver}.h5"));
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'w', libver=({libver:?}, 'latest')) as f:\n\
|
|
\x20 d = f.create_dataset('b', shape=(4, {N}), maxshape=(None, None), chunks=(1, 1), dtype='<i4')\n\
|
|
\x20 d[0, 0:5] = [1, 2, 3, 4, 5]\n\
|
|
\x20 d[0, 900000000000] = 6\n\
|
|
\x20 d[1, {N} - 1] = 7\n\
|
|
\x20 d[2, 7] = 8\n\
|
|
\x20 d[3, 500000000000] = 9\n\
|
|
\x20 assert d.id.get_num_chunks() == 9\n",
|
|
p = path.to_str().unwrap(),
|
|
));
|
|
let start = std::time::Instant::now();
|
|
let mut ed = FileEditor::open(&path).unwrap();
|
|
ed.resize("b", &[2, 600_000_000_000]).unwrap();
|
|
drop(ed);
|
|
let took = start.elapsed();
|
|
assert!(
|
|
took < std::time::Duration::from_secs(30),
|
|
"{libver}: shrink took {took:?}"
|
|
);
|
|
py(&format!(
|
|
"import h5py\n\
|
|
with h5py.File({p:?}, 'r') as f:\n\
|
|
\x20 d = f['b']\n\
|
|
\x20 assert d.shape == (2, 600000000000), d.shape\n\
|
|
\x20 assert d.id.get_num_chunks() == 5, d.id.get_num_chunks()\n\
|
|
\x20 assert list(d[0, 0:6]) == [1, 2, 3, 4, 5, 0], d[0, 0:6]\n\
|
|
\x20 assert d[1, 599999999999] == 0\n\
|
|
with h5py.File({p:?}, 'r+') as f:\n\
|
|
\x20 d = f['b']\n\
|
|
\x20 d.resize((4, {N}))\n\
|
|
\x20 assert d[0, 900000000000] == 0 and d[1, {N} - 1] == 0\n\
|
|
\x20 assert d[2, 7] == 0 and d[3, 500000000000] == 0\n",
|
|
p = path.to_str().unwrap(),
|
|
));
|
|
let f = File::open(&path).unwrap();
|
|
let ds = f.dataset("b").unwrap();
|
|
let got = ds.read_selection(&block(&[0, 0], &[2, 6])).unwrap();
|
|
let got: Vec<i32> = got
|
|
.as_chunks::<4>()
|
|
.0
|
|
.iter()
|
|
.map(|c| i32::from_le_bytes(*c))
|
|
.collect();
|
|
assert_eq!(got, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0], "{libver}");
|
|
}
|
|
}
|