edit: version-2 B-tree chunk indexes, shrinking, early allocation
FileEditor can now: - add, move and resize chunks of datasets with two or more unlimited dimensions (version-2 B-tree chunk index, record types 10/11). The new edit/btree2.rs follows libhdf5's H5B2 code: H5B2_update (modify, or insert into a leaf with room, or fall back to H5B2__insert), the preemptive split/redistribute loop with its two retries, split1, split_root (depth growth, node geometry per depth), redistribute2/3, cumulative record counts and the pointer widths H5B2__hdr_init derives, a checksum per node. Removal (H5B2_remove: merge2/3, redistribution, root collapse, the internal-record swap with a leaf) is there too. A dataset without an index yet gets one from the layout message's node size and split/merge percentages. - shrink a chunked dataset along any dimension (resize to a smaller shape), as H5D__set_extent / H5D__chunk_prune_by_extent do: the same chunks visited in the same order; chunks wholly outside the new extent are removed from the index (version-1 B-tree: H5B_remove with its sibling key and link fix-ups and the empty-root case; version-2 B-tree; Fixed/Extensible Array elements reset to the fill element; an implicit index keeps its chunks, as libhdf5 does) and their space noted as free; the part of each partial edge chunk outside the extent is overwritten with the fill value, so elements that come back after a later growth read as fill. - under early allocation, allocate and fill the chunks a growth brings in (H5D__chunk_allocate), which an implicit index needs: libhdf5 refills them, and they may hold the data of chunks pruned earlier. Tests (crates/clawhdf5-tools/tests/edit_coverage_interop.rs): growth in both dimensions of v110/latest files, unfiltered and deflated, gives node-for-node the version-2 B-tree libhdf5 builds (h5py with its chunk cache off, so chunks enter the index in the editor's order), through a depth increase; random chunk order; 60 random shrink/grow/write steps on Extensible Array, version-2 B-tree, Fixed Array, 1-D and implicit datasets (earliest/v110/latest, with and without gzip+shuffle) give the values h5py gets doing the same and the same index shape (version-1 and version-2 B-tree node shapes, Extensible Array statistics); h5py r+ continues on every result; h5dump and h5rs check accept them. edit_interop's version-2 B-tree case now appends instead of expecting a refusal; shrinking is no longer an error in edit_tests. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,832 @@
|
||||
//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
|
||||
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
|
||||
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
|
||||
//! (and moved there), and space reused within an editing session. Files are
|
||||
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
|
||||
//! clawhdf5; after each edit h5py must read exactly what a model says,
|
||||
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
|
||||
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
|
||||
//! same operations can be done by libhdf5, the structures it builds (B-tree
|
||||
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
|
||||
//!
|
||||
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
|
||||
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Output};
|
||||
|
||||
use clawhdf5::{Error, File, FileEditor, Selection};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn available(cmd: &str, args: &[&str]) -> bool {
|
||||
Command::new(cmd)
|
||||
.args(args)
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether the tools are here; false (skip) or a panic when interop is
|
||||
/// required.
|
||||
fn tools_ok() -> bool {
|
||||
let ok =
|
||||
available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
|
||||
if !ok {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
|
||||
);
|
||||
eprintln!("SKIP: h5py/numpy or h5dump not available");
|
||||
}
|
||||
ok
|
||||
}
|
||||
|
||||
fn py(script: &str) -> String {
|
||||
let o = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("run python");
|
||||
assert!(
|
||||
o.status.success(),
|
||||
"python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&o.stdout),
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&o.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
fn text(o: &Output) -> String {
|
||||
format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&o.stdout),
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
)
|
||||
}
|
||||
|
||||
/// The file is valid for libhdf5's tools and for `h5rs check --data`.
|
||||
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
|
||||
fn check_tools(path: &Path, h5dump: bool) {
|
||||
let p = path.to_str().unwrap();
|
||||
let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
||||
.args(["check", "--data", "-q", p])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
|
||||
if h5dump {
|
||||
let o = Command::new("h5dump")
|
||||
.args(["-o", "/dev/null", p])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
o.status.success() && o.stderr.is_empty(),
|
||||
"h5dump {p}:\n{}",
|
||||
text(&o)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A dataset's expected contents.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Model {
|
||||
shape: Vec<u64>,
|
||||
/// Row-major values.
|
||||
data: Vec<i32>,
|
||||
}
|
||||
|
||||
impl Model {
|
||||
fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
|
||||
let n: u64 = shape.iter().product();
|
||||
Self {
|
||||
shape: shape.to_vec(),
|
||||
data: (0..n).map(f).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn index(&self, c: &[u64]) -> usize {
|
||||
c.iter()
|
||||
.zip(&self.shape)
|
||||
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
||||
}
|
||||
|
||||
/// Change the extent to `shape`: elements inside both keep their
|
||||
/// values, new ones are `fill`.
|
||||
fn resize(&mut self, shape: &[u64], fill: i32) {
|
||||
let old = self.clone();
|
||||
*self = Self::new(shape, |_| fill);
|
||||
let n: u64 = old.shape.iter().product();
|
||||
for flat in 0..n {
|
||||
let mut c = vec![0u64; old.shape.len()];
|
||||
let mut r = flat;
|
||||
for d in (0..c.len()).rev() {
|
||||
c[d] = r % old.shape[d];
|
||||
r /= old.shape[d];
|
||||
}
|
||||
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
||||
let i = self.index(&c);
|
||||
self.data[i] = old.data[flat as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a hyperslab write of `vals` (row-major over the block).
|
||||
fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
|
||||
let n: u64 = count.iter().product();
|
||||
for flat in 0..n {
|
||||
let mut c = vec![0u64; count.len()];
|
||||
let mut r = flat;
|
||||
for d in (0..c.len()).rev() {
|
||||
c[d] = start[d] + r % count[d];
|
||||
r /= count[d];
|
||||
}
|
||||
let i = self.index(&c);
|
||||
self.data[i] = vals[flat as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn block(start: &[u64], count: &[u64]) -> Selection {
|
||||
Selection::Hyperslab {
|
||||
start: start.to_vec(),
|
||||
stride: vec![1; start.len()],
|
||||
count: count.to_vec(),
|
||||
block: vec![1; start.len()],
|
||||
}
|
||||
}
|
||||
|
||||
/// h5py and our reader both read `m` from dataset `name`.
|
||||
fn verify(path: &Path, name: &str, m: &Model) {
|
||||
let f = File::open(path).unwrap();
|
||||
let ds = f.dataset(name).unwrap();
|
||||
assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
|
||||
let got = ds.read_i32().unwrap();
|
||||
if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
|
||||
panic!(
|
||||
"our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
|
||||
path.display(),
|
||||
m.shape,
|
||||
got[i],
|
||||
m.data[i]
|
||||
);
|
||||
}
|
||||
let exp = path.with_extension("expect");
|
||||
let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
std::fs::write(&exp, bytes).unwrap();
|
||||
let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
a = f[{name:?}][()]\n\
|
||||
e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
|
||||
assert a.shape == e.shape, (a.shape, e.shape)\n\
|
||||
bad = np.argwhere(a != e)\n\
|
||||
assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
|
||||
p = path.to_str().unwrap(),
|
||||
e = exp.to_str().unwrap(),
|
||||
s = shape.join(",")
|
||||
));
|
||||
}
|
||||
|
||||
/// A deterministic pseudo-random sequence (SplitMix64).
|
||||
struct Rng(u64);
|
||||
impl Rng {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.0;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
fn below(&mut self, n: u64) -> u64 {
|
||||
self.next() % n
|
||||
}
|
||||
}
|
||||
|
||||
fn tmpdir() -> tempfile::TempDir {
|
||||
let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
|
||||
tempfile::TempDir::new_in(base).unwrap()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
||||
match r {
|
||||
Err(Error::Unsupported(_)) => {}
|
||||
other => panic!("expected Error::Unsupported, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn le(b: &[u8]) -> u64 {
|
||||
b.iter()
|
||||
.take(8)
|
||||
.enumerate()
|
||||
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
||||
}
|
||||
|
||||
fn enc_size(v: u64) -> usize {
|
||||
((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
|
||||
}
|
||||
|
||||
/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
|
||||
/// (8-byte addresses and lengths): its depth, record count, and every
|
||||
/// node's (depth, records) breadth-first.
|
||||
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
|
||||
let h = &b[hdr..];
|
||||
assert_eq!(&h[0..4], b"BTHD");
|
||||
let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
|
||||
let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
|
||||
let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
|
||||
let root = le(&h[16..24]);
|
||||
let root_n = le(&h[24..26]);
|
||||
let total = le(&h[26..34]);
|
||||
// Node geometry, as H5B2__hdr_init computes it.
|
||||
let leaf_max = (node_size - 10) / rs;
|
||||
let nrec_w = enc_size(leaf_max);
|
||||
let mut cum = vec![(leaf_max, 0usize)];
|
||||
for d in 1..=u64::from(depth) {
|
||||
let (below, below_w) = cum[d as usize - 1];
|
||||
let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
|
||||
let max = (node_size - 10 - ptr) / (rs + ptr);
|
||||
let c = (max + 1) * below + max;
|
||||
cum.push((c, enc_size(c)));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut level = vec![(root, root_n)];
|
||||
let mut dep = depth;
|
||||
loop {
|
||||
let mut next = Vec::new();
|
||||
for &(addr, n) in &level {
|
||||
out.push((dep, n));
|
||||
if dep == 0 {
|
||||
assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
|
||||
continue;
|
||||
}
|
||||
let node = &b[addr as usize..];
|
||||
assert_eq!(&node[0..4], b"BTIN");
|
||||
let mut q = 6 + (n * rs) as usize;
|
||||
let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
|
||||
for _ in 0..=n {
|
||||
let a = le(&node[q..q + 8]);
|
||||
let c = le(&node[q + 8..q + 8 + nrec_w]);
|
||||
q += 8 + nrec_w + all_w;
|
||||
next.push((a, c));
|
||||
}
|
||||
}
|
||||
if dep == 0 {
|
||||
break;
|
||||
}
|
||||
dep -= 1;
|
||||
level = next;
|
||||
}
|
||||
(depth, total, out)
|
||||
}
|
||||
|
||||
/// The shape of the file's only chunk B-tree (record type 10 or 11).
|
||||
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
|
||||
let b = std::fs::read(path).unwrap();
|
||||
let hdrs: Vec<usize> = b
|
||||
.windows(6)
|
||||
.enumerate()
|
||||
.filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
|
||||
bt2_shape_at(&b, hdrs[0])
|
||||
}
|
||||
|
||||
/// Grow a dataset with two unlimited dimensions in both directions many
|
||||
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
|
||||
/// chunk enters the index as it is written, in the same order as the
|
||||
/// editor's) and with the editor: the same values, and the same B-tree —
|
||||
/// depth, record count, and every node's record count — through leaf and
|
||||
/// internal splits, redistributions and a depth increase.
|
||||
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
|
||||
// Extents: both dimensions grow; new columns fall between existing
|
||||
// chunks in the index's key order (row first).
|
||||
let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
|
||||
let val = |r: u64, c: u64| (r * 1000 + c) as i32;
|
||||
let create = |p: &Path, write: bool| {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
||||
\x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
|
||||
dtype='<i4'{extra})\n\
|
||||
\x20 if {write}:\n\
|
||||
\x20 r0, c0 = 0, 0\n\
|
||||
\x20 for r, c in {steps:?}:\n\
|
||||
\x20 d.resize((r, c))\n\
|
||||
\x20 g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
|
||||
\x20 if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
|
||||
\x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\
|
||||
\x20 r0, c0 = r, c\n",
|
||||
p = p.to_str().unwrap(),
|
||||
write = if write { "True" } else { "False" },
|
||||
))
|
||||
};
|
||||
create(&a, true);
|
||||
create(&b, false);
|
||||
let mut m = Model::new(&[0, 0], |_| 0);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
let (mut r0, mut c0) = (0u64, 0u64);
|
||||
for &(r, c) in &steps {
|
||||
ed.resize("x", &[r, c]).unwrap();
|
||||
m.resize(&[r, c], 0);
|
||||
if c > c0 && r0 > 0 {
|
||||
let vals: Vec<i32> = (0..r0)
|
||||
.flat_map(|i| (c0..c).map(move |j| val(i, j)))
|
||||
.collect();
|
||||
ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[0, c0], &[r0, c - c0], &vals);
|
||||
}
|
||||
if r > r0 {
|
||||
let vals: Vec<i32> = (r0..r)
|
||||
.flat_map(|i| (0..c).map(move |j| val(i, j)))
|
||||
.collect();
|
||||
ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[r0, 0], &[r - r0, c], &vals);
|
||||
}
|
||||
(r0, c0) = (r, c);
|
||||
}
|
||||
drop(ed);
|
||||
verify(&b, "x", &m);
|
||||
verify(&a, "x", &m);
|
||||
check_tools(&b, h5dump);
|
||||
let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
|
||||
assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
|
||||
assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
|
||||
// libhdf5 goes on growing what the editor built.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 d = f['x']\n\
|
||||
\x20 r, c = d.shape\n\
|
||||
\x20 d.resize((r + 3, c + 2))\n\
|
||||
\x20 d[:, c:] = -1\n\
|
||||
\x20 d[r:, :] = -2\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
let (r, c) = (m.shape[0], m.shape[1]);
|
||||
m.resize(&[r + 3, c + 2], 0);
|
||||
m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
|
||||
m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
|
||||
verify(&b, "x", &m);
|
||||
check_tools(&b, h5dump);
|
||||
}
|
||||
|
||||
/// Dataset `name`'s layout: its chunk index type (0 for a version-1
|
||||
/// B-tree), index address and chunk rank (element-size dimension
|
||||
/// included).
|
||||
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
let f = File::open(path).unwrap();
|
||||
let sb = f.superblock();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
|
||||
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
||||
let m = oh
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::DataLayout)
|
||||
.unwrap();
|
||||
match DataLayout::parse(&m.data, os, ls).unwrap() {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
btree_address,
|
||||
chunk_index_type,
|
||||
..
|
||||
} => (
|
||||
chunk_index_type.unwrap_or(0),
|
||||
btree_address,
|
||||
chunk_dimensions.len(),
|
||||
),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A version-1 chunk B-tree's shape: every node's (level, entries),
|
||||
/// breadth-first from the root.
|
||||
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
|
||||
let ks = 8 + 8 * ndims;
|
||||
let mut out = Vec::new();
|
||||
let mut level = vec![root];
|
||||
while !level.is_empty() {
|
||||
let mut next = Vec::new();
|
||||
for a in level {
|
||||
let n = &b[a as usize..];
|
||||
assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
|
||||
let lv = n[5];
|
||||
let e = le(&n[6..8]);
|
||||
out.push((lv, e));
|
||||
if lv > 0 {
|
||||
for i in 0..e as usize {
|
||||
next.push(le(
|
||||
&n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The shape of dataset `name`'s chunk index, for comparing with the one
|
||||
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
|
||||
/// nothing to compare (Fixed Array, implicit).
|
||||
fn index_shape(path: &Path, name: &str) -> String {
|
||||
let b = std::fs::read(path).unwrap();
|
||||
match chunk_index(path, name) {
|
||||
(_, None, _) => "none".into(),
|
||||
(0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
|
||||
(5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
|
||||
(4, Some(a), _) => {
|
||||
let s: Vec<u64> = (0..6)
|
||||
.map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
|
||||
.collect();
|
||||
format!("ea {s:?}")
|
||||
}
|
||||
(t, Some(_), _) => format!("type {t}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// One step of a resize workload.
|
||||
#[derive(Debug, Clone)]
|
||||
enum Op {
|
||||
Resize(Vec<u64>),
|
||||
/// A block write; values from `vals`.
|
||||
Write(Vec<u64>, Vec<u64>, u64),
|
||||
}
|
||||
|
||||
fn op_vals(seed: u64, n: u64) -> Vec<i32> {
|
||||
(0..n)
|
||||
.map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Random resizes (shrinking and growing any dimension within `max`)
|
||||
/// and block writes.
|
||||
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
|
||||
let mut shape = start.to_vec();
|
||||
let mut ops = Vec::new();
|
||||
for k in 0..n {
|
||||
if k % 3 == 0 {
|
||||
shape = (0..shape.len())
|
||||
.map(|d| {
|
||||
let cap = max[d].min(shape[d] * 2 + 6);
|
||||
rng.below(cap + 1)
|
||||
})
|
||||
.collect();
|
||||
ops.push(Op::Resize(shape.clone()));
|
||||
} else if shape.iter().all(|&s| s > 0) {
|
||||
let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
|
||||
let cnt: Vec<u64> = (0..shape.len())
|
||||
.map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
|
||||
.collect();
|
||||
ops.push(Op::Write(st, cnt, rng.next() % 1000));
|
||||
}
|
||||
}
|
||||
ops
|
||||
}
|
||||
|
||||
/// The python statements applying `ops` to dataset `d`.
|
||||
fn py_ops(ops: &[Op]) -> String {
|
||||
let mut s = String::new();
|
||||
for op in ops {
|
||||
match op {
|
||||
Op::Resize(shape) => {
|
||||
s += &format!("\x20 d.resize({shape:?})\n")
|
||||
.replace('[', "(")
|
||||
.replace(']', ",)");
|
||||
}
|
||||
Op::Write(st, cnt, seed) => {
|
||||
let n: u64 = cnt.iter().product();
|
||||
let sl: Vec<String> = st
|
||||
.iter()
|
||||
.zip(cnt)
|
||||
.map(|(a, c)| format!("{a}:{}", a + c))
|
||||
.collect();
|
||||
s += &format!(
|
||||
"\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
|
||||
.astype('<i4').reshape({cnt:?})\n",
|
||||
sl.join(", ")
|
||||
)
|
||||
.replace("reshape([", "reshape((")
|
||||
.replace("])\n", ",))\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// The same resize workload — shrinking and growing along every
|
||||
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
|
||||
/// editor on dataset `x` created by `create` (python, `f` open): the same
|
||||
/// values as a model where elements that come back after a shrink read as
|
||||
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
|
||||
/// check` accept, and h5py can go on.
|
||||
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
|
||||
let mk = |p: &Path| {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
from h5py import h5p, h5d, h5s, h5t\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
||||
{create}",
|
||||
p = p.to_str().unwrap()
|
||||
))
|
||||
};
|
||||
mk(&a);
|
||||
mk(&b);
|
||||
let (start, max) = py(&format!(
|
||||
"import h5py\n\
|
||||
d = h5py.File({p:?}, 'r')['x']\n\
|
||||
print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
|
||||
p = b.to_str().unwrap()
|
||||
))
|
||||
.split_once("] [")
|
||||
.map(|(x, y)| {
|
||||
let p = |s: &str| -> Vec<u64> {
|
||||
s.trim_matches(|c| c == '[' || c == ']')
|
||||
.split(',')
|
||||
.filter(|t| !t.trim().is_empty())
|
||||
.map(|t| t.trim().parse().unwrap())
|
||||
.collect()
|
||||
};
|
||||
(p(x), p(y))
|
||||
})
|
||||
.unwrap();
|
||||
let initial = py(&format!(
|
||||
"import h5py\n\
|
||||
d = h5py.File({p:?}, 'r')['x']\n\
|
||||
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
let mut m = Model {
|
||||
shape: start.clone(),
|
||||
data: initial
|
||||
.split_whitespace()
|
||||
.map(|v| v.parse().unwrap())
|
||||
.collect(),
|
||||
};
|
||||
let mut rng = Rng(seed);
|
||||
let ops = random_resize_ops(&mut rng, &start, &max, 60);
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
|
||||
\x20 d = f['x']\n{o}",
|
||||
p = a.to_str().unwrap(),
|
||||
o = py_ops(&ops)
|
||||
));
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (k, op) in ops.iter().enumerate() {
|
||||
match op {
|
||||
Op::Resize(s) => {
|
||||
ed.resize("x", s)
|
||||
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
||||
m.resize(s, fill);
|
||||
}
|
||||
Op::Write(st, cnt, seed) => {
|
||||
let vals = op_vals(*seed, cnt.iter().product());
|
||||
ed.write_values("x", &block(st, cnt), &vals)
|
||||
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
||||
m.write_block(st, cnt, &vals);
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
if tag == "implicit" {
|
||||
// An implicit index cannot drop chunks: libhdf5 leaves the data of
|
||||
// chunks wholly outside a shrunk extent in place and does not
|
||||
// refill those that come back unless they lie past the extent it
|
||||
// grows from, so they can show old values. The editor must match
|
||||
// libhdf5, not the model.
|
||||
let got = py(&format!(
|
||||
"import h5py\n\
|
||||
d = h5py.File({p:?}, 'r')['x']\n\
|
||||
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
||||
p = a.to_str().unwrap()
|
||||
));
|
||||
m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
|
||||
}
|
||||
verify(&a, "x", &m);
|
||||
verify(&b, "x", &m);
|
||||
check_tools(&b, h5dump);
|
||||
assert_eq!(
|
||||
index_shape(&b, "x"),
|
||||
index_shape(&a, "x"),
|
||||
"{tag}: chunk index differs from libhdf5's"
|
||||
);
|
||||
// libhdf5 goes on: grow back to the start shape and write everything.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 d = f['x']\n\
|
||||
\x20 d.resize({s:?})\n\
|
||||
\x20 d[...] = 3\n",
|
||||
p = b.to_str().unwrap(),
|
||||
s = start
|
||||
));
|
||||
let n: u64 = start.iter().product();
|
||||
verify(
|
||||
&b,
|
||||
"x",
|
||||
&Model {
|
||||
shape: start.clone(),
|
||||
data: vec![3; n as usize],
|
||||
},
|
||||
);
|
||||
check_tools(&b, h5dump);
|
||||
}
|
||||
|
||||
/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
|
||||
/// dimension, mixed with growth and writes, on every chunk index: the
|
||||
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
|
||||
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
|
||||
/// and implicit (early allocation), unfiltered and deflated.
|
||||
#[test]
|
||||
fn shrink_matches_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let kinds: &[(&str, &str, i32)] = &[
|
||||
(
|
||||
"ea",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
|
||||
maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
|
||||
-1,
|
||||
),
|
||||
(
|
||||
"bt2",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
|
||||
maxshape=(None, None), chunks=(3, 2){z})\n",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"fa",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
|
||||
maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
|
||||
5,
|
||||
),
|
||||
(
|
||||
"1d",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
|
||||
maxshape=(None,), chunks=(4,){z})\n",
|
||||
0,
|
||||
),
|
||||
];
|
||||
let mut seed = 100;
|
||||
for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
for (kind, create, fill) in kinds {
|
||||
for (zi, z) in ["", ", compression='gzip', shuffle=True"]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
seed += 1;
|
||||
let tag = format!("{kind}_{li}_{zi}");
|
||||
shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Implicit index: early allocation, no filters, fixed maximum shape.
|
||||
shrink_workload(
|
||||
"implicit",
|
||||
"'latest'",
|
||||
"\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
||||
\x20 dcpl.set_chunk((3, 4))\n\
|
||||
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
||||
\x20 dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
|
||||
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
|
||||
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
||||
7,
|
||||
false,
|
||||
99,
|
||||
);
|
||||
}
|
||||
|
||||
/// An implicit chunk index (early allocation, no filters, fixed maximum
|
||||
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
|
||||
/// by its position in the maximum chunk grid. The reader once used the
|
||||
/// current grid, and returned other chunks' values for every row after the
|
||||
/// first.
|
||||
#[test]
|
||||
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("implicit_max.h5");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
from h5py import h5p, h5d, h5s, h5t\n\
|
||||
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
||||
\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
||||
\x20 dcpl.set_chunk((3, 4))\n\
|
||||
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
||||
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
|
||||
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
|
||||
verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
|
||||
// The editor places new values by the same grid.
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
|
||||
.unwrap();
|
||||
drop(ed);
|
||||
let mut m = Model::new(&[8, 9], |i| i as i32);
|
||||
m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn btree2_chunk_index_growth_matches_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
|
||||
bt2_growth(lv, *dump, "", &format!("plain{i}"));
|
||||
bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Random writes into a dataset with two unlimited dimensions (chunks
|
||||
/// created in any order) and growth, with a small node size so the tree
|
||||
/// gets deep, against a model; then h5py continues.
|
||||
#[test]
|
||||
fn btree2_random_chunk_order() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("bt2_random.h5");
|
||||
// A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
|
||||
// hundred chunks).
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
from h5py import h5p, h5d, h5s, h5t\n\
|
||||
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
||||
\x20 f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
|
||||
dtype='<i4', fillvalue=-7)\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let mut m = Model::new(&[40, 40], |_| -7);
|
||||
let mut rng = Rng(42);
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
for step in 0..400 {
|
||||
if step % 50 == 49 {
|
||||
let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
|
||||
ed.resize("x", &[r, c]).unwrap();
|
||||
m.resize(&[r, c], -7);
|
||||
}
|
||||
let r0 = rng.below(m.shape[0]);
|
||||
let c0 = rng.below(m.shape[1]);
|
||||
let cnt = [
|
||||
1 + rng.below((m.shape[0] - r0).min(3)),
|
||||
1 + rng.below((m.shape[1] - c0).min(3)),
|
||||
];
|
||||
let vals: Vec<i32> = (0..cnt[0] * cnt[1])
|
||||
.map(|_| (rng.next() % 1_000_000) as i32)
|
||||
.collect();
|
||||
ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[r0, c0], &cnt, &vals);
|
||||
if step % 100 == 99 {
|
||||
drop(ed);
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
ed = FileEditor::open(&path).unwrap();
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
verify(&path, "x", &m);
|
||||
let shape = chunk_bt2_shape(&path);
|
||||
assert!(shape.0 >= 1, "{shape:?}");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 f['x'][...] = 5\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let n = m.data.len();
|
||||
m.data = vec![5; n];
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
Reference in New Issue
Block a user