From e9c71e5d2e7c237b7105caf9125d508be75d3aa2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:35:26 -0500 Subject: [PATCH] edit: version-2 B-tree chunk indexes, shrinking, early allocation FileEditor can now: - add, move and resize chunks of datasets with two or more unlimited dimensions (version-2 B-tree chunk index, record types 10/11). The new edit/btree2.rs follows libhdf5's H5B2 code: H5B2_update (modify, or insert into a leaf with room, or fall back to H5B2__insert), the preemptive split/redistribute loop with its two retries, split1, split_root (depth growth, node geometry per depth), redistribute2/3, cumulative record counts and the pointer widths H5B2__hdr_init derives, a checksum per node. Removal (H5B2_remove: merge2/3, redistribution, root collapse, the internal-record swap with a leaf) is there too. A dataset without an index yet gets one from the layout message's node size and split/merge percentages. - shrink a chunked dataset along any dimension (resize to a smaller shape), as H5D__set_extent / H5D__chunk_prune_by_extent do: the same chunks visited in the same order; chunks wholly outside the new extent are removed from the index (version-1 B-tree: H5B_remove with its sibling key and link fix-ups and the empty-root case; version-2 B-tree; Fixed/Extensible Array elements reset to the fill element; an implicit index keeps its chunks, as libhdf5 does) and their space noted as free; the part of each partial edge chunk outside the extent is overwritten with the fill value, so elements that come back after a later growth read as fill. - under early allocation, allocate and fill the chunks a growth brings in (H5D__chunk_allocate), which an implicit index needs: libhdf5 refills them, and they may hold the data of chunks pruned earlier. Tests (crates/clawhdf5-tools/tests/edit_coverage_interop.rs): growth in both dimensions of v110/latest files, unfiltered and deflated, gives node-for-node the version-2 B-tree libhdf5 builds (h5py with its chunk cache off, so chunks enter the index in the editor's order), through a depth increase; random chunk order; 60 random shrink/grow/write steps on Extensible Array, version-2 B-tree, Fixed Array, 1-D and implicit datasets (earliest/v110/latest, with and without gzip+shuffle) give the values h5py gets doing the same and the same index shape (version-1 and version-2 B-tree node shapes, Extensible Array statistics); h5py r+ continues on every result; h5dump and h5rs check accept them. edit_interop's version-2 B-tree case now appends instead of expecting a refusal; shrinking is no longer an error in edit_tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 832 ++++++++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 25 +- crates/clawhdf5/src/edit/btree1.rs | 167 ++- crates/clawhdf5/src/edit/btree2.rs | 1128 +++++++++++++++++ crates/clawhdf5/src/edit/earray.rs | 29 +- crates/clawhdf5/src/edit/farray.rs | 15 +- crates/clawhdf5/src/edit/image.rs | 17 + crates/clawhdf5/src/edit/mod.rs | 582 ++++++++- crates/clawhdf5/tests/edit_tests.rs | 32 +- 9 files changed, 2738 insertions(+), 89 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/edit_coverage_interop.rs create mode 100644 crates/clawhdf5/src/edit/btree2.rs diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs new file mode 100644 index 0000000..f2c27fd --- /dev/null +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -0,0 +1,832 @@ +//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends, +//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or +//! more unlimited dimensions), datasets shrunk, attributes in dense storage +//! (and moved there), and space reused within an editing session. Files are +//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by +//! clawhdf5; after each edit h5py must read exactly what a model says, +//! h5dump must read the file, `h5rs check --data` must find nothing, and our +//! reader must agree; h5py then opens the file `r+` and goes on. Where the +//! same operations can be done by libhdf5, the structures it builds (B-tree +//! shapes, heap and free-space bookkeeping) are compared with the editor's. +//! +//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips +//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::{Command, Output}; + +use clawhdf5::{Error, File, FileEditor, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn available(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the tools are here; false (skip) or a panic when interop is +/// required. +fn tools_ok() -> bool { + let ok = + available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]); + if !ok { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available" + ); + eprintln!("SKIP: h5py/numpy or h5dump not available"); + } + ok +} + +fn py(script: &str) -> String { + let o = Command::new(python()) + .args(["-c", script]) + .output() + .expect("run python"); + assert!( + o.status.success(), + "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8_lossy(&o.stdout).trim().to_string() +} + +fn text(o: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ) +} + +/// The file is valid for libhdf5's tools and for `h5rs check --data`. +/// HDF5 2.0 `latest` files are beyond h5dump 1.14. +fn check_tools(path: &Path, h5dump: bool) { + let p = path.to_str().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + if h5dump { + let o = Command::new("h5dump") + .args(["-o", "/dev/null", p]) + .output() + .unwrap(); + assert!( + o.status.success() && o.stderr.is_empty(), + "h5dump {p}:\n{}", + text(&o) + ); + } +} + +/// A dataset's expected contents. +#[derive(Clone, Debug)] +struct Model { + shape: Vec, + /// Row-major values. + data: Vec, +} + +impl Model { + fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self { + let n: u64 = shape.iter().product(); + Self { + shape: shape.to_vec(), + data: (0..n).map(f).collect(), + } + } + + fn index(&self, c: &[u64]) -> usize { + c.iter() + .zip(&self.shape) + .fold(0u64, |a, (&x, &d)| a * d + x) as usize + } + + /// Change the extent to `shape`: elements inside both keep their + /// values, new ones are `fill`. + fn resize(&mut self, shape: &[u64], fill: i32) { + let old = self.clone(); + *self = Self::new(shape, |_| fill); + let n: u64 = old.shape.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; old.shape.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % old.shape[d]; + r /= old.shape[d]; + } + if c.iter().zip(shape).all(|(x, s)| x < s) { + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } + } + } + + /// Apply a hyperslab write of `vals` (row-major over the block). + fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) { + let n: u64 = count.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; count.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = start[d] + r % count[d]; + r /= count[d]; + } + let i = self.index(&c); + self.data[i] = vals[flat as usize]; + } + } +} + +fn block(start: &[u64], count: &[u64]) -> Selection { + Selection::Hyperslab { + start: start.to_vec(), + stride: vec![1; start.len()], + count: count.to_vec(), + block: vec![1; start.len()], + } +} + +/// h5py and our reader both read `m` from dataset `name`. +fn verify(path: &Path, name: &str, m: &Model) { + let f = File::open(path).unwrap(); + let ds = f.dataset(name).unwrap(); + assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}"); + let got = ds.read_i32().unwrap(); + if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) { + panic!( + "our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}", + path.display(), + m.shape, + got[i], + m.data[i] + ); + } + let exp = path.with_extension("expect"); + let bytes: Vec = m.data.iter().flat_map(|v| v.to_le_bytes()).collect(); + std::fs::write(&exp, bytes).unwrap(); + let shape: Vec = m.shape.iter().map(|d| d.to_string()).collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + a = f[{name:?}][()]\n\ + e = np.fromfile({e:?}, dtype=' u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +fn tmpdir() -> tempfile::TempDir { + let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + tempfile::TempDir::new_in(base).unwrap() +} + +#[allow(dead_code)] +fn unsupported(r: Result) { + match r { + Err(Error::Unsupported(_)) => {} + other => panic!("expected Error::Unsupported, got {other:?}"), + } +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} + +fn enc_size(v: u64) -> usize { + ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize +} + +/// The shape of the version-2 B-tree whose header is at `hdr` in `b` +/// (8-byte addresses and lengths): its depth, record count, and every +/// node's (depth, records) breadth-first. +fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) { + let h = &b[hdr..]; + assert_eq!(&h[0..4], b"BTHD"); + let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap())); + let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap())); + let depth = u16::from_le_bytes(h[12..14].try_into().unwrap()); + let root = le(&h[16..24]); + let root_n = le(&h[24..26]); + let total = le(&h[26..34]); + // Node geometry, as H5B2__hdr_init computes it. + let leaf_max = (node_size - 10) / rs; + let nrec_w = enc_size(leaf_max); + let mut cum = vec![(leaf_max, 0usize)]; + for d in 1..=u64::from(depth) { + let (below, below_w) = cum[d as usize - 1]; + let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 }; + let max = (node_size - 10 - ptr) / (rs + ptr); + let c = (max + 1) * below + max; + cum.push((c, enc_size(c))); + } + let mut out = Vec::new(); + let mut level = vec![(root, root_n)]; + let mut dep = depth; + loop { + let mut next = Vec::new(); + for &(addr, n) in &level { + out.push((dep, n)); + if dep == 0 { + assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF"); + continue; + } + let node = &b[addr as usize..]; + assert_eq!(&node[0..4], b"BTIN"); + let mut q = 6 + (n * rs) as usize; + let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 }; + for _ in 0..=n { + let a = le(&node[q..q + 8]); + let c = le(&node[q + 8..q + 8 + nrec_w]); + q += 8 + nrec_w + all_w; + next.push((a, c)); + } + } + if dep == 0 { + break; + } + dep -= 1; + level = next; + } + (depth, total, out) +} + +/// The shape of the file's only chunk B-tree (record type 10 or 11). +fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) { + let b = std::fs::read(path).unwrap(); + let hdrs: Vec = b + .windows(6) + .enumerate() + .filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11)) + .map(|(i, _)| i) + .collect(); + assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display()); + bt2_shape_at(&b, hdrs[0]) +} + +/// Grow a dataset with two unlimited dimensions in both directions many +/// times, writing the new parts, with libhdf5 (its chunk cache off, so each +/// chunk enters the index as it is written, in the same order as the +/// editor's) and with the editor: the same values, and the same B-tree — +/// depth, record count, and every node's record count — through leaf and +/// internal splits, redistributions and a depth increase. +fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) { + let dir = tmpdir(); + let a = dir.path().join(format!("bt2_{tag}_h5py.h5")); + let b = dir.path().join(format!("bt2_{tag}_edit.h5")); + // Extents: both dimensions grow; new columns fall between existing + // chunks in the index's key order (row first). + let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect(); + let val = |r: u64, c: u64| (r * 1000 + c) as i32; + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\ + \x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \ + dtype=' c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\ + \x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\ + \x20 r0, c0 = r, c\n", + p = p.to_str().unwrap(), + write = if write { "True" } else { "False" }, + )) + }; + create(&a, true); + create(&b, false); + let mut m = Model::new(&[0, 0], |_| 0); + let mut ed = FileEditor::open(&b).unwrap(); + let (mut r0, mut c0) = (0u64, 0u64); + for &(r, c) in &steps { + ed.resize("x", &[r, c]).unwrap(); + m.resize(&[r, c], 0); + if c > c0 && r0 > 0 { + let vals: Vec = (0..r0) + .flat_map(|i| (c0..c).map(move |j| val(i, j))) + .collect(); + ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals) + .unwrap(); + m.write_block(&[0, c0], &[r0, c - c0], &vals); + } + if r > r0 { + let vals: Vec = (r0..r) + .flat_map(|i| (0..c).map(move |j| val(i, j))) + .collect(); + ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals) + .unwrap(); + m.write_block(&[r0, 0], &[r - r0, c], &vals); + } + (r0, c0) = (r, c); + } + drop(ed); + verify(&b, "x", &m); + verify(&a, "x", &m); + check_tools(&b, h5dump); + let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b)); + assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}"); + assert_eq!(sb, sa, "B-tree shape differs from libhdf5's"); + // libhdf5 goes on growing what the editor built. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 r, c = d.shape\n\ + \x20 d.resize((r + 3, c + 2))\n\ + \x20 d[:, c:] = -1\n\ + \x20 d[r:, :] = -2\n", + p = b.to_str().unwrap() + )); + let (r, c) = (m.shape[0], m.shape[1]); + m.resize(&[r + 3, c + 2], 0); + m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]); + m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]); + verify(&b, "x", &m); + check_tools(&b, h5dump); +} + +/// Dataset `name`'s layout: its chunk index type (0 for a version-1 +/// B-tree), index address and chunk rank (element-size dimension +/// included). +fn chunk_index(path: &Path, name: &str) -> (u8, Option, usize) { + use clawhdf5_format::data_layout::DataLayout; + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + let f = File::open(path).unwrap(); + let sb = f.superblock(); + let (os, ls) = (sb.offset_size, sb.length_size); + let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap(); + let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap(); + let m = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + .unwrap(); + match DataLayout::parse(&m.data, os, ls).unwrap() { + DataLayout::Chunked { + chunk_dimensions, + btree_address, + chunk_index_type, + .. + } => ( + chunk_index_type.unwrap_or(0), + btree_address, + chunk_dimensions.len(), + ), + other => panic!("{other:?}"), + } +} + +/// A version-1 chunk B-tree's shape: every node's (level, entries), +/// breadth-first from the root. +fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> { + let ks = 8 + 8 * ndims; + let mut out = Vec::new(); + let mut level = vec![root]; + while !level.is_empty() { + let mut next = Vec::new(); + for a in level { + let n = &b[a as usize..]; + assert_eq!(&n[0..5], b"TREE\x01", "node at {a}"); + let lv = n[5]; + let e = le(&n[6..8]); + out.push((lv, e)); + if lv > 0 { + for i in 0..e as usize { + next.push(le( + &n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8] + )); + } + } + } + level = next; + } + out +} + +/// The shape of dataset `name`'s chunk index, for comparing with the one +/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or +/// nothing to compare (Fixed Array, implicit). +fn index_shape(path: &Path, name: &str) -> String { + let b = std::fs::read(path).unwrap(); + match chunk_index(path, name) { + (_, None, _) => "none".into(), + (0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)), + (5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)), + (4, Some(a), _) => { + let s: Vec = (0..6) + .map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k])) + .collect(); + format!("ea {s:?}") + } + (t, Some(_), _) => format!("type {t}"), + } +} + +/// One step of a resize workload. +#[derive(Debug, Clone)] +enum Op { + Resize(Vec), + /// A block write; values from `vals`. + Write(Vec, Vec, u64), +} + +fn op_vals(seed: u64, n: u64) -> Vec { + (0..n) + .map(|i| ((i * 31 + seed * 7919) % 100_000) as i32) + .collect() +} + +/// Random resizes (shrinking and growing any dimension within `max`) +/// and block writes. +fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec { + let mut shape = start.to_vec(); + let mut ops = Vec::new(); + for k in 0..n { + if k % 3 == 0 { + shape = (0..shape.len()) + .map(|d| { + let cap = max[d].min(shape[d] * 2 + 6); + rng.below(cap + 1) + }) + .collect(); + ops.push(Op::Resize(shape.clone())); + } else if shape.iter().all(|&s| s > 0) { + let st: Vec = shape.iter().map(|&s| rng.below(s)).collect(); + let cnt: Vec = (0..shape.len()) + .map(|d| 1 + rng.below((shape[d] - st[d]).min(6))) + .collect(); + ops.push(Op::Write(st, cnt, rng.next() % 1000)); + } + } + ops +} + +/// The python statements applying `ops` to dataset `d`. +fn py_ops(ops: &[Op]) -> String { + let mut s = String::new(); + for op in ops { + match op { + Op::Resize(shape) => { + s += &format!("\x20 d.resize({shape:?})\n") + .replace('[', "(") + .replace(']', ",)"); + } + Op::Write(st, cnt, seed) => { + let n: u64 = cnt.iter().product(); + let sl: Vec = st + .iter() + .zip(cnt) + .map(|(a, c)| format!("{a}:{}", a + c)) + .collect(); + s += &format!( + "\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\ + .astype(' Vec { + s.trim_matches(|c| c == '[' || c == ']') + .split(',') + .filter(|t| !t.trim().is_empty()) + .map(|t| t.trim().parse().unwrap()) + .collect() + }; + (p(x), p(y)) + }) + .unwrap(); + let initial = py(&format!( + "import h5py\n\ + d = h5py.File({p:?}, 'r')['x']\n\ + print(' '.join(str(v) for v in d[()].ravel()))\n", + p = b.to_str().unwrap() + )); + let mut m = Model { + shape: start.clone(), + data: initial + .split_whitespace() + .map(|v| v.parse().unwrap()) + .collect(), + }; + let mut rng = Rng(seed); + let ops = random_resize_ops(&mut rng, &start, &max, 60); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\ + \x20 d = f['x']\n{o}", + p = a.to_str().unwrap(), + o = py_ops(&ops) + )); + let mut ed = FileEditor::open(&b).unwrap(); + for (k, op) in ops.iter().enumerate() { + match op { + Op::Resize(s) => { + ed.resize("x", s) + .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}")); + m.resize(s, fill); + } + Op::Write(st, cnt, seed) => { + let vals = op_vals(*seed, cnt.iter().product()); + ed.write_values("x", &block(st, cnt), &vals) + .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}")); + m.write_block(st, cnt, &vals); + } + } + } + drop(ed); + if tag == "implicit" { + // An implicit index cannot drop chunks: libhdf5 leaves the data of + // chunks wholly outside a shrunk extent in place and does not + // refill those that come back unless they lie past the extent it + // grows from, so they can show old values. The editor must match + // libhdf5, not the model. + let got = py(&format!( + "import h5py\n\ + d = h5py.File({p:?}, 'r')['x']\n\ + print(' '.join(str(v) for v in d[()].ravel()))\n", + p = a.to_str().unwrap() + )); + m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect(); + } + verify(&a, "x", &m); + verify(&b, "x", &m); + check_tools(&b, h5dump); + assert_eq!( + index_shape(&b, "x"), + index_shape(&a, "x"), + "{tag}: chunk index differs from libhdf5's" + ); + // libhdf5 goes on: grow back to the start shape and write everything. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 d.resize({s:?})\n\ + \x20 d[...] = 3\n", + p = b.to_str().unwrap(), + s = start + )); + let n: u64 = start.iter().product(); + verify( + &b, + "x", + &Model { + shape: start.clone(), + data: vec![3; n as usize], + }, + ); + check_tools(&b, h5dump); +} + +/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every +/// dimension, mixed with growth and writes, on every chunk index: the +/// version-1 B-tree (`earliest`), Extensible Array (one unlimited +/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape) +/// and implicit (early allocation), unfiltered and deflated. +#[test] +fn shrink_matches_libhdf5() { + if !tools_ok() { + return; + } + let kinds: &[(&str, &str, i32)] = &[ + ( + "ea", + "\x20 d = f.create_dataset('x', data=np.arange(40, dtype=' = (0..cnt[0] * cnt[1]) + .map(|_| (rng.next() % 1_000_000) as i32) + .collect(); + ed.write_values("x", &block(&[r0, c0], &cnt), &vals) + .unwrap(); + m.write_block(&[r0, c0], &cnt, &vals); + if step % 100 == 99 { + drop(ed); + verify(&path, "x", &m); + check_tools(&path, false); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "x", &m); + let shape = chunk_bt2_shape(&path); + assert!(shape.0 >= 1, "{shape:?}"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['x'][...] = 5\n", + p = path.to_str().unwrap() + )); + let n = m.data.len(); + m.data = vec![5; n]; + verify(&path, "x", &m); + check_tools(&path, false); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 85d4d0c..6a2dbf6 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -752,19 +752,20 @@ fn overwrite_every_layout() { } check_tools(&path, *dump); } - // A version-2 B-tree index can take new chunks only from libhdf5 for - // now: growing works, writing the new chunks is refused and changes - // nothing. + // A version-2 B-tree index takes new chunks too. if *lv != "'earliest'" { let mut ed = FileEditor::open(&path).unwrap(); - ed.resize("bt2", &[8, 6]).unwrap(); - let before = std::fs::read(&path).unwrap(); - unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12])); - assert!( - std::fs::read(&path).unwrap() == before, - "a refused edit changed the file" - ); - models[12].resize(&[8, 6], 0); + ed.resize("bt2", &[8, 7]).unwrap(); + models[12].resize(&[8, 7], 0); + let vals: Vec = (0..23).collect(); + ed.write_values("bt2", &block(&[6, 0], &[2, 7]), &vals[..14]) + .unwrap(); + models[12].write_block(&[6, 0], &[2, 7], &vals[..14]); + ed.write_values("bt2", &block(&[0, 6], &[6, 1]), &vals[14..20]) + .unwrap(); + models[12].write_block(&[0, 6], &[6, 1], &vals[14..20]); + drop(ed); + verify(&path, "bt2", &models[12]); } // libhdf5 goes on modifying what we wrote. py(&format!( @@ -1027,7 +1028,7 @@ fn refused_edits_change_nothing() { let before = std::fs::read(&path).unwrap(); let mut ed = FileEditor::open(&path).unwrap(); unsupported(ed.write_all("s", &[0u8; 32])); - unsupported(ed.resize("x", &[4])); + unsupported(ed.resize("c", &[4])); unsupported( ed.resize("c", &[6, 1]) .map_err(|_| Error::Unsupported(String::new())), diff --git a/crates/clawhdf5/src/edit/btree1.rs b/crates/clawhdf5/src/edit/btree1.rs index 7289c3f..25c626a 100644 --- a/crates/clawhdf5/src/edit/btree1.rs +++ b/crates/clawhdf5/src/edit/btree1.rs @@ -56,6 +56,15 @@ fn bad(why: &str) -> Error { )) } +/// What a removal did below a node (`H5B_ins_t`), with the removed chunk's +/// address and size. +enum Rm { + NotFound, + Noop((u64, u32)), + /// The child is gone: the parent must drop it. + Remove((u64, u32)), +} + enum Ins { Done, /// The node split; the new right sibling and its first key. @@ -144,7 +153,14 @@ impl BTree1 { put_uint(&mut d[8 + osz..], node.right, os); let ks = self.key_size(); let mut p = 8 + 2 * osz; - for (i, k) in node.keys.iter().enumerate() { + // An empty node (a root whose last chunk was removed) stores no + // keys, as libhdf5 writes it. + let nkeys = if node.children.is_empty() { + 0 + } else { + node.keys.len() + }; + for (i, k) in node.keys.iter().take(nkeys).enumerate() { d[p..p + 4].copy_from_slice(&k.size.to_le_bytes()); d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes()); for (j, o) in k.offs.iter().enumerate() { @@ -210,6 +226,18 @@ impl BTree1 { return Err(bad("bad chunk key")); } let root = self.read(img, self.root)?; + if root.children.is_empty() { + // Every chunk was removed (H5B__insert_helper's first + // insertion): the root, a leaf again, takes it. + let right = self.right_key_after(&key); + let node = Node { + level: 0, + keys: vec![key, right], + children: vec![addr], + ..root + }; + return self.write(img, &node); + } if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? { // The root split: move its (left) half to a new node so the root // keeps its address, then make the root the parent of both. @@ -372,6 +400,143 @@ impl BTree1 { cmp(&key.offs, &right.keys[0].offs) != Ordering::Less } + /// Remove the chunk at offsets `offs` (element-size coordinate 0), as + /// `H5B_remove` does for the chunk index (whose critical key is the + /// left one): no rebalancing; a node left without children is deleted + /// and its siblings relinked (the left one takes over its right key), + /// a root left empty becomes an empty leaf. Returns the chunk's address + /// and stored size, or `None` when the tree has no such chunk (nothing + /// changes then). Deleted nodes are freed in `img`. + pub(crate) fn remove( + &mut self, + img: &mut Image<'_>, + offs: &[u64], + ) -> Result, Error> { + if offs.len() != self.ndims { + return Err(bad("bad chunk key")); + } + let mut lt = None; + match self.remove_at(img, self.root, 0, offs, &mut lt, 64)? { + Rm::NotFound => Ok(None), + Rm::Noop(c) | Rm::Remove(c) => Ok(Some(c)), + } + } + + fn remove_at( + &self, + img: &mut Image<'_>, + addr: u64, + level: usize, + offs: &[u64], + lt_out: &mut Option, + depth: u8, + ) -> Result { + if depth == 0 { + return Err(bad("tree too deep")); + } + let mut node = self.read(img, addr)?; + let n = node.children.len(); + // H5D__btree_cmp3 over (keys[i], keys[i + 1]), binary search. + let (mut lo, mut hi, mut idx) = (0usize, n, 0usize); + let mut c = 1i32; + while lo < hi && c != 0 { + idx = (lo + hi) / 2; + c = if cmp(offs, &node.keys[idx + 1].offs) != Ordering::Less { + 1 + } else if cmp(offs, &node.keys[idx].offs) == Ordering::Less { + -1 + } else { + 0 + }; + if c < 0 { + hi = idx; + } else { + lo = idx + 1; + } + } + if c != 0 { + return Ok(Rm::NotFound); + } + let mut lt_changed = None; + let res = if node.level > 0 { + let child = self.read(img, node.children[idx])?; + if usize::from(child.level) + 1 != usize::from(node.level) { + return Err(bad("inconsistent node levels")); + } + self.remove_at( + img, + node.children[idx], + level + 1, + offs, + &mut lt_changed, + depth - 1, + )? + } else { + if node.keys[idx].offs != offs { + return Ok(Rm::NotFound); + } + Rm::Remove((node.children[idx], node.keys[idx].size)) + }; + let chunk = match res { + Rm::NotFound => return Ok(Rm::NotFound), + Rm::Noop(c) | Rm::Remove(c) => c, + }; + let mut dirty = false; + if let Some(k) = lt_changed { + node.keys[idx] = k; + dirty = true; + if idx == 0 { + *lt_out = Some(node.keys[0].clone()); + } + } + let out = Rm::Noop(chunk); + if let Rm::Remove(_) = res { + let undefined = undef(img.os); + if n == 1 { + if level > 0 { + if node.left != undefined { + let mut sib = self.read(img, node.left)?; + let last = sib.children.len(); + sib.keys[last] = node.keys[1].clone(); + sib.right = node.right; + self.write(img, &sib)?; + } + if node.right != undefined { + let mut sib = self.read(img, node.right)?; + sib.left = node.left; + self.write(img, &sib)?; + } + img.free(addr, self.node_size(img.os) as u64); + return Ok(Rm::Remove(chunk)); + } + node.children.clear(); + node.keys.truncate(1); + node.level = 0; + } else if idx == 0 { + node.keys.remove(0); + node.children.remove(0); + *lt_out = Some(node.keys[0].clone()); + } else { + // Right-most or middle child: its left key goes, the next + // key becomes the following child's left key. + node.keys.remove(idx); + node.children.remove(idx); + } + dirty = true; + } + if dirty { + self.write(img, &node)?; + } + // The left sibling's right key follows a changed left key. + if lt_out.is_some() && node.left != undef(img.os) && level > 0 { + let mut sib = self.read(img, node.left)?; + let last = sib.children.len(); + sib.keys[last] = node.keys[0].clone(); + self.write(img, &sib)?; + } + Ok(out) + } + fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) { let n = node.children.len(); if node.level == 0 { diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs new file mode 100644 index 0000000..8ff7205 --- /dev/null +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -0,0 +1,1128 @@ +//! Changing a version-2 B-tree (`BTHD`/`BTIN`/`BTLF`) in place, as libhdf5's +//! `H5B2` code does: the same insertion (`H5B2_update`, falling back to +//! `H5B2__insert` when the leaf is full), node splits and redistributions +//! (`H5B2__split1`, `H5B2__split_root`, `H5B2__redistribute2/3`), and removal +//! with merges and redistributions (`H5B2_remove`, `H5B2__merge2/3`, root +//! collapse, the internal-record swap with a leaf) — so a tree the editor +//! changes has the nodes libhdf5 would have made for the same operations. +//! +//! Split and merge thresholds come from the header's split and merge +//! percentages; each node pointer carries the child's record count and, +//! below the first internal level, the child subtree's total, in the widths +//! libhdf5 derives from the node size (`H5B2__hdr_init`). Every node written +//! gets its checksum; the header's depth, root and counts are updated by +//! [`Bt2::finish`]. +//! +//! Nodes are loaded on first use, changed in memory and written back by +//! [`Bt2::finish`]; nodes a merge empties are dropped and their space handed +//! to the image's free list ([`Image::free`]). + +use std::cmp::Ordering; +use std::collections::{BTreeSet, HashMap}; + +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +/// A comparison of the record being looked for against a stored record +/// (`Less`: the target sorts before the stored record). +pub(crate) type Cmp<'c> = dyn FnMut(&Image<'_>, &[u8]) -> Result + 'c; + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("version-2 B-tree: {why}"), + )) +} + +/// A pointer to a node, as its parent (or the header, for the root) holds +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Ptr { + addr: u64, + nrec: u16, + all: u64, +} + +#[derive(Debug, Clone)] +struct Node { + depth: u16, + recs: Vec>, + /// `recs.len() + 1` children (internal nodes only). + ptrs: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct Level { + max_nrec: u64, + split_nrec: u64, + merge_nrec: u64, + cum_max_nrec: u64, + cum_max_nrec_size: usize, +} + +/// `H5VM_limit_enc_size`: bytes to encode values up to `v`. +fn enc_size(v: u64) -> usize { + ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize +} + +pub(crate) struct Bt2 { + addr: u64, + tree_type: u8, + node_size: u32, + rec_size: usize, + split_pct: u8, + merge_pct: u8, + depth: u16, + root: Ptr, + levels: Vec, + max_nrec_size: usize, + nodes: HashMap, + dirty: BTreeSet, + hdr_dirty: bool, +} + +/// What `update` did below a node. +#[derive(Debug, PartialEq, Eq)] +enum Status { + Modified, + Inserted, + ChildFull, +} + +const MAX_DEPTH: u16 = 32; + +impl Bt2 { + fn header_len(os: u8, ls: u8) -> usize { + 16 + os as usize + 2 + ls as usize + 4 + } + + /// Open the tree whose header is at `addr`. + pub(crate) fn open(img: &Image<'_>, addr: u64) -> Result { + let len = Self::header_len(img.os, img.ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"BTHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let os = img.os as usize; + let mut t = Self { + addr, + tree_type: d[5], + node_size: u32::from_le_bytes([d[6], d[7], d[8], d[9]]), + rec_size: usize::from(u16::from_le_bytes([d[10], d[11]])), + depth: u16::from_le_bytes([d[12], d[13]]), + split_pct: d[14], + merge_pct: d[15], + root: Ptr { + addr: get_uint(&d[16..], img.os), + nrec: u16::from_le_bytes([d[16 + os], d[17 + os]]), + all: get_uint(&d[18 + os..], img.ls), + }, + levels: Vec::new(), + max_nrec_size: 0, + nodes: HashMap::new(), + dirty: BTreeSet::new(), + hdr_dirty: false, + }; + if t.depth > MAX_DEPTH || t.rec_size == 0 || t.split_pct == 0 || t.split_pct > 100 { + return Err(bad("unsupported header parameters")); + } + t.compute_levels(img.os)?; + Ok(t) + } + + /// Create an empty tree (header only, undefined root), as + /// `H5B2_create` does. + pub(crate) fn create( + img: &mut Image<'_>, + tree_type: u8, + node_size: u32, + rec_size: usize, + split_pct: u8, + merge_pct: u8, + ) -> Result { + let addr = img.alloc(Self::header_len(img.os, img.ls) as u64)?; + let mut t = Self { + addr, + tree_type, + node_size, + rec_size, + split_pct, + merge_pct, + depth: 0, + root: Ptr { + addr: undef(img.os), + nrec: 0, + all: 0, + }, + levels: Vec::new(), + max_nrec_size: 0, + nodes: HashMap::new(), + dirty: BTreeSet::new(), + hdr_dirty: true, + }; + t.compute_levels(img.os)?; + t.write_header(img)?; + Ok(t) + } + + pub(crate) fn address(&self) -> u64 { + self.addr + } + + pub(crate) fn record_size(&self) -> usize { + self.rec_size + } + + pub(crate) fn tree_type(&self) -> u8 { + self.tree_type + } + + /// Records in the whole tree. + pub(crate) fn len(&self) -> u64 { + self.root.all + } + + /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, + /// `H5B2__split_root`). + fn compute_levels(&mut self, os: u8) -> Result<(), Error> { + let ns = u64::from(self.node_size); + let rs = self.rec_size as u64; + let leaf_max = ns.saturating_sub(10) / rs; + if leaf_max < 2 || leaf_max > u64::from(u16::MAX) { + return Err(bad("node size holds too few or too many records")); + } + self.max_nrec_size = enc_size(leaf_max); + let pct = |n: u64, p: u8| n * u64::from(p) / 100; + let mut levels = vec![Level { + max_nrec: leaf_max, + split_nrec: pct(leaf_max, self.split_pct), + merge_nrec: pct(leaf_max, self.merge_pct), + cum_max_nrec: leaf_max, + cum_max_nrec_size: 0, + }]; + for d in 1..=self.depth { + let below = levels[usize::from(d) - 1]; + let ptr = u64::from(os) + + self.max_nrec_size as u64 + + if d > 1 { + below.cum_max_nrec_size as u64 + } else { + 0 + }; + let max = ns.saturating_sub(10 + ptr) / (rs + ptr); + if max < 2 { + return Err(bad("internal nodes hold too few records")); + } + let cum = max + .saturating_add(1) + .saturating_mul(below.cum_max_nrec) + .saturating_add(max); + levels.push(Level { + max_nrec: max, + split_nrec: pct(max, self.split_pct), + merge_nrec: pct(max, self.merge_pct), + cum_max_nrec: cum, + cum_max_nrec_size: enc_size(cum), + }); + } + if levels.iter().any(|l| l.split_nrec == 0) { + return Err(bad("split percentage too small")); + } + self.levels = levels; + Ok(()) + } + + fn ptr_size(&self, os: u8, depth: u16) -> usize { + os as usize + + self.max_nrec_size + + if depth > 1 { + self.levels[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + } + } + + /// Load the node `p` points at (a node of `depth`). + fn load(&mut self, img: &Image<'_>, p: Ptr, depth: u16) -> Result<(), Error> { + if let Some(n) = self.nodes.get(&p.addr) { + if n.depth != depth || n.recs.len() != usize::from(p.nrec) { + return Err(bad("node does not match its pointer")); + } + return Ok(()); + } + if p.addr == undef(img.os) { + return Err(bad("undefined node address")); + } + let nrec = usize::from(p.nrec); + if nrec as u64 > self.levels[usize::from(depth)].max_nrec { + return Err(bad("node holds more records than it can")); + } + let ps = if depth > 0 { + self.ptr_size(img.os, depth) + } else { + 0 + }; + let body = 6 + nrec * self.rec_size + if depth > 0 { (nrec + 1) * ps } else { 0 }; + if body + 4 > self.node_size as usize { + return Err(bad("node overflows its size")); + } + let d = img.read(p.addr, body + 4)?; + let sig: &[u8] = if depth == 0 { b"BTLF" } else { b"BTIN" }; + if &d[0..4] != sig || d[4] != 0 || d[5] != self.tree_type { + return Err(bad("bad node signature")); + } + let stored = u32::from_le_bytes(d[body..body + 4].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..body]) != stored { + return Err(bad("node checksum mismatch")); + } + let mut q = 6; + let mut recs = Vec::with_capacity(nrec); + for _ in 0..nrec { + recs.push(d[q..q + self.rec_size].to_vec()); + q += self.rec_size; + } + let mut ptrs = Vec::new(); + if depth > 0 { + let all_w = if depth > 1 { + self.levels[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for _ in 0..=nrec { + let addr = get_uint(&d[q..], img.os); + q += img.os as usize; + let n = le(&d[q..q + self.max_nrec_size]); + q += self.max_nrec_size; + let all = if depth > 1 { + let a = le(&d[q..q + all_w]); + q += all_w; + a + } else { + n + }; + let n = u16::try_from(n).map_err(|_| bad("record count"))?; + ptrs.push(Ptr { addr, nrec: n, all }); + } + } + self.nodes.insert(p.addr, Node { depth, recs, ptrs }); + Ok(()) + } + + fn node(&mut self, addr: u64) -> &mut Node { + self.dirty.insert(addr); + self.nodes.get_mut(&addr).expect("node loaded") + } + + fn peek(&self, addr: u64) -> &Node { + self.nodes.get(&addr).expect("node loaded") + } + + fn new_node(&mut self, img: &mut Image<'_>, depth: u16) -> Result { + let addr = img.alloc_reusing(u64::from(self.node_size))?; + self.nodes.insert( + addr, + Node { + depth, + recs: Vec::new(), + ptrs: Vec::new(), + }, + ); + self.dirty.insert(addr); + Ok(addr) + } + + fn drop_node(&mut self, img: &mut Image<'_>, addr: u64) { + self.nodes.remove(&addr); + self.dirty.remove(&addr); + img.free(addr, u64::from(self.node_size)); + } + + /// `H5B2__locate_record`: binary search; the index and the comparison + /// there. + fn locate( + img: &Image<'_>, + recs: &[Vec], + cmp: &mut Cmp<'_>, + ) -> Result<(usize, Ordering), Error> { + let (mut lo, mut hi) = (0usize, recs.len()); + let mut idx = 0; + let mut c = Ordering::Less; + while lo < hi && c != Ordering::Equal { + idx = (lo + hi) / 2; + c = cmp(img, &recs[idx])?; + if c == Ordering::Less { + hi = idx; + } else { + lo = idx + 1; + } + } + Ok((idx, c)) + } + + /// The record matching `cmp`, if any. + pub(crate) fn find( + &mut self, + img: &Image<'_>, + cmp: &mut Cmp<'_>, + ) -> Result>, Error> { + if self.root.addr == undef(img.os) || self.root.nrec == 0 { + return Ok(None); + } + let mut p = self.root; + let mut depth = self.depth; + loop { + self.load(img, p, depth)?; + let n = self.peek(p.addr); + let (idx, c) = Self::locate(img, &n.recs, cmp)?; + if c == Ordering::Equal { + return Ok(Some(n.recs[idx].clone())); + } + if depth == 0 { + return Ok(None); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + p = n.ptrs[i]; + depth -= 1; + } + } + + /// Every record, in key order. + pub(crate) fn records(&mut self, img: &Image<'_>) -> Result>, Error> { + let mut out = Vec::new(); + if self.root.addr != undef(img.os) && self.root.nrec > 0 { + self.walk(img, self.root, self.depth, &mut out)?; + } + Ok(out) + } + + fn walk( + &mut self, + img: &Image<'_>, + p: Ptr, + depth: u16, + out: &mut Vec>, + ) -> Result<(), Error> { + self.load(img, p, depth)?; + let n = self.peek(p.addr).clone(); + for i in 0..=n.recs.len() { + if depth > 0 { + self.walk(img, n.ptrs[i], depth - 1, out)?; + } + if i < n.recs.len() { + out.push(n.recs[i].clone()); + } + } + Ok(()) + } + + /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). + pub(crate) fn update( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + if rec.len() != self.rec_size { + return Err(bad("record of the wrong size")); + } + if self.root.addr == undef(img.os) { + let a = self.new_node(img, 0)?; + self.root = Ptr { + addr: a, + nrec: 0, + all: 0, + }; + self.hdr_dirty = true; + } + let mut root = self.root; + let st = self.update_at(img, &mut root, self.depth, cmp, rec)?; + self.root = root; + self.hdr_dirty = true; + if st == Status::ChildFull { + self.insert(img, cmp, rec)?; + } + Ok(()) + } + + fn update_at( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result { + self.load(img, *p, depth)?; + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c == Ordering::Equal { + self.node(p.addr).recs[idx] = rec.to_vec(); + return Ok(Status::Modified); + } + if depth == 0 { + if u64::from(p.nrec) == self.levels[0].split_nrec { + return Ok(Status::ChildFull); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + self.node(p.addr).recs.insert(i, rec.to_vec()); + p.nrec += 1; + p.all += 1; + return Ok(Status::Inserted); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + let mut child = self.peek(p.addr).ptrs[i]; + let st = self.update_at(img, &mut child, depth - 1, cmp, rec)?; + match st { + Status::Modified => Ok(st), + Status::Inserted => { + self.node(p.addr).ptrs[i] = child; + p.all += 1; + Ok(st) + } + Status::ChildFull => { + // H5B2__update_internal: a full child is split or + // redistributed here, unless this node is full too and the + // child and a neighbour are close to full (the split could + // reach this node): then the caller deals with it. + let d = usize::from(depth); + let (nrec, ptrs) = { + let n = self.peek(p.addr); + (n.recs.len(), n.ptrs.clone()) + }; + if nrec as u64 == self.levels[d].split_nrec { + let near = (self.levels[d - 1].split_nrec * 2).saturating_sub(1); + let pair = |a: usize, b: usize| { + u64::from(ptrs[a].nrec) + u64::from(ptrs[b].nrec) >= near + }; + let could_split = if i == 0 { + pair(0, 1) + } else if i == nrec { + pair(i - 1, i) + } else { + pair(i - 1, i) || pair(i, i + 1) + }; + if could_split { + return Ok(Status::ChildFull); + } + } + self.insert_internal(img, p, depth, cmp, rec)?; + Ok(Status::Inserted) + } + } + } + + /// `H5B2__insert`: insert a record that is not in the tree, splitting + /// or redistributing full nodes on the way down. + pub(crate) fn insert( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + if rec.len() != self.rec_size { + return Err(bad("record of the wrong size")); + } + if self.root.addr == undef(img.os) { + let a = self.new_node(img, 0)?; + self.root = Ptr { + addr: a, + nrec: 0, + all: 0, + }; + } else if u64::from(self.root.nrec) == self.levels[usize::from(self.depth)].split_nrec { + self.split_root(img)?; + } + self.hdr_dirty = true; + let mut root = self.root; + if self.depth > 0 { + self.insert_internal(img, &mut root, self.depth, cmp, rec)?; + } else { + self.insert_leaf(img, &mut root, cmp, rec)?; + } + self.root = root; + Ok(()) + } + + fn insert_leaf( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + self.load(img, *p, 0)?; + let mut i = 0; + if p.nrec > 0 { + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c == Ordering::Equal { + return Err(bad("record is already in the tree")); + } + i = if c == Ordering::Greater { idx + 1 } else { idx }; + } + if u64::from(p.nrec) >= self.levels[0].max_nrec { + return Err(bad("leaf full")); + } + self.node(p.addr).recs.insert(i, rec.to_vec()); + p.nrec += 1; + p.all += 1; + Ok(()) + } + + fn insert_internal( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + self.load(img, *p, depth)?; + let child_split = self.levels[usize::from(depth) - 1].split_nrec; + let mut i = self.child_index(img, p.addr, cmp)?; + // Preemptively split or redistribute the child to enter; a + // redistribution can move the insertion point into another full + // child, so after two tries it splits. + let mut retries = 2u32; + while u64::from(self.peek(p.addr).ptrs[i].nrec) == child_split { + let nrec = self.peek(p.addr).recs.len(); + let ptrs = self.peek(p.addr).ptrs.clone(); + let room = |j: usize| retries > 0 && u64::from(ptrs[j].nrec) < child_split; + if i == 0 { + if room(1) { + self.redistribute2(img, p.addr, depth, 0)?; + } else { + self.split1(img, p, depth, 0)?; + } + } else if i == nrec { + if room(i - 1) { + self.redistribute2(img, p.addr, depth, i - 1)?; + } else { + self.split1(img, p, depth, i)?; + } + } else if room(i + 1) || room(i - 1) { + self.redistribute3(img, p.addr, depth, i)?; + } else { + self.split1(img, p, depth, i)?; + } + i = self.child_index(img, p.addr, cmp)?; + retries = retries.saturating_sub(1); + } + let mut child = self.peek(p.addr).ptrs[i]; + if depth > 1 { + self.insert_internal(img, &mut child, depth - 1, cmp, rec)?; + } else { + self.insert_leaf(img, &mut child, cmp, rec)?; + } + self.node(p.addr).ptrs[i] = child; + p.all += 1; + Ok(()) + } + + /// The child of internal node `addr` to descend into for an insertion; + /// refuses a record that is already there. + fn child_index( + &mut self, + img: &Image<'_>, + addr: u64, + cmp: &mut Cmp<'_>, + ) -> Result { + let n = self.peek(addr); + if n.recs.is_empty() { + return Ok(0); + } + let (idx, c) = Self::locate(img, &n.recs, cmp)?; + match c { + Ordering::Equal => Err(bad("record is already in the tree")), + Ordering::Greater => Ok(idx + 1), + Ordering::Less => Ok(idx), + } + } + + /// Load the children `lo..=hi` of internal node `parent` (at `depth`). + fn load_children( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + lo: usize, + hi: usize, + ) -> Result<(), Error> { + for i in lo..=hi { + let c = self.peek(parent).ptrs[i]; + self.load(img, c, depth - 1)?; + } + Ok(()) + } + + /// Total records under a node with these records and children. + fn subtree_total(nrec: usize, ptrs: &[Ptr]) -> u64 { + nrec as u64 + ptrs.iter().map(|p| p.all).sum::() + } + + /// Lay `recs`/`ptrs` (the concatenation of several siblings and the + /// parent records between them) out over the children `lo..` of + /// `parent`, with `counts[k]` records in the k-th; the records between + /// them go back into the parent. + fn relayout( + &mut self, + parent: u64, + depth: u16, + lo: usize, + addrs: &[u64], + mut recs: Vec>, + mut ptrs: Vec, + counts: &[usize], + ) { + let internal = depth > 1; + let mut new_ptrs = Vec::with_capacity(addrs.len()); + let mut seps = Vec::with_capacity(addrs.len().saturating_sub(1)); + for (k, &a) in addrs.iter().enumerate() { + let n = counts[k]; + let rest_r = recs.split_off(n); + let mine_r = std::mem::replace(&mut recs, rest_r); + let mine_p = if internal { + let rest_p = ptrs.split_off(n + 1); + std::mem::replace(&mut ptrs, rest_p) + } else { + Vec::new() + }; + if k + 1 < addrs.len() { + seps.push(recs.remove(0)); + } + let all = if internal { + Self::subtree_total(n, &mine_p) + } else { + n as u64 + }; + new_ptrs.push(Ptr { + addr: a, + nrec: n as u16, + all, + }); + let node = self.node(a); + node.recs = mine_r; + node.ptrs = mine_p; + } + let pn = self.node(parent); + for (k, s) in seps.into_iter().enumerate() { + pn.recs[lo + k] = s; + } + for (k, p) in new_ptrs.into_iter().enumerate() { + pn.ptrs[lo + k] = p; + } + } + + /// The children `lo..=hi` of `parent` and the parent records between + /// them, concatenated. + fn gather(&self, parent: u64, lo: usize, hi: usize) -> (Vec, Vec>, Vec) { + let pn = self.peek(parent); + let mut addrs = Vec::new(); + let mut recs = Vec::new(); + let mut ptrs = Vec::new(); + for i in lo..=hi { + let c = self.peek(pn.ptrs[i].addr); + addrs.push(pn.ptrs[i].addr); + recs.extend(c.recs.iter().cloned()); + ptrs.extend(c.ptrs.iter().copied()); + if i < hi { + recs.push(pn.recs[i].clone()); + } + } + (addrs, recs, ptrs) + } + + /// `H5B2__redistribute2`: even out children `idx` and `idx + 1`. + fn redistribute2( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + self.load_children(img, parent, depth, idx, idx + 1)?; + let (l, r) = { + let pn = self.peek(parent); + ( + usize::from(pn.ptrs[idx].nrec), + usize::from(pn.ptrs[idx + 1].nrec), + ) + }; + let (nl, nr) = if l < r { + let nr = (l + r) / 2; + (l + r - nr, nr) + } else { + let nl = (l + r) / 2; + (nl, l + r - nl) + }; + let (addrs, recs, ptrs) = self.gather(parent, idx, idx + 1); + self.relayout(parent, depth, idx, &addrs, recs, ptrs, &[nl, nr]); + Ok(()) + } + + /// `H5B2__redistribute3`: even out children `idx - 1`, `idx`, `idx + 1`. + fn redistribute3( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + self.load_children(img, parent, depth, idx - 1, idx + 1)?; + let total = { + let pn = self.peek(parent); + (idx - 1..=idx + 1) + .map(|i| usize::from(pn.ptrs[i].nrec)) + .sum::() + + 2 + }; + let nm = (total - 2) / 3; + let nl = ((total - 2) - nm) / 2; + let nr = (total - 2) - (nl + nm); + let (addrs, recs, ptrs) = self.gather(parent, idx - 1, idx + 1); + self.relayout(parent, depth, idx - 1, &addrs, recs, ptrs, &[nl, nm, nr]); + Ok(()) + } + + /// `H5B2__split1`: split child `idx` of the node `p` points at, which + /// gains a record and a child. + fn split1( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx, idx)?; + let right = self.new_node(img, depth - 1)?; + let old = usize::from(self.peek(parent).ptrs[idx].nrec); + let mid = old / 2; + { + let pn = self.node(parent); + pn.recs.insert(idx, Vec::new()); + pn.ptrs.insert( + idx + 1, + Ptr { + addr: right, + nrec: 0, + all: 0, + }, + ); + } + let left = self.peek(parent).ptrs[idx].addr; + let (recs, ptrs) = { + let l = self.peek(left); + (l.recs.clone(), l.ptrs.clone()) + }; + self.relayout( + parent, + depth, + idx, + &[left, right], + recs, + ptrs, + &[mid, old - mid - 1], + ); + p.nrec += 1; + Ok(()) + } + + /// `H5B2__split_root`: the tree grows a level. + fn split_root(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + if self.depth >= MAX_DEPTH { + return Err(bad("tree too deep")); + } + self.depth += 1; + self.compute_levels(img.os)?; + let old = self.root; + let new_root = self.new_node(img, self.depth)?; + self.node(new_root).ptrs.push(old); + let mut root = Ptr { + addr: new_root, + nrec: 0, + all: old.all, + }; + let depth = self.depth; + self.split1(img, &mut root, depth, 0)?; + self.root = root; + self.hdr_dirty = true; + Ok(()) + } + + /// Remove the record `cmp` matches (`H5B2_remove`); returns it, or + /// `None` when the tree has no such record (nothing changes then). + pub(crate) fn remove( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + ) -> Result>, Error> { + let Some(found) = self.find(img, cmp)? else { + return Ok(None); + }; + let mut root = self.root; + if self.depth > 0 { + let mut decreased = false; + self.remove_internal(img, &mut root, self.depth, true, None, cmp, &mut decreased)?; + if decreased { + self.depth -= 1; + self.compute_levels(img.os)?; + } + } else { + self.remove_leaf(img, &mut root, cmp)?; + } + root.all -= 1; + self.root = root; + self.hdr_dirty = true; + Ok(Some(found)) + } + + fn remove_leaf( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + cmp: &mut Cmp<'_>, + ) -> Result<(), Error> { + self.load(img, *p, 0)?; + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c != Ordering::Equal { + return Err(bad("record to remove not found")); + } + self.node(p.addr).recs.remove(idx); + if self.peek(p.addr).recs.is_empty() { + self.drop_node(img, p.addr); + p.addr = undef(img.os); + } + p.nrec -= 1; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn remove_internal( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + is_root: bool, + mut swap: Option<(u64, usize)>, + cmp: &mut Cmp<'_>, + decreased: &mut bool, + ) -> Result<(), Error> { + self.load(img, *p, depth)?; + let merge = self.levels[usize::from(depth) - 1].merge_nrec; + let addr = p.addr; + let pn = self.peek(addr); + // libhdf5 checks every level; only a root can hold a single record. + if is_root + && pn.recs.len() == 1 + && u64::from(pn.ptrs[0].nrec) + u64::from(pn.ptrs[1].nrec) <= merge * 2 + 1 + { + // Collapse the root: merge its two children into the left one, + // which becomes the root. + let mut dummy = *p; + self.merge2(img, &mut dummy, depth, 0)?; + let child = self.peek(addr).ptrs[0]; + self.drop_node(img, addr); + p.addr = child.addr; + p.nrec = child.nrec; + *decreased = true; + if depth > 1 { + return self.remove_internal(img, p, depth - 1, true, swap, cmp, decreased); + } + return self.remove_leaf(img, p, cmp); + } + let mut idx; + let mut c = Ordering::Less; + if swap.is_some() { + idx = 0; + } else { + let (i, cc) = Self::locate(img, &pn.recs, cmp)?; + c = cc; + idx = if cc != Ordering::Less { i + 1 } else { i }; + } + let mut retries = 2u32; + while u64::from(self.peek(addr).ptrs[idx].nrec) == merge { + let n = self.peek(addr).recs.len(); + if n == 0 { + return Err(bad("internal node without records")); + } + let ptrs = self.peek(addr).ptrs.clone(); + let spare = |j: usize| retries > 0 && u64::from(ptrs[j].nrec) > merge; + if idx == 0 { + if spare(1) { + self.redistribute2(img, addr, depth, 0)?; + } else { + self.merge2(img, p, depth, 0)?; + } + } else if idx == n { + if spare(idx - 1) { + self.redistribute2(img, addr, depth, idx - 1)?; + } else { + self.merge2(img, p, depth, idx - 1)?; + } + } else if spare(idx + 1) || spare(idx - 1) { + self.redistribute3(img, addr, depth, idx)?; + } else { + self.merge3(img, p, depth, idx)?; + } + if swap.is_some() { + idx = 0; + } else { + let (i, cc) = Self::locate(img, &self.peek(addr).recs, cmp)?; + c = cc; + idx = if cc != Ordering::Less { i + 1 } else { i }; + } + retries = retries.saturating_sub(1); + } + if swap.is_none() && c == Ordering::Equal { + swap = Some((addr, idx - 1)); + } + if let (Some((sa, si)), 1) = (swap, depth) { + // H5B2__swap_leaf: the record to delete trades places with the + // leaf's first record. + let child = self.peek(addr).ptrs[idx]; + self.load(img, child, 0)?; + let leaf_first = self.peek(child.addr).recs[0].clone(); + let deleted = std::mem::replace(&mut self.node(sa).recs[si], leaf_first); + self.node(child.addr).recs[0] = deleted; + } + let mut child = self.peek(addr).ptrs[idx]; + if depth > 1 { + self.remove_internal(img, &mut child, depth - 1, false, swap, cmp, decreased)?; + } else { + self.remove_leaf(img, &mut child, cmp)?; + } + child.all -= 1; + self.node(addr).ptrs[idx] = child; + Ok(()) + } + + /// `H5B2__merge2`: merge child `idx + 1` of the node `p` points at into + /// child `idx`. + fn merge2( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx, idx + 1)?; + let (addrs, recs, ptrs) = self.gather(parent, idx, idx + 1); + let n = recs.len(); + { + let pn = self.node(parent); + pn.recs.remove(idx); + pn.ptrs.remove(idx + 1); + } + self.relayout(parent, depth, idx, &addrs[..1], recs, ptrs, &[n]); + self.drop_node(img, addrs[1]); + p.nrec -= 1; + Ok(()) + } + + /// `H5B2__merge3`: merge children `idx - 1`, `idx`, `idx + 1` of the + /// node `p` points at into two. + fn merge3( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx - 1, idx + 1)?; + let (addrs, recs, ptrs) = self.gather(parent, idx - 1, idx + 1); + let total = recs.len(); + let nl = (total - 1) / 2; + { + let pn = self.node(parent); + pn.recs.remove(idx); + pn.ptrs.remove(idx + 1); + } + self.relayout( + parent, + depth, + idx - 1, + &addrs[..2], + recs, + ptrs, + &[nl, total - 1 - nl], + ); + self.drop_node(img, addrs[2]); + p.nrec -= 1; + Ok(()) + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let len = Self::header_len(img.os, img.ls); + let os = img.os as usize; + let mut d = vec![0u8; len]; + d[0..4].copy_from_slice(b"BTHD"); + d[5] = self.tree_type; + d[6..10].copy_from_slice(&self.node_size.to_le_bytes()); + d[10..12].copy_from_slice(&(self.rec_size as u16).to_le_bytes()); + d[12..14].copy_from_slice(&self.depth.to_le_bytes()); + d[14] = self.split_pct; + d[15] = self.merge_pct; + put_uint(&mut d[16..], self.root.addr, img.os); + d[16 + os..18 + os].copy_from_slice(&self.root.nrec.to_le_bytes()); + put_uint(&mut d[18 + os..], self.root.all, img.ls); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]); + d[len - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d)?; + self.hdr_dirty = false; + Ok(()) + } + + /// Write every changed node and the header. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for addr in std::mem::take(&mut self.dirty) { + let Some(n) = self.nodes.get(&addr) else { + continue; + }; + let mut d = Vec::with_capacity(self.node_size as usize); + d.extend_from_slice(if n.depth == 0 { b"BTLF" } else { b"BTIN" }); + d.push(0); + d.push(self.tree_type); + for r in &n.recs { + d.extend_from_slice(r); + } + if n.depth > 0 { + let all_w = if n.depth > 1 { + self.levels[usize::from(n.depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for p in &n.ptrs { + let mut a = vec![0u8; img.os as usize]; + put_uint(&mut a, p.addr, img.os); + d.extend_from_slice(&a); + d.extend_from_slice(&u64::from(p.nrec).to_le_bytes()[..self.max_nrec_size]); + if n.depth > 1 { + d.extend_from_slice(&p.all.to_le_bytes()[..all_w]); + } + } + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + if d.len() > self.node_size as usize { + return Err(bad("node overflows its size")); + } + d.resize(self.node_size as usize, 0); + img.write(addr, &d)?; + } + if self.hdr_dirty { + self.write_header(img)?; + } + Ok(()) + } +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} diff --git a/crates/clawhdf5/src/edit/earray.rs b/crates/clawhdf5/src/edit/earray.rs index 49d9621..bae196f 100644 --- a/crates/clawhdf5/src/edit/earray.rs +++ b/crates/clawhdf5/src/edit/earray.rs @@ -383,12 +383,23 @@ impl Ea { } /// Set element `idx` to `e`. - pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + /// Set element `idx` to `e`, or back to the fill element (`None`: a + /// removed chunk, `H5D__earray_idx_remove`), which creates no block. + pub(crate) fn set( + &mut self, + img: &mut Image<'_>, + idx: u64, + e: Option, + ) -> Result<(), Error> { let os = img.os; let osz = u64::from(os); let es = self.slot_size(os) as u64; - let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let enc = encode_elem(e, self.filtered, self.elem_size, os)?; + let clear = e.is_none(); if self.iblock == undef(os) { + if clear { + return Ok(()); + } self.create_iblock(img)?; } let ib = self.iblock; @@ -420,6 +431,9 @@ impl Ea { let dblk_idx = l.start_dblk + local; let slot = dblks_at + dblk_idx * osz; let mut addr = get_uint(&img.read(slot, os as usize)?, os); + if addr == undef(os) && clear { + return Ok(()); + } if addr == undef(os) { // libhdf5 records start_idx + (global data block index) // * nelmts here (H5EA__lookup_elmt), not the block's @@ -449,6 +463,9 @@ impl Ea { let sb_prefix = self.dblk_prefix_len(os); let sb_len = sb_prefix + bitmap_len + l.ndblks * osz; let mut sb = get_uint(&img.read(sslot, os as usize)?, os); + if sb == undef(os) && clear { + return Ok(()); + } if sb == undef(os) { let mut d = self.block_prefix(b"EASB", l.start_idx, os); d.resize(d.len() + bitmap_len as usize, 0); @@ -471,6 +488,9 @@ impl Ea { let local = (rel - l.start_idx) / l.dblk_nelmts; let dslot = sb + sb_prefix + bitmap_len + local * osz; let mut addr = get_uint(&img.read(dslot, os as usize)?, os); + if addr == undef(os) && clear { + return Ok(()); + } if addr == undef(os) { let off = l.start_idx + local * l.dblk_nelmts; addr = self.create_dblock(img, l.dblk_nelmts, off)?; @@ -491,6 +511,9 @@ impl Ea { let bpos = sb + sb_prefix + bit / 8; let mut byte = img.read(bpos, 1)?[0]; let mask = 0x80u8 >> (bit % 8); + if byte & mask == 0 && clear { + return Ok(()); + } if byte & mask == 0 { let fill = self.fill_elems(page, os)?; img.write(page_at, &fill)?; @@ -503,7 +526,7 @@ impl Ea { } } } - if idx + 1 > self.stats[4] { + if !clear && idx + 1 > self.stats[4] { self.stats[4] = idx + 1; self.dirty_hdr = true; } diff --git a/crates/clawhdf5/src/edit/farray.rs b/crates/clawhdf5/src/edit/farray.rs index 0a25fe7..e7a1696 100644 --- a/crates/clawhdf5/src/edit/farray.rs +++ b/crates/clawhdf5/src/edit/farray.rs @@ -137,13 +137,19 @@ impl Fa { Ok((fa, hdr)) } - /// Set element `idx` to `e`. - pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + /// Set element `idx` to `e`, or back to the fill element (`None`: a + /// removed chunk, `H5D__farray_idx_remove`), which creates no page. + pub(crate) fn set( + &mut self, + img: &mut Image<'_>, + idx: u64, + e: Option, + ) -> Result<(), Error> { let os = img.os; if idx >= self.nelmts { return Err(bad("index beyond the array")); } - let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let enc = encode_elem(e, self.filtered, self.elem_size, os)?; let es = self.slot(os); let prefix = 6 + u64::from(os); let page = self.page(); @@ -162,6 +168,9 @@ impl Fa { let bpos = self.dblk + prefix + p / 8; let mut byte = img.read(bpos, 1)?[0]; let mask = 0x80u8 >> (p % 8); + if byte & mask == 0 && e.is_none() { + return Ok(()); + } if byte & mask == 0 { let fill = encode_elem(None, self.filtered, self.elem_size, os)?; img.write(page_at, &fill.repeat(count as usize))?; diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index 254abcd..722534d 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -35,6 +35,8 @@ pub(crate) struct Image<'a> { /// Width of addresses and lengths in the file. pub(crate) os: u8, pub(crate) ls: u8, + /// Space the edit stopped using. + freed: Vec<(u64, u64)>, } impl<'a> Image<'a> { @@ -47,6 +49,7 @@ impl<'a> Image<'a> { old_eoa: eoa, os, ls, + freed: Vec::new(), } } @@ -77,6 +80,20 @@ impl<'a> Image<'a> { Ok(addr) } + /// Allocate `size` bytes for metadata or data, from space an earlier + /// edit of this session freed when there is a block that fits, + /// otherwise at the end of the file. + pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result { + self.alloc(size) + } + + /// Note that the edit no longer uses `[addr, addr + len)`. + pub(crate) fn free(&mut self, addr: u64, len: u64) { + if len > 0 { + self.freed.push((addr, len)); + } + } + /// If `[addr, addr + old_len)` is the last allocated space, grow it to /// `new_len` bytes (a structure at the end of the file can grow where /// it is) and return true. diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 4903672..b14244a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -16,6 +16,7 @@ //! is alive while the file changes (see `image`). mod btree1; +mod btree2; mod earray; mod farray; mod image; @@ -39,6 +40,7 @@ use crate::error::Error; use crate::reader::File; use crate::types::AttrValue; use btree1::{BTree1, Key}; +use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; use image::{Image, get_uint, put_uint, undef}; @@ -249,6 +251,14 @@ impl Target { fn dims(&self) -> &[u64] { &self.ds.dimensions } + + /// The chunk index (or implicit chunks') address. + fn layout_address(&self) -> Option { + match &self.layout { + DataLayout::Chunked { btree_address, .. } => *btree_address, + _ => None, + } + } } /// Datatypes whose stored bytes point elsewhere in the file (variable-length @@ -331,7 +341,7 @@ enum IndexEdit { Implicit, Fixed(Option), Extensible(Option), - BTree2, + BTree2(Option), } struct ChunkedEdit<'t> { @@ -398,7 +408,9 @@ impl<'t> ChunkedEdit<'t> { (_, Some(4)) => { IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?) } - (_, Some(5)) => IndexEdit::BTree2, + (_, Some(5)) => { + IndexEdit::BTree2(btree_address.map(|a| Bt2::open(img, a)).transpose()?) + } (v, i) => { return Err(Error::Unsupported(format!( "chunked layout version {v}, index type {i:?}" @@ -520,7 +532,7 @@ impl<'t> ChunkedEdit<'t> { self.patch_layout_addr(img, hdr_addr)?; } if let IndexEdit::Fixed(Some(fa)) = &mut self.index { - fa.set(img, idx, e)?; + fa.set(img, idx, Some(e))?; } } IndexEdit::Extensible(ea) => { @@ -549,30 +561,165 @@ impl<'t> ChunkedEdit<'t> { self.patch_layout_addr(img, hdr_addr)?; } if let IndexEdit::Extensible(Some(ea)) = &mut self.index { - ea.set(img, idx, e)?; + ea.set(img, idx, Some(e))?; } } - IndexEdit::BTree2 => { - return Err(Error::Unsupported( - "adding, moving or resizing a chunk in a version-2 B-tree chunk index \ - (datasets with more than one unlimited dimension)" - .into(), - )); + IndexEdit::BTree2(tree) => { + if tree.is_none() { + let at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("B-tree v2 parameters".into()))?; + let d = self.hdr.data(img, self.layout_msg)?; + let node_size = u32::from_le_bytes([d[at], d[at + 1], d[at + 2], d[at + 3]]); + let size_len = if filtered { + earray::chunk_size_len(self.chunk_bytes as u64, self.lpos.version) + 4 + } else { + 0 + }; + let rec_size = img.os as usize + size_len + 8 * self.cd.len(); + let new = Bt2::create( + img, + if filtered { 11 } else { 10 }, + node_size, + rec_size, + d[at + 4], + d[at + 5], + )?; + let a = new.address(); + *tree = Some(new); + self.patch_layout_addr(img, a)?; + } + let IndexEdit::BTree2(Some(tree)) = &mut self.index else { + unreachable!("created above") + }; + let rec = bt2_chunk_record(tree, scaled, e, filtered, img.os)?; + let mut cmp = bt2_chunk_cmp(scaled, tree.record_size()); + tree.update(img, &mut cmp, &rec)?; } } Ok(()) } + /// Remove chunk `scaled` (stored at `info`) from the index, as the + /// index's `remove` operation does when libhdf5 prunes a dataset, and + /// free its space. An implicit index cannot drop chunks: libhdf5 leaves + /// them (and their data) where they are, and so does this. + fn remove( + &mut self, + img: &mut Image<'_>, + scaled: &[u64], + info: &ChunkInfo, + ) -> Result<(), Error> { + let t = self.t; + match &mut self.index { + IndexEdit::Implicit => return Ok(()), + IndexEdit::Single => { + return Err(Error::Unsupported( + "removing the chunk of a single-chunk index".into(), + )); + } + IndexEdit::BTree1(Some(tree)) => { + let mut offs: Vec = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect(); + offs.push(0); + if tree.remove(img, &offs)?.is_none() { + return Err(Error::Unsupported("chunk missing from its B-tree".into())); + } + } + IndexEdit::BTree2(Some(tree)) => { + let mut cmp = bt2_chunk_cmp(scaled, tree.record_size()); + if tree.remove(img, &mut cmp)?.is_none() { + return Err(Error::Unsupported("chunk missing from its B-tree".into())); + } + } + IndexEdit::Fixed(Some(fa)) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, false)?; + fa.set(img, idx, None)?; + } + IndexEdit::Extensible(Some(ea)) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, true)?; + ea.set(img, idx, None)?; + } + _ => return Err(Error::Unsupported("chunk index missing".into())), + } + img.free(info.address, u64::from(info.chunk_size)); + Ok(()) + } + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { match &mut self.index { IndexEdit::Extensible(Some(ea)) => ea.finish(img)?, IndexEdit::Fixed(Some(fa)) => fa.finish(img)?, + IndexEdit::BTree2(Some(t)) => t.finish(img)?, _ => {} } self.hdr.finish(img) } } +/// A version-2 B-tree chunk record (type 10, or 11 when filtered): the +/// chunk's address, [its stored size and filter mask,] and its scaled +/// offsets. +fn bt2_chunk_record( + tree: &Bt2, + scaled: &[u64], + e: Elem, + filtered: bool, + os: u8, +) -> Result, Error> { + let rs = tree.record_size(); + let want_type = if filtered { 11 } else { 10 }; + let osz = os as usize; + let size_len = rs + .checked_sub(osz + 8 * scaled.len()) + .and_then(|n| { + if filtered { + n.checked_sub(4).filter(|&w| (1..=8).contains(&w)) + } else { + (n == 0).then_some(0) + } + }) + .filter(|_| tree.tree_type() == want_type) + .ok_or_else(|| Error::Unsupported("version-2 B-tree chunk record layout".into()))?; + let mut r = vec![0u8; rs]; + put_uint(&mut r, e.addr, os); + let mut q = osz; + if filtered { + if size_len < 8 && e.size >> (8 * size_len) != 0 { + return Err(Error::Unsupported(format!( + "filtered chunk of {} bytes does not fit the index's {size_len}-byte size field", + e.size + ))); + } + r[q..q + size_len].copy_from_slice(&e.size.to_le_bytes()[..size_len]); + q += size_len; + r[q..q + 4].copy_from_slice(&e.mask.to_le_bytes()); + q += 4; + } + for &s in scaled { + r[q..q + 8].copy_from_slice(&s.to_le_bytes()); + q += 8; + } + Ok(r) +} + +/// Compare chunk `scaled` with a chunk record (`H5D__bt2_compare`: the +/// scaled offsets, the first dimension most significant). +fn bt2_chunk_cmp( + scaled: &[u64], + rec_size: usize, +) -> impl FnMut(&Image<'_>, &[u8]) -> Result + '_ { + move |_, r| { + let at = rec_size - 8 * scaled.len(); + let theirs = (0..scaled.len()).map(|d| { + u64::from_le_bytes(r[at + 8 * d..at + 8 * d + 8].try_into().unwrap_or([0; 8])) + }); + Ok(scaled.iter().copied().cmp(theirs)) + } +} + /// The chunk B-tree's K: the superblock's (version 1), the superblock /// extension's (versions 2 and 3), or libhdf5's default of 32. fn chunk_btree_k(f: &File) -> Result { @@ -694,10 +841,15 @@ impl FileEditor { }) } - /// Change a chunked dataset's current dimensions to `shape`, which may - /// only grow each dimension, up to the dataset's maximum dimensions - /// (h5py's `Dataset.resize`). New elements read as the fill value until - /// written. Shrinking is [`Error::Unsupported`]. + /// Change a chunked dataset's current dimensions to `shape`: each + /// dimension may grow up to the dataset's maximum or shrink (h5py's + /// `Dataset.resize`), as `H5Dset_extent` does. New elements read as the + /// fill value until written (with early allocation their chunks are + /// allocated and filled now). Shrinking removes the chunks wholly + /// outside the new extent from the chunk index and frees them, and + /// overwrites the part of each partial edge chunk outside it with the + /// fill value, so elements that come back after a later growth read as + /// the fill value (`H5D__chunk_prune_by_extent`). pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> { self.edit(|f, img| { let t = Target::load(f, path)?; @@ -720,18 +872,16 @@ impl FileEditor { shape[d], max[d] ))); } - if shape[d] < dims[d] { - return Err(Error::Unsupported("shrinking a dataset".into())); - } } if !matches!(t.layout, DataLayout::Chunked { .. }) { return Err(Error::Unsupported( "resizing a dataset that is not chunked".into(), )); } - // Only the dataspace changes: every chunk index is keyed - // independently of the current extent (the array indexes by the - // maximum dimensions), and new chunks come with the writes. + // The dataspace changes; every chunk index is keyed + // independently of the current extent (the arrays index by the + // maximum dimensions). Growth allocates chunks only under early + // allocation, shrinking prunes. let mut hdr = Header::load(img, t.addr)?; let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage( clawhdf5_format::message_type::MessageType::Dataspace, @@ -754,7 +904,15 @@ impl FileEditor { put_uint(&mut dims_bytes[d * ls..], n, img.ls); } hdr.patch(img, i, first, &dims_bytes)?; - hdr.finish(img) + let fill = fill_info(img, &hdr)?; + hdr.finish(img)?; + let expand = shape.iter().zip(&dims).any(|(n, o)| n > o); + let shrink = shape.iter().zip(&dims).any(|(n, o)| n < o); + let early = expand && fill.alloc_time == ALLOC_EARLY; + if early || shrink { + resize_chunks(f, img, &t, &dims, shape, &fill, early, shrink)?; + } + Ok(()) }) } @@ -1187,52 +1345,7 @@ fn write_selection( Ok(()) })?; for (scaled, buf) in bufs { - // As libhdf5 does: an optional filter that fails (LZF that - // does not shrink the chunk) is skipped and its mask bit set. - let (bytes, mask) = match &t.pipeline { - Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?, - None => (buf, 0u32), - }; - let len = bytes.len() as u64; - let placed = match existing.get(&scaled) { - Some(info) if t.pipeline.is_none() => { - if u64::from(info.chunk_size) != len { - return Err(Error::Unsupported( - "unfiltered chunk stored at an unexpected size".into(), - )); - } - img.write(info.address, &bytes)?; - None - } - // Rewritten where it is when it still fits, or when it - // is the last thing in the file (the chunk an append - // keeps rewriting usually is) and can grow there. - Some(info) - if len <= u64::from(info.chunk_size) - || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => - { - img.write(info.address, &bytes)?; - (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some( - Elem { - addr: info.address, - size: len, - mask, - }, - ) - } - _ => { - let a = img.alloc(len)?; - img.write(a, &bytes)?; - Some(Elem { - addr: a, - size: len, - mask, - }) - } - }; - if let Some(e) = placed { - ce.set(img, &scaled, e)?; - } + store_chunk(img, &mut ce, existing.get(&scaled), &scaled, buf)?; } ce.finish(img) } @@ -1240,6 +1353,341 @@ fn write_selection( } } +/// `H5D_ALLOC_TIME_EARLY`. +const ALLOC_EARLY: u8 = 1; + +/// When a dataset's storage is allocated and filled, from its fill value +/// message (`H5O__fill_new_decode`); a dataset without one uses the chunked +/// defaults (incremental allocation, fill if set). +struct FillInfo { + alloc_time: u8, + /// 0 on allocation, 1 never, 2 if set. + fill_time: u8, + /// The fill value is undefined (`H5D_FILL_VALUE_UNDEFINED`). + undefined: bool, +} + +fn fill_info(img: &Image<'_>, hdr: &Header) -> Result { + let mut fi = FillInfo { + alloc_time: 3, + fill_time: 2, + undefined: false, + }; + let Some(i) = hdr.find(0x05) else { + return Ok(fi); + }; + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared fill value message".into())); + } + let d = hdr.data(img, i)?; + let short = || Error::Unsupported("short fill value message".into()); + match d.first() { + Some(1) | Some(2) => { + fi.alloc_time = *d.get(1).ok_or_else(short)?; + fi.fill_time = *d.get(2).ok_or_else(short)?; + fi.undefined = d[0] == 2 && *d.get(3).ok_or_else(short)? == 0; + } + Some(3) => { + let flags = *d.get(1).ok_or_else(short)?; + fi.alloc_time = flags & 0x03; + fi.fill_time = (flags >> 2) & 0x03; + fi.undefined = flags & 0x10 != 0; + } + _ => return Err(Error::Unsupported("fill value message version".into())), + } + Ok(fi) +} + +/// Visit the chunks libhdf5 visits, in its order, when the extent changes +/// from `old` to `new`: for allocation (`H5D__chunk_allocate`), every +/// chunk of the new extent outside the old one; `f` gets each scaled +/// offset. +fn for_each_new_chunk( + old: &[u64], + new: &[u64], + cd: &[u64], + mut f: impl FnMut(&[u64]) -> Result<(), Error>, +) -> Result<(), Error> { + let rank = new.len(); + if rank == 0 || new.contains(&0) { + return Ok(()); + } + let min: Vec = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect(); + let mut max: Vec = (0..rank).map(|d| (new[d] - 1) / cd[d]).collect(); + for op in 0..rank { + if min[op] > max[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min[op]; + loop { + f(&scaled)?; + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max[i] { + scaled[i] = if i == op { min[i] } else { 0 }; + } else { + carry = false; + break; + } + } + if carry { + break; + } + } + if min[op] == 0 { + break; + } + max[op] = min[op] - 1; + } + Ok(()) +} + +/// What pruning does to one chunk. +enum Prune { + /// Overwrite the part outside the new extent with the fill value. + Fill, + /// Drop it from the index. + Remove, +} + +/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks +/// from `old` to `new`, in its order. +fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { + let rank = new.len(); + let mut out = Vec::new(); + if old.contains(&0) { + return out; + } + let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); + let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_fill: Vec = (0..rank) + .map(|d| { + if new[d] == 0 { + -1 + } else { + ((new[d].min(old[d]) - 1) / cd[d]) as i64 + } + }) + .collect(); + let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); + let fill_dim: Vec = (0..rank) + .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) + .collect(); + for op in 0..rank { + if !shrunk[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min_mod[op]; + let mut outside: Vec = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); + let mut n_out = outside.iter().filter(|&&o| o).count(); + loop { + if n_out == 0 { + out.push((scaled.clone(), Prune::Fill)); + } else { + out.push((scaled.clone(), Prune::Remove)); + } + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max_mod[i] { + if i == op { + scaled[i] = min_mod[i]; + if outside[i] && fill_dim[i] { + outside[i] = false; + n_out -= 1; + } + } else { + scaled[i] = 0; + if outside[i] && max_fill[i] >= 0 { + outside[i] = false; + n_out -= 1; + } + } + } else { + if !outside[i] && scaled[i] as i64 > max_fill[i] { + outside[i] = true; + n_out += 1; + } + carry = false; + break; + } + } + if carry { + break; + } + } + if min_mod[op] == 0 { + // Every chunk was visited (the dimension shrank to nothing). + break; + } + max_mod[op] = min_mod[op] - 1; + } + out +} + +/// The chunk work of a resize: under early allocation, allocate and fill +/// the chunks the growth brings in (`H5D__chunk_allocate`); when a +/// dimension shrank, prune (`H5D__chunk_prune_by_extent`). +#[allow(clippy::too_many_arguments)] +fn resize_chunks( + f: &File, + img: &mut Image<'_>, + t: &Target, + old: &[u64], + new: &[u64], + fill: &FillInfo, + early: bool, + shrink: bool, +) -> Result<(), Error> { + let mut ce = ChunkedEdit::new(f, img, t)?; + let cd = ce.cd.clone(); + let es = t.es; + let chunk_bytes = ce.chunk_bytes; + let existing: HashMap, ChunkInfo> = match &t.layout { + DataLayout::Chunked { + btree_address: Some(_), + .. + } => { + let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?; + chunks + .into_iter() + .map(|c| { + let s: Vec = c.offsets.iter().zip(&cd).map(|(o, c)| o / c).collect(); + (s, c) + }) + .collect() + } + _ => HashMap::new(), + }; + let fill_chunk = t.fill.repeat(chunk_bytes / es); + if early { + let should_fill = + fill.fill_time == 0 || (fill.fill_time == 2 && !fill.undefined) || t.pipeline.is_some(); + let implicit = matches!(ce.index, IndexEdit::Implicit); + let base = t.layout_address(); + for_each_new_chunk(old, new, &cd, |scaled| { + if implicit { + if should_fill { + let base = + base.ok_or_else(|| Error::Unsupported("implicit index address".into()))?; + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, new, max, &cd, false)?; + let at = idx + .checked_mul(chunk_bytes as u64) + .and_then(|o| o.checked_add(base)) + .ok_or_else(|| Error::Unsupported("chunk address overflows".into()))?; + img.write(at, &fill_chunk)?; + } + Ok(()) + } else if existing.contains_key(scaled) { + Ok(()) + } else { + store_chunk(img, &mut ce, None, scaled, fill_chunk.clone()) + } + })?; + } + if shrink && !existing.is_empty() { + for (scaled, what) in prune_plan(old, new, &cd) { + let Some(info) = existing.get(&scaled) else { + continue; + }; + match what { + Prune::Remove => ce.remove(img, &scaled, info)?, + Prune::Fill => { + let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?; + // Keep [0, count) in each dimension; fill the rest. + let count: Vec = (0..cd.len()) + .map(|d| cd[d].min(new[d] - scaled[d] * cd[d])) + .collect(); + let n: u64 = cd.iter().product(); + for flat in 0..n { + let mut r = flat; + let mut inside = true; + for d in (0..cd.len()).rev() { + if r % cd[d] >= count[d] { + inside = false; + } + r /= cd[d]; + } + if !inside { + let at = flat as usize * es; + buf[at..at + es].copy_from_slice(&t.fill); + } + } + store_chunk(img, &mut ce, Some(info), &scaled, buf)?; + } + } + } + } + ce.finish(img) +} + +/// Encode chunk `scaled` (decoded bytes `buf`) and store it: an existing +/// unfiltered chunk in place; a filtered one in place when it still fits +/// (or can grow at the end of the file), else in new space, the old space +/// freed; a new chunk in new space. The index is updated when the chunk's +/// address, size or filter mask changed. +fn store_chunk( + img: &mut Image<'_>, + ce: &mut ChunkedEdit<'_>, + existing: Option<&ChunkInfo>, + scaled: &[u64], + buf: Vec, +) -> Result<(), Error> { + let t = ce.t; + // As libhdf5 does: an optional filter that fails (LZF that does not + // shrink the chunk) is skipped and its mask bit set. + let (bytes, mask) = match &t.pipeline { + Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, t.es as u32)?, + None => (buf, 0u32), + }; + let len = bytes.len() as u64; + let placed = match existing { + Some(info) if t.pipeline.is_none() => { + if u64::from(info.chunk_size) != len { + return Err(Error::Unsupported( + "unfiltered chunk stored at an unexpected size".into(), + )); + } + img.write(info.address, &bytes)?; + None + } + // Rewritten where it is when it still fits, or when it is the last + // thing in the file (the chunk an append keeps rewriting usually is) + // and can grow there. + Some(info) + if len <= u64::from(info.chunk_size) + || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => + { + img.write(info.address, &bytes)?; + (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(Elem { + addr: info.address, + size: len, + mask, + }) + } + _ => { + let a = img.alloc_reusing(len)?; + img.write(a, &bytes)?; + if let Some(info) = existing { + img.free(info.address, u64::from(info.chunk_size)); + } + Some(Elem { + addr: a, + size: len, + mask, + }) + } + }; + if let Some(e) = placed { + ce.set(img, scaled, e)?; + } + Ok(()) +} + fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> { let start = usize::try_from(info.address) .map_err(|_| Error::Unsupported("chunk address out of range".into()))?; diff --git a/crates/clawhdf5/tests/edit_tests.rs b/crates/clawhdf5/tests/edit_tests.rs index 2467256..9190a2c 100644 --- a/crates/clawhdf5/tests/edit_tests.rs +++ b/crates/clawhdf5/tests/edit_tests.rs @@ -98,8 +98,8 @@ fn errors_leave_the_file_untouched() { let mut ed = FileEditor::open(&path).unwrap(); assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); assert!(ed.write_all("missing", &[0; 4]).is_err()); - // Wrong length, wrong type, outside the extent, beyond maxshape, - // shrinking, a rank change. + // Wrong length, wrong type, outside the extent, beyond maxshape, a + // rank change, resizing a dataset that is not chunked. assert!(matches!( ed.write_all("flat", &[0; 7]), Err(Error::InvalidArgument(_)) @@ -116,7 +116,10 @@ fn errors_leave_the_file_untouched() { ed.resize("raw", &[3, 5]), Err(Error::InvalidArgument(_)) )); - assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_)))); + assert!(matches!( + ed.resize("flat", &[2]), + Err(Error::Unsupported(_)) + )); assert!(matches!( ed.resize("ext", &[4, 1]), Err(Error::InvalidArgument(_)) @@ -136,3 +139,26 @@ fn errors_leave_the_file_untouched() { drop(ed); assert!(std::fs::read(&path).unwrap() == before); } + +/// Shrinking and growing again on a file clawhdf5 wrote: elements that come +/// back read as the fill value, the ones kept keep their values. +#[test] +fn shrink_then_grow_reads_fill() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + { + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("ext", &[2]).unwrap(); + ed.resize("ext", &[9]).unwrap(); + ed.resize("raw", &[1, 4]).unwrap(); + ed.resize("raw", &[3, 4]).unwrap(); + } + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("ext").unwrap().read_i32().unwrap(), + [0, 1, 0, 0, 0, 0, 0, 0, 0] + ); + let mut raw = vec![0.0f64; 12]; + raw[..4].fill(0.5); + assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw); +}