From 3c89a31df06b613c6f082691912b3b9b34b5de60 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:35:06 -0500 Subject: [PATCH 1/7] clawhdf5: FileEditor modifies existing files in place New clawhdf5::FileEditor opens an HDF5 file (h5py-written at any libver, HDF5 2.0 format included, or clawhdf5-written) under an exclusive flock and changes only what an edit touches: - write_selection/write_all/write_values: compact, contiguous (also late-allocated) and chunked datasets, any selection. Chunks are decoded, updated and re-encoded; a filtered chunk that no longer fits moves to the end of the file unless it is the file's last structure, which grows in place. New chunks go into v1 B-tree, Extensible Array (paged data blocks included), Fixed Array and single-chunk indexes, created on first use. - resize: grow chunked datasets up to maxshape. - set_attr: add/replace compact attributes, in a NIL slot or a new continuation chunk. Each edit is planned in an in-memory image and refused whole (Error::Unsupported) when any part is unsupported (v2 B-tree / implicit new chunks, shrinking, vlen/reference data, dense or order-tracked attributes, cache images, paged/persistent free space). Commit writes and syncs new space before patching existing bytes. Layout v5 (HDF5 2.0) array indexes use 8-byte filtered chunk sizes, as libhdf5 does. Error gains Unsupported/InvalidArgument/Locked and is #[non_exhaustive]; the Python bindings map them. build_attr_message is public. Tests (h5py, h5dump, h5rs check --data after every round; h5py r+ afterwards): appends crossing EA super/data blocks and B-tree splits, the same B-tree node counts and EA statistics as libhdf5 for the same writes (in order, reversed and shuffled; paged blocks), every layout and chunk index overwritten under random selections, attributes to continuation chunks, random operations against a model, refused edits leave the file byte-identical, locking. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/type_builders.rs | 3 +- crates/clawhdf5-py/src/lib.rs | 11 +- crates/clawhdf5-tools/tests/edit_interop.rs | 1189 ++++++++++++++++++ crates/clawhdf5/src/edit/btree1.rs | 403 ++++++ crates/clawhdf5/src/edit/earray.rs | 512 ++++++++ crates/clawhdf5/src/edit/farray.rs | 185 +++ crates/clawhdf5/src/edit/image.rs | 317 +++++ crates/clawhdf5/src/edit/mod.rs | 1213 +++++++++++++++++++ crates/clawhdf5/src/edit/ohdr.rs | 504 ++++++++ crates/clawhdf5/src/edit/select.rs | 142 +++ crates/clawhdf5/src/error.rs | 16 + crates/clawhdf5/src/lib.rs | 18 + crates/clawhdf5/src/reader.rs | 13 +- crates/clawhdf5/tests/edit_tests.rs | 138 +++ 14 files changed, 4657 insertions(+), 7 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/edit_interop.rs create mode 100644 crates/clawhdf5/src/edit/btree1.rs create mode 100644 crates/clawhdf5/src/edit/earray.rs create mode 100644 crates/clawhdf5/src/edit/farray.rs create mode 100644 crates/clawhdf5/src/edit/image.rs create mode 100644 crates/clawhdf5/src/edit/mod.rs create mode 100644 crates/clawhdf5/src/edit/ohdr.rs create mode 100644 crates/clawhdf5/src/edit/select.rs create mode 100644 crates/clawhdf5/tests/edit_tests.rs diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 3019daa..9d32e5c 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -296,7 +296,8 @@ impl EnumTypeBuilder { // ---- Attribute helper ---- -pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage { +/// The attribute message the writers store for `value` under `name`. +pub fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage { match value { AttrValue::F64(v) => AttributeMessage { name: name.to_string(), diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index e3b5619..4055b82 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -66,7 +66,9 @@ fn _panic_for_test() -> PyResult<()> { /// - I/O errors -> `PyIOError` /// - Format/parsing errors -> `PyValueError` /// - Missing dataset/path errors -> `PyKeyError` -/// - Other errors -> `PyOSError` +/// - Invalid arguments -> `PyValueError` +/// - Unsupported operations -> `PyNotImplementedError` +/// - Other errors (a locked file, ...) -> `PyOSError` pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { use clawhdf5_rs::Error; match &e { @@ -79,9 +81,14 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { | Error::ZeroCopyNotContiguous | Error::ZeroCopyNonNativeEndian | Error::ZeroCopyTypeMismatch { .. } - | Error::ZeroCopyUnaligned { .. } => { + | Error::ZeroCopyUnaligned { .. } + | Error::InvalidArgument(_) => { PyErr::new::(e.to_string()) } + Error::Unsupported(_) => { + PyErr::new::(e.to_string()) + } + _ => PyErr::new::(e.to_string()), } } diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs new file mode 100644 index 0000000..2df54d7 --- /dev/null +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -0,0 +1,1189 @@ +//! `clawhdf5::FileEditor` against libhdf5: files written by h5py (default +//! `libver`, `v114`, and HDF5 2.0's `latest`) and by clawhdf5 are modified +//! in place — values overwritten, datasets grown and appended to many +//! times (crossing Extensible Array super-block / data-block boundaries and +//! version-1 B-tree node splits), attributes added and replaced — and after +//! each round h5py must read exactly the expected values, h5dump must read +//! the file, `h5rs check --data` must find nothing, and our reader must +//! agree. At the end h5py opens the file `r+` and modifies it further. +//! +//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips +//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::{Command, Output}; + +use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn available(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the tools are here; false (skip) or a panic when interop is +/// required. +fn tools_ok() -> bool { + let ok = + available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]); + if !ok { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available" + ); + eprintln!("SKIP: h5py/numpy or h5dump not available"); + } + ok +} + +fn py(script: &str) -> String { + let o = Command::new(python()) + .args(["-c", script]) + .output() + .expect("run python"); + assert!( + o.status.success(), + "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8_lossy(&o.stdout).trim().to_string() +} + +fn text(o: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ) +} + +/// The file is valid for libhdf5's tools and for `h5rs check --data`. +/// HDF5 2.0 `latest` files are beyond h5dump 1.14. +fn check_tools(path: &Path, h5dump: bool) { + let p = path.to_str().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + if h5dump { + let o = Command::new("h5dump") + .args(["-o", "/dev/null", p]) + .output() + .unwrap(); + assert!( + o.status.success() && o.stderr.is_empty(), + "h5dump {p}:\n{}", + text(&o) + ); + } +} + +/// A dataset's expected contents. +#[derive(Clone, Debug)] +struct Model { + shape: Vec, + /// Row-major values. + data: Vec, +} + +impl Model { + fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self { + let n: u64 = shape.iter().product(); + Self { + shape: shape.to_vec(), + data: (0..n).map(f).collect(), + } + } + + fn index(&self, c: &[u64]) -> usize { + c.iter() + .zip(&self.shape) + .fold(0u64, |a, (&x, &d)| a * d + x) as usize + } + + /// Grow to `shape`, new elements `fill`. + fn resize(&mut self, shape: &[u64], fill: i32) { + let old = self.clone(); + *self = Self::new(shape, |_| fill); + let n: u64 = old.shape.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; old.shape.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % old.shape[d]; + r /= old.shape[d]; + } + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } + } + + /// Apply a hyperslab write of `vals` (row-major over the block). + fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) { + let n: u64 = count.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; count.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = start[d] + r % count[d]; + r /= count[d]; + } + let i = self.index(&c); + self.data[i] = vals[flat as usize]; + } + } +} + +fn block(start: &[u64], count: &[u64]) -> Selection { + Selection::Hyperslab { + start: start.to_vec(), + stride: vec![1; start.len()], + count: count.to_vec(), + block: vec![1; start.len()], + } +} + +/// h5py and our reader both read `m` from dataset `name`. +fn verify(path: &Path, name: &str, m: &Model) { + let f = File::open(path).unwrap(); + let ds = f.dataset(name).unwrap(); + assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}"); + assert!(ds.read_i32().unwrap() == m.data, "our values of {name}"); + let exp = path.with_extension("expect"); + let bytes: Vec = m.data.iter().flat_map(|v| v.to_le_bytes()).collect(); + std::fs::write(&exp, bytes).unwrap(); + let shape: Vec = m.shape.iter().map(|d| d.to_string()).collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + a = f[{name:?}][()]\n\ + e = np.fromfile({e:?}, dtype=' u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// `h5py.File(..., libver=...)` argument for each flavour tested; the last +/// is HDF5 2.0's own format, which h5dump 1.14 cannot read. +const LIBVERS: &[(&str, bool)] = &[("'earliest'", true), ("'v114'", true), ("'latest'", false)]; + +fn tmpdir() -> tempfile::TempDir { + let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + tempfile::TempDir::new_in(base).unwrap() +} + +/// Append to a 1-D unlimited dataset hundreds of times, in runs of random +/// length: enough chunks to fill an Extensible Array's index block, direct +/// data blocks and several super blocks (paged data blocks included), and +/// to split version-1 B-tree nodes several levels deep. +fn append_many(libver: &str, h5dump: bool, compression: &str, tag: &str) { + let dir = tmpdir(); + let path = dir.path().join(format!("append_{tag}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 f.create_dataset('x', shape=(5,), maxshape=(None,), chunks=(3,), dtype=' = (0..add).map(|k| (n + k) as i32 * 7 - 3).collect(); + ed.write_values("x", &block(&[n], &[add]), &vals).unwrap(); + m.write_block(&[n], &[add], &vals); + // Occasionally rewrite something earlier (in-place, or a filtered + // chunk that has to move). + if round % 7 == 0 { + let s = rng.below(m.shape[0]); + let c = 1 + rng.below((m.shape[0] - s).min(9)); + let vals: Vec = (0..c).map(|_| rng.next() as i32).collect(); + ed.write_values("x", &block(&[s], &[c]), &vals).unwrap(); + m.write_block(&[s], &[c], &vals); + } + if round % 150 == 149 { + drop(ed); + verify(&path, "x", &m); + check_tools(&path, h5dump); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "x", &m); + check_tools(&path, h5dump); + // h5py can go on appending to what we wrote. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 n = d.shape[0]\n\ + \x20 d.resize((n + 50,))\n\ + \x20 d[n:] = np.arange(50, dtype=' = (0..50).map(|k| 1000 + k).collect(); + m.write_block(&[n], &[50], &vals); + m.data[0] = -1; + verify(&path, "x", &m); + check_tools(&path, h5dump); +} + +#[test] +fn append_many_unfiltered() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + append_many(lv, *dump, "", &format!("plain{i}")); + } +} + +#[test] +fn append_many_gzip() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + append_many( + lv, + *dump, + ", compression='gzip', shuffle=True", + &format!("gzip{i}"), + ); + } +} + +/// Random operations — grow, hyperslab writes, point writes, attributes — +/// on a 2-D dataset with one unlimited dimension, checked against a model +/// after every few operations. +fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { + let dir = tmpdir(); + let path = dir.path().join(format!("rand_{tag}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \ + dtype=' = Vec::new(); + let mut rng = Rng(seed); + let mut ed = FileEditor::open(&path).unwrap(); + for step in 0..120 { + match rng.below(10) { + 0..=1 => { + let rows = m.shape[0] + 1 + rng.below(5); + ed.resize("m", &[rows, 7]).unwrap(); + m.resize(&[rows, 7], -9); + } + 2..=6 => { + let r0 = rng.below(m.shape[0]); + let c0 = rng.below(7); + let cnt = [ + 1 + rng.below((m.shape[0] - r0).min(6)), + 1 + rng.below(7 - c0), + ]; + let n = cnt[0] * cnt[1]; + let vals: Vec = (0..n).map(|_| (rng.next() % 100_000) as i32).collect(); + ed.write_values("m", &block(&[r0, c0], &cnt), &vals) + .unwrap(); + m.write_block(&[r0, c0], &cnt, &vals); + } + 7 => { + let pts: Vec> = (0..1 + rng.below(4)) + .map(|_| vec![rng.below(m.shape[0]), rng.below(7)]) + .collect(); + let vals: Vec = pts.iter().map(|_| rng.next() as i32).collect(); + ed.write_values("m", &Selection::Points(pts.clone()), &vals) + .unwrap(); + for (p, v) in pts.iter().zip(&vals) { + let i = m.index(p); + m.data[i] = *v; + } + } + _ => { + let k = rng.below(6); + let name = format!("a{k}"); + let v = rng.next() as i64; + match ed.set_attr("m", &name, &AttrValue::I64(v)) { + Ok(()) => { + attrs.retain(|(n, _)| *n != name); + attrs.push((name, v)); + } + Err(e) => panic!("set_attr {name}: {e}"), + } + } + } + if step % 30 == 29 { + drop(ed); + verify(&path, "m", &m); + check_tools(&path, h5dump); + check_attrs(&path, "m", &attrs); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "m", &m); + check_attrs(&path, "m", &attrs); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['m']\n\ + \x20 n = d.shape[0]\n\ + \x20 d.resize((n + 3, 7))\n\ + \x20 d[n:, :] = 42\n\ + \x20 d.attrs['from_h5py'] = 1.5\n", + p = path.to_str().unwrap() + )); + let n = m.shape[0]; + m.resize(&[n + 3, 7], -9); + m.write_block(&[n, 0], &[3, 7], &[42; 21]); + verify(&path, "m", &m); + check_tools(&path, h5dump); +} + +fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) { + let f = File::open(path).unwrap(); + let got = f.dataset(obj).unwrap().attrs().unwrap(); + for (n, v) in attrs { + match got.get(n) { + Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"), + other => panic!("attribute {n}: {other:?}"), + } + } + let want: Vec = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect(); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + want = {{{w}}}\n\ + got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\ + assert got == want, (got, want)\n", + p = path.to_str().unwrap(), + w = want.join(", ") + )); +} + +#[test] +fn random_operations_match_a_model() { + if !tools_ok() { + return; + } + let mut seed = 1; + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + // h5dump has no LZF decoder (h5py's own filter). + for (j, (extra, lzf)) in [ + ("", false), + (", compression='gzip', shuffle=True, fletcher32=True", false), + (", compression='lzf'", true), + ] + .iter() + .enumerate() + { + seed += 1; + random_ops(lv, *dump && !lzf, extra, &format!("{i}_{j}"), seed); + } + } +} + +fn unsupported(r: Result) { + match r { + Err(Error::Unsupported(_)) => {} + other => panic!("expected Error::Unsupported, got {other:?}"), + } +} + +/// The statistics an Extensible Array header keeps (super blocks and their +/// bytes, data blocks and their bytes, one past the highest index set, +/// elements realised), for the file's only Extensible Array. +fn ea_stats(path: &Path) -> [u64; 6] { + let b = std::fs::read(path).unwrap(); + let at = b + .windows(4) + .position(|w| w == b"EAHD") + .expect("an EA header"); + let mut s = [0u64; 6]; + for (k, v) in s.iter_mut().enumerate() { + let o = at + 12 + 8 * k; + *v = u64::from_le_bytes(b[o..o + 8].try_into().unwrap()); + } + s +} + +/// Version-1 B-tree nodes in the file: (leaves, internal nodes). +fn btree1_nodes(path: &Path) -> (usize, usize) { + let b = std::fs::read(path).unwrap(); + let mut out = (0, 0); + for i in 0..b.len().saturating_sub(6) { + if &b[i..i + 4] == b"TREE" && b[i + 4] == 1 { + if b[i + 5] == 0 { + out.0 += 1; + } else { + out.1 += 1; + } + } + } + out +} + +/// Growth one chunk at a time and in large steps: the editor's version-1 +/// B-tree must split the way libhdf5's does (the same number of leaves and +/// internal nodes for the same insertions). +#[test] +fn btree1_splits_match_libhdf5() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let mut steps: Vec = (1..=300).collect(); + steps.extend([1000, 1001, 5000, 9000]); + let a = dir.path().join("bt_h5py.h5"); + let b = dir.path().join("bt_edit.h5"); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 d = f.create_dataset('x', shape=(0,), maxshape=(None,), chunks=(2,), dtype=' = (n..s).map(|v| v as i32).collect(); + ed.write_values("x", &block(&[n], &[s - n]), &vals).unwrap(); + n = s; + } + drop(ed); + verify(&b, "x", &Model::new(&[n], |i| i as i32)); + check_tools(&b, true); + assert_eq!( + btree1_nodes(&b), + btree1_nodes(&a), + "B-tree shape differs from libhdf5's" + ); +} + +/// Enough chunks that the Extensible Array reaches its paged data blocks +/// (the first one holds element 131060 with h5py's parameters): the same +/// growth done by libhdf5 and by the editor must create the same blocks. +#[test] +fn extensible_array_paged_blocks_match_libhdf5() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let steps = [5u64, 131_000, 131_070, 133_000, 140_000]; + let a = dir.path().join("paged_h5py.h5"); + let b = dir.path().join("paged_edit.h5"); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v114') as f:\n\ + \x20 d = f.create_dataset('x', shape=(0,), maxshape=(None,), chunks=(1,), dtype=' = (n..s).map(|v| v as i32).collect(); + ed.write_values("x", &block(&[n], &[s - n]), &vals).unwrap(); + n = s; + } + drop(ed); + let m = Model::new(&[n], |i| i as i32); + verify(&b, "x", &m); + check_tools(&b, true); + assert_eq!( + ea_stats(&b), + ea_stats(&a), + "EA statistics differ from libhdf5's" + ); +} + +/// Every element of `sel` over `shape`, in selection order. +/// Every index of the box `ext`, row-major. +fn odometer(ext: &[u64]) -> Vec> { + let n: u64 = ext.iter().product(); + (0..n) + .map(|flat| { + let mut c = vec![0u64; ext.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % ext[d]; + r /= ext[d]; + } + c + }) + .collect() +} + +fn sel_coords(sel: &Selection, shape: &[u64]) -> Vec> { + match sel { + Selection::All => odometer(shape), + Selection::None => Vec::new(), + Selection::Points(p) => p.clone(), + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let ext: Vec = count.iter().zip(block).map(|(c, b)| c * b).collect(); + odometer(&ext) + .into_iter() + .map(|j| { + (0..j.len()) + .map(|d| start[d] + (j[d] / block[d]) * stride[d] + j[d] % block[d]) + .collect() + }) + .collect() + } + } +} + +impl Model { + fn write_sel(&mut self, sel: &Selection, vals: &[i32]) { + let cs = sel_coords(sel, &self.shape); + assert_eq!(cs.len(), vals.len()); + for (c, v) in cs.iter().zip(vals) { + let i = self.index(c); + self.data[i] = *v; + } + } +} + +/// A random selection of `shape`: a block, a strided hyperslab, points, +/// or everything. +fn random_sel(rng: &mut Rng, shape: &[u64]) -> Selection { + let rank = shape.len(); + match rng.below(4) { + 0 => Selection::All, + 1 => { + let pts = (0..1 + rng.below(5)) + .map(|_| shape.iter().map(|&d| rng.below(d)).collect()) + .collect(); + Selection::Points(pts) + } + _ => { + let mut start = vec![0; rank]; + let mut stride = vec![1; rank]; + let mut count = vec![1; rank]; + let mut block = vec![1; rank]; + for d in 0..rank { + start[d] = rng.below(shape[d]); + let room = shape[d] - start[d]; + block[d] = 1 + rng.below(room.min(3)); + stride[d] = block[d] + rng.below(3); + // count * stride may overshoot as long as the last block fits. + count[d] = 1 + (room - block[d]) / stride[d]; + count[d] = 1 + rng.below(count[d]); + } + Selection::Hyperslab { + start, + stride, + count, + block, + } + } + } +} + +/// Datasets of every layout and chunk index h5py writes, overwritten under +/// random selections: contiguous (allocated, and never written — late +/// allocation), compact, chunked with a Fixed Array (paged, too), single +/// chunk (unfiltered and filtered), implicit (early allocation), version-1 +/// B-tree (every chunked layout of the default libver), and version-2 +/// B-tree (existing unfiltered chunks only). +#[test] +fn overwrite_every_layout() { + if !tools_ok() { + return; + } + let names: &[(&str, &[u64])] = &[ + ("contig", &[10, 4]), + ("contig_late", &[6]), + ("compact", &[3, 4]), + ("fixed", &[10, 7]), + ("fixed_gz", &[10, 7]), + ("fa_paged", &[3000]), + ("single", &[5, 6]), + ("single_gz", &[5, 6]), + ("implicit", &[8, 8]), + ("fa_empty", &[10, 7]), + ("fa_gz_empty", &[3000]), + ("ea_gz_empty", &[20]), + ("bt2", &[6, 6]), + ]; + for (li, (lv, dump)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("overwrite_{li}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + from h5py import h5p, h5d, h5s, h5t\n\ + def ar(*s): return np.arange(int(np.prod(s)), dtype=' = vec![ + Model::new(&[10, 4], |i| i as i32), + Model::new(&[6], |_| 7), + Model::new(&[3, 4], |i| i as i32), + { + let mut m = Model::new(&[10, 7], |_| -1); + m.write_block(&[0, 0], &[3, 4], &[1; 12]); + m + }, + { + let mut m = Model::new(&[10, 7], |_| 0); + m.write_block(&[4, 3], &[6, 4], &(0..24).collect::>()); + m + }, + Model::new(&[3000], |i| if i % 7 == 0 { 3 } else { 0 }), + Model::new(&[5, 6], |_| 0), + Model::new(&[5, 6], |i| i as i32), + Model::new(&[8, 8], |_| 0), + Model::new(&[10, 7], |_| 4), + Model::new(&[3000], |_| 0), + Model::new(&[20], |_| 0), + Model::new(&[6, 6], |i| i as i32), + ]; + let mut rng = Rng(77 + li as u64); + for round in 0..3 { + let mut ed = FileEditor::open(&path).unwrap(); + for ((name, shape), m) in names.iter().zip(models.iter_mut()) { + for _ in 0..6 { + let sel = random_sel(&mut rng, shape); + let n = sel_coords(&sel, shape).len(); + let vals: Vec = (0..n).map(|_| rng.next() as i32).collect(); + ed.write_values(name, &sel, &vals) + .unwrap_or_else(|e| panic!("{name} round {round} {sel:?}: {e}")); + m.write_sel(&sel, &vals); + } + } + drop(ed); + for ((name, _), m) in names.iter().zip(&models) { + verify(&path, name, m); + } + check_tools(&path, *dump); + } + // A version-2 B-tree index can take new chunks only from libhdf5 for + // now: growing works, writing the new chunks is refused and changes + // nothing. + if *lv != "'earliest'" { + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("bt2", &[8, 6]).unwrap(); + let before = std::fs::read(&path).unwrap(); + unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12])); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); + models[12].resize(&[8, 6], 0); + } + // libhdf5 goes on modifying what we wrote. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for n in {names:?}:\n\ + \x20 d = f[n]\n\ + \x20 d[(0,) * d.ndim] = 123\n", + p = path.to_str().unwrap(), + names = names.iter().map(|(n, _)| *n).collect::>() + )); + for ((name, _), m) in names.iter().zip(models.iter_mut()) { + m.data[0] = 123; + verify(&path, name, m); + } + check_tools(&path, *dump); + } +} + +/// Attributes added to and replaced on the root group, a group and a +/// dataset, until the headers need continuation chunks; then h5py adds, +/// changes and deletes attributes in the same headers. +#[test] +fn attributes_in_place() { + if !tools_ok() { + return; + } + for (li, (lv, dump)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("attrs_{li}.h5")); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.attrs['title'] = 'root'\n\ + \x20 g = f.create_group('g')\n\ + \x20 g.attrs['n'] = 3\n\ + \x20 d = f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + let mut ed = FileEditor::open(&path).unwrap(); + // v2 headers hold 8 compact attributes by default. + let many = if *lv == "'earliest'" { 20 } else { 6 }; + for obj in ["/", "g", "d"] { + for i in 0..many { + let name = format!("x{i}"); + let v = AttrValue::I64Array((0..=i as i64).collect()); + ed.set_attr(obj, &name, &v).unwrap(); + want.push((obj, name, v)); + } + // Replace one with something much larger (moves it). + let big = AttrValue::String("z".repeat(700)); + ed.set_attr(obj, "x1", &big).unwrap(); + want.retain(|(o, n, _)| !(*o == obj && n == "x1")); + want.push((obj, "x1".into(), big)); + // And one smaller, in place. + ed.set_attr(obj, "x0", &AttrValue::F64(2.5)).unwrap(); + want.retain(|(o, n, _)| !(*o == obj && n == "x0")); + want.push((obj, "x0".into(), AttrValue::F64(2.5))); + } + ed.set_attr("d", "units", &AttrValue::String("km".into())) + .unwrap(); + want.push(("d", "units".into(), AttrValue::String("km".into()))); + if *lv != "'earliest'" { + // Up to the compact limit (8) and no further; attributes in + // dense storage and tracked creation order are refused, and a + // refused edit writes nothing. + ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); + want.push(("g", "eighth".into(), AttrValue::I64(8))); + let before = std::fs::read(&path).unwrap(); + unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); + unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); + unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); + } + drop(ed); + check_tools(&path, *dump); + let f = File::open(&path).unwrap(); + let expect_py: Vec = want + .iter() + .map(|(o, n, v)| { + let got = if *o == "/" { + f.root().attrs().unwrap() + } else if *o == "d" { + f.dataset(o).unwrap().attrs().unwrap() + } else { + f.group(o).unwrap().attrs().unwrap() + }; + let g = got.get(n).unwrap_or_else(|| panic!("{o}/{n} missing")); + assert_eq!(format!("{g:?}"), format!("{v:?}"), "{o}/{n}"); + let pv = match v { + AttrValue::I64Array(a) => format!("{a:?}"), + AttrValue::String(s) => format!("{s:?}"), + AttrValue::F64(x) => format!("{x:?}"), + AttrValue::I64(x) => format!("{x}"), + other => panic!("{other:?}"), + }; + format!("({o:?}, {n:?}, {pv})") + }) + .collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + for o, n, v in [{w}]:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 assert a == v, (o, n, a, v)\n\ + assert f['d'][()].tolist() == [0, 1, 2, 3]\n\ + assert f['g'].attrs['n'] == 3 and f.attrs['title'] == 'root'\n", + w = expect_py.join(", ") + )); + // libhdf5 modifies the headers we changed. + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'd']:\n\ + \x20 f[o].attrs['from_h5py'] = 'yes'\n\ + \x20 f[o].attrs['x0'] = 9\n\ + \x20 del f[o].attrs['x1']\n" + )); + check_tools(&path, *dump); + let f = File::open(&path).unwrap(); + for attrs in [ + f.root().attrs().unwrap(), + f.dataset("d").unwrap().attrs().unwrap(), + ] { + assert!(matches!(attrs.get("from_h5py"), Some(AttrValue::String(s)) if s == "yes")); + assert!(matches!(attrs.get("x0"), Some(AttrValue::I64(9)))); + assert!(!attrs.contains_key("x1")); + assert!(attrs.contains_key("x2")); + } + } +} + +/// Files written by clawhdf5's own writer: appended to, overwritten and +/// given attributes, then read by h5py. +#[test] +fn edit_clawhdf5_written_files() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("ours.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("ext") + .with_i32_data(&(0..10).collect::>()) + .with_shape(&[10]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[4]) + .with_deflate(4); + b.create_dataset("plain").with_i32_data(&[1, 2, 3, 4, 5, 6]); + b.create_dataset("grid") + .with_i32_data(&(0..24).collect::>()) + .with_shape(&[4, 6]) + .with_maxshape(&[u64::MAX, 6]) + .with_chunks(&[2, 3]); + b.set_attr("version", AttrValue::I64(1)); + b.write(&path).unwrap(); + let mut ext = Model::new(&[10], |i| i as i32); + let mut plain = Model::new(&[6], |i| i as i32 + 1); + let mut grid = Model::new(&[4, 6], |i| i as i32); + let mut ed = FileEditor::open(&path).unwrap(); + let mut rng = Rng(9); + for _ in 0..200 { + let n = ext.shape[0]; + let add = 1 + rng.below(9); + ed.resize("ext", &[n + add]).unwrap(); + ext.resize(&[n + add], 0); + let vals: Vec = (0..add).map(|_| rng.next() as i32).collect(); + ed.write_values("ext", &block(&[n], &[add]), &vals).unwrap(); + ext.write_block(&[n], &[add], &vals); + } + for _ in 0..30 { + let rows = grid.shape[0] + rng.below(3); + ed.resize("grid", &[rows, 6]).unwrap(); + grid.resize(&[rows, 6], 0); + let sel = random_sel(&mut rng, &grid.shape.clone()); + let vals: Vec = (0..sel_coords(&sel, &grid.shape).len()) + .map(|_| rng.next() as i32) + .collect(); + ed.write_values("grid", &sel, &vals).unwrap(); + grid.write_sel(&sel, &vals); + } + ed.write_values( + "plain", + &Selection::Points(vec![vec![5], vec![0]]), + &[60, 10], + ) + .unwrap(); + plain.write_sel(&Selection::Points(vec![vec![5], vec![0]]), &[60, 10]); + ed.set_attr("/", "version", &AttrValue::I64(2)).unwrap(); + ed.set_attr("ext", "note", &AttrValue::String("appended".into())) + .unwrap(); + drop(ed); + verify(&path, "ext", &ext); + verify(&path, "plain", &plain); + verify(&path, "grid", &grid); + check_tools(&path, true); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + assert f.attrs['version'] == 2\n\ + assert f['ext'].attrs['note'] in ('appended', b'appended')\n", + p = path.to_str().unwrap() + )); +} + +/// One writer at a time: a second editor (or libhdf5 with file locking) +/// is refused until the first is dropped. +#[test] +fn editor_locks_the_file() { + let dir = tmpdir(); + let path = dir.path().join("lock.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("x").with_i32_data(&[1, 2, 3]); + b.write(&path).unwrap(); + let ed = FileEditor::open(&path).unwrap(); + assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); + if available(&python(), &["-c", "import h5py"]) { + let o = Command::new(python()) + .args([ + "-c", + &format!("import h5py; h5py.File({:?}, 'r+')", path.to_str().unwrap()), + ]) + .env_remove("HDF5_USE_FILE_LOCKING") + .output() + .unwrap(); + assert!(!o.status.success(), "h5py opened a locked file r+"); + } + drop(ed); + FileEditor::open(&path).unwrap(); +} + +/// Refused edits leave the file byte for byte as it was. +#[test] +fn refused_edits_change_nothing() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("refuse.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 f.create_dataset('s', data=['a', 'bb'], dtype=h5py.string_dtype())\n\ + \x20 f.create_dataset('x', data=np.arange(6, dtype=' = (900..len).rev().collect(); + let mut rest: Vec = (0..900).collect(); + let mut rng = Rng(3); + for i in (1..rest.len()).rev() { + rest.swap(i, rng.below(i as u64 + 1) as usize); + } + order.extend(rest.iter().take(500)); + for (lv, tag) in [("'earliest'", "bt"), ("'v114'", "ea")] { + let dir = tmpdir(); + let a = dir.path().join(format!("ooo_{tag}_h5py.h5")); + let b = dir.path().join(format!("ooo_{tag}_edit.h5")); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 d = f.create_dataset('x', shape=({len},), maxshape=(None,), chunks=(1,), dtype=' = (n..n + add) + .map(|i| ((i as f64 / 50.0).sin() * 1000.0).round() / 1000.0) + .collect(); + ed.write_values("x", &block(&[n], &[add]), &vals).unwrap(); + } + drop(ed); + let size = |p: &Path| std::fs::metadata(p).unwrap().len(); + let repacked = |p: &Path| { + let out = p.with_extension("repacked.h5"); + let _ = std::fs::remove_file(&out); + let o = Command::new("h5repack") + .args([p.to_str().unwrap(), out.to_str().unwrap()]) + .output() + .unwrap(); + assert!(o.status.success(), "{}", text(&o)); + size(&out) + }; + println!( + "chunks {chunk}{comp}, {rounds} appends of {add}: editor {} bytes \ + (repacked {}), libhdf5 {} bytes (repacked {})", + size(&b), + repacked(&b), + size(&a), + repacked(&a) + ); + } +} diff --git a/crates/clawhdf5/src/edit/btree1.rs b/crates/clawhdf5/src/edit/btree1.rs new file mode 100644 index 0000000..7289c3f --- /dev/null +++ b/crates/clawhdf5/src/edit/btree1.rs @@ -0,0 +1,403 @@ +//! Inserting into (and updating) a version-1 B-tree chunk index (node type +//! 1; layout versions 1-3), as libhdf5's `H5B_insert` does: +//! +//! - keys compare lexicographically over the chunk offsets *and* the +//! element-size coordinate (0 in a chunk's own key), so a node's final +//! ("right") key after an append is the last chunk's offsets with the +//! element-size coordinate set to the element size — the smallest key +//! greater than that chunk, which is what libhdf5 writes; +//! - a full node (2K children) splits before the insertion: the right-most +//! node of a level keeps 90% of its children, the left-most 10%, any other +//! half (libhdf5's default split ratios); siblings are relinked; +//! - a full root splits by moving its left half to a new node, so the root +//! keeps its address (the layout message never changes). +//! +//! Version-1 B-tree nodes carry no checksum. + +use std::cmp::Ordering; + +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) size: u32, + pub(crate) mask: u32, + /// Offsets in every dimension, the element-size one last. + pub(crate) offs: Vec, +} + +fn cmp(a: &[u64], b: &[u64]) -> Ordering { + a.cmp(b) +} + +#[derive(Debug, Clone)] +struct Node { + addr: u64, + level: u8, + left: u64, + right: u64, + /// `children.len() + 1` keys. + keys: Vec, + children: Vec, +} + +pub(crate) struct BTree1 { + root: u64, + /// Children per node at most (2K). + two_k: usize, + ndims: usize, + elem_size: u64, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("chunk B-tree: {why}"), + )) +} + +enum Ins { + Done, + /// The node split; the new right sibling and its first key. + Split(Key, u64), +} + +impl BTree1 { + /// `k` is the file's chunk B-tree K (children per node are 2K); + /// `ndims` counts the element-size dimension. + pub(crate) fn new(root: u64, k: u16, ndims: usize, elem_size: u64) -> Result { + if k == 0 || ndims < 2 { + return Err(bad("bad parameters")); + } + Ok(Self { + root, + two_k: 2 * k as usize, + ndims, + elem_size, + }) + } + + fn key_size(&self) -> usize { + 8 + 8 * self.ndims + } + + fn node_size(&self, os: u8) -> usize { + let os = os as usize; + 8 + 2 * os + (self.two_k + 1) * self.key_size() + self.two_k * os + } + + fn read(&self, img: &Image<'_>, addr: u64) -> Result { + let os = img.os; + let osz = os as usize; + let d = img.read(addr, 8 + 2 * osz)?; + if &d[0..4] != b"TREE" || d[4] != 1 { + return Err(bad("not a chunk B-tree node")); + } + let level = d[5]; + let n = u16::from_le_bytes([d[6], d[7]]) as usize; + if n > self.two_k { + return Err(bad("node holds more children than 2K")); + } + let left = get_uint(&d[8..], os); + let right = get_uint(&d[8 + osz..], os); + let ks = self.key_size(); + let body = img.read(addr + 8 + 2 * osz as u64, (n + 1) * ks + n * osz)?; + let mut keys = Vec::with_capacity(n + 1); + let mut children = Vec::with_capacity(n); + let mut p = 0; + for i in 0..=n { + let k = &body[p..p + ks]; + keys.push(Key { + size: u32::from_le_bytes([k[0], k[1], k[2], k[3]]), + mask: u32::from_le_bytes([k[4], k[5], k[6], k[7]]), + offs: (0..self.ndims) + .map(|d| { + u64::from_le_bytes(k[8 + 8 * d..16 + 8 * d].try_into().unwrap_or([0; 8])) + }) + .collect(), + }); + p += ks; + if i < n { + children.push(get_uint(&body[p..], os)); + p += osz; + } + } + Ok(Node { + addr, + level, + left, + right, + keys, + children, + }) + } + + fn write(&self, img: &mut Image<'_>, node: &Node) -> Result<(), Error> { + let os = img.os; + let osz = os as usize; + let mut d = vec![0u8; self.node_size(os)]; + d[0..4].copy_from_slice(b"TREE"); + d[4] = 1; + d[5] = node.level; + d[6..8].copy_from_slice(&(node.children.len() as u16).to_le_bytes()); + put_uint(&mut d[8..], node.left, os); + put_uint(&mut d[8 + osz..], node.right, os); + let ks = self.key_size(); + let mut p = 8 + 2 * osz; + for (i, k) in node.keys.iter().enumerate() { + d[p..p + 4].copy_from_slice(&k.size.to_le_bytes()); + d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes()); + for (j, o) in k.offs.iter().enumerate() { + d[p + 8 + 8 * j..p + 16 + 8 * j].copy_from_slice(&o.to_le_bytes()); + } + p += ks; + if i < node.children.len() { + put_uint(&mut d[p..], node.children[i], os); + p += osz; + } + } + // Unused key/child slots stay zero, as libhdf5 leaves them. + img.write(node.addr, &d) + } + + /// Create a tree holding one chunk; returns it (its root is a new leaf). + pub(crate) fn create( + img: &mut Image<'_>, + k: u16, + ndims: usize, + elem_size: u64, + key: Key, + addr: u64, + ) -> Result { + let mut t = Self::new(0, k, ndims, elem_size)?; + let root = img.alloc(t.node_size(img.os) as u64)?; + t.root = root; + let right = t.right_key_after(&key); + let node = Node { + addr: root, + level: 0, + left: undef(img.os), + right: undef(img.os), + keys: vec![key, right], + children: vec![addr], + }; + t.write(img, &node)?; + Ok(t) + } + + pub(crate) fn root(&self) -> u64 { + self.root + } + + /// The smallest key above chunk `key`: its offsets with the element-size + /// coordinate one element in (what libhdf5 writes as a right key). + fn right_key_after(&self, key: &Key) -> Key { + let mut offs = key.offs.clone(); + if let Some(last) = offs.last_mut() { + *last = self.elem_size; + } + Key { + size: 0, + mask: 0, + offs, + } + } + + /// Insert chunk `key` at address `addr`, or update it when the tree + /// already has a chunk at those offsets. + pub(crate) fn insert(&mut self, img: &mut Image<'_>, key: Key, addr: u64) -> Result<(), Error> { + if key.offs.len() != self.ndims || key.offs[self.ndims - 1] != 0 { + return Err(bad("bad chunk key")); + } + let root = self.read(img, self.root)?; + if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? { + // The root split: move its (left) half to a new node so the root + // keeps its address, then make the root the parent of both. + let old = self.read(img, self.root)?; + let right = self.read(img, right_addr)?; + let new_left = img.alloc(self.node_size(img.os) as u64)?; + let mut moved = old.clone(); + moved.addr = new_left; + self.write(img, &moved)?; + let mut right = right; + right.left = new_left; + self.write(img, &right)?; + let first = old.keys[0].clone(); + let last = right + .keys + .last() + .cloned() + .ok_or_else(|| bad("empty node"))?; + let new_root = Node { + addr: self.root, + level: old.level + 1, + left: undef(img.os), + right: undef(img.os), + keys: vec![first, mid, last], + children: vec![new_left, right_addr], + }; + self.write(img, &new_root)?; + } + Ok(()) + } + + fn insert_at( + &self, + img: &mut Image<'_>, + mut node: Node, + key: &Key, + addr: u64, + depth: u8, + ) -> Result { + if depth == 0 { + return Err(bad("tree too deep")); + } + let n = node.children.len(); + if n == 0 { + return Err(bad("empty node")); + } + // The child whose range holds the key: the last i with + // keys[i] <= key (the first child when the key is below them all). + let mut i = node + .keys + .iter() + .take(n) + .rposition(|k| cmp(&k.offs, &key.offs) != Ordering::Greater) + .unwrap_or(0); + if node.level == 0 { + if node.keys[i].offs == key.offs { + node.keys[i].size = key.size; + node.keys[i].mask = key.mask; + node.children[i] = addr; + self.write(img, &node)?; + return Ok(Ins::Done); + } + // Insert after child i unless the key is below every child. + let pos = if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less { + 0 + } else { + i + 1 + }; + return self.add_child(img, node, pos, key.clone(), addr); + } + let child = self.read(img, node.children[i])?; + if child.level + 1 != node.level { + return Err(bad("inconsistent node levels")); + } + let ins = self.insert_at(img, child, key, addr, depth - 1)?; + let mut changed = false; + if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less && i == 0 { + node.keys[0] = key.clone(); + changed = true; + } + if cmp(&key.offs, &node.keys[n].offs) != Ordering::Less { + node.keys[n] = self.right_key_after(key); + changed = true; + } + match ins { + Ins::Done => { + if changed { + self.write(img, &node)?; + } + Ok(Ins::Done) + } + Ins::Split(mid, right) => { + i += 1; + self.add_child(img, node, i, mid, right) + } + } + } + + /// Insert child `addr` with left key `key` at position `pos` of `node` + /// (splitting it first when full), and write what changed. + fn add_child( + &self, + img: &mut Image<'_>, + mut node: Node, + pos: usize, + key: Key, + addr: u64, + ) -> Result { + let n = node.children.len(); + if n < self.two_k { + Self::insert_child(self, &mut node, pos, key, addr); + self.write(img, &node)?; + return Ok(Ins::Done); + } + // Split first (H5B__split): how many children stay left. + let undefined = undef(img.os); + let mut nleft = if node.right == undefined { + (self.two_k as f64 * 0.9) as usize + } else if node.left == undefined { + (self.two_k as f64 * 0.1) as usize + } else { + self.two_k / 2 + }; + if pos < nleft && nleft == self.two_k { + nleft -= 1; + } else if pos >= nleft && nleft == 0 { + nleft += 1; + } + let right_addr = img.alloc(self.node_size(img.os) as u64)?; + let mut right = Node { + addr: right_addr, + level: node.level, + left: node.addr, + right: node.right, + keys: node.keys[nleft..].to_vec(), + children: node.children[nleft..].to_vec(), + }; + if node.right != undefined { + let mut sib = self.read(img, node.right)?; + sib.left = right_addr; + self.write(img, &sib)?; + } + node.keys.truncate(nleft + 1); + node.children.truncate(nleft); + node.right = right_addr; + if pos <= nleft && !(pos == nleft && nleft < n && self.goes_right(&key, &right)) { + self.insert_child(&mut node, pos, key, addr); + } else { + self.insert_child(&mut right, pos - nleft, key, addr); + } + self.write(img, &node)?; + self.write(img, &right)?; + let mid = right.keys[0].clone(); + Ok(Ins::Split(mid, right_addr)) + } + + /// For an insertion exactly at the split point: whether the key belongs + /// to the right half (it is not below the right half's first key). + fn goes_right(&self, key: &Key, right: &Node) -> bool { + cmp(&key.offs, &right.keys[0].offs) != Ordering::Less + } + + fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) { + let n = node.children.len(); + if node.level == 0 { + // A leaf: the new chunk's key goes at `pos`. At the end, the + // node's right key moves up to stay above the new chunk. + if pos == n { + let right = self.right_key_after(&key); + let last = node.keys.len() - 1; + if cmp(&node.keys[last].offs, &right.offs) == Ordering::Less { + node.keys[last] = right; + } + node.keys.insert(n, key); + } else { + node.keys.insert(pos, key); + } + } else { + // An internal node: `key` is the new child's left key, taking + // position `pos` (the child's range starts there). + if pos == n { + // A child split off the last child: its right key is the + // parent's right key already. + node.keys.insert(n, key); + } else { + node.keys.insert(pos, key); + } + } + node.children.insert(pos, addr); + } +} diff --git a/crates/clawhdf5/src/edit/earray.rs b/crates/clawhdf5/src/edit/earray.rs new file mode 100644 index 0000000..49d9621 --- /dev/null +++ b/crates/clawhdf5/src/edit/earray.rs @@ -0,0 +1,512 @@ +//! Setting elements of an Extensible Array chunk index (layout v4, index +//! type 4), creating the index block, super blocks, data blocks and data +//! block pages the element needs, exactly as `H5EA__lookup_elmt` creates +//! them — including the header statistics libhdf5 keeps (blocks created, +//! their bytes, elements realised, one past the highest index set) and the +//! "block offset" each data block records. + +use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef}; +use crate::error::Error; + +/// A chunk index element. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Elem { + pub(crate) addr: u64, + pub(crate) size: u64, + pub(crate) mask: u32, +} + +/// Encode an element: the address, and for a filtered array the stored +/// size (in `elem_size - os - 4` bytes) and filter mask. `None` is the +/// fill element (undefined address, zero size and mask). +pub(crate) fn encode_elem( + e: Option, + filtered: bool, + elem_size: usize, + os: u8, +) -> Result, Error> { + let osz = os as usize; + let mut b = vec![0u8; if filtered { elem_size } else { osz }]; + let addr = e.map_or(undef(os), |e| e.addr); + put_uint(&mut b, addr, os); + if filtered { + let width = elem_size - osz - 4; + if let Some(e) = e { + if width < 8 && e.size >> (8 * width) != 0 { + return Err(Error::Unsupported(format!( + "filtered chunk of {} bytes does not fit the index's {width}-byte size field", + e.size + ))); + } + b[osz..osz + width].copy_from_slice(&e.size.to_le_bytes()[..width]); + b[osz + width..].copy_from_slice(&e.mask.to_le_bytes()); + } + } + Ok(b) +} + +/// The width libhdf5 gives the stored-size field of a filtered chunk index +/// element for chunks of `chunk_bytes` bytes (`H5D__earray_idx_create`, +/// `H5D__farray_idx_create`): one byte more than the nominal size needs — +/// except under layout message version 5 (HDF5 2.0's own format), which +/// always uses 8 bytes. +pub(crate) fn chunk_size_len(chunk_bytes: u64, layout_version: u8) -> usize { + if layout_version >= 5 { + return 8; + } + let log2 = if chunk_bytes <= 1 { + 0 + } else { + 63 - chunk_bytes.leading_zeros() + }; + (1 + ((log2 + 8) / 8) as usize).min(8) +} + +/// Creation parameters, in the layout message's order. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EaParams { + pub(crate) max_nelmts_bits: u8, + pub(crate) idx_blk_elmts: u8, + pub(crate) sup_blk_min_data_ptrs: u8, + pub(crate) data_blk_min_elmts: u8, + pub(crate) max_dblk_page_nelmts_bits: u8, +} + +#[derive(Debug, Clone, Copy)] +struct Level { + ndblks: u64, + dblk_nelmts: u64, + /// First element of the level, counted after the index block's own. + start_idx: u64, + /// Number of data blocks in the levels before this one. + start_dblk: u64, +} + +/// An open Extensible Array. +pub(crate) struct Ea { + hdr: u64, + filtered: bool, + elem_size: usize, + p: EaParams, + /// nsuper_blks, super_blk_size, ndata_blks, data_blk_size, + /// max_idx_set, nelmts. + stats: [u64; 6], + iblock: u64, + levels: Vec, + /// Levels whose data blocks the index block addresses directly. + direct_levels: usize, + ndblk_addrs: usize, + nsblk_addrs: usize, + dirty_hdr: bool, + /// Checksummed ranges changed by `set` (start -> checksum position), + /// recomputed once by `finish`. + dirty: std::collections::BTreeMap, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("Extensible Array: {why}"), + )) +} + +impl Ea { + fn layout(p: EaParams) -> Result<(Vec, usize, usize, usize), Error> { + let dmin = u64::from(p.data_blk_min_elmts); + if dmin == 0 || !dmin.is_power_of_two() || p.max_nelmts_bits > 64 { + return Err(bad("bad creation parameters")); + } + let nsblks = + 1 + (p.max_nelmts_bits as usize).saturating_sub(dmin.trailing_zeros() as usize); + let mut levels = Vec::with_capacity(nsblks); + let (mut start_idx, mut start_dblk) = (0u64, 0u64); + for u in 0..nsblks { + let ndblks = 1u64.checked_shl((u / 2) as u32).unwrap_or(u64::MAX); + let dblk_nelmts = dmin.checked_shl(u.div_ceil(2) as u32).unwrap_or(u64::MAX); + levels.push(Level { + ndblks, + dblk_nelmts, + start_idx, + start_dblk, + }); + start_idx = start_idx.saturating_add(ndblks.saturating_mul(dblk_nelmts)); + start_dblk = start_dblk.saturating_add(ndblks); + } + let ndblk_addrs = 2 * (p.sup_blk_min_data_ptrs as usize).saturating_sub(1); + let mut direct_levels = 0; + let mut n = 0u64; + while n < ndblk_addrs as u64 { + if direct_levels >= levels.len() { + return Err(bad("index block holds more data blocks than the array")); + } + n += levels[direct_levels].ndblks; + direct_levels += 1; + } + if n != ndblk_addrs as u64 { + return Err(bad("index block ends mid super block")); + } + Ok((levels, direct_levels, ndblk_addrs, nsblks - direct_levels)) + } + + fn arr_off_size(&self) -> usize { + (self.p.max_nelmts_bits as usize).div_ceil(8) + } + + fn page_nelmts(&self) -> u64 { + 1u64.checked_shl(u32::from(self.p.max_dblk_page_nelmts_bits)) + .unwrap_or(u64::MAX) + } + + fn slot_size(&self, os: u8) -> usize { + if self.filtered { + self.elem_size + } else { + os as usize + } + } + + /// Open the array whose header is at `hdr`. + pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result { + let os = img.os; + let ls = img.ls as usize; + let size = 12 + 6 * ls + os as usize + 4; + let d = img.read(hdr, size)?; + if &d[0..4] != b"EAHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let filtered = match d[5] { + 0 => false, + 1 => true, + _ => return Err(bad("unknown client")), + }; + let elem_size = d[6] as usize; + if filtered && elem_size < os as usize + 5 { + return Err(bad("element too small")); + } + let p = EaParams { + max_nelmts_bits: d[7], + idx_blk_elmts: d[8], + data_blk_min_elmts: d[9], + sup_blk_min_data_ptrs: d[10], + max_dblk_page_nelmts_bits: d[11], + }; + let mut stats = [0u64; 6]; + for (k, s) in stats.iter_mut().enumerate() { + *s = get_uint(&d[12 + k * ls..], img.ls); + } + let iblock = get_uint(&d[12 + 6 * ls..], os); + let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?; + Ok(Self { + hdr, + filtered, + elem_size, + p, + stats, + iblock, + levels, + direct_levels, + ndblk_addrs, + nsblk_addrs, + dirty_hdr: false, + dirty: Default::default(), + }) + } + + /// Create an empty array (header only; the index block comes with the + /// first element) and return it. + pub(crate) fn create( + img: &mut Image<'_>, + p: EaParams, + filtered: bool, + chunk_bytes: u64, + layout_version: u8, + ) -> Result { + let os = img.os; + let elem_size = if filtered { + os as usize + chunk_size_len(chunk_bytes, layout_version) + 4 + } else { + os as usize + }; + let size = 12 + 6 * img.ls as usize + os as usize + 4; + let hdr = img.alloc(size as u64)?; + let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?; + let mut ea = Self { + hdr, + filtered, + elem_size, + p, + stats: [0; 6], + iblock: undef(os), + levels, + direct_levels, + ndblk_addrs, + nsblk_addrs, + dirty_hdr: true, + dirty: Default::default(), + }; + ea.write_header(img)?; + Ok(ea) + } + + pub(crate) fn header_address(&self) -> u64 { + self.hdr + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + let ls = img.ls as usize; + let size = 12 + 6 * ls + os as usize + 4; + let mut d = vec![0u8; size]; + d[0..4].copy_from_slice(b"EAHD"); + d[4] = 0; + d[5] = u8::from(self.filtered); + d[6] = self.elem_size as u8; + d[7] = self.p.max_nelmts_bits; + d[8] = self.p.idx_blk_elmts; + d[9] = self.p.data_blk_min_elmts; + d[10] = self.p.sup_blk_min_data_ptrs; + d[11] = self.p.max_dblk_page_nelmts_bits; + for (k, s) in self.stats.iter().enumerate() { + put_uint(&mut d[12 + k * ls..], *s, img.ls); + } + put_uint(&mut d[12 + 6 * ls..], self.iblock, os); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]); + d[size - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(self.hdr, &d)?; + self.dirty_hdr = false; + Ok(()) + } + + /// Recompute the checksums of the blocks `set` changed; store changed + /// header statistics. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for (start, end) in std::mem::take(&mut self.dirty) { + rechecksum(img, start, end)?; + } + if self.dirty_hdr { + self.write_header(img)?; + } + Ok(()) + } + + fn fill_elems(&self, n: u64, os: u8) -> Result, Error> { + let one = encode_elem(None, self.filtered, self.elem_size, os)?; + let n = usize::try_from(n).map_err(|_| bad("block too large"))?; + Ok(one.repeat(n)) + } + + fn iblock_prefix(&self, os: u8) -> u64 { + 6 + u64::from(os) + } + + fn iblock_len(&self, os: u8) -> u64 { + let osz = os as u64; + self.iblock_prefix(os) + + u64::from(self.p.idx_blk_elmts) * self.slot_size(os) as u64 + + (self.ndblk_addrs + self.nsblk_addrs) as u64 * osz + } + + fn create_iblock(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + let len = self.iblock_len(os); + let addr = img.alloc(len + 4)?; + let mut d = Vec::with_capacity(len as usize + 4); + d.extend_from_slice(b"EAIB"); + d.push(0); + d.push(u8::from(self.filtered)); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.hdr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&self.fill_elems(u64::from(self.p.idx_blk_elmts), os)?); + let u = undef(os).to_le_bytes(); + for _ in 0..self.ndblk_addrs + self.nsblk_addrs { + d.extend_from_slice(&u[..os as usize]); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(addr, &d)?; + self.iblock = addr; + self.stats[5] += u64::from(self.p.idx_blk_elmts); + self.dirty_hdr = true; + Ok(()) + } + + fn block_prefix(&self, sig: &[u8; 4], off: u64, os: u8) -> Vec { + let mut d = Vec::new(); + d.extend_from_slice(sig); + d.push(0); + d.push(u8::from(self.filtered)); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.hdr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&off.to_le_bytes()[..self.arr_off_size()]); + d + } + + fn dblk_prefix_len(&self, os: u8) -> u64 { + 6 + u64::from(os) + self.arr_off_size() as u64 + } + + /// Create a data block of `nelmts` elements whose recorded block offset + /// is `off`; returns its address. + fn create_dblock(&mut self, img: &mut Image<'_>, nelmts: u64, off: u64) -> Result { + let os = img.os; + let es = self.slot_size(os) as u64; + let page = self.page_nelmts(); + let prefix = self.block_prefix(b"EADB", off, os); + let (size, body) = if nelmts > page { + // Paged: only the prefix (and its checksum) is written now; each + // page is written when an element in it is first set. + let npages = nelmts / page; + let size = prefix.len() as u64 + 4 + npages * (page * es + 4); + let mut d = prefix; + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + (size, d) + } else { + let mut d = prefix; + d.extend_from_slice(&self.fill_elems(nelmts, os)?); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + (d.len() as u64, d) + }; + let addr = img.alloc(size)?; + img.write(addr, &body)?; + self.stats[2] += 1; + self.stats[3] += size; + self.stats[5] += nelmts; + self.dirty_hdr = true; + Ok(addr) + } + + /// Set element `idx` to `e`. + pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + let os = img.os; + let osz = u64::from(os); + let es = self.slot_size(os) as u64; + let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + if self.iblock == undef(os) { + self.create_iblock(img)?; + } + let ib = self.iblock; + let ib_len = self.iblock_len(os); + let idx_blk = u64::from(self.p.idx_blk_elmts); + if idx < idx_blk { + img.write(ib + self.iblock_prefix(os) + idx * es, &enc)?; + self.dirty.insert(ib, ib + ib_len); + } else { + let rel = idx - idx_blk; + let u = self + .levels + .iter() + .position(|l| { + rel < l + .start_idx + .saturating_add(l.ndblks.saturating_mul(l.dblk_nelmts)) + }) + .ok_or_else(|| bad("index beyond the array's maximum"))?; + let l = self.levels[u]; + let dblks_at = ib + self.iblock_prefix(os) + idx_blk * es; + if u < self.direct_levels { + if l.dblk_nelmts > self.page_nelmts() { + return Err(Error::Unsupported( + "Extensible Array index block addressing a paged data block".into(), + )); + } + let local = (rel - l.start_idx) / l.dblk_nelmts; + let dblk_idx = l.start_dblk + local; + let slot = dblks_at + dblk_idx * osz; + let mut addr = get_uint(&img.read(slot, os as usize)?, os); + if addr == undef(os) { + // libhdf5 records start_idx + (global data block index) + // * nelmts here (H5EA__lookup_elmt), not the block's + // own first element; kept for byte-for-byte parity. + let off = l.start_idx + dblk_idx * l.dblk_nelmts; + addr = self.create_dblock(img, l.dblk_nelmts, off)?; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, addr, os); + img.write(slot, &a)?; + self.dirty.insert(ib, ib + ib_len); + } + let within = (rel - l.start_idx) % l.dblk_nelmts; + let at = addr + self.dblk_prefix_len(os) + within * es; + img.write(at, &enc)?; + self.dirty + .insert(addr, addr + self.dblk_prefix_len(os) + l.dblk_nelmts * es); + } else { + let s = (u - self.direct_levels) as u64; + let sslot = dblks_at + self.ndblk_addrs as u64 * osz + s * osz; + let page = self.page_nelmts(); + let npages = if l.dblk_nelmts > page { + l.dblk_nelmts / page + } else { + 0 + }; + let bitmap_len = npages.div_ceil(8) * l.ndblks; + let sb_prefix = self.dblk_prefix_len(os); + let sb_len = sb_prefix + bitmap_len + l.ndblks * osz; + let mut sb = get_uint(&img.read(sslot, os as usize)?, os); + if sb == undef(os) { + let mut d = self.block_prefix(b"EASB", l.start_idx, os); + d.resize(d.len() + bitmap_len as usize, 0); + let u8s = undef(os).to_le_bytes(); + for _ in 0..l.ndblks { + d.extend_from_slice(&u8s[..os as usize]); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + sb = img.alloc(d.len() as u64)?; + img.write(sb, &d)?; + self.stats[0] += 1; + self.stats[1] += d.len() as u64; + self.dirty_hdr = true; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, sb, os); + img.write(sslot, &a)?; + self.dirty.insert(ib, ib + ib_len); + } + let local = (rel - l.start_idx) / l.dblk_nelmts; + let dslot = sb + sb_prefix + bitmap_len + local * osz; + let mut addr = get_uint(&img.read(dslot, os as usize)?, os); + if addr == undef(os) { + let off = l.start_idx + local * l.dblk_nelmts; + addr = self.create_dblock(img, l.dblk_nelmts, off)?; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, addr, os); + img.write(dslot, &a)?; + self.dirty.insert(sb, sb + sb_len); + } + let within = (rel - l.start_idx) % l.dblk_nelmts; + let dprefix = self.dblk_prefix_len(os); + if npages == 0 { + img.write(addr + dprefix + within * es, &enc)?; + self.dirty.insert(addr, addr + dprefix + l.dblk_nelmts * es); + } else { + let pg = within / page; + let page_at = addr + dprefix + 4 + pg * (page * es + 4); + let bit = local * npages + pg; + let bpos = sb + sb_prefix + bit / 8; + let mut byte = img.read(bpos, 1)?[0]; + let mask = 0x80u8 >> (bit % 8); + if byte & mask == 0 { + let fill = self.fill_elems(page, os)?; + img.write(page_at, &fill)?; + byte |= mask; + img.write(bpos, &[byte])?; + self.dirty.insert(sb, sb + sb_len); + } + img.write(page_at + (within % page) * es, &enc)?; + self.dirty.insert(page_at, page_at + page * es); + } + } + } + if idx + 1 > self.stats[4] { + self.stats[4] = idx + 1; + self.dirty_hdr = true; + } + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/farray.rs b/crates/clawhdf5/src/edit/farray.rs new file mode 100644 index 0000000..0a25fe7 --- /dev/null +++ b/crates/clawhdf5/src/edit/farray.rs @@ -0,0 +1,185 @@ +//! Setting elements of a Fixed Array chunk index (layout v4, index type 3), +//! creating the array (header and data block) when the dataset has none +//! yet, and a data block page when an element in it is first set. + +use crate::edit::earray::{Elem, chunk_size_len, encode_elem}; +use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef}; +use crate::error::Error; + +pub(crate) struct Fa { + filtered: bool, + elem_size: usize, + page_bits: u8, + nelmts: u64, + dblk: u64, + /// Checksummed ranges changed by `set`, recomputed by `finish`. + dirty: std::collections::BTreeMap, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("Fixed Array: {why}"), + )) +} + +impl Fa { + fn slot(&self, os: u8) -> u64 { + if self.filtered { + self.elem_size as u64 + } else { + u64::from(os) + } + } + + fn page(&self) -> u64 { + 1u64.checked_shl(u32::from(self.page_bits)) + .unwrap_or(u64::MAX) + } + + /// Open the array whose header is at `hdr`. + pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result { + let os = img.os; + let size = 8 + img.ls as usize + os as usize + 4; + let d = img.read(hdr, size)?; + if &d[0..4] != b"FAHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let filtered = match d[5] { + 0 => false, + 1 => true, + _ => return Err(bad("unknown client")), + }; + let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let fa = Self { + filtered, + elem_size: d[6] as usize, + page_bits: d[7], + nelmts: get_uint(&d[8..], img.ls), + dblk: get_uint(&d[8 + img.ls as usize..], os), + dirty: Default::default(), + }; + if fa.filtered && fa.elem_size < os as usize + 5 { + return Err(bad("element too small")); + } + if fa.page_bits >= 64 || fa.dblk == undef(os) { + return Err(bad("bad header fields")); + } + Ok(fa) + } + + /// Create an array of `nelmts` fill elements; returns it and its + /// header address. + pub(crate) fn create( + img: &mut Image<'_>, + nelmts: u64, + page_bits: u8, + filtered: bool, + chunk_bytes: u64, + layout_version: u8, + ) -> Result<(Self, u64), Error> { + let os = img.os; + let osz = os as usize; + let elem_size = if filtered { + osz + chunk_size_len(chunk_bytes, layout_version) + 4 + } else { + osz + }; + let mut fa = Self { + filtered, + elem_size, + page_bits, + nelmts, + dblk: 0, + dirty: Default::default(), + }; + let hsize = 8 + img.ls as usize + osz + 4; + let hdr = img.alloc(hsize as u64)?; + // Data block. + let fill = encode_elem(None, filtered, elem_size, os)?; + let mut d = Vec::new(); + d.extend_from_slice(b"FADB"); + d.push(0); + d.push(u8::from(filtered)); + let mut a = vec![0u8; osz]; + put_uint(&mut a, hdr, os); + d.extend_from_slice(&a); + let page = fa.page(); + let n = usize::try_from(nelmts).map_err(|_| bad("too many elements"))?; + let total = if nelmts > page { + let npages = nelmts.div_ceil(page); + d.resize(d.len() + npages.div_ceil(8) as usize, 0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + // Pages are written when first used; their space is reserved. + d.len() as u64 + nelmts * fa.slot(os) + npages * 4 + } else { + d.extend_from_slice(&fill.repeat(n)); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + d.len() as u64 + }; + let dblk = img.alloc(total)?; + img.write(dblk, &d)?; + fa.dblk = dblk; + let mut h = vec![0u8; hsize]; + h[0..4].copy_from_slice(b"FAHD"); + h[5] = u8::from(filtered); + h[6] = elem_size as u8; + h[7] = page_bits; + put_uint(&mut h[8..], nelmts, img.ls); + put_uint(&mut h[8 + img.ls as usize..], dblk, os); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&h[..hsize - 4]); + h[hsize - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(hdr, &h)?; + Ok((fa, hdr)) + } + + /// Set element `idx` to `e`. + pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + let os = img.os; + if idx >= self.nelmts { + return Err(bad("index beyond the array")); + } + let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let es = self.slot(os); + let prefix = 6 + u64::from(os); + let page = self.page(); + if self.nelmts <= page { + img.write(self.dblk + prefix + idx * es, &enc)?; + self.dirty + .insert(self.dblk, self.dblk + prefix + self.nelmts * es); + return Ok(()); + } + let npages = self.nelmts.div_ceil(page); + let bitmap_len = npages.div_ceil(8); + let pages_at = self.dblk + prefix + bitmap_len + 4; + let p = idx / page; + let count = page.min(self.nelmts - p * page); + let page_at = pages_at + p * (page * es + 4); + let bpos = self.dblk + prefix + p / 8; + let mut byte = img.read(bpos, 1)?[0]; + let mask = 0x80u8 >> (p % 8); + if byte & mask == 0 { + let fill = encode_elem(None, self.filtered, self.elem_size, os)?; + img.write(page_at, &fill.repeat(count as usize))?; + byte |= mask; + img.write(bpos, &[byte])?; + self.dirty + .insert(self.dblk, self.dblk + prefix + bitmap_len); + } + img.write(page_at + (idx % page) * es, &enc)?; + self.dirty.insert(page_at, page_at + count * es); + Ok(()) + } + + /// Recompute the checksums of the blocks and pages `set` changed. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for (start, end) in std::mem::take(&mut self.dirty) { + rechecksum(img, start, end)?; + } + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs new file mode 100644 index 0000000..a100b66 --- /dev/null +++ b/crates/clawhdf5/src/edit/image.rs @@ -0,0 +1,317 @@ +//! The file as one edit sees it: the bytes on disk plus the edit's pending +//! writes, and an allocator that hands out space at the end of the file. +//! +//! An edit never writes to the file while it is being planned. Every change +//! is recorded here first (reads see them), so an edit that fails half-way — +//! a filter that cannot encode, a chunk index this code does not handle — +//! leaves the file exactly as it was. [`Image::commit`] then writes the +//! changes in an order that keeps the old metadata valid for as long as +//! possible (see there). + +use std::collections::BTreeMap; +use std::io::{Seek, SeekFrom, Write}; + +use crate::error::Error; + +/// Pending writes over the file's bytes, addressed as HDF5 addresses +/// (relative to the superblock). +pub(crate) struct Image<'a> { + /// The file from the superblock to its recorded end of allocation. + base: &'a [u8], + /// Pending writes: start address -> bytes. Never overlapping. + patches: BTreeMap>, + /// End of allocated space (grows with [`Self::alloc`]). + eoa: u64, + /// The end of allocated space when the edit started. + old_eoa: u64, + /// Width of addresses and lengths in the file. + pub(crate) os: u8, + pub(crate) ls: u8, +} + +impl<'a> Image<'a> { + pub(crate) fn new(base: &'a [u8], os: u8, ls: u8) -> Self { + let eoa = base.len() as u64; + Self { + base, + patches: BTreeMap::new(), + eoa, + old_eoa: eoa, + os, + ls, + } + } + + pub(crate) fn eoa(&self) -> u64 { + self.eoa + } + + pub(crate) fn old_eoa(&self) -> u64 { + self.old_eoa + } + + /// Whether the edit changes anything. + pub(crate) fn is_dirty(&self) -> bool { + !self.patches.is_empty() || self.eoa != self.old_eoa + } + + /// Allocate `size` bytes at the end of the file. The space reads as + /// zeros until written. Nothing is ever freed: space an edit stops + /// using (a relocated chunk, say) is leaked, as there is no free-space + /// manager. + pub(crate) fn alloc(&mut self, size: u64) -> Result { + let addr = self.eoa; + let end = addr + .checked_add(size) + .filter(|&e| self.os >= 8 || e < (1u64 << (8 * u32::from(self.os))) - 1) + .ok_or_else(|| Error::Unsupported("file would exceed its address size".into()))?; + self.eoa = end; + Ok(addr) + } + + /// If `[addr, addr + old_len)` is the last allocated space, grow it to + /// `new_len` bytes (a structure at the end of the file can grow where + /// it is) and return true. + pub(crate) fn grow_tail( + &mut self, + addr: u64, + old_len: u64, + new_len: u64, + ) -> Result { + if addr.checked_add(old_len) != Some(self.eoa) || new_len < old_len { + return Ok(false); + } + let old_end = self.eoa; + self.eoa = addr; + if let Err(e) = self.alloc(new_len) { + self.eoa = old_end; + return Err(e); + } + Ok(true) + } + + /// `len` bytes at `addr`, with the pending writes applied. + pub(crate) fn read(&self, addr: u64, len: usize) -> Result, Error> { + let end = addr + .checked_add(len as u64) + .filter(|&e| e <= self.eoa) + .ok_or_else(|| { + Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof { + expected: addr.saturating_add(len as u64) as usize, + available: self.eoa as usize, + }) + })?; + let mut out = vec![0u8; len]; + let base_len = self.base.len() as u64; + if addr < base_len { + let b_end = end.min(base_len); + out[..(b_end - addr) as usize] + .copy_from_slice(&self.base[addr as usize..b_end as usize]); + } + // Patches overlapping [addr, end): the last one starting before + // `end`, walking back while they still reach `addr`. + for (&p_start, bytes) in self.patches.range(..end).rev() { + let p_end = p_start + bytes.len() as u64; + if p_end <= addr { + break; + } + let lo = p_start.max(addr); + let hi = p_end.min(end); + out[(lo - addr) as usize..(hi - addr) as usize] + .copy_from_slice(&bytes[(lo - p_start) as usize..(hi - p_start) as usize]); + } + Ok(out) + } + + /// Record a write of `bytes` at `addr` (inside allocated space). + pub(crate) fn write(&mut self, addr: u64, bytes: &[u8]) -> Result<(), Error> { + if bytes.is_empty() { + return Ok(()); + } + let end = addr + .checked_add(bytes.len() as u64) + .filter(|&e| e <= self.eoa) + .ok_or_else(|| Error::Unsupported("write past the end of allocated space".into()))?; + // Fast path: inside, or extending, the one patch that starts at or + // before `addr` and reaches it (sequential writes into a block, and + // chunks allocated back to back, stay linear). + if let Some((&p_start, p)) = self.patches.range_mut(..=addr).next_back() + && p_start + p.len() as u64 >= addr + && self + .patches + .range(addr + 1..end.max(addr + 1)) + .next() + .is_none() + { + let p = self.patches.get_mut(&p_start).expect("found above"); + let off = (addr - p_start) as usize; + if off + bytes.len() > p.len() { + p.resize(off + bytes.len(), 0); + } + p[off..off + bytes.len()].copy_from_slice(bytes); + return Ok(()); + } + // Patches that overlap or touch [addr, end) merge into one. + let touching: Vec = self + .patches + .range(..=end) + .rev() + .take_while(|(s, b)| **s + b.len() as u64 >= addr) + .map(|(s, _)| *s) + .collect(); + if touching.is_empty() { + self.patches.insert(addr, bytes.to_vec()); + return Ok(()); + } + let lo = touching.iter().copied().min().map_or(addr, |s| s.min(addr)); + let hi = touching + .iter() + .map(|s| s + self.patches[s].len() as u64) + .max() + .map_or(end, |e| e.max(end)); + let mut merged = self.read(lo, (hi - lo) as usize)?; + merged[(addr - lo) as usize..(end - lo) as usize].copy_from_slice(bytes); + for s in touching { + self.patches.remove(&s); + } + self.patches.insert(lo, merged); + Ok(()) + } + + /// Write the edit to `file`, whose superblock is at `user_block`. + /// + /// Order: first everything in newly allocated space (new chunks, new + /// index blocks, relocated structures), which nothing on disk refers to + /// yet, then a sync; then the changes to existing bytes — raw data + /// overwritten in place and the metadata that links the new space in + /// (superblock end of file, chunk index entries, object header + /// messages) — then a sync. A crash during the first phase leaves the + /// file as it was (plus unreferenced bytes past its end of file); a + /// crash during the second can leave it inconsistent, as with libhdf5 + /// without SWMR: there is no journal. + pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> { + let old_eoa = self.old_eoa; + let mut in_place: Vec<(u64, &[u8])> = Vec::new(); + for (&addr, bytes) in &self.patches { + // A patch may run from existing bytes into new space (writes + // merge); its new part goes with the new space. + let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize; + let (old, new) = bytes.split_at(split); + if !new.is_empty() { + write_at(file, user_block + addr + split as u64, new)?; + } + if !old.is_empty() { + in_place.push((addr, old)); + } + } + if self.eoa > old_eoa { + let want = user_block + self.eoa; + if file.metadata()?.len() < want { + file.set_len(want)?; + } + } + file.sync_data()?; + for (addr, bytes) in in_place { + write_at(file, user_block + addr, bytes)?; + } + file.sync_all()?; + Ok(()) + } +} + +fn write_at(file: &mut std::fs::File, pos: u64, bytes: &[u8]) -> Result<(), Error> { + file.seek(SeekFrom::Start(pos))?; + file.write_all(bytes)?; + Ok(()) +} + +/// Little-endian encode of `v` in `width` bytes. +pub(crate) fn put_uint(buf: &mut [u8], v: u64, width: u8) { + let w = width as usize; + buf[..w].copy_from_slice(&v.to_le_bytes()[..w]); +} + +/// Little-endian decode of `width` bytes. +pub(crate) fn get_uint(buf: &[u8], width: u8) -> u64 { + let mut b = [0u8; 8]; + b[..width as usize].copy_from_slice(&buf[..width as usize]); + u64::from_le_bytes(b) +} + +/// The undefined address for `os`-byte addresses. +pub(crate) fn undef(os: u8) -> u64 { + if os >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(os))) - 1 + } +} + +/// Recompute the Jenkins checksum over `[start, end)` and store it at `end`. +pub(crate) fn rechecksum(img: &mut Image<'_>, start: u64, end: u64) -> Result<(), Error> { + let bytes = img.read(start, (end - start) as usize)?; + let sum = clawhdf5_format::checksum::jenkins_lookup3(&bytes); + img.write(end, &sum.to_le_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_see_writes_and_merges() { + let base = vec![1u8; 32]; + let mut img = Image::new(&base, 8, 8); + img.write(4, &[9, 9]).unwrap(); + img.write(8, &[7]).unwrap(); + img.write(5, &[3, 3, 3]).unwrap(); // extends the first up to the second + assert_eq!(img.read(3, 7).unwrap(), vec![1, 9, 3, 3, 3, 7, 1]); + img.write(2, &[4, 4, 4, 4, 4, 4, 4, 4]).unwrap(); // covers both: merged + assert_eq!(img.patches.len(), 1); + assert_eq!(img.read(1, 10).unwrap(), vec![1, 4, 4, 4, 4, 4, 4, 4, 4, 1]); + let a = img.alloc(10).unwrap(); + assert_eq!(a, 32); + assert_eq!(img.read(30, 4).unwrap(), vec![1, 1, 0, 0]); + img.write(40, &[5]).unwrap(); + assert_eq!(img.read(39, 3).unwrap(), vec![0, 5, 0]); + assert!(img.write(42, &[1]).is_err()); + } + + /// Random reads and writes against a flat copy of the bytes. + #[test] + fn matches_a_flat_model() { + let base: Vec = (0..200u32).map(|i| i as u8).collect(); + let mut img = Image::new(&base, 8, 8); + img.alloc(100).unwrap(); + let mut flat = base.clone(); + flat.resize(300, 0); + let mut x = 12345u64; + let mut next = |n: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x % n + }; + for step in 0..5000 { + let at = next(300); + let len = 1 + next(20).min(299 - at); + if step % 3 == 0 { + assert_eq!( + img.read(at, len as usize).unwrap(), + flat[at as usize..(at + len) as usize] + ); + } else { + let bytes: Vec = (0..len).map(|_| next(256) as u8).collect(); + img.write(at, &bytes).unwrap(); + flat[at as usize..(at + len) as usize].copy_from_slice(&bytes); + } + } + assert_eq!(img.read(0, 300).unwrap(), flat); + // Patches never overlap. + let mut end = 0; + for (s, b) in &img.patches { + assert!(*s >= end); + end = s + b.len() as u64; + } + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs new file mode 100644 index 0000000..b135a3f --- /dev/null +++ b/crates/clawhdf5/src/edit/mod.rs @@ -0,0 +1,1213 @@ +//! In-place modification of an existing HDF5 file: [`FileEditor`]. +//! +//! [`FileBuilder`](crate::FileBuilder) builds a whole file in memory; the +//! editor instead opens a file libhdf5 (or clawhdf5) wrote and changes it +//! where it lies: dataset values are overwritten in place, chunks are added +//! or relocated at the end of the file, the chunk index, dataspace and +//! object headers are patched, and every checksum of a structure it touched +//! is recomputed. +//! +//! Each operation is planned in memory first ([`image::Image`]): if any part +//! of it is unsupported, nothing is written. The plan is then committed with +//! the new space (new chunks, new index blocks) written and synced before +//! the existing bytes that link it in, then synced again. + +mod btree1; +mod earray; +mod farray; +mod image; +mod ohdr; +mod select; + +use std::collections::{BTreeMap, HashMap}; +use std::fs::{OpenOptions, TryLockError}; +use std::path::{Path, PathBuf}; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::NativeElement; +use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::filter_pipeline::FilterPipeline; +use clawhdf5_format::selection::Selection; + +use crate::error::Error; +use crate::reader::File; +use crate::types::AttrValue; +use btree1::{BTree1, Key}; +use earray::{Ea, EaParams, Elem}; +use farray::Fa; +use image::{Image, get_uint, put_uint, undef}; +use ohdr::{Header, MSG_ATTRIBUTE}; + +const MSG_DATASPACE: u16 = 0x01; +const MSG_LAYOUT: u16 = 0x08; +const MSG_EXTERNAL: u16 = 0x07; +const MSG_ATTR_INFO: u16 = 0x15; +/// Message flag: the message is shared (stored elsewhere). +const MSG_FLAG_SHARED: u8 = 0x02; + +/// An HDF5 file opened for in-place modification. +/// +/// Opening takes an exclusive advisory lock on the file (`flock`, the lock +/// libhdf5 itself takes when file locking is on), so a second editor, or +/// h5py opening the file for writing, fails until the editor is dropped. +/// Readers that do not lock ([`File`]) can still open it, but see a file +/// that may be mid-update. +/// +/// Every method is one self-contained edit: it re-reads the file's +/// metadata, applies the change, and syncs the file before returning. +/// +/// # What it can change +/// +/// - [`write_selection`](Self::write_selection) / +/// [`write_all`](Self::write_all): overwrite values of a compact, +/// contiguous or chunked dataset with values of the dataset's own +/// datatype, under any selection. Chunks are decoded, updated and +/// re-encoded; an unfiltered chunk is rewritten in place, a filtered one +/// in place when it still fits and otherwise at the end of the file. New +/// chunks are added to the chunk index: version-1 B-tree (layout v1-v3, +/// what h5py's default `libver` writes), Extensible Array, Fixed Array and +/// single-chunk indexes. A version-2 B-tree index (two or more unlimited +/// dimensions) can only have existing chunks overwritten in place, and an +/// implicit index only in place. +/// - [`resize`](Self::resize): grow a chunked dataset up to its maximum +/// dimensions (h5py's `Dataset.resize`); new chunks come with the writes. +/// - [`set_attr`](Self::set_attr): add or replace an attribute of any +/// object whose attributes are stored in its object header. +/// +/// Anything else is an [`Error::Unsupported`] and leaves the file untouched. +/// +/// # Crash safety and space +/// +/// There is no journal (libhdf5 has none either, outside SWMR). An edit +/// writes all of its new space — chunks and index blocks past the old end +/// of file — and syncs it before it changes any existing byte, so a crash +/// before that point leaves the file as it was; a crash while the existing +/// structures are being patched can leave the file inconsistent. +/// +/// Space is never reused: a filtered chunk that grows moves to the end of +/// the file and its old bytes are leaked, as are index blocks that are +/// replaced. `h5repack` reclaims such space. +#[derive(Debug)] +pub struct FileEditor { + path: PathBuf, + file: std::fs::File, +} + +/// Where a layout message keeps the fields an edit may change (offsets in +/// the message body). +#[derive(Debug, Default, Clone, Copy)] +struct LayoutPos { + version: u8, + /// Contiguous data address, or the chunk index address. + addr: Option, + /// Filtered single chunk: its stored size and filter mask. + single_size: Option, + single_mask: Option, + /// First chunk index creation parameter (Fixed/Extensible Array). + params: Option, + /// Compact raw data. + compact_data: Option, +} + +fn layout_pos(d: &[u8], os: u8, ls: u8) -> Result { + let short = || Error::Unsupported("layout message too short".into()); + let os = os as usize; + let ls = ls as usize; + let version = *d.first().ok_or_else(short)?; + let mut p = LayoutPos { + version, + ..LayoutPos::default() + }; + match version { + 1 | 2 => { + let nd = *d.get(1).ok_or_else(short)? as usize; + match *d.get(2).ok_or_else(short)? { + 0 => p.compact_data = Some(8 + nd * 4 + 4), + _ => p.addr = Some(8), + } + } + 3 => match *d.get(1).ok_or_else(short)? { + 0 => p.compact_data = Some(4), + 1 => p.addr = Some(2), + _ => p.addr = Some(3), + }, + 4 | 5 => match *d.get(1).ok_or_else(short)? { + 0 => p.compact_data = Some(4), + 1 => p.addr = Some(2), + 2 => { + let flags = *d.get(2).ok_or_else(short)?; + let nd = *d.get(3).ok_or_else(short)? as usize; + let enc = *d.get(4).ok_or_else(short)? as usize; + let mut q = 5 + nd * enc; + let itype = *d.get(q).ok_or_else(short)?; + q += 1; + match itype { + 1 if flags & 0x02 != 0 => { + p.single_size = Some(q); + p.single_mask = Some(q + ls); + p.addr = Some(q + ls + 4); + } + 1 | 2 => p.addr = Some(q), + 3 => { + p.params = Some(q); + p.addr = Some(q + 1); + } + 4 => { + p.params = Some(q); + p.addr = Some(q + 5); + } + 5 => { + p.params = Some(q); + p.addr = Some(q + 6); + } + _ => return Err(Error::Unsupported(format!("chunk index type {itype}"))), + } + } + c => return Err(Error::Unsupported(format!("layout class {c}"))), + }, + v => return Err(Error::Unsupported(format!("layout message version {v}"))), + } + let end = p + .addr + .map_or(0, |a| a + os) + .max(p.compact_data.unwrap_or(0)); + if end > d.len() { + return Err(short()); + } + Ok(p) +} + +/// A dataset as an edit sees it. +struct Target { + addr: u64, + dt: Datatype, + es: usize, + ds: Dataspace, + layout: DataLayout, + pipeline: Option, + /// One element of fill value. + fill: Vec, +} + +impl Target { + fn load(f: &File, path: &str) -> Result { + let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let d = f.dataset_at(addr)?; + let dt = d.datatype()?; + let es = dt.type_size() as usize; + if es == 0 { + return Err(Error::Unsupported("datatype of size 0".into())); + } + let ds = d.dataspace()?; + let layout = d.data_layout()?; + let pipeline = d.filter_pipeline()?.filter(|p| !p.filters.is_empty()); + let os = f.superblock().offset_size; + let ls = f.superblock().length_size; + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + f.as_bytes(), + &d.header().messages, + os, + ls, + )?; + let fill = match fill { + None => vec![0u8; es], + Some(v) if v.len() == es => v, + Some(_) => { + return Err(Error::Unsupported( + "fill value size differs from the element size".into(), + )); + } + }; + if d.header() + .messages + .iter() + .any(|m| m.msg_type.to_u16() == MSG_EXTERNAL) + { + return Err(Error::Unsupported( + "dataset stored in external files".into(), + )); + } + Ok(Self { + addr, + dt, + es, + ds, + layout, + pipeline, + fill, + }) + } + + fn dims(&self) -> &[u64] { + &self.ds.dimensions + } +} + +/// Datatypes whose stored bytes point elsewhere in the file (variable-length +/// data in a global heap, references) cannot be written as raw values. +fn check_plain(dt: &Datatype) -> Result<(), Error> { + match dt { + Datatype::VariableLength { .. } => { + Err(Error::Unsupported("writing variable-length data".into())) + } + Datatype::Reference { .. } => Err(Error::Unsupported("writing references".into())), + Datatype::Compound { members, .. } => { + members.iter().try_for_each(|m| check_plain(&m.datatype)) + } + Datatype::Array { base_type, .. } => check_plain(base_type), + Datatype::Enumeration { base_type, .. } => check_plain(base_type), + _ => Ok(()), + } +} + +fn row_major(coords: &[u64], dims: &[u64]) -> u64 { + coords + .iter() + .zip(dims) + .fold(0u64, |acc, (&c, &d)| acc * d + c) +} + +/// Linear index of a chunk in a Fixed Array (`swizzle` false) or Extensible +/// Array (`swizzle` true) index; see `clawhdf5_format`'s `chunk_grid`. +fn array_index( + scaled: &[u64], + dims: &[u64], + max: Option<&[u64]>, + cd: &[u64], + swizzle: bool, +) -> Result { + let rank = cd.len(); + let max_chunks: Vec = (0..rank) + .map(|d| { + let m = max.map_or(dims[d], |m| m[d]); + if m == u64::MAX { + u64::MAX + } else { + m.div_ceil(cd[d]).max(dims[d].div_ceil(cd[d])) + } + }) + .collect(); + let mut order: Vec = (0..rank).collect(); + if swizzle && let Some(u) = max_chunks.iter().position(|&m| m == u64::MAX) { + order.remove(u); + order.insert(0, u); + } + let mut idx = 0u64; + let mut down = 1u64; + for p in (0..rank).rev() { + let d = order[p]; + idx = scaled[d] + .checked_mul(down) + .and_then(|v| idx.checked_add(v)) + .ok_or_else(|| Error::Unsupported("chunk index overflows".into()))?; + if p > 0 { + if max_chunks[d] == u64::MAX { + return Err(Error::Unsupported( + "array chunk index with more than one unlimited dimension".into(), + )); + } + down = down + .checked_mul(max_chunks[d]) + .ok_or_else(|| Error::Unsupported("chunk index overflows".into()))?; + } + } + Ok(idx) +} + +/// The chunk index of one dataset, opened for changes. Index structures a +/// dataset does not have yet (no chunk was ever written) are created on the +/// first insertion, and the layout message is pointed at them. +enum IndexEdit { + BTree1(Option), + Single, + Implicit, + Fixed(Option), + Extensible(Option), + BTree2, +} + +struct ChunkedEdit<'t> { + t: &'t Target, + /// Spatial chunk dimensions. + cd: Vec, + chunk_bytes: usize, + index: IndexEdit, + /// The dataset's object header (for layout message changes). + hdr: Header, + layout_msg: usize, + lpos: LayoutPos, + btree_k: u16, +} + +impl<'t> ChunkedEdit<'t> { + fn new(f: &File, img: &Image<'_>, t: &'t Target) -> Result { + let DataLayout::Chunked { + chunk_dimensions, + btree_address, + version, + chunk_index_type, + dont_filter_partial_edge_chunks, + .. + } = &t.layout + else { + return Err(Error::Unsupported("not a chunked dataset".into())); + }; + if *dont_filter_partial_edge_chunks { + return Err(Error::Unsupported( + "datasets whose partial edge chunks are not filtered".into(), + )); + } + let rank = t.dims().len(); + if chunk_dimensions.len() < rank { + return Err(Error::Unsupported( + "chunk rank differs from the dataspace".into(), + )); + } + let cd: Vec = chunk_dimensions[..rank] + .iter() + .map(|&c| u64::from(c)) + .collect(); + let chunk_bytes = cd + .iter() + .try_fold(t.es as u64, |a, &c| a.checked_mul(c)) + .and_then(|b| usize::try_from(b).ok()) + .filter(|&b| b <= u32::MAX as usize) + .ok_or_else(|| Error::Unsupported("chunk larger than 4 GiB".into()))?; + let hdr = Header::load(img, t.addr)?; + let layout_msg = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, layout_msg)?, img.os, img.ls)?; + let index = match (*version, *chunk_index_type) { + (3, _) => IndexEdit::BTree1( + btree_address + .map(|a| BTree1::new(a, chunk_btree_k(f)?, rank + 1, t.es as u64)) + .transpose()?, + ), + (_, Some(1)) => IndexEdit::Single, + (_, Some(2)) => IndexEdit::Implicit, + (_, Some(3)) => IndexEdit::Fixed(btree_address.map(|a| Fa::open(img, a)).transpose()?), + (_, Some(4)) => { + IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?) + } + (_, Some(5)) => IndexEdit::BTree2, + (v, i) => { + return Err(Error::Unsupported(format!( + "chunked layout version {v}, index type {i:?}" + ))); + } + }; + Ok(Self { + t, + cd, + chunk_bytes, + index, + hdr, + layout_msg, + lpos, + btree_k: chunk_btree_k(f)?, + }) + } + + fn patch_layout_addr(&mut self, img: &mut Image<'_>, addr: u64) -> Result<(), Error> { + let at = self + .lpos + .addr + .ok_or_else(|| Error::Unsupported("layout message has no address".into()))?; + if self.lpos.version < 3 { + return Err(Error::Unsupported( + "creating a chunk index in a version 1/2 layout message".into(), + )); + } + let mut a = vec![0u8; img.os as usize]; + put_uint(&mut a, addr, img.os); + self.hdr.patch(img, self.layout_msg, at, &a) + } + + /// Record chunk `scaled` at `e` in the index (new, moved or resized). + fn set(&mut self, img: &mut Image<'_>, scaled: &[u64], e: Elem) -> Result<(), Error> { + let t = self.t; + let filtered = t.pipeline.is_some(); + match &mut self.index { + IndexEdit::BTree1(tree) => { + let size = u32::try_from(e.size) + .map_err(|_| Error::Unsupported("chunk larger than 4 GiB".into()))?; + let mut offs: Vec = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect(); + offs.push(0); + let key = Key { + size, + mask: e.mask, + offs, + }; + match tree { + Some(tree) => tree.insert(img, key, e.addr)?, + None => { + let new = BTree1::create( + img, + self.btree_k, + scaled.len() + 1, + t.es as u64, + key, + e.addr, + )?; + let root = new.root(); + *tree = Some(new); + self.patch_layout_addr(img, root)?; + } + } + } + IndexEdit::Single => { + if scaled.iter().any(|&s| s != 0) { + return Err(Error::Unsupported( + "single-chunk index with a second chunk".into(), + )); + } + match (self.lpos.single_size, self.lpos.single_mask) { + (Some(s), Some(m)) => { + let mut b = vec![0u8; img.ls as usize]; + put_uint(&mut b, e.size, img.ls); + self.hdr.patch(img, self.layout_msg, s, &b)?; + self.hdr + .patch(img, self.layout_msg, m, &e.mask.to_le_bytes())?; + } + _ if filtered => { + return Err(Error::Unsupported( + "filtered single chunk without a filtered-size field".into(), + )); + } + _ => {} + } + self.patch_layout_addr(img, e.addr)?; + } + IndexEdit::Implicit => { + return Err(Error::Unsupported( + "adding a chunk to an implicit chunk index".into(), + )); + } + IndexEdit::Fixed(fa) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, false)?; + if fa.is_none() { + let pbits_at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("Fixed Array parameters".into()))?; + let page_bits = self.hdr.data(img, self.layout_msg)?[pbits_at]; + let nelmts = (0..self.cd.len()) + .map(|d| { + let m = max.map_or(t.dims()[d], |m| m[d]); + m.div_ceil(self.cd[d]) + }) + .try_fold(1u64, |a, n| a.checked_mul(n)) + .ok_or_else(|| Error::Unsupported("Fixed Array too large".into()))?; + let (new, hdr_addr) = Fa::create( + img, + nelmts, + page_bits, + filtered, + self.chunk_bytes as u64, + self.lpos.version, + )?; + *fa = Some(new); + self.patch_layout_addr(img, hdr_addr)?; + } + if let IndexEdit::Fixed(Some(fa)) = &mut self.index { + fa.set(img, idx, e)?; + } + } + IndexEdit::Extensible(ea) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, true)?; + if ea.is_none() { + let at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("Extensible Array parameters".into()))?; + let d = self.hdr.data(img, self.layout_msg)?; + // Layout message order: max_nelmts_bits, idx_blk_elmts, + // sup_blk_min_data_ptrs, data_blk_min_elmts, + // max_dblk_page_nelmts_bits. + let p = EaParams { + max_nelmts_bits: d[at], + idx_blk_elmts: d[at + 1], + sup_blk_min_data_ptrs: d[at + 2], + data_blk_min_elmts: d[at + 3], + max_dblk_page_nelmts_bits: d[at + 4], + }; + let new = + Ea::create(img, p, filtered, self.chunk_bytes as u64, self.lpos.version)?; + let hdr_addr = new.header_address(); + *ea = Some(new); + self.patch_layout_addr(img, hdr_addr)?; + } + if let IndexEdit::Extensible(Some(ea)) = &mut self.index { + ea.set(img, idx, e)?; + } + } + IndexEdit::BTree2 => { + return Err(Error::Unsupported( + "adding, moving or resizing a chunk in a version-2 B-tree chunk index \ + (datasets with more than one unlimited dimension)" + .into(), + )); + } + } + Ok(()) + } + + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + match &mut self.index { + IndexEdit::Extensible(Some(ea)) => ea.finish(img)?, + IndexEdit::Fixed(Some(fa)) => fa.finish(img)?, + _ => {} + } + self.hdr.finish(img) + } +} + +/// The chunk B-tree's K: the superblock's (version 1), the superblock +/// extension's (versions 2 and 3), or libhdf5's default of 32. +fn chunk_btree_k(f: &File) -> Result { + let sb = f.superblock(); + if let Some(k) = sb.indexed_storage_internal_node_k { + return Ok(k); + } + let ext = clawhdf5_format::superblock_ext::read_superblock_extension(f.as_bytes(), sb)?; + Ok(ext + .and_then(|e| e.btree_k) + .map_or(32, |(chunk, _, _)| chunk)) +} + +impl FileEditor { + /// Open `path` for modification, locking it exclusively. + /// + /// Refused ([`Error::Unsupported`]) for files the editor cannot keep + /// consistent: a metadata cache image, paged or persistent free-space + /// management, a multi-file driver, a file another writer has marked + /// open (superblock version 3 consistency flags). + pub fn open>(path: P) -> Result { + let path = path.as_ref().to_path_buf(); + let file = OpenOptions::new().read(true).write(true).open(&path)?; + match file.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(Error::Locked(format!( + "{} is open for writing elsewhere", + path.display() + ))); + } + Err(TryLockError::Error(e)) => return Err(Error::Io(e)), + } + let ed = Self { path, file }; + let f = File::open(&ed.path)?; + check_editable(&f)?; + Ok(ed) + } + + /// The file's path. + pub fn path(&self) -> &Path { + &self.path + } + + fn edit( + &mut self, + op: impl FnOnce(&File, &mut Image<'_>) -> Result, + ) -> Result { + let f = File::open(&self.path)?; + check_editable(&f)?; + let sb = f.superblock().clone(); + let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); + let r = op(&f, &mut img)?; + if img.is_dirty() { + if img.eoa() != img.old_eoa() { + set_superblock_eof(&mut img, &sb)?; + } + img.commit(&mut self.file, f.user_block_size())?; + } + Ok(r) + } + + /// Overwrite the dataset's values under `selection` with `data`: the + /// selected elements' bytes in the dataset's own datatype and byte + /// order, in selection order (row-major over a hyperslab, the listed + /// order for points). + pub fn write_selection( + &mut self, + path: &str, + selection: &Selection, + data: &[u8], + ) -> Result<(), Error> { + self.edit(|f, img| write_selection(f, img, path, selection, data)) + } + + /// Overwrite every value of the dataset (see + /// [`write_selection`](Self::write_selection)). + pub fn write_all(&mut self, path: &str, data: &[u8]) -> Result<(), Error> { + self.write_selection(path, &Selection::All, data) + } + + /// [`write_selection`](Self::write_selection) with typed values; the + /// dataset's datatype must be `T`'s native representation. + pub fn write_values( + &mut self, + path: &str, + selection: &Selection, + values: &[T], + ) -> Result<(), Error> { + // SAFETY: `NativeElement` types have no padding and no invalid bit + // patterns, so their memory is plain bytes. + let bytes = unsafe { + std::slice::from_raw_parts(values.as_ptr().cast::(), std::mem::size_of_val(values)) + }; + self.edit(|f, img| { + let t = Target::load(f, path)?; + if !T::is_native(&t.dt) { + return Err(Error::InvalidArgument(format!( + "dataset {path} does not store {}", + std::any::type_name::() + ))); + } + write_selection(f, img, path, selection, bytes) + }) + } + + /// Change a chunked dataset's current dimensions to `shape`, which may + /// only grow each dimension, up to the dataset's maximum dimensions + /// (h5py's `Dataset.resize`). New elements read as the fill value until + /// written. Shrinking is [`Error::Unsupported`]. + pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> { + self.edit(|f, img| { + let t = Target::load(f, path)?; + let dims = t.dims().to_vec(); + if shape.len() != dims.len() { + return Err(Error::InvalidArgument(format!( + "rank {} for a dataset of rank {}", + shape.len(), + dims.len() + ))); + } + if shape == dims.as_slice() { + return Ok(()); + } + let max = t.ds.max_dimensions.clone().unwrap_or_else(|| dims.clone()); + for d in 0..dims.len() { + if shape[d] > max[d] { + return Err(Error::InvalidArgument(format!( + "dimension {d}: {} exceeds the maximum {}", + shape[d], max[d] + ))); + } + if shape[d] < dims[d] { + return Err(Error::Unsupported("shrinking a dataset".into())); + } + } + if !matches!(t.layout, DataLayout::Chunked { .. }) { + return Err(Error::Unsupported( + "resizing a dataset that is not chunked".into(), + )); + } + // Only the dataspace changes: every chunk index is keyed + // independently of the current extent (the array indexes by the + // maximum dimensions), and new chunks come with the writes. + let mut hdr = Header::load(img, t.addr)?; + let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::Dataspace, + ))?; + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared dataspace message".into())); + } + let body = hdr.data(img, i)?; + let first = match body.first() { + Some(1) => 8, + Some(2) => 4, + _ => return Err(Error::Unsupported("dataspace message version".into())), + }; + let ls = img.ls as usize; + let mut dims_bytes = vec![0u8; shape.len() * ls]; + for (d, &n) in shape.iter().enumerate() { + if ls < 8 && n >> (8 * ls) != 0 { + return Err(Error::InvalidArgument("dimension too large".into())); + } + put_uint(&mut dims_bytes[d * ls..], n, img.ls); + } + hdr.patch(img, i, first, &dims_bytes)?; + hdr.finish(img) + }) + } + + /// Set attribute `name` of the object at `path` (a group or dataset; + /// `"/"` is the root group) to `value`, replacing an attribute of that + /// name. The attribute goes into free space in the object header, or a + /// new header continuation chunk at the end of the file. + /// + /// [`Error::Unsupported`] for an object whose attributes are in dense + /// storage (or would have to move there: more than the object's + /// compact-attribute limit), one that tracks attribute creation order, + /// or one with shared attribute messages. + pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { + if name.is_empty() { + return Err(Error::InvalidArgument("empty attribute name".into())); + } + self.edit(|f, img| { + let addr = + clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let mut hdr = Header::load(img, addr)?; + if hdr.version == 2 && hdr.flags & 0x04 != 0 { + return Err(Error::Unsupported( + "object tracks attribute creation order".into(), + )); + } + if let Some(i) = hdr.find(MSG_ATTR_INFO) { + let d = hdr.data(img, i)?; + // version(1) flags(1) [max creation index(2)] fractal heap + // address, name index address, [order index address]. + let mut p = 2; + if d.get(1).is_some_and(|f| f & 0x01 != 0) { + p += 2; + } + let os = img.os as usize; + if d.len() < p + os { + return Err(Error::Unsupported("short attribute info message".into())); + } + if get_uint(&d[p..], img.os) != undef(img.os) { + return Err(Error::Unsupported( + "object with attributes in dense storage".into(), + )); + } + } + let mut existing = None; + let mut count = 0usize; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + count += 1; + if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { + existing = Some(i); + } + } + if existing.is_none() && hdr.version == 2 { + let max_compact = max_compact_attrs(img, &hdr)?; + if count + 1 > usize::from(max_compact) { + return Err(Error::Unsupported(format!( + "object already has {count} compact attributes (its limit is \ + {max_compact}); more need dense storage" + ))); + } + } + let msg = clawhdf5_format::type_builders::build_attr_message(name, value); + check_plain(&msg.datatype)?; + let body = if hdr.version == 1 { + encode_attr_v1(&msg, img.ls) + } else { + let mut b = msg.serialize_v3(img.ls); + if !name.is_ascii() { + b[8] = 1; // UTF-8 name + } + b + }; + if let Some(i) = existing { + hdr.delete(img, i)?; + } + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + hdr.finish(img) + }) + } +} + +/// libhdf5's checks before it opens a file for writing, and what this +/// editor cannot keep consistent. +fn check_editable(f: &File) -> Result<(), Error> { + let sb = f.superblock(); + if sb.version > 3 { + return Err(Error::Unsupported(format!( + "superblock version {}", + sb.version + ))); + } + if sb.version >= 3 && (sb.is_write_access() || sb.is_swmr_write()) { + return Err(Error::Unsupported( + "the file is marked as open for writing by another process".into(), + )); + } + if sb + .driver_info_address + .is_some_and(|a| a != undef(sb.offset_size)) + { + return Err(Error::Unsupported("files with a driver info block".into())); + } + if let Some(ext) = clawhdf5_format::superblock_ext::read_superblock_extension(f.as_bytes(), sb)? + { + if ext.cache_image.is_some() { + return Err(Error::Unsupported( + "files with a metadata cache image".into(), + )); + } + // Strategy 1 is H5F_FSPACE_STRATEGY_PAGE. + if let Some(fs) = ext.file_space_info + && (fs.persist || fs.strategy == 1) + { + return Err(Error::Unsupported( + "files with paged or persistent free-space management".into(), + )); + } + } + Ok(()) +} + +/// Store the new end of file in the superblock. +fn set_superblock_eof( + img: &mut Image<'_>, + sb: &clawhdf5_format::superblock::Superblock, +) -> Result<(), Error> { + let os = u64::from(sb.offset_size); + let (at, sum) = match sb.version { + 0 => (24 + 2 * os, None), + 1 => (28 + 2 * os, None), + 2 | 3 => (12 + 2 * os, Some(12 + 4 * os)), + v => return Err(Error::Unsupported(format!("superblock version {v}"))), + }; + let eof = sb + .base_address + .checked_add(img.eoa()) + .ok_or_else(|| Error::Unsupported("end of file overflows".into()))?; + let mut b = vec![0u8; os as usize]; + put_uint(&mut b, eof, sb.offset_size); + img.write(at, &b)?; + if let Some(end) = sum { + let bytes = img.read(0, end as usize)?; + let s = clawhdf5_format::checksum::jenkins_lookup3(&bytes); + img.write(end, &s.to_le_bytes())?; + } + Ok(()) +} + +/// The name bytes of an attribute message body (without the NUL). +fn attr_name(d: &[u8]) -> Result<&[u8], Error> { + let bad = || Error::Unsupported("malformed attribute message".into()); + let (len, at) = match d.first() { + Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), + Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), + _ => return Err(bad()), + }; + let name = d.get(at..at + len).ok_or_else(bad)?; + Ok(name.split(|&b| b == 0).next().unwrap_or(name)) +} + +/// A version-2 header's limit on compact attributes: stored when its flags +/// say so, else libhdf5's default of 8. +fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { + if hdr.flags & 0x10 == 0 { + return Ok(8); + } + let mut p = hdr.addr + 6; + if hdr.flags & 0x20 != 0 { + p += 16; + } + let b = img.read(p, 2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) +} + +/// A version-1 attribute message (what libhdf5 writes in a version-1 object +/// header): name, datatype and dataspace each padded to 8 bytes, the +/// dataspace as a version-1 dataspace message. +fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { + let mut name = a.name.as_bytes().to_vec(); + name.push(0); + let dt = a.datatype.serialize(); + let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; + if a.dataspace.space_type == DataspaceType::Simple { + for &d in &a.dataspace.dimensions { + let mut b = vec![0u8; ls as usize]; + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + } else { + ds[1] = 0; + } + let mut out = vec![1u8, 0]; + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); + out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); + for part in [&name, &dt, &ds] { + out.extend_from_slice(part); + out.resize(out.len().next_multiple_of(8), 0); + } + out.extend_from_slice(&a.raw_data); + out +} + +fn write_selection( + f: &File, + img: &mut Image<'_>, + path: &str, + sel: &Selection, + data: &[u8], +) -> Result<(), Error> { + let t = Target::load(f, path)?; + check_plain(&t.dt)?; + if t.ds.space_type == DataspaceType::Null { + return Err(Error::InvalidArgument( + "dataset has a null dataspace".into(), + )); + } + let dims = t.dims().to_vec(); + clawhdf5_format::partial_read::validate(sel, &dims) + .map_err(|e| Error::InvalidArgument(e.to_string()))?; + if let Selection::Hyperslab { + stride, + count, + block, + .. + } = sel + && (0..dims.len()).any(|d| count[d] > 1 && stride[d] < block[d]) + { + return Err(Error::InvalidArgument( + "overlapping hyperslab blocks".into(), + )); + } + let n = select::for_each_run(sel, &dims, |_, _, _| Ok(()))?; + let want = n + .checked_mul(t.es as u64) + .ok_or_else(|| Error::InvalidArgument("selection too large".into()))?; + if data.len() as u64 != want { + return Err(Error::InvalidArgument(format!( + "{} bytes for {n} elements of {} bytes", + data.len(), + t.es + ))); + } + if n == 0 { + return Ok(()); + } + let es = t.es; + match &t.layout { + DataLayout::Compact { data: stored } => { + let mut hdr = Header::load(img, t.addr)?; + let li = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, li)?, img.os, img.ls)?; + let at = lpos + .compact_data + .ok_or_else(|| Error::Unsupported("compact layout".into()))?; + let total = dims.iter().product::() as usize * es; + if stored.len() < total { + return Err(Error::Unsupported( + "compact data shorter than the dataset".into(), + )); + } + select::for_each_run(sel, &dims, |c, len, src| { + let off = at + row_major(c, &dims) as usize * es; + let s = src as usize * es; + hdr.patch(img, li, off, &data[s..s + len as usize * es]) + })?; + hdr.finish(img) + } + DataLayout::Contiguous { address, size } => { + let total = dims + .iter() + .try_fold(es as u64, |a, &d| a.checked_mul(d)) + .ok_or_else(|| Error::Unsupported("dataset too large".into()))?; + let base = match address { + Some(a) => { + if *size < total { + return Err(Error::Unsupported("contiguous storage too small".into())); + } + *a + } + None => { + // Never written (late allocation): allocate it now, as + // libhdf5 does on the first write, with the fill value. + let mut hdr = Header::load(img, t.addr)?; + let li = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, li)?, img.os, img.ls)?; + if lpos.version < 3 { + return Err(Error::Unsupported( + "allocating storage for a version 1/2 layout message".into(), + )); + } + let at = lpos + .addr + .ok_or_else(|| Error::Unsupported("layout".into()))?; + let a = img.alloc(total)?; + if t.fill.iter().any(|&b| b != 0) { + let buf = t.fill.repeat(total as usize / es); + img.write(a, &buf)?; + } + let mut ab = vec![0u8; img.os as usize]; + put_uint(&mut ab, a, img.os); + hdr.patch(img, li, at, &ab)?; + // The size field follows the address. + let mut sb = vec![0u8; img.ls as usize]; + put_uint(&mut sb, total, img.ls); + hdr.patch(img, li, at + img.os as usize, &sb)?; + hdr.finish(img)?; + a + } + }; + select::for_each_run(sel, &dims, |c, len, src| { + let off = base + row_major(c, &dims) * es as u64; + let s = src as usize * es; + img.write(off, &data[s..s + len as usize * es]) + })?; + Ok(()) + } + DataLayout::Chunked { btree_address, .. } => { + let mut ce = ChunkedEdit::new(f, img, &t)?; + let existing: HashMap, ChunkInfo> = if btree_address.is_some() { + let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?; + chunks + .into_iter() + .map(|c| { + let s: Vec = + c.offsets.iter().zip(&ce.cd).map(|(o, cd)| o / cd).collect(); + (s, c) + }) + .collect() + } else { + HashMap::new() + }; + let cd = ce.cd.clone(); + let chunk_bytes = ce.chunk_bytes; + let rank = dims.len(); + let mut bufs: BTreeMap, Vec> = BTreeMap::new(); + let load = |scaled: &Vec| -> Result, Error> { + match existing.get(scaled) { + Some(info) => decode_chunk(img_read(f, info)?, &t, info, chunk_bytes), + None => Ok(t.fill.repeat(chunk_bytes / es)), + } + }; + select::for_each_run(sel, &dims, |coords, len, src| { + let mut c = coords.to_vec(); + let mut left = len; + let mut s = src as usize; + let l = rank.saturating_sub(1); + while left > 0 { + let scaled: Vec = c.iter().zip(&cd).map(|(x, d)| x / d).collect(); + let within: Vec = c.iter().zip(&cd).map(|(x, d)| x % d).collect(); + let n = if rank == 0 { + left + } else { + left.min(cd[l] - within[l]) + }; + if !bufs.contains_key(&scaled) { + let b = load(&scaled)?; + bufs.insert(scaled.clone(), b); + } + let buf = bufs.get_mut(&scaled).expect("inserted above"); + let at = row_major(&within, &cd) as usize * es; + let bytes = n as usize * es; + buf[at..at + bytes].copy_from_slice(&data[s * es..s * es + bytes]); + s += n as usize; + left -= n; + if rank > 0 { + c[l] += n; + } + } + Ok(()) + })?; + for (scaled, buf) in bufs { + let (bytes, mask) = match &t.pipeline { + Some(p) => ( + clawhdf5_format::filters::compress_chunk(&buf, p, es as u32)?, + 0u32, + ), + None => (buf, 0u32), + }; + let len = bytes.len() as u64; + let placed = match existing.get(&scaled) { + Some(info) if t.pipeline.is_none() => { + if u64::from(info.chunk_size) != len { + return Err(Error::Unsupported( + "unfiltered chunk stored at an unexpected size".into(), + )); + } + img.write(info.address, &bytes)?; + None + } + // Rewritten where it is when it still fits, or when it + // is the last thing in the file (the chunk an append + // keeps rewriting usually is) and can grow there. + Some(info) + if len <= u64::from(info.chunk_size) + || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => + { + img.write(info.address, &bytes)?; + (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some( + Elem { + addr: info.address, + size: len, + mask, + }, + ) + } + _ => { + let a = img.alloc(len)?; + img.write(a, &bytes)?; + Some(Elem { + addr: a, + size: len, + mask, + }) + } + }; + if let Some(e) = placed { + ce.set(img, &scaled, e)?; + } + } + ce.finish(img) + } + DataLayout::Virtual { .. } => Err(Error::Unsupported("writing a virtual dataset".into())), + } +} + +fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> { + let start = usize::try_from(info.address) + .map_err(|_| Error::Unsupported("chunk address out of range".into()))?; + f.as_bytes() + .get(start..start + info.chunk_size as usize) + .ok_or_else(|| { + Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof { + expected: start + info.chunk_size as usize, + available: f.as_bytes().len(), + }) + }) +} + +fn decode_chunk( + raw: &[u8], + t: &Target, + info: &ChunkInfo, + chunk_bytes: usize, +) -> Result, Error> { + let out = match &t.pipeline { + Some(p) if !clawhdf5_format::filters::all_filters_skipped(p, info.filter_mask) => { + clawhdf5_format::filters::decompress_chunk_masked( + raw, + p, + chunk_bytes, + t.es as u32, + info.filter_mask, + )? + } + _ => raw.to_vec(), + }; + if out.len() != chunk_bytes { + return Err(Error::Format( + clawhdf5_format::error::FormatError::ChunkedReadError(format!( + "chunk decodes to {} bytes, expected {chunk_bytes}", + out.len() + )), + )); + } + Ok(out) +} diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs new file mode 100644 index 0000000..5446702 --- /dev/null +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -0,0 +1,504 @@ +//! An object header as an edit sees it: every chunk and every message +//! (NIL and continuation messages included) with its position in the file, +//! so single messages can be changed in place, deleted (turned into NIL +//! messages) and added (into a NIL message big enough, or into a new +//! continuation chunk at the end of the file). +//! +//! Version-2 chunks carry a checksum, recomputed by [`Header::finish`] for +//! every chunk the edit touched; a version-1 header's message count is kept +//! up to date there too. + +use std::collections::BTreeSet; + +use crate::edit::image::{Image, get_uint, put_uint, rechecksum}; +use crate::error::Error; +use clawhdf5_format::error::FormatError; + +pub(crate) const MSG_NIL: u16 = 0x00; +pub(crate) const MSG_CONTINUATION: u16 = 0x10; +pub(crate) const MSG_ATTRIBUTE: u16 = 0x0C; + +/// One message: where its header and body are, and what it is. +#[derive(Debug, Clone)] +pub(crate) struct Msg { + pub(crate) chunk: usize, + pub(crate) hdr_pos: u64, + pub(crate) data_pos: u64, + pub(crate) size: usize, + pub(crate) mtype: u16, + pub(crate) flags: u8, + pub(crate) corder: Option, +} + +/// One chunk of the header. +#[derive(Debug, Clone)] +struct Chunk { + /// Where the checksummed bytes start (the `OHDR`/`OCHK` signature). + start: u64, + /// Where the checksum is (version 2 only). + checksum_at: Option, + /// Where the chunk's messages end. + end: u64, + /// Bytes at the end too few for a message header (version 2 only). + /// libhdf5 refuses a chunk with both a gap and a NIL message, so a + /// NIL message made in such a chunk must absorb the gap. + gap: u64, +} + +#[derive(Debug)] +pub(crate) struct Header { + pub(crate) addr: u64, + pub(crate) version: u8, + /// Version-2 header flags (0 for version 1). + pub(crate) flags: u8, + chunks: Vec, + pub(crate) msgs: Vec, + dirty: BTreeSet, + /// Messages added (a split NIL message, a new chunk's messages), for a + /// version-1 header's message count. + added: usize, +} + +const MAX_CHUNKS: usize = 1024; + +fn corrupt(why: &'static str) -> Error { + Error::Format(FormatError::InvalidObjectHeader(why)) +} + +impl Header { + /// Locate every chunk and message of the header at `addr`. + pub(crate) fn load(img: &Image<'_>, addr: u64) -> Result { + let sig = img.read(addr, 4)?; + let mut h = Header { + addr, + version: 0, + flags: 0, + chunks: Vec::new(), + msgs: Vec::new(), + dirty: BTreeSet::new(), + added: 0, + }; + let mut pending: Vec<(u64, u64)> = Vec::new(); + if sig == b"OHDR" { + let pre = img.read(addr, 6)?; + if pre[4] != 2 { + return Err(corrupt("bad object header version")); + } + h.version = 2; + h.flags = pre[5]; + let mut pos = addr + 6; + if h.flags & 0x20 != 0 { + pos += 16; + } + if h.flags & 0x10 != 0 { + pos += 4; + } + let w = 1u8 << (h.flags & 0x03); + let size = get_uint(&img.read(pos, w as usize)?, w); + pos += u64::from(w); + h.chunks.push(Chunk { + start: addr, + checksum_at: Some(pos + size), + end: pos + size, + gap: 0, + }); + h.scan(img, 0, pos, pos + size, &mut pending)?; + } else { + let pre = img.read(addr, 16)?; + if pre[0] != 1 { + return Err(corrupt("bad object header version")); + } + h.version = 1; + let size = u64::from(u32::from_le_bytes([pre[8], pre[9], pre[10], pre[11]])); + h.chunks.push(Chunk { + start: addr, + checksum_at: None, + end: addr + 16 + size, + gap: 0, + }); + h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; + } + while let Some((caddr, clen)) = pending.pop() { + if h.chunks.len() >= MAX_CHUNKS { + return Err(corrupt("too many object header chunks")); + } + let idx = h.chunks.len(); + if h.version == 2 { + if clen < 8 || img.read(caddr, 4)? != b"OCHK" { + return Err(corrupt("bad continuation chunk")); + } + h.chunks.push(Chunk { + start: caddr, + checksum_at: Some(caddr + clen - 4), + end: caddr + clen - 4, + gap: 0, + }); + h.scan(img, idx, caddr + 4, caddr + clen - 4, &mut pending)?; + } else { + h.chunks.push(Chunk { + start: caddr, + checksum_at: None, + end: caddr + clen, + gap: 0, + }); + h.scan(img, idx, caddr, caddr + clen, &mut pending)?; + } + } + Ok(h) + } + + /// Size of a message header in this object header. + pub(crate) fn hsize(&self) -> usize { + match (self.version, self.flags & 0x04 != 0) { + (1, _) => 8, + (_, true) => 6, + _ => 4, + } + } + + fn scan( + &mut self, + img: &Image<'_>, + chunk: usize, + start: u64, + end: u64, + pending: &mut Vec<(u64, u64)>, + ) -> Result<(), Error> { + let hs = self.hsize() as u64; + let bytes = img.read(start, (end - start) as usize)?; + let mut p = 0usize; + while (p as u64) + hs <= end - start { + let b = &bytes[p..]; + let (mtype, size, flags, corder) = if self.version == 1 { + ( + u16::from_le_bytes([b[0], b[1]]), + u16::from_le_bytes([b[2], b[3]]) as usize, + b[4], + None, + ) + } else { + ( + u16::from(b[0]), + u16::from_le_bytes([b[1], b[2]]) as usize, + b[3], + (hs == 6).then(|| u16::from_le_bytes([b[4], b[5]])), + ) + }; + let data_off = p + hs as usize; + if data_off + size > bytes.len() { + return Err(corrupt("message size exceeds buffer end")); + } + if mtype == MSG_CONTINUATION { + let d = &bytes[data_off..data_off + size]; + let os = img.os as usize; + let ls = img.ls as usize; + if d.len() < os + ls { + return Err(corrupt("short continuation message")); + } + pending.push((get_uint(d, img.os), get_uint(&d[os..], img.ls))); + } + self.msgs.push(Msg { + chunk, + hdr_pos: start + p as u64, + data_pos: start + data_off as u64, + size, + mtype, + flags, + corder, + }); + p = data_off + size; + } + self.chunks[chunk].gap = (end - start) - p as u64; + Ok(()) + } + + /// After message `i` became a NIL message: if its chunk ends in a gap, + /// grow the NIL message over it (it must be the chunk's last message). + fn absorb_gap(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> { + let m = self.msgs[i].clone(); + let c = &self.chunks[m.chunk]; + if c.gap == 0 { + return Ok(()); + } + if m.data_pos + m.size as u64 + c.gap != c.end { + return Err(Error::Unsupported( + "object header chunk ends in a gap that a free message cannot absorb".into(), + )); + } + let new_size = m.size + c.gap as usize; + if new_size > usize::from(u16::MAX) { + return Err(Error::Unsupported("object header message too large".into())); + } + self.write_msg_header(img, m.hdr_pos, MSG_NIL, new_size, 0, m.corder)?; + img.write(m.data_pos, &vec![0u8; new_size])?; + self.msgs[i].size = new_size; + self.chunks[m.chunk].gap = 0; + Ok(()) + } + + /// The first message of type `mtype`. + pub(crate) fn find(&self, mtype: u16) -> Option { + self.msgs.iter().position(|m| m.mtype == mtype) + } + + pub(crate) fn data(&self, img: &Image<'_>, i: usize) -> Result, Error> { + let m = &self.msgs[i]; + img.read(m.data_pos, m.size) + } + + /// Overwrite bytes of message `i`'s body, from `offset`. + pub(crate) fn patch( + &mut self, + img: &mut Image<'_>, + i: usize, + offset: usize, + bytes: &[u8], + ) -> Result<(), Error> { + let m = &self.msgs[i]; + if offset + bytes.len() > m.size { + return Err(Error::Unsupported( + "change does not fit the header message".into(), + )); + } + img.write(m.data_pos + offset as u64, bytes)?; + self.dirty.insert(m.chunk); + Ok(()) + } + + fn write_msg_header( + &mut self, + img: &mut Image<'_>, + hdr_pos: u64, + mtype: u16, + size: usize, + flags: u8, + corder: Option, + ) -> Result<(), Error> { + let mut h = vec![0u8; self.hsize()]; + if self.version == 1 { + h[0..2].copy_from_slice(&mtype.to_le_bytes()); + h[2..4].copy_from_slice(&(size as u16).to_le_bytes()); + h[4] = flags; + } else { + h[0] = mtype as u8; + h[1..3].copy_from_slice(&(size as u16).to_le_bytes()); + h[3] = flags; + if h.len() == 6 { + h[4..6].copy_from_slice(&corder.unwrap_or(0).to_le_bytes()); + } + } + img.write(hdr_pos, &h) + } + + /// Turn message `i` into a NIL message (its space becomes free). + pub(crate) fn delete(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> { + let m = self.msgs[i].clone(); + self.write_msg_header(img, m.hdr_pos, MSG_NIL, m.size, 0, m.corder)?; + img.write(m.data_pos, &vec![0u8; m.size])?; + self.msgs[i].mtype = MSG_NIL; + self.msgs[i].flags = 0; + self.dirty.insert(m.chunk); + self.absorb_gap(img, i) + } + + /// Body size a message of `len` bytes occupies (version 1 pads to 8). + fn padded(&self, len: usize) -> usize { + if self.version == 1 { + len.next_multiple_of(8) + } else { + len + } + } + + /// Whether a free slot of `slot` bytes can take a body of `need` bytes: + /// exactly, or with room left for a NIL message after it. + fn fits(&self, slot: usize, need: usize) -> bool { + slot == need || slot >= need + self.hsize() + } + + /// The smallest NIL message that can take `need` body bytes. + fn best_nil(&self, need: usize) -> Option { + self.msgs + .iter() + .enumerate() + .filter(|(_, m)| m.mtype == MSG_NIL && self.fits(m.size, need)) + .min_by_key(|(_, m)| m.size) + .map(|(i, _)| i) + } + + /// Put a message into slot `i` (a NIL message, or a message being + /// moved away), splitting off the rest as a NIL message. + fn place( + &mut self, + img: &mut Image<'_>, + i: usize, + mtype: u16, + flags: u8, + data: &[u8], + corder: Option, + ) -> Result<(), Error> { + let slot = self.msgs[i].clone(); + let need = self.padded(data.len()); + debug_assert!(self.fits(slot.size, need)); + let mut body = data.to_vec(); + body.resize(need, 0); + self.write_msg_header(img, slot.hdr_pos, mtype, need, flags, corder)?; + img.write(slot.data_pos, &body)?; + self.msgs[i] = Msg { + size: need, + mtype, + flags, + corder, + ..slot.clone() + }; + if slot.size > need { + let hs = self.hsize(); + let nil_hdr = slot.data_pos + need as u64; + let nil_size = slot.size - need - hs; + self.write_msg_header(img, nil_hdr, MSG_NIL, nil_size, 0, Some(0))?; + img.write(nil_hdr + hs as u64, &vec![0u8; nil_size])?; + self.msgs.push(Msg { + chunk: slot.chunk, + hdr_pos: nil_hdr, + data_pos: nil_hdr + hs as u64, + size: nil_size, + mtype: MSG_NIL, + flags: 0, + corder: (hs == 6).then_some(0), + }); + self.added += 1; + let nil = self.msgs.len() - 1; + self.absorb_gap(img, nil)?; + } + self.dirty.insert(slot.chunk); + Ok(()) + } + + /// Add a message: into free space in the header when there is some, + /// else into a new continuation chunk at the end of the file (whose + /// continuation message takes a NIL slot, or the slot of another + /// message — an attribute if possible — that moves into the new chunk + /// with it). + pub(crate) fn insert( + &mut self, + img: &mut Image<'_>, + mtype: u16, + flags: u8, + data: &[u8], + corder: Option, + ) -> Result<(), Error> { + if data.len() > usize::from(u16::MAX) { + return Err(Error::Unsupported( + "message larger than 64 KiB (would need dense storage)".into(), + )); + } + let need = self.padded(data.len()); + if let Some(i) = self.best_nil(need) { + return self.place(img, i, mtype, flags, data, corder); + } + let os = img.os as usize; + let ls = img.ls as usize; + let cont_need = self.padded(os + ls); + // Where the continuation message goes, and the message (if any) + // that moves out of that slot into the new chunk. + let (slot, moved) = match self.best_nil(cont_need) { + Some(i) => (i, None), + None => { + // Any message but a continuation can live in any chunk; + // prefer moving an attribute, then the smallest that fits. + let i = self + .msgs + .iter() + .enumerate() + .filter(|(_, m)| { + m.mtype != MSG_NIL + && m.mtype != MSG_CONTINUATION + && self.fits(m.size, cont_need) + }) + .min_by_key(|(_, m)| (m.mtype != MSG_ATTRIBUTE, m.size)) + .map(|(i, _)| i) + .ok_or_else(|| { + Error::Unsupported( + "no room in the object header for a continuation message".into(), + ) + })?; + let m = self.msgs[i].clone(); + let body = img.read(m.data_pos, m.size)?; + (i, Some((m, body))) + } + }; + + // The new chunk: [moved message] + new message + a NIL message + // holding spare room for later additions. + let hs = self.hsize(); + let spare = 64usize; + let mut payload = hs + need; + if let Some((m, _)) = &moved { + payload += hs + m.size; + } + let msgs_len = payload + hs + spare; + let (prefix, suffix) = if self.version == 2 { (4, 4) } else { (0, 0) }; + let chunk_len = prefix + msgs_len + suffix; + let caddr = img.alloc(chunk_len as u64)?; + if self.version == 2 { + img.write(caddr, b"OCHK")?; + } + let cidx = self.chunks.len(); + self.chunks.push(Chunk { + start: caddr, + checksum_at: (self.version == 2).then_some(caddr + (prefix + msgs_len) as u64), + end: caddr + (prefix + msgs_len) as u64, + gap: 0, + }); + let first = caddr + prefix as u64; + // Lay the chunk out as one NIL message, then place into it. + self.write_msg_header(img, first, MSG_NIL, msgs_len - hs, 0, Some(0))?; + self.msgs.push(Msg { + chunk: cidx, + hdr_pos: first, + data_pos: first + hs as u64, + size: msgs_len - hs, + mtype: MSG_NIL, + flags: 0, + corder: (hs == 6).then_some(0), + }); + self.added += 1; + if let Some((m, body)) = &moved { + let nil = self.msgs.len() - 1; + self.place(img, nil, m.mtype, m.flags, body, m.corder)?; + } + let nil = self.msgs.len() - 1; + self.place(img, nil, mtype, flags, data, corder)?; + + // Link it in. + let mut cont = vec![0u8; os + ls]; + put_uint(&mut cont, caddr, img.os); + put_uint(&mut cont[os..], chunk_len as u64, img.ls); + if moved.is_some() { + self.msgs[slot].mtype = MSG_NIL; // its content now lives in the new chunk + } + self.place(img, slot, MSG_CONTINUATION, 0, &cont, Some(0))?; + self.dirty.insert(cidx); + Ok(()) + } + + /// Recompute the checksum of every changed version-2 chunk; store a + /// version-1 header's new message count. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for &c in &self.dirty { + if let Some(at) = self.chunks[c].checksum_at { + rechecksum(img, self.chunks[c].start, at)?; + } + } + if self.version == 1 && self.added > 0 { + let old = u16::from_le_bytes(img.read(self.addr + 2, 2)?.try_into().unwrap_or([0; 2])); + let new = usize::from(old) + self.added; + let new = u16::try_from(new) + .map_err(|_| Error::Unsupported("too many object header messages".into()))?; + img.write(self.addr + 2, &new.to_le_bytes())?; + } + self.dirty.clear(); + self.added = 0; + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/select.rs b/crates/clawhdf5/src/edit/select.rs new file mode 100644 index 0000000..4c34336 --- /dev/null +++ b/crates/clawhdf5/src/edit/select.rs @@ -0,0 +1,142 @@ +//! A selection as runs of consecutive elements along the last dimension, in +//! the order the selection's elements are numbered (row-major over a +//! hyperslab, as h5py and libhdf5 number them; a point list in its order). + +use clawhdf5_format::selection::Selection; + +use crate::error::Error; + +/// Call `f(coords, len, src)` for each run: `len` elements starting at +/// `coords` (consecutive in the last dimension), which are elements +/// `src..src + len` of the selection. Returns the number of elements. +/// The selection must already be validated against `dims`. +pub(crate) fn for_each_run( + sel: &Selection, + dims: &[u64], + mut f: impl FnMut(&[u64], u64, u64) -> Result<(), Error>, +) -> Result { + let rank = dims.len(); + let mut src = 0u64; + match sel { + Selection::None => {} + Selection::Points(pts) => { + for p in pts { + f(p, 1, src)?; + src += 1; + } + } + Selection::All => { + if rank == 0 { + f(&[], 1, 0)?; + return Ok(1); + } + if dims.contains(&0) { + return Ok(0); + } + let last = dims[rank - 1]; + let mut coords = vec![0u64; rank]; + loop { + f(&coords, last, src)?; + src += last; + if !advance(&mut coords[..rank - 1], &dims[..rank - 1]) { + break; + } + } + } + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + if rank == 0 { + return Err(Error::InvalidArgument( + "hyperslab selection on a scalar dataset".into(), + )); + } + if (0..rank).any(|d| count[d] == 0 || block[d] == 0) { + return Ok(0); + } + // Per-dimension extent of the selection: j in 0..count*block. + let ext: Vec = (0..rank).map(|d| count[d] * block[d]).collect(); + let coord = |d: usize, j: u64| start[d] + (j / block[d]) * stride[d] + j % block[d]; + let l = rank - 1; + // Along the last dimension, blocks merge when they touch. + let merged = stride[l] == block[l] || count[l] == 1; + let mut js = vec![0u64; rank - 1]; + let mut coords = vec![0u64; rank]; + loop { + for (d, &j) in js.iter().enumerate() { + coords[d] = coord(d, j); + } + if merged { + coords[l] = start[l]; + f(&coords, ext[l], src)?; + src += ext[l]; + } else { + for c in 0..count[l] { + coords[l] = start[l] + c * stride[l]; + f(&coords, block[l], src)?; + src += block[l]; + } + } + if !advance(&mut js, &ext[..l]) { + break; + } + } + } + } + Ok(src) +} + +/// Odometer step over `0..lim[d]`; false when it wraps around. +fn advance(v: &mut [u64], lim: &[u64]) -> bool { + for d in (0..v.len()).rev() { + v[d] += 1; + if v[d] < lim[d] { + return true; + } + v[d] = 0; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collect(sel: &Selection, dims: &[u64]) -> Vec<(Vec, u64, u64)> { + let mut out = Vec::new(); + for_each_run(sel, dims, |c, n, s| { + out.push((c.to_vec(), n, s)); + Ok(()) + }) + .unwrap(); + out + } + + #[test] + fn runs() { + assert_eq!( + collect(&Selection::All, &[2, 3]), + vec![(vec![0, 0], 3, 0), (vec![1, 0], 3, 3)] + ); + let h = Selection::Hyperslab { + start: vec![1, 0], + stride: vec![2, 3], + count: vec![2, 2], + block: vec![1, 2], + }; + assert_eq!( + collect(&h, &[5, 6]), + vec![ + (vec![1, 0], 2, 0), + (vec![1, 3], 2, 2), + (vec![3, 0], 2, 4), + (vec![3, 3], 2, 6) + ] + ); + assert_eq!(collect(&Selection::All, &[]), vec![(vec![], 1, 0)]); + assert_eq!(collect(&Selection::All, &[0, 4]), vec![]); + } +} diff --git a/crates/clawhdf5/src/error.rs b/crates/clawhdf5/src/error.rs index de34355..110abe3 100644 --- a/crates/clawhdf5/src/error.rs +++ b/crates/clawhdf5/src/error.rs @@ -6,7 +6,10 @@ use clawhdf5_format::error::FormatError; use clawhdf5_format::message_type::MessageType; /// Errors that can occur when using the high-level API. +/// +/// Non-exhaustive: new kinds of failure may be added. #[derive(Debug)] +#[non_exhaustive] pub enum Error { /// I/O error from the filesystem. Io(std::io::Error), @@ -36,6 +39,16 @@ pub enum Error { /// Actual alignment of the data pointer. actual: usize, }, + /// The requested change is valid but not supported (by + /// [`FileEditor`](crate::FileEditor): a chunk index, filter or header + /// layout it cannot modify). Nothing was written. + Unsupported(String), + /// An argument does not fit the object (a selection outside the + /// dataset, a buffer of the wrong length, a shrinking resize, ...). + InvalidArgument(String), + /// The file is locked by another writer (another [`FileEditor`](crate::FileEditor), + /// or libhdf5 with file locking on). + Locked(String), } impl fmt::Display for Error { @@ -58,6 +71,9 @@ impl fmt::Display for Error { "zero-copy type mismatch: expected {expected}, got {actual}" ) } + Error::Unsupported(msg) => write!(f, "unsupported: {msg}"), + Error::InvalidArgument(msg) => write!(f, "invalid argument: {msg}"), + Error::Locked(msg) => write!(f, "file is locked: {msg}"), Error::ZeroCopyUnaligned { required, actual } => { write!( f, diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 222e432..280f64b 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -23,8 +23,25 @@ //! builder.set_attr("version", AttrValue::I64(1)); //! builder.write("output.h5").unwrap(); //! ``` +//! +//! # Modifying a file in place +//! +//! ```no_run +//! use clawhdf5::{FileEditor, Selection}; +//! +//! let mut ed = FileEditor::open("data.h5").unwrap(); +//! ed.resize("series", &[1100]).unwrap(); // a chunked dataset, maxshape (None,) +//! let tail = Selection::Hyperslab { +//! start: vec![1000], +//! stride: vec![1], +//! count: vec![100], +//! block: vec![1], +//! }; +//! ed.write_values("series", &tail, &[0.5f64; 100]).unwrap(); +//! ``` mod cache_image; +mod edit; pub mod error; pub mod lazy; #[cfg(feature = "mmap")] @@ -34,6 +51,7 @@ pub mod types; pub mod vlen; pub mod writer; +pub use edit::FileEditor; pub use error::Error; pub use lazy::{LazyDataset, LazyFile, LazyGroup}; #[cfg(feature = "mmap")] diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 1b20dd6..3a11e20 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -1104,13 +1104,18 @@ impl<'f> Dataset<'f> { .ok_or(Error::MissingMessage(msg_type)) } - fn datatype(&self) -> Result { + /// The dataset's object header as parsed. + pub(crate) fn header(&self) -> &ObjectHeader { + &self.header + } + + pub(crate) fn datatype(&self) -> Result { let data = self.required_payload(MessageType::Datatype)?; let (dt, _) = Datatype::parse_in_header(&data, self.header.version)?; Ok(dt) } - fn dataspace(&self) -> Result { + pub(crate) fn dataspace(&self) -> Result { let data = self.required_payload(MessageType::Dataspace)?; let mut ds = Dataspace::parse(&data, self.file.length_size())?; // libhdf5 reports a virtual dataset with unlimited or printf-style @@ -1130,7 +1135,7 @@ impl<'f> Dataset<'f> { Ok(ds) } - fn data_layout(&self) -> Result { + pub(crate) fn data_layout(&self) -> Result { let msg = find_message(&self.header, MessageType::DataLayout)?; Ok(DataLayout::parse( &msg.data, @@ -1143,7 +1148,7 @@ impl<'f> Dataset<'f> { /// that is present but unparseable is an error: treating it as "no /// filters" would hand the caller the still-compressed bytes as if they /// were the data. - fn filter_pipeline(&self) -> Result, Error> { + pub(crate) fn filter_pipeline(&self) -> Result, Error> { self.message_payload(MessageType::FilterPipeline)? .map(|data| FilterPipeline::parse(&data).map_err(Error::Format)) .transpose() diff --git a/crates/clawhdf5/tests/edit_tests.rs b/crates/clawhdf5/tests/edit_tests.rs new file mode 100644 index 0000000..2467256 --- /dev/null +++ b/crates/clawhdf5/tests/edit_tests.rs @@ -0,0 +1,138 @@ +//! `FileEditor` on files clawhdf5 writes, read back with our own reader +//! (libhdf5 interop is in `clawhdf5-tools/tests/edit_interop.rs`, which can +//! also run `h5rs check`). + +use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection}; + +fn block(start: u64, count: u64) -> Selection { + Selection::Hyperslab { + start: vec![start], + stride: vec![1], + count: vec![count], + block: vec![1], + } +} + +fn sample(dir: &std::path::Path) -> std::path::PathBuf { + let path = dir.join("f.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("ext") + .with_i32_data(&[0, 1, 2, 3, 4]) + .with_shape(&[5]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[4]) + .with_deflate(6); + b.create_dataset("raw") + .with_f64_data(&[0.5; 8]) + .with_shape(&[2, 4]) + .with_maxshape(&[u64::MAX, 4]) + .with_chunks(&[1, 4]); + b.create_dataset("flat").with_i64_data(&[1, 2, 3]); + b.set_attr("title", AttrValue::String("t".into())); + b.write(&path).unwrap(); + path +} + +#[test] +fn append_overwrite_and_attributes_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + let len_before = std::fs::metadata(&path).unwrap().len(); + let mut expect: Vec = (0..5).collect(); + let mut raw = vec![0.5f64; 8]; + { + let mut ed = FileEditor::open(&path).unwrap(); + for k in 0..300u64 { + let n = expect.len() as u64; + let add = 1 + k % 5; + ed.resize("ext", &[n + add]).unwrap(); + let vals: Vec = (0..add).map(|j| (n + j) as i32 * 2).collect(); + ed.write_values("ext", &block(n, add), &vals).unwrap(); + expect.extend(&vals); + } + // A filtered chunk rewritten with data that compresses worse moves. + let noisy: Vec = (0..4).map(|i| i * 7_919_993).collect(); + ed.write_values("ext", &block(0, 4), &noisy).unwrap(); + expect[..4].copy_from_slice(&noisy); + + ed.resize("raw", &[5, 4]).unwrap(); + raw.resize(20, 0.0); + let sel = Selection::Hyperslab { + start: vec![1, 1], + stride: vec![2, 2], + count: vec![2, 2], + block: vec![1, 1], + }; + ed.write_values("raw", &sel, &[1.0f64, 2.0, 3.0, 4.0]) + .unwrap(); + for (i, (r, c)) in [(1, 1), (1, 3), (3, 1), (3, 3)].iter().enumerate() { + raw[r * 4 + c] = i as f64 + 1.0; + } + ed.write_values("flat", &Selection::Points(vec![vec![2]]), &[30i64]) + .unwrap(); + ed.set_attr("/", "title", &AttrValue::String("a longer title".into())) + .unwrap(); + ed.set_attr("ext", "count", &AttrValue::I64(expect.len() as i64)) + .unwrap(); + } + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("ext").unwrap().read_i32().unwrap(), expect); + assert_eq!(f.dataset("raw").unwrap().shape().unwrap(), vec![5, 4]); + assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw); + assert_eq!( + f.dataset("flat").unwrap().read_i64().unwrap(), + vec![1, 2, 30] + ); + let root = f.root().attrs().unwrap(); + assert!(matches!(root.get("title"), Some(AttrValue::String(s)) if s == "a longer title")); + let ext = f.dataset("ext").unwrap().attrs().unwrap(); + assert!(matches!(ext.get("count"), Some(AttrValue::I64(n)) if *n == expect.len() as i64)); + assert!(std::fs::metadata(&path).unwrap().len() > len_before); +} + +#[test] +fn errors_leave_the_file_untouched() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + let before = std::fs::read(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); + assert!(ed.write_all("missing", &[0; 4]).is_err()); + // Wrong length, wrong type, outside the extent, beyond maxshape, + // shrinking, a rank change. + assert!(matches!( + ed.write_all("flat", &[0; 7]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.write_values("flat", &Selection::All, &[1i32, 2, 3]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.write_values("ext", &block(4, 2), &[1i32, 2]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.resize("raw", &[3, 5]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_)))); + assert!(matches!( + ed.resize("ext", &[4, 1]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.resize("flat", &[4]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.set_attr("/", "", &AttrValue::I64(1)), + Err(Error::InvalidArgument(_)) + )); + // No-ops write nothing. + ed.resize("ext", &[5]).unwrap(); + ed.write_values("ext", &Selection::None, &[] as &[i32]) + .unwrap(); + drop(ed); + assert!(std::fs::read(&path).unwrap() == before); +} From 677dc5ec7c44be4c0a6ac402ddffc94537bbfbb8 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:35:06 -0500 Subject: [PATCH 2/7] =?UTF-8?q?docs:=20FileEditor=20=E2=80=94=20changelog,?= =?UTF-8?q?=20limits=20and=20leaked=20space,=20README=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit known-issues records what the editor refuses, that freed space is never reused (append-workload file sizes measured 2026-09-26 on tank with the ignored measure_append_waste test; sizes are deterministic), and that there is no journal. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 46 ++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 7 +++++++ README.md | 17 ++++++++++++++++ docs/known-issues.md | 35 +++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f4316..06069f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,52 @@ ## Unreleased +### In-place modification (2026-09-26) +- **`clawhdf5::FileEditor` modifies an existing file where it lies.** + `FileBuilder` builds whole files in memory; the editor opens a file + written by libhdf5 (any `libver`, including HDF5 2.0's own format) or by + clawhdf5 and changes only what an edit touches, recomputing the checksum + of every structure it changes. It takes an exclusive `flock` on the file + (the lock libhdf5 takes), so a second editor gets `Error::Locked`. + - `write_selection` / `write_all` / `write_values`: overwrite values of a + compact, contiguous (also never-written, late-allocated) or chunked + dataset, in its own datatype, under any selection. Chunks are decoded, + updated and re-encoded through the dataset's filters; a chunk that no + longer fits moves to the end of the file. New chunks are added to + version-1 B-tree (every chunked dataset of h5py's default `libver`), + Extensible Array, Fixed Array and single-chunk indexes — creating the + index, its data blocks, super blocks and pages, and splitting B-tree + nodes, as libhdf5 does: after the same sequence of writes the B-tree has + the same number of nodes per level and the Extensible Array header the + same block statistics as libhdf5's (tested). + - `resize`: grow a chunked dataset up to its maximum dimensions (h5py's + `Dataset.resize`). + - `set_attr`: add or replace an attribute in an object header, in free + space or in a new continuation chunk at the end of the file. + - Each edit is planned in memory and refused as a whole + (`Error::Unsupported`, file untouched) when any part is not supported: + new chunks in a version-2 B-tree index (two or more unlimited + dimensions) or an implicit index, shrinking, variable-length and + reference data, attributes in dense storage, past an object's compact + limit or with tracked creation order, files with a metadata cache + image, paged or persistent free space, or marked open by another + writer. New error variants `Error::Unsupported`, + `Error::InvalidArgument`, `Error::Locked`, and `clawhdf5::Error` is now + `#[non_exhaustive]` — a breaking change for code that matches it + exhaustively (the Python bindings map the new variants to + `NotImplementedError`, `ValueError` and `OSError`). + - Durability: the new space (chunks, index blocks) is written and synced + before any existing byte changes, then the metadata that links it in, + then a second sync. There is no journal: a crash during the second + phase can leave the file inconsistent (as with libhdf5 without SWMR). + Freed space is not reused (see `docs/known-issues.md`). + - Tests: `crates/clawhdf5-tools/tests/edit_interop.rs` (h5py `earliest`, + `v114` and `latest` files and clawhdf5 files; after every round h5py + reads the expected values, h5dump and `h5rs check --data` accept the + file, and h5py `r+` modifies it further; random operations against a + model) and `crates/clawhdf5/tests/edit_tests.rs`. +- `clawhdf5_format::type_builders::build_attr_message` is public. + ### Chunked full reads (2026-09-26) - **Chunks are decoded straight into the output, into reused buffers.** A full read of a chunked dataset faulted in about three times its size in diff --git a/CLAUDE.md b/CLAUDE.md index b029c5d..2482401 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,6 +150,13 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F Alerts never block a save — drain them with `HDF5Memory::take_anomaly_alerts`. `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). +- In-place modification: `clawhdf5::FileEditor` (`crates/clawhdf5/src/edit/`) + overwrites values, grows chunked datasets and sets attributes in existing + files (h5py- or clawhdf5-written) without rewriting them; anything it + cannot do safely is `Error::Unsupported` before any write (limits in + `docs/known-issues.md`). Test changes with + `cargo test -p clawhdf5-tools --test edit_interop` (h5py, h5dump, + `h5rs check`). - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. diff --git a/README.md b/README.md index 25f43a7..b772985 100644 --- a/README.md +++ b/README.md @@ -433,6 +433,23 @@ b.write("groups.h5")?; A group holds at most 65 535 links; more is an error, as is a link over 65 515 bytes (a very long soft-link target) in a group of more than 8 links. +### Modifying an existing file + +```rust +use clawhdf5::{AttrValue, FileEditor, Selection}; + +// A file from h5py or clawhdf5, dataset "x" chunked with maxshape=(None,). +let mut ed = FileEditor::open("data.h5")?; // exclusive lock, like libhdf5 +ed.resize("x", &[1100])?; // h5py: ds.resize((1100,)) +let sel = Selection::Hyperslab { start: vec![1000], stride: vec![1], count: vec![100], block: vec![1] }; +ed.write_values("x", &sel, &[0.5f64; 100])?; // ds[1000:1100] = 0.5 +ed.set_attr("x", "units", &AttrValue::String("m/s".into()))?; +``` + +Each call changes the file in place (no rewrite) and syncs it. What it +cannot change safely is refused before anything is written; see +[known issues](docs/known-issues.md) for the limits. + ### Python `crates/clawhdf5-py` is a Python package (PyO3 + numpy) that reads HDF5 with diff --git a/docs/known-issues.md b/docs/known-issues.md index 287e501..4d49f0e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,41 @@ deleting it. --- +## In-place modification (`FileEditor`) limits + +**Status:** open (documented 2026-09-26). `clawhdf5::FileEditor` refuses, +with `Error::Unsupported` and without writing anything: +- new, moved or resized chunks in a **version-2 B-tree** chunk index (what + libhdf5 uses for two or more unlimited dimensions) — existing unfiltered + chunks, and filtered ones that re-encode to the same size, are + overwritten in place; `resize` works — and new chunks in an **implicit** + index (it has all of its chunks from the start); +- **shrinking** a dataset; +- variable-length and reference data; +- attributes of an object in **dense storage**, past its compact limit (8 + by default) or with tracked **creation order**; +- partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external + raw data files, virtual datasets; +- files with a metadata cache image, paged or persistent free-space + management, a driver info block, or version-3 consistency flags set. + +**Space is never reused.** There is no free-space manager: the old bytes of +a filtered chunk that grows and has to move, and of an attribute that is +replaced by a larger one, are leaked (`h5repack` reclaims them). A chunk +that is the last thing in the file grows in place instead, which covers the +usual append. Measured 2026-09-26 on tank with +`cargo test --release -p clawhdf5-tools --test edit_interop -- --ignored +--nocapture measure_append_waste` (file sizes are deterministic): 1000 +appends of 100 `f8` values to a 1-D dataset with 1024-element chunks give +810 504 bytes unfiltered, as libhdf5's file, and 307 210 bytes with gzip +(libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values with +4096-element gzip chunks give 119 684 bytes against libhdf5's 50 292 +(`h5repack`: 49 930), because the chunk being appended to is followed by +new index blocks and moves each time it grows. + +**No journal.** A crash while an edit patches existing structures can leave +the file inconsistent; see the `FileEditor` documentation. + ## Selection reads that decode more than the selection **Status:** open (documented 2026-09-26). `Dataset::read_selection` (and so From f7e2ab12f27b10677de8385cd623d39b9b673944 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:09:34 -0500 Subject: [PATCH 3/7] clawhdf5: FileEditor skips optional filters that fail, as libhdf5 does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The editor stored every chunk through the whole pipeline with filter mask 0. For LZF that did not shrink a chunk, h5py instead stores it raw with the filter's mask bit set. A chunk the editor stored LZF-encoded at exactly the raw size was then rewritten raw by libhdf5 at the same size; libhdf5 does not touch the index entry when the size is unchanged, so the stale mask 0 stayed and h5py (and h5dump) could no longer read the dataset. clawhdf5_format::filters::compress_chunk_masked runs the pipeline as H5Z_pipeline does: an optional filter (H5Z_FLAG_OPTIONAL) that fails is skipped and its bit set, a mandatory one fails the write, and LZF/Blosc output no smaller than the input counts as failure, as in the reference filters (their output buffer is the input's size). Deflate, LZ4, Zstd, bitshuffle and bzip2 never fail on size in libhdf5 and are kept as before. Test: edit_interop optional_filters_that_fail_are_skipped — the reviewer's repro at every libver: the editor stores the chunk exactly as h5py does (mask 1, size 5; shuffle+LZF+fletcher32 mask 2), h5py r+ rewrites and extends the datasets, and h5py, h5dump and our reader read every value. Fails on the previous editor (mask 0; h5dump cannot read /u8). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 +- crates/clawhdf5-format/src/filters.rs | 120 ++++++++++++++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 128 ++++++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 7 +- 4 files changed, 260 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06069f7..f070cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,15 @@ index, its data blocks, super blocks and pages, and splitting B-tree nodes, as libhdf5 does: after the same sequence of writes the B-tree has the same number of nodes per level and the Extensible Array header the - same block statistics as libhdf5's (tested). + same block statistics as libhdf5's (tested). Filters run as libhdf5's + `H5Z_pipeline` runs them (new + `clawhdf5_format::filters::compress_chunk_masked`): an optional filter + that fails — LZF or Blosc output no smaller than the chunk — is skipped + and its filter-mask bit set, so the chunk is stored exactly as h5py + stores it; a mandatory filter that fails fails the edit. (Storing such + a chunk LZF-encoded at the raw size with a clear mask let a later + libhdf5 rewrite of it keep the stale mask, and h5py could no longer + read the dataset.) - `resize`: grow a chunked dataset up to its maximum dimensions (h5py's `Dataset.resize`). - `set_attr`: add or replace an attribute in an object header, in free diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 40d6cd9..f187746 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -346,6 +346,72 @@ pub fn compress_chunk( Ok(result) } +/// Filter flag bit 0: `H5Z_FLAG_OPTIONAL`. +const FILTER_FLAG_OPTIONAL: u16 = 0x0001; + +/// Filters whose reference HDF5 filter (h5py's `lzf_filter.c`, +/// hdf5-blosc's `blosc_filter.c`) gives the encoder an output buffer only as +/// large as its input, so output that is not smaller than the input is a +/// failure there. +const FAIL_UNLESS_SMALLER: &[u16] = &[ + crate::filter_pipeline::FILTER_LZF, + crate::filter_pipeline::FILTER_BLOSC, +]; + +/// Run a chunk through a filter pipeline for writing the way libhdf5's +/// `H5Z_pipeline` does, returning the bytes to store and the chunk's filter +/// mask (bit `i` set: filter `i` was skipped). +/// +/// A filter that fails is skipped if the pipeline marks it optional +/// (`H5Z_FLAG_OPTIONAL`): its mask bit is set and the next filter gets the +/// same input. A mandatory filter that fails fails the write. Failure +/// includes what the reference filter counts as failure: LZF and Blosc +/// output that is not smaller than the input (h5py then stores the chunk +/// unfiltered with the bit set; storing it filtered with a clear mask can +/// leave a stale mask once libhdf5 rewrites the chunk at the same size). +/// +/// A filter this build cannot encode is [`FormatError::UnsupportedFilter`] +/// even when optional: libhdf5 skips an optional filter only when its own +/// build lacks it, and every libhdf5 has the ones clawhdf5 cannot encode. +pub fn compress_chunk_masked( + data: &[u8], + pipeline: &FilterPipeline, + element_size: u32, +) -> Result<(Vec, u32), FormatError> { + if pipeline.filters.len() > 32 { + return Err(FormatError::CompressionError( + "more than 32 filters in a pipeline".into(), + )); + } + let mut result = data.to_vec(); + let mut mask = 0u32; + for (i, filter) in pipeline.filters.iter().enumerate() { + let ctx = FilterContext { + filter, + element_size: element_size as usize, + max_output: 0, + }; + let out = match filter_registry::encode(&result, &ctx) { + Ok(out) + if FAIL_UNLESS_SMALLER.contains(&filter.filter_id) && out.len() >= result.len() => + { + Err(FormatError::CompressionError(format!( + "filter {} did not shrink the chunk", + filter.filter_id + ))) + } + r => r, + }; + match out { + Ok(out) => result = out, + Err(e @ FormatError::UnsupportedFilter(_)) => return Err(e), + Err(_) if filter.flags & FILTER_FLAG_OPTIONAL != 0 => mask |= 1 << i, + Err(e) => return Err(e), + } + } + Ok((result, mask)) +} + /// The filters compiled into this build, sorted by ID (see /// [`crate::filter_registry`]). A filter whose cargo feature is off is left /// out, so it fails as [`FormatError::UnsupportedFilter`] like any unknown ID. @@ -2099,6 +2165,60 @@ mod tests { } } + /// `compress_chunk_masked` follows `H5Z_pipeline`: an optional LZF that + /// does not shrink the chunk is skipped with its mask bit set (h5py + /// stores `[182, 0, 0, 0, 0]` raw with mask 1), a mandatory one fails, + /// and filters that grow the data (deflate) are kept, as libhdf5 keeps + /// them. + #[test] + #[cfg(all(feature = "lzf", feature = "deflate"))] + fn masked_compression_skips_optional_filters_that_fail() { + use crate::filter_pipeline::FILTER_LZF; + let opt = |id: u16| FilterDescription { + flags: FILTER_FLAG_OPTIONAL, + ..filter(id) + }; + let pl = |filters: Vec| FilterPipeline { + version: 2, + filters, + }; + let raw = [182u8, 0, 0, 0, 0]; + let (out, mask) = compress_chunk_masked(&raw, &pl(vec![opt(FILTER_LZF)]), 1).unwrap(); + assert_eq!((out.as_slice(), mask), (&raw[..], 1)); + let (out, mask) = compress_chunk_masked( + &raw, + &pl(vec![ + opt(FILTER_SHUFFLE), + opt(FILTER_LZF), + filter(FILTER_FLETCHER32), + ]), + 1, + ) + .unwrap(); + assert_eq!((out.len(), mask), (raw.len() + 4, 2)); + assert_eq!( + decompress_chunk_masked( + &out, + &pl(vec![ + opt(FILTER_SHUFFLE), + opt(FILTER_LZF), + filter(FILTER_FLETCHER32) + ]), + raw.len(), + 1, + mask + ) + .unwrap(), + raw + ); + assert!(compress_chunk_masked(&raw, &pl(vec![filter(FILTER_LZF)]), 1).is_err()); + let zeros = [0u8; 256]; + let (out, mask) = compress_chunk_masked(&zeros, &pl(vec![opt(FILTER_LZF)]), 1).unwrap(); + assert!(out.len() < zeros.len() && mask == 0); + let (out, mask) = compress_chunk_masked(&raw, &pl(vec![opt(FILTER_DEFLATE)]), 1).unwrap(); + assert!(out.len() > raw.len() && mask == 0); + } + #[test] #[cfg(feature = "deflate")] fn filter_mask_skips_only_the_masked_filters() { diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 2df54d7..fe44aa8 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1187,3 +1187,131 @@ fn measure_append_waste() { ); } } + +/// Optional filters that fail are skipped as libhdf5 skips them: an LZF +/// that does not shrink a chunk leaves it stored unfiltered with its mask +/// bit set, exactly as h5py stores it. Storing the LZF stream with a clear +/// mask at the raw chunk's size once let a later libhdf5 rewrite of the +/// chunk (raw, same size) keep the stale mask, and h5py then failed to read +/// the dataset. After the editor, h5py r+ rewrites and extends the datasets +/// and h5py, h5dump (on the chunks it can decode: LZF is not in h5dump) and +/// our reader read every value. +#[test] +fn optional_filters_that_fail_are_skipped() { + if !tools_ok() { + return; + } + for (li, (lv, _)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("optional_{li}.h5")); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.create_dataset('u8', shape=(5,), dtype='u1', chunks=(5,), maxshape=(None,), compression='lzf')\n\ + \x20 f.create_dataset('mix', shape=(32,), dtype=' = (0..32) + .map(|i| { + if (i / 8) % 2 == 0 { + rng.next() as i32 + } else { + 7 + } + }) + .collect(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.write_all("u8", &[182, 0, 0, 0, 0]).unwrap(); + ed.write_values("mix", &Selection::All, &mix).unwrap(); + drop(ed); + // Stored as h5py stores it: raw, LZF's bit (0; 1 behind shuffle) set. + let masks = py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + i = lambda d, k: f[d].id.get_chunk_info(k)\n\ + print(i('u8', 0).filter_mask, i('u8', 0).size, i('ref', 0).filter_mask, i('ref', 0).size,\n\ + \x20 *[(i('mix', k).filter_mask, i('mix', k).size < 36) for k in range(4)])\n" + )); + assert_eq!( + masks, "1 5 1 5 (2, False) (0, True) (2, False) (0, True)", + "{lv}" + ); + let dump = |ds: &str| { + let o = Command::new("h5dump") + .args(["-d", ds, "-y", "-w", "0", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5dump -d {ds} {p}:\n{}", text(&o)); + let s = String::from_utf8_lossy(&o.stdout).into_owned(); + s.split_once("DATA {") + .and_then(|(_, r)| r.split_once('}')) + .map(|(d, _)| { + d.split(|c: char| c == ',' || c.is_whitespace()) + .filter(|t| !t.is_empty()) + .map(|t| t.parse::().unwrap()) + .collect::>() + }) + .unwrap_or_default() + }; + if *lv != "'latest'" { + assert_eq!(dump("/u8"), [182, 0, 0, 0, 0]); + } + // libhdf5 rewrites the chunk raw at the same size, then extends the + // datasets with more incompressible data. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['u8'][...] = [182, 0, 0, 0, 1]\n\ + \x20 f['u8'].resize((12,))\n\ + \x20 f['u8'][5:] = [9, 200, 3, 77, 1, 250, 42]\n\ + \x20 m = f['mix']\n\ + \x20 m[8:16] = np.arange(8, dtype=' = vec![182, 0, 0, 0, 1, 9, 200, 3, 77, 1, 250, 42]; + let mut mix_want = mix.clone(); + mix_want[0..8].fill(5); + for (k, v) in mix_want[8..16].iter_mut().enumerate() { + *v = k as i32 * 104_729 + 12_345; + } + mix_want.extend([11; 8]); + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("u8") + .unwrap() + .read_selection(&Selection::All) + .unwrap(), + u8_want + ); + assert_eq!(f.dataset("mix").unwrap().read_i32().unwrap(), mix_want); + drop(f); + verify( + &path, + "mix", + &Model { + shape: vec![40], + data: mix_want, + }, + ); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + assert f['u8'][()].tolist() == {u8_want:?}, f['u8'][()]\n" + )); + if *lv != "'latest'" { + let got: Vec = dump("/u8").into_iter().map(|v| v as u8).collect(); + assert_eq!(got, u8_want); + } + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index b135a3f..c0a5d07 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -1116,11 +1116,10 @@ fn write_selection( Ok(()) })?; for (scaled, buf) in bufs { + // As libhdf5 does: an optional filter that fails (LZF that + // does not shrink the chunk) is skipped and its mask bit set. let (bytes, mask) = match &t.pipeline { - Some(p) => ( - clawhdf5_format::filters::compress_chunk(&buf, p, es as u32)?, - 0u32, - ), + Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?, None => (buf, 0u32), }; let len = bytes.len() as u64; From 485bea0f4f3444f414bdb65224014661c62a6310 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:12:35 -0500 Subject: [PATCH 4/7] clawhdf5: set_attr adds the Attribute Info message a version-2 header needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libhdf5 counts a version-2 object header's attributes through its Attribute Info message (0x15) and reports none when the header has none. set_attr gave v110/latest groups, the root group and datasets without attributes an attribute message only, so h5py listed the attribute but len(obj.attrs) and H5Oget_info's num_attrs said 0, and stayed wrong after h5py r+ added more. Like H5O__attr_create, the edit now adds the message when a version-2 header lacks it, in the same planned edit: version 0, the header's creation-order track/index flags, maximum creation index 0, undefined fractal heap and B-tree addresses, message flag DONTSHARE — byte for byte what libhdf5 writes. It goes before the attribute (libhdf5's order) when free space holds both, else after it, so a continuation chunk made for the attribute also takes it. Test: edit_interop attribute_count_in_version_2_headers — v110 and latest files, attributes set on the root group, groups and datasets with and without existing attributes: h5py's len/num_attrs/list/values, h5dump -A and our reader agree, also after h5py r+ adds attributes up to and past the compact limit. Fails on the previous editor (h5py len 0). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 +- crates/clawhdf5-tools/tests/edit_interop.rs | 87 +++++++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 39 +++++++++ crates/clawhdf5/src/edit/ohdr.rs | 5 ++ 4 files changed, 136 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f070cd5..2516491 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,11 @@ - `resize`: grow a chunked dataset up to its maximum dimensions (h5py's `Dataset.resize`). - `set_attr`: add or replace an attribute in an object header, in free - space or in a new continuation chunk at the end of the file. + space or in a new continuation chunk at the end of the file. A + version-2 header (h5py `libver='v110'` and later) without an Attribute + Info message gets one, as libhdf5's `H5O__attr_create` adds it: libhdf5 + counts such a header's attributes through that message, and without it + h5py reported `len(obj.attrs) == 0` while listing them. - Each edit is planned in memory and refused as a whole (`Error::Unsupported`, file untouched) when any part is not supported: new chunks in a version-2 B-tree index (two or more unlimited diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index fe44aa8..7587126 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1315,3 +1315,90 @@ fn optional_filters_that_fail_are_skipped() { assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); } } + +/// libhdf5 counts a version-2 object header's attributes through its +/// Attribute Info message and reports none without one. The editor adds +/// that message, as `H5O__attr_create` does, when it gives a version-2 +/// header (h5py `libver='v110'`/`'latest'`) its first attribute: afterwards +/// h5py lists, counts and reads every attribute, new and existing, h5dump +/// agrees, and h5py `r+` can add more (past the compact limit, into dense +/// storage) with the count still right. +#[test] +fn attribute_count_in_version_2_headers() { + if !tools_ok() { + return; + } + for (lv, dump) in [("'v110'", true), ("'latest'", false)] { + let dir = tmpdir(); + let path = dir.path().join("acount.h5"); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.create_group('g')\n\ + \x20 f.create_group('has').attrs['old'] = 7\n\ + \x20 f.create_dataset('d', data=np.arange(3, dtype=' f.root().attrs().unwrap(), + "d" | "e" => f.dataset(o).unwrap().attrs().unwrap(), + _ => f.group(o).unwrap().attrs().unwrap(), + }; + assert_eq!(attrs.len(), n + extra, "{o}"); + assert!(matches!(attrs.get("a1"), Some(AttrValue::F64(x)) if *x == 1.5)); + } + if dump { + // Every object's attributes: 3 set by the editor on each of + // the five, 2 existing, `extra` from h5py on each. + let out = Command::new("h5dump").args(["-A", p]).output().unwrap(); + assert!(out.status.success(), "h5dump -A:\n{}", text(&out)); + let s = String::from_utf8_lossy(&out.stdout); + assert_eq!( + s.matches("ATTRIBUTE \"").count(), + 5 * (3 + extra) + 2, + "h5dump -A:\n{s}" + ); + } + }; + check(0); + // libhdf5 adds more: to 7 or 8 (compact), then past its limit. + for extra in [4usize, 10] { + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'g', 'has', 'd', 'e']:\n\ + \x20 for i in range({extra}): f[o].attrs[f'h{{i}}'] = i\n" + )); + check(extra); + check_tools(&path, dump); + } + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index c0a5d07..15237fc 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -47,6 +47,8 @@ const MSG_EXTERNAL: u16 = 0x07; const MSG_ATTR_INFO: u16 = 0x15; /// Message flag: the message is shared (stored elsewhere). const MSG_FLAG_SHARED: u8 = 0x02; +/// Message flag: the message must not be shared (`H5O_MSG_FLAG_DONTSHARE`). +const MSG_FLAG_DONTSHARE: u8 = 0x04; /// An HDF5 file opened for in-place modification. /// @@ -815,7 +817,24 @@ impl FileEditor { if let Some(i) = existing { hdr.delete(img, i)?; } + // libhdf5 counts a version-2 header's attributes through its + // Attribute Info message and reports none without one; like + // H5O__attr_create, add it when missing: before the attribute + // when free space holds both (libhdf5's order), else after it, + // so that a new continuation chunk made for the attribute has + // room for it too. + let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none()) + .then(|| attr_info_message(hdr.flags, img.os)); + let ainfo_first = ainfo + .as_ref() + .is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len())); + if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; + } hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; + } hdr.finish(img) }) } @@ -900,6 +919,26 @@ fn attr_name(d: &[u8]) -> Result<&[u8], Error> { Ok(name.split(|&b| b == 0).next().unwrap_or(name)) } +/// A new Attribute Info message for a version-2 header with flags +/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order +/// tracked / indexed as the header's flags say, maximum creation index 0, +/// and no dense storage (undefined fractal heap and B-tree addresses). +fn attr_info_message(hdr_flags: u8, os: u8) -> Vec { + let track = hdr_flags & 0x04 != 0; + let index = hdr_flags & 0x08 != 0; + let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; + if track { + b.extend_from_slice(&0u16.to_le_bytes()); + } + let undef_addr = vec![0xffu8; os as usize]; + b.extend_from_slice(&undef_addr); + b.extend_from_slice(&undef_addr); + if index { + b.extend_from_slice(&undef_addr); + } + b +} + /// A version-2 header's limit on compact attributes: stored when its flags /// say so, else libhdf5's default of 8. fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs index 5446702..f3abfe0 100644 --- a/crates/clawhdf5/src/edit/ohdr.rs +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -326,6 +326,11 @@ impl Header { .map(|(i, _)| i) } + /// Whether free space in the header can take a body of `len` bytes. + pub(crate) fn has_free(&self, len: usize) -> bool { + self.best_nil(self.padded(len)).is_some() + } + /// Put a message into slot `i` (a NIL message, or a message being /// moved away), splitting off the rest as a NIL message. fn place( From b668878129dfe2245099ac80b4475c46949fb769 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:13:45 -0500 Subject: [PATCH 5/7] clawhdf5: FileEditor reports filters it cannot run as Error::Unsupported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dataset whose filter this build cannot encode (scale-offset, N-Bit, SZIP; a plugin filter the build lacks) failed with Error::Format("unsupported filter: 6"), although the editor documents every refused edit as Error::Unsupported, and the Python bindings raised ValueError rather than NotImplementedError. Every edit now maps FormatError::UnsupportedFilter to Error::Unsupported; the file is left untouched as before. Test: edit_interop unencodable_filters_are_unsupported — h5py scale-offset datasets (integer with chunks, integer never written, float D-scale): Error::Unsupported naming the filter, and the file byte for byte unchanged. Fails on the previous editor (Format(UnsupportedFilter(6))). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 ++-- crates/clawhdf5-tools/tests/edit_interop.rs | 33 +++++++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 16 +++++++++- docs/known-issues.md | 7 ++++- 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2516491..f959f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,8 +40,10 @@ (`Error::Unsupported`, file untouched) when any part is not supported: new chunks in a version-2 B-tree index (two or more unlimited dimensions) or an implicit index, shrinking, variable-length and - reference data, attributes in dense storage, past an object's compact - limit or with tracked creation order, files with a metadata cache + reference data, chunks through a filter this build cannot encode + (scale-offset, N-Bit, SZIP), attributes in dense storage, past an + object's compact limit or with tracked creation order, files with a + metadata cache image, paged or persistent free space, or marked open by another writer. New error variants `Error::Unsupported`, `Error::InvalidArgument`, `Error::Locked`, and `clawhdf5::Error` is now diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 7587126..85d4d0c 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1402,3 +1402,36 @@ fn attribute_count_in_version_2_headers() { } } } + +/// A dataset whose filters this build cannot encode (scale-offset) +/// is `Error::Unsupported`, as the editor documents, not a format error, +/// and the file is left byte for byte as it was. +#[test] +fn unencodable_filters_are_unsupported() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("unencodable.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 f.create_dataset('so', data=np.arange(16, dtype=' assert!(msg.contains("filter"), "{ds}: {msg}"), + other => panic!("{ds}: {other:?}"), + } + } + drop(ed); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 15237fc..f6a590e 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -622,7 +622,7 @@ impl FileEditor { check_editable(&f)?; let sb = f.superblock().clone(); let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); - let r = op(&f, &mut img)?; + let r = op(&f, &mut img).map_err(unsupported_filter)?; if img.is_dirty() { if img.eoa() != img.old_eoa() { set_superblock_eof(&mut img, &sb)?; @@ -840,6 +840,20 @@ impl FileEditor { } } +/// A filter this build cannot run (scale-offset, N-Bit and SZIP have no +/// encoder; a plugin filter may be missing) is something the editor does not +/// support, not a malformed file. +fn unsupported_filter(e: Error) -> Error { + match e { + Error::Format(clawhdf5_format::error::FormatError::UnsupportedFilter(id)) => { + Error::Unsupported(format!( + "datasets with filter {id}, which this build cannot run" + )) + } + e => e, + } +} + /// libhdf5's checks before it opens a file for writing, and what this /// editor cannot keep consistent. fn check_editable(f: &File) -> Result<(), Error> { diff --git a/docs/known-issues.md b/docs/known-issues.md index 4d49f0e..818a487 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -13,11 +13,16 @@ deleting it. with `Error::Unsupported` and without writing anything: - new, moved or resized chunks in a **version-2 B-tree** chunk index (what libhdf5 uses for two or more unlimited dimensions) — existing unfiltered - chunks, and filtered ones that re-encode to the same size, are + chunks, and filtered ones that re-encode to the same size and filter + mask, are overwritten in place; `resize` works — and new chunks in an **implicit** index (it has all of its chunks from the start); - **shrinking** a dataset; - variable-length and reference data; +- chunks through a filter this build cannot encode (scale-offset, N-Bit, + SZIP, or a plugin filter it lacks), even an optional one: libhdf5 skips + an optional filter only when its own build lacks it, which none does for + these; - attributes of an object in **dense storage**, past its compact limit (8 by default) or with tracked **creation order**; - partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external From fe377266e1cdbb8923af2260da6bab295f8e8194 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:14:58 -0500 Subject: [PATCH 6/7] clawhdf5: FileEditor unmaps the file before an edit writes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each edit planned over the reader's memory map of the file and committed while that File, and the Image's &[u8] over the mapping, were still alive, writing the same file through the editor's descriptor. Nothing read the mapping during the writes, but a shared slice whose memory changes underneath it is undefined behaviour under Rust's aliasing rules. Image::into_plan now detaches the edit's writes (patches, end of allocation) into a Plan that owns all of its bytes and borrows nothing; edit() takes the user-block size, drops the File — unmapping the file — and only then commits the Plan. The invariant is documented in the image module and the editor's module docs. Test: edit::tests::file_is_not_mapped_while_an_edit_writes_it checks /proc/self/maps at the moment each commit starts (write, resize, set_attr): never mapped. With the commit moved back before the reader is dropped (the previous order) it reports all three commits with the file mapped. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/src/edit/image.rs | 32 +++++++++++- crates/clawhdf5/src/edit/mod.rs | 82 +++++++++++++++++++++++++++++-- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index a100b66..254abcd 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -4,9 +4,17 @@ //! An edit never writes to the file while it is being planned. Every change //! is recorded here first (reads see them), so an edit that fails half-way — //! a filter that cannot encode, a chunk index this code does not handle — -//! leaves the file exactly as it was. [`Image::commit`] then writes the -//! changes in an order that keeps the old metadata valid for as long as +//! leaves the file exactly as it was. [`Image::into_plan`] then detaches the +//! changes from the bytes they were planned over, and [`Plan::commit`] +//! writes them in an order that keeps the old metadata valid for as long as //! possible (see there). +//! +//! **Invariant:** the base bytes an image reads are the reader's view of the +//! file — a memory map when the `mmap` feature is on. Nothing may write the +//! file while that view is alive: a write through another descriptor would +//! change memory behind a live `&[u8]`, which Rust's aliasing rules forbid. +//! So a [`Plan`] owns everything it writes and borrows nothing, and the +//! editor drops the reader (unmapping the file) before it commits. use std::collections::BTreeMap; use std::io::{Seek, SeekFrom, Write}; @@ -178,6 +186,26 @@ impl<'a> Image<'a> { Ok(()) } + /// The edit's writes, detached from the base bytes (see the module's + /// invariant: the reader that owns them can then be dropped before + /// anything is written). + pub(crate) fn into_plan(self) -> Plan { + Plan { + patches: self.patches, + eoa: self.eoa, + old_eoa: self.old_eoa, + } + } +} + +/// The writes of a planned edit, owning all of their bytes. +pub(crate) struct Plan { + patches: BTreeMap>, + eoa: u64, + old_eoa: u64, +} + +impl Plan { /// Write the edit to `file`, whose superblock is at `user_block`. /// /// Order: first everything in newly allocated space (new chunks, new diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index f6a590e..4903672 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -10,7 +10,10 @@ //! Each operation is planned in memory first ([`image::Image`]): if any part //! of it is unsupported, nothing is written. The plan is then committed with //! the new space (new chunks, new index blocks) written and synced before -//! the existing bytes that link it in, then synced again. +//! the existing bytes that link it in, then synced again. The plan owns the +//! bytes it writes; the reader it was planned with (a memory map of the +//! file) is dropped before the first write, so no `&[u8]` over the mapping +//! is alive while the file changes (see `image`). mod btree1; mod earray; @@ -614,6 +617,12 @@ impl FileEditor { &self.path } + /// Plan an edit over the file's current bytes, then commit it. + /// + /// The reader (a memory map of the file, with the `mmap` feature) is + /// dropped before anything is written: the plan owns every byte it + /// writes, so no slice over the mapping is alive while the file changes + /// underneath it (see `image`'s invariant). fn edit( &mut self, op: impl FnOnce(&File, &mut Image<'_>) -> Result, @@ -621,13 +630,22 @@ impl FileEditor { let f = File::open(&self.path)?; check_editable(&f)?; let sb = f.superblock().clone(); + let user_block = f.user_block_size(); let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); let r = op(&f, &mut img).map_err(unsupported_filter)?; - if img.is_dirty() { + let plan = if img.is_dirty() { if img.eoa() != img.old_eoa() { set_superblock_eof(&mut img, &sb)?; } - img.commit(&mut self.file, f.user_block_size())?; + Some(img.into_plan()) + } else { + None + }; + drop(f); + if let Some(plan) = plan { + #[cfg(test)] + tests::note_commit(&self.path); + plan.commit(&mut self.file, user_block)?; } Ok(r) } @@ -1263,3 +1281,61 @@ fn decode_chunk( } Ok(out) } + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::path::Path; + + use super::FileEditor; + use crate::{AttrValue, FileBuilder}; + + thread_local! { + /// Commits seen on this thread, and how many found the file mapped. + static COMMITS: Cell<(usize, usize)> = const { Cell::new((0, 0)) }; + } + + /// Called just before an edit writes the file: whether this process + /// still maps it (`/proc/self/maps` lists every mapping by path). + pub(super) fn note_commit(path: &Path) { + let path = std::fs::canonicalize(path).unwrap(); + let maps = std::fs::read_to_string("/proc/self/maps").unwrap_or_default(); + let mapped = maps + .lines() + .any(|l| l.ends_with(&format!(" {}", path.display()))); + COMMITS.with(|c| { + let (n, m) = c.get(); + c.set((n + 1, m + usize::from(mapped))); + }); + } + + /// No edit writes the file while the reader's memory map of it (and so + /// a `&[u8]` over it) is alive. + #[test] + #[cfg(all(target_os = "linux", feature = "mmap"))] + fn file_is_not_mapped_while_an_edit_writes_it() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("mapped.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("x") + .with_i32_data(&[1, 2, 3, 4]) + .with_shape(&[4]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[2]) + .with_deflate(4); + b.write(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.write_values("x", &crate::Selection::All, &[5i32, 6, 7, 8]) + .unwrap(); + ed.resize("x", &[6]).unwrap(); + ed.set_attr("x", "a", &AttrValue::I64(1)).unwrap(); + drop(ed); + let (commits, mapped) = COMMITS.with(Cell::get); + assert_eq!((commits, mapped), (3, 0)); + let f = crate::File::open(&path).unwrap(); + assert_eq!( + f.dataset("x").unwrap().read_i32().unwrap(), + [5, 6, 7, 8, 0, 0] + ); + } +} From 0e8522cfad34fd12ae6109abf6f94951b470d62f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 14:45:51 -0500 Subject: [PATCH 7/7] clawhdf5-format: the writer skips optional filters that fail, as libhdf5 does FileBuilder stored every chunk of an LZF or Blosc dataset through the filter with filter mask 0. libhdf5 counts LZF and Blosc output no smaller than the chunk as a failure of the optional filter and stores the chunk raw with the filter's mask bit set. For an LZF chunk whose stream was exactly the chunk's size, the first libhdf5 rewrite stored raw data at the same size and kept our stale mask 0 in the index, and h5py could no longer read the dataset. precompress_chunks now runs chunks through compress_chunk_masked (as FileEditor does since f7e2ab1), sequentially and on the parallel path, and build_chunked_data_from_precompressed records each chunk's real mask in every index the writer builds: single chunk (layout field), Fixed Array and Extensible Array filtered elements, and version-2 B-tree type 11 records (create_datasets_parallel goes through the same path). The writer builds no version-1 B-tree or implicit index. PrecompressedChunks::chunks gains the mask. Files whose chunks all compress are byte-identical. Latent only in the unreleased LZF/Blosc writer (added 2026-09-26); no tagged release writes either filter. Tests: - plugin_filters_interop skipped_optional_filters_are_masked_as_libhdf5_masks_them: LZF, shuffle+LZF+fletcher32 and Blosc over random, compressible and alternating chunks in every index; masks equal an h5py-written twin's; h5py r+ rewrites and extends them; h5py, h5dump and our reader read every value. Before: 20 of 24 datasets had masks other than h5py's, and with that check disabled h5py failed to read the rewritten datasets ("filter returned failure during read"). - plugin_filters_interop files_whose_chunks_all_compress_are_unchanged: pins the pre-fix bytes of five all-compressing files. - chunked_write skipped_lzf_chunks_are_masked_in_every_index (fails before: mask 0, want 2). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 31 ++ crates/clawhdf5-format/src/chunked_write.rs | 123 ++++- .../clawhdf5/tests/plugin_filters_interop.rs | 454 ++++++++++++++++++ docs/known-issues.md | 17 + 4 files changed, 613 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f959f39..eea7a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1062,6 +1062,37 @@ and fails their objects (see below). - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- **`FileBuilder` stored LZF and Blosc chunks with filter mask 0 even when + the filter had not shrunk them** (fixed 2026-09-26). Latent in the + unreleased LZF/Blosc writer only (added 2026-09-26, "Plugin filters"): + no tagged release writes LZF or Blosc, so v2.7.0 and earlier are + unaffected. libhdf5 treats LZF and Blosc output no smaller than the chunk + as a filter failure and, both being optional filters, stores such a chunk + unfiltered with the filter's mask bit set. clawhdf5 stored the filter's + output with a clear mask. For an LZF chunk whose stream was exactly the + chunk's size (h5py stores `[182, 0, 0, 0, 0]` in a 5-byte chunk raw), + the first libhdf5 rewrite of that chunk stored the new data raw at the + same size and, the size being unchanged, kept the stale mask 0: h5py + then failed to read the dataset ("filter returned failure during read"). + The whole-file writer now runs chunks through the pipeline as libhdf5 + does (`clawhdf5_format::filters::compress_chunk_masked`, as `FileEditor` + already did) and records each chunk's real mask in every chunk index it + builds (single chunk, Fixed Array, Extensible Array, version-2 B-tree; + it builds no version-1 B-tree or implicit index), in the sequential and + `parallel` paths and `create_datasets_parallel`. Files whose chunks all + compress are byte-identical to before. `PrecompressedChunks::chunks` is + now `(raw size, stored bytes, filter mask)` (**breaking** for code that + reads it). Files written before the fix read correctly; rewrite them + (with this build or `h5repack`) before modifying them with libhdf5. + Tests: `plugin_filters_interop` + `skipped_optional_filters_are_masked_as_libhdf5_masks_them` (LZF, + shuffle+LZF+fletcher32 and Blosc, random, compressible and alternating + chunks, every index: masks equal an h5py-written twin's; after h5py r+ + rewrites and extends the datasets, h5py, h5dump and our reader read every + value — before the fix 20 of 24 datasets had other masks than h5py's, + and h5py could not read the rewritten `[x, 0, 0, 0, 0]` datasets) and + `files_whose_chunks_all_compress_are_unchanged`; `chunked_write` + `skipped_lzf_chunks_are_masked_in_every_index`. - **Scale-offset data read wrong values in every release that decoded it (v2.2.0 to v2.7.0), silently, on ordinary h5py files** (fixed 2026-09-26). Of 1480 scale-offset datasets h5py writes across every diff --git a/crates/clawhdf5-format/src/chunked_write.rs b/crates/clawhdf5-format/src/chunked_write.rs index 2a8bd09..b93142d 100644 --- a/crates/clawhdf5-format/src/chunked_write.rs +++ b/crates/clawhdf5-format/src/chunked_write.rs @@ -17,7 +17,7 @@ use crate::filter_pipeline::{ FILTER_LZF, FILTER_PCODEC, FILTER_PCODEC_NAME, FILTER_SHUFFLE, FILTER_ZSTD, FilterDescription, FilterPipeline, }; -use crate::filters::compress_chunk; +use crate::filters::compress_chunk_masked; /// Round a file offset up to the next cache-line boundary. /// /// This ensures chunk data starts at an address that is a multiple of the @@ -489,7 +489,12 @@ pub fn split_into_chunks( #[cfg(feature = "parallel")] const PARALLEL_COMPRESS_THRESHOLD: usize = 2; -/// Compress all chunks, using parallel compression when beneficial. +/// Compress all chunks, using parallel compression when beneficial, and +/// return each chunk's stored bytes with its filter mask. +/// +/// Chunks run through the pipeline as libhdf5 runs them +/// ([`compress_chunk_masked`]): an optional filter that fails — LZF or Blosc +/// output no smaller than its input — is skipped and its mask bit set. /// /// With the `parallel` feature and more than [`PARALLEL_COMPRESS_THRESHOLD`] /// filtered chunks, compression runs across rayon threads; otherwise it is @@ -499,7 +504,7 @@ fn compress_all_chunks( chunks: &[(Vec, Vec)], pipeline: &Option, element_size: u32, -) -> Result>, FormatError> { +) -> Result, u32)>, FormatError> { #[cfg(feature = "parallel")] { if let Some(pl) = pipeline @@ -508,7 +513,7 @@ fn compress_all_chunks( use rayon::prelude::*; return chunks .par_iter() - .map(|(_offsets, chunk_bytes)| compress_chunk(chunk_bytes, pl, element_size)) + .map(|(_offsets, chunk_bytes)| compress_chunk_masked(chunk_bytes, pl, element_size)) .collect(); } } @@ -518,9 +523,9 @@ fn compress_all_chunks( .iter() .map(|(_offsets, chunk_bytes)| { if let Some(pl) = pipeline { - compress_chunk(chunk_bytes, pl, element_size) + compress_chunk_masked(chunk_bytes, pl, element_size) } else { - Ok(chunk_bytes.clone()) + Ok((chunk_bytes.clone(), 0)) } }) .collect() @@ -798,8 +803,10 @@ pub fn build_fixed_array_at( /// writer passes eliminates the double-compression that the two-pass layout /// algorithm previously performed. pub struct PrecompressedChunks { - /// Per-chunk: (raw_size_bytes, compressed_bytes). - pub chunks: Vec<(u64, Vec)>, + /// Per-chunk: (raw_size_bytes, stored_bytes, filter_mask). Bit `i` of + /// the mask is set when filter `i` was skipped (an optional filter that + /// failed); 0 for every chunk of an unfiltered dataset. + pub chunks: Vec<(u64, Vec, u32)>, pub has_filters: bool, pub element_size: usize, pub shape: Vec, @@ -834,7 +841,7 @@ pub fn precompress_chunks( let chunks = raw_chunks .into_iter() .zip(compressed) - .map(|((_offsets, raw_bytes), c)| (raw_bytes.len() as u64, c)) + .map(|((_offsets, raw_bytes), (c, mask))| (raw_bytes.len() as u64, c, mask)) .collect(); Ok(PrecompressedChunks { @@ -867,7 +874,7 @@ pub fn build_chunked_data_from_precompressed( let mut data_buf = Vec::new(); let mut written_chunks = Vec::with_capacity(num_chunks); - for (raw_size, compressed) in &pre.chunks { + for (raw_size, compressed, filter_mask) in &pre.chunks { let aligned_offset = align_to_cache_line(data_buf.len()); if aligned_offset > data_buf.len() { data_buf.resize(aligned_offset, 0u8); @@ -879,7 +886,7 @@ pub fn build_chunked_data_from_precompressed( address, compressed_size, raw_size: *raw_size, - filter_mask: 0, + filter_mask: *filter_mask, }); } @@ -916,7 +923,7 @@ pub fn build_chunked_data_from_precompressed( } else { None }; - let filter_mask = if pre.has_filters { Some(0u32) } else { None }; + let filter_mask = pre.has_filters.then_some(written_chunks[0].filter_mask); serialize_v4_single_chunk( &chunk_dims_u32, chunk_addr, @@ -1943,6 +1950,98 @@ mod tests { bytes_to_f64(&output) } + /// Every chunk index the writer builds records each chunk's real filter + /// mask: LZF output no smaller than the chunk is skipped (bit 1, behind + /// shuffle) and the chunk stored shuffled only; compressible chunks keep + /// mask 0. The data reads back through both kinds of chunk. + #[cfg(feature = "lzf")] + #[test] + fn skipped_lzf_chunks_are_masked_in_every_index() { + let c = 64usize; + // Chunks alternate: random bytes (LZF cannot shrink them), then 7s. + let mut state = 0x1234_5678_u64; + let data: Vec = (0..4 * c) + .map(|i| { + if (i / c).is_multiple_of(2) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + f64::from_bits(state) + } else { + 7.0 + } + }) + .collect(); + let raw = f64_to_bytes(&data); + let options = ChunkOptions { + plugin: Some(PluginFilter::Lzf), + ..Default::default() + }; + let c64 = c as u64; + #[allow(clippy::type_complexity)] + let cases: [(&[u64], &[u64], Option<&[u64]>, u8, &[u32]); 4] = [ + (&[c64], &[c64], None, 1, &[2]), + (&[4 * c64], &[c64], None, 3, &[2, 0, 2, 0]), + (&[4 * c64], &[c64], Some(&[u64::MAX]), 4, &[2, 0, 2, 0]), + ( + &[2, 2 * c64], + &[1, c64], + Some(&[u64::MAX, u64::MAX]), + 5, + &[2, 0, 2, 0], + ), + ]; + let base = 0x1000u64; + for (shape, chunks, maxshape, index_type, want_masks) in cases { + let n: u64 = shape.iter().product(); + let raw = &raw[..n as usize * 8]; + let result = + build_chunked_data_at_ext(raw, shape, chunks, 8, &options, base, maxshape).unwrap(); + let mut file = vec![0u8; base as usize]; + file.extend_from_slice(&result.data_bytes); + let layout = DataLayout::parse(&result.layout_message, 8, 8).unwrap(); + assert!( + matches!(&layout, DataLayout::Chunked { chunk_index_type, .. } + if *chunk_index_type == Some(index_type)), + "{layout:?}" + ); + let dataspace = Dataspace { + space_type: DataspaceType::Simple, + rank: shape.len() as u8, + dimensions: shape.to_vec(), + max_dimensions: maxshape.map(<[u64]>::to_vec), + }; + let (mut infos, _) = + crate::chunked_read::list_chunks(&file, &layout, &dataspace, 8, 8, 8).unwrap(); + infos.sort_by(|a, b| a.offsets.cmp(&b.offsets)); + let masks: Vec = infos.iter().map(|i| i.filter_mask).collect(); + assert_eq!(masks, want_masks, "index type {index_type}"); + for info in &infos { + // Skipped chunks are stored at the chunk's size (shuffled). + assert_eq!( + info.chunk_size == (c * 8) as u32, + info.filter_mask != 0, + "{info:?}" + ); + } + let pipeline = crate::filter_pipeline::FilterPipeline::parse( + result.pipeline_message.as_ref().unwrap(), + ) + .unwrap(); + let out = read_chunked_data( + &file, + &layout, + &dataspace, + &make_f64_type(), + Some(&pipeline), + 8, + 8, + ) + .unwrap(); + assert_eq!(out, raw, "index type {index_type}"); + } + } + #[test] fn ea_roundtrip_1d_inline_only() { let values: Vec = (0..10).map(|i| i as f64).collect(); diff --git a/crates/clawhdf5/tests/plugin_filters_interop.rs b/crates/clawhdf5/tests/plugin_filters_interop.rs index 00911c6..ffe2fb1 100644 --- a/crates/clawhdf5/tests/plugin_filters_interop.rs +++ b/crates/clawhdf5/tests/plugin_filters_interop.rs @@ -586,3 +586,457 @@ with h5py.File(sys.argv[1], 'w') as f: let want: Vec = (0..16).collect(); assert_eq!(file.dataset("lzf_ok").unwrap().read_i32().unwrap(), want); } + +/// A family of datasets for the filter-mask tests: element type, chunk +/// length along the last dimension, the h5py `create_dataset` keywords of +/// the same filters, and how chunk `k` is filled. +#[cfg(feature = "lzf")] +struct MaskFamily { + name: &'static str, + /// 1 (`u1`) or 4 (` Vec<(&'static str, Vec, Vec, Option>)> { + vec![ + ("single", vec![c], vec![c], None), + ("fixed", vec![4 * c], vec![c], None), + ("ea", vec![4 * c], vec![c], Some(vec![u64::MAX])), + ( + "bt2", + vec![2, 2 * c], + vec![1, c], + Some(vec![u64::MAX, u64::MAX]), + ), + ] +} + +/// Raw little-endian bytes of the dataset `fam` fills over `shape`. +#[cfg(feature = "lzf")] +fn mask_data(fam: &MaskFamily, shape: &[u64], seed: u64) -> Vec { + let c = fam.chunk as usize; + let cols = *shape.last().unwrap() as usize; + let n: usize = shape.iter().product::() as usize; + let chunks_per_row = cols.div_ceil(c); + let mut state = seed; + let mut noise = move || { + state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + }; + let mut out = Vec::with_capacity(n * fam.elem); + for i in 0..n { + let (row, col) = (i / cols, i % cols); + let k = row * chunks_per_row + col / c; + let v: u64 = match fam.fill { + MaskFill::FiveBytes if col % c == 0 => 182 - k as u64, + MaskFill::FiveBytes => 0, + MaskFill::Alternating if k.is_multiple_of(2) => noise(), + MaskFill::Alternating | MaskFill::Compressible => 7, + }; + out.extend_from_slice(&v.to_le_bytes()[..fam.elem]); + } + out +} + +#[cfg(feature = "lzf")] +fn mask_families() -> Vec { + #[cfg_attr(not(feature = "blosc"), allow(unused_mut))] + let mut v = vec![ + MaskFamily { + name: "lzf5", + elem: 1, + chunk: 5, + h5py_kw: "dict(compression='lzf')", + build: |d| { + d.with_lzf().without_shuffle(); + }, + fill: MaskFill::FiveBytes, + }, + MaskFamily { + name: "lzf", + elem: 4, + chunk: 64, + h5py_kw: "dict(compression='lzf')", + build: |d| { + d.with_lzf().without_shuffle(); + }, + fill: MaskFill::Alternating, + }, + MaskFamily { + name: "mix", + elem: 4, + chunk: 8, + h5py_kw: "dict(compression='lzf', shuffle=True, fletcher32=True)", + build: |d| { + d.with_lzf().with_shuffle().with_fletcher32(); + }, + fill: MaskFill::Alternating, + }, + MaskFamily { + name: "lzfc", + elem: 4, + chunk: 64, + h5py_kw: "dict(compression='lzf')", + build: |d| { + d.with_lzf().without_shuffle(); + }, + fill: MaskFill::Compressible, + }, + ]; + #[cfg(feature = "blosc")] + { + use clawhdf5_format::chunked_write::{BloscCodec, BloscShuffle}; + v.push(MaskFamily { + name: "blosc", + elem: 4, + chunk: 64, + h5py_kw: "hdf5plugin.Blosc(cname='lz4', clevel=5, shuffle=hdf5plugin.Blosc.SHUFFLE)", + build: |d| { + d.with_blosc(BloscCodec::Lz4, 5, BloscShuffle::Byte); + }, + fill: MaskFill::Alternating, + }); + v.push(MaskFamily { + name: "blosc0", + elem: 4, + chunk: 64, + h5py_kw: "hdf5plugin.Blosc(cname='lz4', clevel=0, shuffle=hdf5plugin.Blosc.SHUFFLE)", + build: |d| { + d.with_blosc(BloscCodec::Lz4, 0, BloscShuffle::Byte); + }, + fill: MaskFill::Compressible, + }); + } + v +} + +/// Builds h5py twins of our datasets and prints, per dataset, the filter +/// masks by chunk offset in our file and in the twin. +const MASK_TWIN: &str = r#" +import sys, numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +ours, twin, spec = sys.argv[1], sys.argv[2], eval(sys.argv[3]) +def masks(ds): + return sorted((tuple(ds.id.get_chunk_info(k).chunk_offset), ds.id.get_chunk_info(k).filter_mask) + for k in range(ds.id.get_num_chunks())) +with h5py.File(ours, 'r') as o, h5py.File(twin, 'w', libver='v114') as t: + for name, dt, shape, chunks, maxshape, kw, raw in spec: + want = np.fromfile(raw, dtype=dt).reshape(shape) + assert np.array_equal(o[name][()], want), name + t.create_dataset(name, data=want, chunks=chunks, maxshape=maxshape, **eval(kw)) + print(name, masks(o[name]), '|', masks(t[name])) +"#; + +/// h5py (libhdf5) rewrites every chunk of our datasets — random chunks +/// become compressible and the other way round; the `[x,0,0,0,0]` chunks +/// change in place at the same size — then extends the resizable ones with +/// random data, and saves what each dataset must now hold. Prints the +/// datasets h5dump can decode (no chunk left LZF-encoded: h5dump has no +/// LZF filter). +const MASK_REWRITE: &str = r#" +import sys, numpy as np, h5py +try: + import hdf5plugin +except ImportError: + hdf5plugin = None +ours, spec = sys.argv[1], eval(sys.argv[2]) +rng = np.random.default_rng(3) +def noise(shape, dt): + return rng.integers(0, 256, int(np.prod(shape)) * np.dtype(dt).itemsize, + dtype=np.uint8).view(dt).reshape(shape) +dumpable = [] +with h5py.File(ours, 'r+') as f: + for name, dt, shape, chunks, maxshape, kw, raw in spec: + d = f[name] + want = d[()] + for s in d.iter_chunks(): + blk = want[s] + if dt == 'u1': + blk.flat[-1] = 1 + elif (blk == blk.flat[0]).all(): + blk[...] = noise(blk.shape, dt) + else: + blk[...] = 5 + d[...] = want + if maxshape is not None: + new = tuple(n + c for n, c in zip(shape, chunks)) + grown = noise(new, dt) + grown[tuple(slice(0, n) for n in shape)] = want + d.resize(new) + d[...] = grown + want = grown + want.tofile(raw + '.want') + pl = d.id.get_create_plist() + ids = [pl.get_filter(i)[0] for i in range(pl.get_nfilters())] + if 32000 in ids: + bit = 1 << ids.index(32000) + if not all(d.id.get_chunk_info(k).filter_mask & bit for k in range(d.id.get_num_chunks())): + continue + dumpable.append(name) +with h5py.File(ours, 'r') as f: + for name, dt, shape, chunks, maxshape, kw, raw in spec: + want = np.fromfile(raw + '.want', dtype=dt).reshape(f[name].shape) + assert np.array_equal(f[name][()], want), name +print(' '.join(dumpable)) +"#; + +/// Optional filters that fail are skipped in files `FileBuilder` writes, +/// exactly as libhdf5 skips them: an LZF or Blosc output no smaller than the +/// chunk leaves the chunk stored unfiltered with the filter's mask bit set. +/// The writer used to store every chunk filtered with mask 0. For LZF, a +/// chunk whose LZF stream is exactly the chunk's size (`[x,0,0,0,0]`) was +/// then corrupted by the first libhdf5 rewrite of it: libhdf5 stores the +/// new data raw at the same size and, the size being unchanged, leaves the +/// stale mask in the index, so h5py could no longer read the dataset. +/// +/// For every family × every chunk index the writer builds (single chunk, +/// Fixed Array, Extensible Array, version-2 B-tree): our masks equal those +/// of an h5py-written twin of the same data; then h5py r+ rewrites and +/// extends the datasets, and h5py, h5dump (where it has the filter) and our +/// reader read every value. +#[cfg(feature = "lzf")] +#[test] +fn skipped_optional_filters_are_masked_as_libhdf5_masks_them() { + let modules = if cfg!(feature = "blosc") { + "h5py, numpy, hdf5plugin" + } else { + "h5py, numpy" + }; + if !have_python(modules) { + return; + } + let dir = tempfile::tempdir().unwrap(); + let ours = dir.path().join("ours.h5"); + let twin = dir.path().join("twin.h5"); + let mut fb = clawhdf5::FileBuilder::new(); + let mut spec = Vec::new(); + let mut names = Vec::new(); + for (fi, fam) in mask_families().iter().enumerate() { + for (label, shape, chunks, maxshape) in mask_layouts(fam.chunk) { + let name = format!("{}_{label}", fam.name); + let data = mask_data(fam, &shape, fi as u64 * 31 + shape.len() as u64); + let raw = dir.path().join(format!("{name}.raw")); + std::fs::write(&raw, &data).unwrap(); + let ds = fb.create_dataset(&name); + if fam.elem == 1 { + ds.with_u8_data(&data); + } else { + let v: Vec = data + .as_chunks::<4>() + .0 + .iter() + .map(|&b| i32::from_le_bytes(b)) + .collect(); + ds.with_i32_data(&v); + } + ds.with_shape(&shape).with_chunks(&chunks); + if let Some(ms) = &maxshape { + ds.with_maxshape(ms); + } + (fam.build)(ds); + let py_tuple = |v: &[u64]| { + let items: Vec = v + .iter() + .map(|&d| { + if d == u64::MAX { + "None".into() + } else { + d.to_string() + } + }) + .collect(); + format!("({},)", items.join(",")) + }; + spec.push(format!( + "({name:?}, {:?}, {}, {}, {}, {:?}, {:?})", + if fam.elem == 1 { "u1" } else { "= 12, + "too few datasets with skipped filters:\n{out}" + ); + + // libhdf5 rewrites and extends them; everyone reads the new values. + let dumpable = run_python(MASK_REWRITE, &[ours.to_str().unwrap(), &spec]); + let plugin_path = run_python("import hdf5plugin; print(hdf5plugin.PLUGIN_PATH)", &[]); + let file = File::open(&ours).unwrap(); + for (name, elem) in &names { + let want = std::fs::read(dir.path().join(format!("{name}.raw.want"))).unwrap(); + let got = file + .dataset(name) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + assert!(got == want, "{name}: our reader after h5py r+"); + if !dumpable.split(' ').any(|d| d == name) || Command::new("h5dump").output().is_err() { + continue; + } + let o = Command::new("h5dump") + .env("HDF5_PLUGIN_PATH", &plugin_path) + .args(["-d", name, "-y", "-w", "0", ours.to_str().unwrap()]) + .output() + .unwrap(); + assert!( + o.status.success(), + "h5dump -d {name}: {}", + String::from_utf8_lossy(&o.stderr) + ); + let s = String::from_utf8_lossy(&o.stdout).into_owned(); + let vals: Vec = s + .split_once("DATA {") + .and_then(|(_, r)| r.split_once('}')) + .map(|(d, _)| { + d.split(|c: char| c == ',' || c.is_whitespace()) + .filter(|t| !t.is_empty()) + .map(|t| t.parse::().unwrap()) + .collect() + }) + .unwrap_or_default(); + let want_vals: Vec = want + .chunks_exact(*elem) + .map(|b| { + if *elem == 1 { + i64::from(b[0]) + } else { + i64::from(i32::from_le_bytes(b.try_into().unwrap())) + } + }) + .collect(); + assert_eq!(vals, want_vals, "h5dump -d {name}"); + } + assert!( + dumpable.split(' ').any(|d| d.starts_with("lzf5_")), + "{dumpable}" + ); +} + +/// Files whose chunks all compress are written exactly as before optional +/// filters could be skipped: every mask is 0 and nothing else changed. The +/// hashes are of the files the writer produced before that change. +#[cfg(feature = "lzf")] +#[test] +fn files_whose_chunks_all_compress_are_unchanged() { + use clawhdf5_format::checksum::jenkins_lookup3; + #[allow(clippy::type_complexity)] + #[cfg_attr(not(feature = "blosc"), allow(unused_mut))] + let mut cases: Vec<( + &str, + fn(&mut clawhdf5_format::type_builders::DatasetBuilder), + (usize, u32), + )> = vec![ + ( + "lzf_fixed", + |d| { + d.with_i32_data(&ramp_i32(4000)) + .with_chunks(&[500]) + .with_lzf(); + }, + (3965, 449169442), + ), + ( + "lzf_ea_noshuffle", + |d| { + d.with_i32_data(&ramp_i32(4000)) + .with_chunks(&[700]) + .with_maxshape(&[u64::MAX]) + .with_lzf() + .without_shuffle(); + }, + (7213, 4277403206), + ), + ( + "mix_bt2", + |d| { + d.with_f64_data(&ramp_f64(40 * 60)) + .with_shape(&[40, 60]) + .with_chunks(&[16, 16]) + .with_maxshape(&[u64::MAX, u64::MAX]) + .with_lzf() + .with_fletcher32(); + }, + (11495, 3340532700), + ), + ( + "lzf_single", + |d| { + d.with_u8_data(&ramp_u8(3000)) + .with_chunks(&[3000]) + .with_lzf(); + }, + (546, 690805477), + ), + ]; + #[cfg(feature = "blosc")] + cases.push(( + "blosc_fixed", + |d| { + use clawhdf5_format::chunked_write::{BloscCodec, BloscShuffle}; + d.with_i32_data(&ramp_i32(5000)) + .with_chunks(&[1024]) + .with_blosc(BloscCodec::Lz4, 5, BloscShuffle::Byte); + }, + (2776, 4278611376), + )); + for (name, build, want) in &cases { + let mut fb = clawhdf5::FileBuilder::new(); + build(fb.create_dataset("d")); + let bytes = fb.finish().unwrap(); + assert_eq!( + (bytes.len(), jenkins_lookup3(&bytes)), + *want, + "{name}: (length, lookup3 hash) of the file" + ); + } +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 818a487..44a5344 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,23 @@ deleting it. --- +## LZF/Blosc chunks written with a stale filter mask + +**Status:** fixed 2026-09-26, before any release (the LZF and Blosc writers +were added the same day; v2.7.0 and earlier write neither). + +`FileBuilder` stored every chunk of an LZF or Blosc dataset through the +filter with filter mask 0. libhdf5 counts LZF and Blosc output no smaller +than the chunk as a failure of the (optional) filter and stores the chunk +raw with the filter's mask bit set. When a chunk's LZF stream was exactly +the chunk's size, the first libhdf5 rewrite of it stored raw data at the +same size and left our mask 0 in the index, so h5py could no longer read +the dataset. `FileEditor` had the same bug, fixed earlier the same day. +Both now use `clawhdf5_format::filters::compress_chunk_masked`, and every +chunk index the writer builds records the real mask (see `CHANGELOG.md`). +Files written before the fix read correctly; rewrite them before letting +libhdf5 modify them. + ## In-place modification (`FileEditor`) limits **Status:** open (documented 2026-09-26). `clawhdf5::FileEditor` refuses,