diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index 3019daa..9d32e5c 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -296,7 +296,8 @@ impl EnumTypeBuilder { // ---- Attribute helper ---- -pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage { +/// The attribute message the writers store for `value` under `name`. +pub fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage { match value { AttrValue::F64(v) => AttributeMessage { name: name.to_string(), diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index e3b5619..4055b82 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -66,7 +66,9 @@ fn _panic_for_test() -> PyResult<()> { /// - I/O errors -> `PyIOError` /// - Format/parsing errors -> `PyValueError` /// - Missing dataset/path errors -> `PyKeyError` -/// - Other errors -> `PyOSError` +/// - Invalid arguments -> `PyValueError` +/// - Unsupported operations -> `PyNotImplementedError` +/// - Other errors (a locked file, ...) -> `PyOSError` pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { use clawhdf5_rs::Error; match &e { @@ -79,9 +81,14 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { | Error::ZeroCopyNotContiguous | Error::ZeroCopyNonNativeEndian | Error::ZeroCopyTypeMismatch { .. } - | Error::ZeroCopyUnaligned { .. } => { + | Error::ZeroCopyUnaligned { .. } + | Error::InvalidArgument(_) => { PyErr::new::(e.to_string()) } + Error::Unsupported(_) => { + PyErr::new::(e.to_string()) + } + _ => PyErr::new::(e.to_string()), } } diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs new file mode 100644 index 0000000..2df54d7 --- /dev/null +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -0,0 +1,1189 @@ +//! `clawhdf5::FileEditor` against libhdf5: files written by h5py (default +//! `libver`, `v114`, and HDF5 2.0's `latest`) and by clawhdf5 are modified +//! in place — values overwritten, datasets grown and appended to many +//! times (crossing Extensible Array super-block / data-block boundaries and +//! version-1 B-tree node splits), attributes added and replaced — and after +//! each round h5py must read exactly the expected values, h5dump must read +//! the file, `h5rs check --data` must find nothing, and our reader must +//! agree. At the end h5py opens the file `r+` and modifies it further. +//! +//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips +//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::{Command, Output}; + +use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn available(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the tools are here; false (skip) or a panic when interop is +/// required. +fn tools_ok() -> bool { + let ok = + available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]); + if !ok { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available" + ); + eprintln!("SKIP: h5py/numpy or h5dump not available"); + } + ok +} + +fn py(script: &str) -> String { + let o = Command::new(python()) + .args(["-c", script]) + .output() + .expect("run python"); + assert!( + o.status.success(), + "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8_lossy(&o.stdout).trim().to_string() +} + +fn text(o: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ) +} + +/// The file is valid for libhdf5's tools and for `h5rs check --data`. +/// HDF5 2.0 `latest` files are beyond h5dump 1.14. +fn check_tools(path: &Path, h5dump: bool) { + let p = path.to_str().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + if h5dump { + let o = Command::new("h5dump") + .args(["-o", "/dev/null", p]) + .output() + .unwrap(); + assert!( + o.status.success() && o.stderr.is_empty(), + "h5dump {p}:\n{}", + text(&o) + ); + } +} + +/// A dataset's expected contents. +#[derive(Clone, Debug)] +struct Model { + shape: Vec, + /// Row-major values. + data: Vec, +} + +impl Model { + fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self { + let n: u64 = shape.iter().product(); + Self { + shape: shape.to_vec(), + data: (0..n).map(f).collect(), + } + } + + fn index(&self, c: &[u64]) -> usize { + c.iter() + .zip(&self.shape) + .fold(0u64, |a, (&x, &d)| a * d + x) as usize + } + + /// Grow to `shape`, new elements `fill`. + fn resize(&mut self, shape: &[u64], fill: i32) { + let old = self.clone(); + *self = Self::new(shape, |_| fill); + let n: u64 = old.shape.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; old.shape.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % old.shape[d]; + r /= old.shape[d]; + } + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } + } + + /// Apply a hyperslab write of `vals` (row-major over the block). + fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) { + let n: u64 = count.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; count.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = start[d] + r % count[d]; + r /= count[d]; + } + let i = self.index(&c); + self.data[i] = vals[flat as usize]; + } + } +} + +fn block(start: &[u64], count: &[u64]) -> Selection { + Selection::Hyperslab { + start: start.to_vec(), + stride: vec![1; start.len()], + count: count.to_vec(), + block: vec![1; start.len()], + } +} + +/// h5py and our reader both read `m` from dataset `name`. +fn verify(path: &Path, name: &str, m: &Model) { + let f = File::open(path).unwrap(); + let ds = f.dataset(name).unwrap(); + assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}"); + assert!(ds.read_i32().unwrap() == m.data, "our values of {name}"); + let exp = path.with_extension("expect"); + let bytes: Vec = m.data.iter().flat_map(|v| v.to_le_bytes()).collect(); + std::fs::write(&exp, bytes).unwrap(); + let shape: Vec = m.shape.iter().map(|d| d.to_string()).collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + a = f[{name:?}][()]\n\ + e = np.fromfile({e:?}, dtype=' u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// `h5py.File(..., libver=...)` argument for each flavour tested; the last +/// is HDF5 2.0's own format, which h5dump 1.14 cannot read. +const LIBVERS: &[(&str, bool)] = &[("'earliest'", true), ("'v114'", true), ("'latest'", false)]; + +fn tmpdir() -> tempfile::TempDir { + let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + tempfile::TempDir::new_in(base).unwrap() +} + +/// Append to a 1-D unlimited dataset hundreds of times, in runs of random +/// length: enough chunks to fill an Extensible Array's index block, direct +/// data blocks and several super blocks (paged data blocks included), and +/// to split version-1 B-tree nodes several levels deep. +fn append_many(libver: &str, h5dump: bool, compression: &str, tag: &str) { + let dir = tmpdir(); + let path = dir.path().join(format!("append_{tag}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 f.create_dataset('x', shape=(5,), maxshape=(None,), chunks=(3,), dtype=' = (0..add).map(|k| (n + k) as i32 * 7 - 3).collect(); + ed.write_values("x", &block(&[n], &[add]), &vals).unwrap(); + m.write_block(&[n], &[add], &vals); + // Occasionally rewrite something earlier (in-place, or a filtered + // chunk that has to move). + if round % 7 == 0 { + let s = rng.below(m.shape[0]); + let c = 1 + rng.below((m.shape[0] - s).min(9)); + let vals: Vec = (0..c).map(|_| rng.next() as i32).collect(); + ed.write_values("x", &block(&[s], &[c]), &vals).unwrap(); + m.write_block(&[s], &[c], &vals); + } + if round % 150 == 149 { + drop(ed); + verify(&path, "x", &m); + check_tools(&path, h5dump); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "x", &m); + check_tools(&path, h5dump); + // h5py can go on appending to what we wrote. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 n = d.shape[0]\n\ + \x20 d.resize((n + 50,))\n\ + \x20 d[n:] = np.arange(50, dtype=' = (0..50).map(|k| 1000 + k).collect(); + m.write_block(&[n], &[50], &vals); + m.data[0] = -1; + verify(&path, "x", &m); + check_tools(&path, h5dump); +} + +#[test] +fn append_many_unfiltered() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + append_many(lv, *dump, "", &format!("plain{i}")); + } +} + +#[test] +fn append_many_gzip() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + append_many( + lv, + *dump, + ", compression='gzip', shuffle=True", + &format!("gzip{i}"), + ); + } +} + +/// Random operations — grow, hyperslab writes, point writes, attributes — +/// on a 2-D dataset with one unlimited dimension, checked against a model +/// after every few operations. +fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { + let dir = tmpdir(); + let path = dir.path().join(format!("rand_{tag}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \ + dtype=' = Vec::new(); + let mut rng = Rng(seed); + let mut ed = FileEditor::open(&path).unwrap(); + for step in 0..120 { + match rng.below(10) { + 0..=1 => { + let rows = m.shape[0] + 1 + rng.below(5); + ed.resize("m", &[rows, 7]).unwrap(); + m.resize(&[rows, 7], -9); + } + 2..=6 => { + let r0 = rng.below(m.shape[0]); + let c0 = rng.below(7); + let cnt = [ + 1 + rng.below((m.shape[0] - r0).min(6)), + 1 + rng.below(7 - c0), + ]; + let n = cnt[0] * cnt[1]; + let vals: Vec = (0..n).map(|_| (rng.next() % 100_000) as i32).collect(); + ed.write_values("m", &block(&[r0, c0], &cnt), &vals) + .unwrap(); + m.write_block(&[r0, c0], &cnt, &vals); + } + 7 => { + let pts: Vec> = (0..1 + rng.below(4)) + .map(|_| vec![rng.below(m.shape[0]), rng.below(7)]) + .collect(); + let vals: Vec = pts.iter().map(|_| rng.next() as i32).collect(); + ed.write_values("m", &Selection::Points(pts.clone()), &vals) + .unwrap(); + for (p, v) in pts.iter().zip(&vals) { + let i = m.index(p); + m.data[i] = *v; + } + } + _ => { + let k = rng.below(6); + let name = format!("a{k}"); + let v = rng.next() as i64; + match ed.set_attr("m", &name, &AttrValue::I64(v)) { + Ok(()) => { + attrs.retain(|(n, _)| *n != name); + attrs.push((name, v)); + } + Err(e) => panic!("set_attr {name}: {e}"), + } + } + } + if step % 30 == 29 { + drop(ed); + verify(&path, "m", &m); + check_tools(&path, h5dump); + check_attrs(&path, "m", &attrs); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "m", &m); + check_attrs(&path, "m", &attrs); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['m']\n\ + \x20 n = d.shape[0]\n\ + \x20 d.resize((n + 3, 7))\n\ + \x20 d[n:, :] = 42\n\ + \x20 d.attrs['from_h5py'] = 1.5\n", + p = path.to_str().unwrap() + )); + let n = m.shape[0]; + m.resize(&[n + 3, 7], -9); + m.write_block(&[n, 0], &[3, 7], &[42; 21]); + verify(&path, "m", &m); + check_tools(&path, h5dump); +} + +fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) { + let f = File::open(path).unwrap(); + let got = f.dataset(obj).unwrap().attrs().unwrap(); + for (n, v) in attrs { + match got.get(n) { + Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"), + other => panic!("attribute {n}: {other:?}"), + } + } + let want: Vec = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect(); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + want = {{{w}}}\n\ + got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\ + assert got == want, (got, want)\n", + p = path.to_str().unwrap(), + w = want.join(", ") + )); +} + +#[test] +fn random_operations_match_a_model() { + if !tools_ok() { + return; + } + let mut seed = 1; + for (i, (lv, dump)) in LIBVERS.iter().enumerate() { + // h5dump has no LZF decoder (h5py's own filter). + for (j, (extra, lzf)) in [ + ("", false), + (", compression='gzip', shuffle=True, fletcher32=True", false), + (", compression='lzf'", true), + ] + .iter() + .enumerate() + { + seed += 1; + random_ops(lv, *dump && !lzf, extra, &format!("{i}_{j}"), seed); + } + } +} + +fn unsupported(r: Result) { + match r { + Err(Error::Unsupported(_)) => {} + other => panic!("expected Error::Unsupported, got {other:?}"), + } +} + +/// The statistics an Extensible Array header keeps (super blocks and their +/// bytes, data blocks and their bytes, one past the highest index set, +/// elements realised), for the file's only Extensible Array. +fn ea_stats(path: &Path) -> [u64; 6] { + let b = std::fs::read(path).unwrap(); + let at = b + .windows(4) + .position(|w| w == b"EAHD") + .expect("an EA header"); + let mut s = [0u64; 6]; + for (k, v) in s.iter_mut().enumerate() { + let o = at + 12 + 8 * k; + *v = u64::from_le_bytes(b[o..o + 8].try_into().unwrap()); + } + s +} + +/// Version-1 B-tree nodes in the file: (leaves, internal nodes). +fn btree1_nodes(path: &Path) -> (usize, usize) { + let b = std::fs::read(path).unwrap(); + let mut out = (0, 0); + for i in 0..b.len().saturating_sub(6) { + if &b[i..i + 4] == b"TREE" && b[i + 4] == 1 { + if b[i + 5] == 0 { + out.0 += 1; + } else { + out.1 += 1; + } + } + } + out +} + +/// Growth one chunk at a time and in large steps: the editor's version-1 +/// B-tree must split the way libhdf5's does (the same number of leaves and +/// internal nodes for the same insertions). +#[test] +fn btree1_splits_match_libhdf5() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let mut steps: Vec = (1..=300).collect(); + steps.extend([1000, 1001, 5000, 9000]); + let a = dir.path().join("bt_h5py.h5"); + let b = dir.path().join("bt_edit.h5"); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 d = f.create_dataset('x', shape=(0,), maxshape=(None,), chunks=(2,), dtype=' = (n..s).map(|v| v as i32).collect(); + ed.write_values("x", &block(&[n], &[s - n]), &vals).unwrap(); + n = s; + } + drop(ed); + verify(&b, "x", &Model::new(&[n], |i| i as i32)); + check_tools(&b, true); + assert_eq!( + btree1_nodes(&b), + btree1_nodes(&a), + "B-tree shape differs from libhdf5's" + ); +} + +/// Enough chunks that the Extensible Array reaches its paged data blocks +/// (the first one holds element 131060 with h5py's parameters): the same +/// growth done by libhdf5 and by the editor must create the same blocks. +#[test] +fn extensible_array_paged_blocks_match_libhdf5() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let steps = [5u64, 131_000, 131_070, 133_000, 140_000]; + let a = dir.path().join("paged_h5py.h5"); + let b = dir.path().join("paged_edit.h5"); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v114') as f:\n\ + \x20 d = f.create_dataset('x', shape=(0,), maxshape=(None,), chunks=(1,), dtype=' = (n..s).map(|v| v as i32).collect(); + ed.write_values("x", &block(&[n], &[s - n]), &vals).unwrap(); + n = s; + } + drop(ed); + let m = Model::new(&[n], |i| i as i32); + verify(&b, "x", &m); + check_tools(&b, true); + assert_eq!( + ea_stats(&b), + ea_stats(&a), + "EA statistics differ from libhdf5's" + ); +} + +/// Every element of `sel` over `shape`, in selection order. +/// Every index of the box `ext`, row-major. +fn odometer(ext: &[u64]) -> Vec> { + let n: u64 = ext.iter().product(); + (0..n) + .map(|flat| { + let mut c = vec![0u64; ext.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % ext[d]; + r /= ext[d]; + } + c + }) + .collect() +} + +fn sel_coords(sel: &Selection, shape: &[u64]) -> Vec> { + match sel { + Selection::All => odometer(shape), + Selection::None => Vec::new(), + Selection::Points(p) => p.clone(), + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let ext: Vec = count.iter().zip(block).map(|(c, b)| c * b).collect(); + odometer(&ext) + .into_iter() + .map(|j| { + (0..j.len()) + .map(|d| start[d] + (j[d] / block[d]) * stride[d] + j[d] % block[d]) + .collect() + }) + .collect() + } + } +} + +impl Model { + fn write_sel(&mut self, sel: &Selection, vals: &[i32]) { + let cs = sel_coords(sel, &self.shape); + assert_eq!(cs.len(), vals.len()); + for (c, v) in cs.iter().zip(vals) { + let i = self.index(c); + self.data[i] = *v; + } + } +} + +/// A random selection of `shape`: a block, a strided hyperslab, points, +/// or everything. +fn random_sel(rng: &mut Rng, shape: &[u64]) -> Selection { + let rank = shape.len(); + match rng.below(4) { + 0 => Selection::All, + 1 => { + let pts = (0..1 + rng.below(5)) + .map(|_| shape.iter().map(|&d| rng.below(d)).collect()) + .collect(); + Selection::Points(pts) + } + _ => { + let mut start = vec![0; rank]; + let mut stride = vec![1; rank]; + let mut count = vec![1; rank]; + let mut block = vec![1; rank]; + for d in 0..rank { + start[d] = rng.below(shape[d]); + let room = shape[d] - start[d]; + block[d] = 1 + rng.below(room.min(3)); + stride[d] = block[d] + rng.below(3); + // count * stride may overshoot as long as the last block fits. + count[d] = 1 + (room - block[d]) / stride[d]; + count[d] = 1 + rng.below(count[d]); + } + Selection::Hyperslab { + start, + stride, + count, + block, + } + } + } +} + +/// Datasets of every layout and chunk index h5py writes, overwritten under +/// random selections: contiguous (allocated, and never written — late +/// allocation), compact, chunked with a Fixed Array (paged, too), single +/// chunk (unfiltered and filtered), implicit (early allocation), version-1 +/// B-tree (every chunked layout of the default libver), and version-2 +/// B-tree (existing unfiltered chunks only). +#[test] +fn overwrite_every_layout() { + if !tools_ok() { + return; + } + let names: &[(&str, &[u64])] = &[ + ("contig", &[10, 4]), + ("contig_late", &[6]), + ("compact", &[3, 4]), + ("fixed", &[10, 7]), + ("fixed_gz", &[10, 7]), + ("fa_paged", &[3000]), + ("single", &[5, 6]), + ("single_gz", &[5, 6]), + ("implicit", &[8, 8]), + ("fa_empty", &[10, 7]), + ("fa_gz_empty", &[3000]), + ("ea_gz_empty", &[20]), + ("bt2", &[6, 6]), + ]; + for (li, (lv, dump)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("overwrite_{li}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + from h5py import h5p, h5d, h5s, h5t\n\ + def ar(*s): return np.arange(int(np.prod(s)), dtype=' = vec![ + Model::new(&[10, 4], |i| i as i32), + Model::new(&[6], |_| 7), + Model::new(&[3, 4], |i| i as i32), + { + let mut m = Model::new(&[10, 7], |_| -1); + m.write_block(&[0, 0], &[3, 4], &[1; 12]); + m + }, + { + let mut m = Model::new(&[10, 7], |_| 0); + m.write_block(&[4, 3], &[6, 4], &(0..24).collect::>()); + m + }, + Model::new(&[3000], |i| if i % 7 == 0 { 3 } else { 0 }), + Model::new(&[5, 6], |_| 0), + Model::new(&[5, 6], |i| i as i32), + Model::new(&[8, 8], |_| 0), + Model::new(&[10, 7], |_| 4), + Model::new(&[3000], |_| 0), + Model::new(&[20], |_| 0), + Model::new(&[6, 6], |i| i as i32), + ]; + let mut rng = Rng(77 + li as u64); + for round in 0..3 { + let mut ed = FileEditor::open(&path).unwrap(); + for ((name, shape), m) in names.iter().zip(models.iter_mut()) { + for _ in 0..6 { + let sel = random_sel(&mut rng, shape); + let n = sel_coords(&sel, shape).len(); + let vals: Vec = (0..n).map(|_| rng.next() as i32).collect(); + ed.write_values(name, &sel, &vals) + .unwrap_or_else(|e| panic!("{name} round {round} {sel:?}: {e}")); + m.write_sel(&sel, &vals); + } + } + drop(ed); + for ((name, _), m) in names.iter().zip(&models) { + verify(&path, name, m); + } + check_tools(&path, *dump); + } + // A version-2 B-tree index can take new chunks only from libhdf5 for + // now: growing works, writing the new chunks is refused and changes + // nothing. + if *lv != "'earliest'" { + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("bt2", &[8, 6]).unwrap(); + let before = std::fs::read(&path).unwrap(); + unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12])); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); + models[12].resize(&[8, 6], 0); + } + // libhdf5 goes on modifying what we wrote. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for n in {names:?}:\n\ + \x20 d = f[n]\n\ + \x20 d[(0,) * d.ndim] = 123\n", + p = path.to_str().unwrap(), + names = names.iter().map(|(n, _)| *n).collect::>() + )); + for ((name, _), m) in names.iter().zip(models.iter_mut()) { + m.data[0] = 123; + verify(&path, name, m); + } + check_tools(&path, *dump); + } +} + +/// Attributes added to and replaced on the root group, a group and a +/// dataset, until the headers need continuation chunks; then h5py adds, +/// changes and deletes attributes in the same headers. +#[test] +fn attributes_in_place() { + if !tools_ok() { + return; + } + for (li, (lv, dump)) in LIBVERS.iter().enumerate() { + let dir = tmpdir(); + let path = dir.path().join(format!("attrs_{li}.h5")); + let p = path.to_str().unwrap(); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 f.attrs['title'] = 'root'\n\ + \x20 g = f.create_group('g')\n\ + \x20 g.attrs['n'] = 3\n\ + \x20 d = f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + let mut ed = FileEditor::open(&path).unwrap(); + // v2 headers hold 8 compact attributes by default. + let many = if *lv == "'earliest'" { 20 } else { 6 }; + for obj in ["/", "g", "d"] { + for i in 0..many { + let name = format!("x{i}"); + let v = AttrValue::I64Array((0..=i as i64).collect()); + ed.set_attr(obj, &name, &v).unwrap(); + want.push((obj, name, v)); + } + // Replace one with something much larger (moves it). + let big = AttrValue::String("z".repeat(700)); + ed.set_attr(obj, "x1", &big).unwrap(); + want.retain(|(o, n, _)| !(*o == obj && n == "x1")); + want.push((obj, "x1".into(), big)); + // And one smaller, in place. + ed.set_attr(obj, "x0", &AttrValue::F64(2.5)).unwrap(); + want.retain(|(o, n, _)| !(*o == obj && n == "x0")); + want.push((obj, "x0".into(), AttrValue::F64(2.5))); + } + ed.set_attr("d", "units", &AttrValue::String("km".into())) + .unwrap(); + want.push(("d", "units".into(), AttrValue::String("km".into()))); + if *lv != "'earliest'" { + // Up to the compact limit (8) and no further; attributes in + // dense storage and tracked creation order are refused, and a + // refused edit writes nothing. + ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); + want.push(("g", "eighth".into(), AttrValue::I64(8))); + let before = std::fs::read(&path).unwrap(); + unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); + unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); + unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit changed the file" + ); + } + drop(ed); + check_tools(&path, *dump); + let f = File::open(&path).unwrap(); + let expect_py: Vec = want + .iter() + .map(|(o, n, v)| { + let got = if *o == "/" { + f.root().attrs().unwrap() + } else if *o == "d" { + f.dataset(o).unwrap().attrs().unwrap() + } else { + f.group(o).unwrap().attrs().unwrap() + }; + let g = got.get(n).unwrap_or_else(|| panic!("{o}/{n} missing")); + assert_eq!(format!("{g:?}"), format!("{v:?}"), "{o}/{n}"); + let pv = match v { + AttrValue::I64Array(a) => format!("{a:?}"), + AttrValue::String(s) => format!("{s:?}"), + AttrValue::F64(x) => format!("{x:?}"), + AttrValue::I64(x) => format!("{x}"), + other => panic!("{other:?}"), + }; + format!("({o:?}, {n:?}, {pv})") + }) + .collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + for o, n, v in [{w}]:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 assert a == v, (o, n, a, v)\n\ + assert f['d'][()].tolist() == [0, 1, 2, 3]\n\ + assert f['g'].attrs['n'] == 3 and f.attrs['title'] == 'root'\n", + w = expect_py.join(", ") + )); + // libhdf5 modifies the headers we changed. + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'd']:\n\ + \x20 f[o].attrs['from_h5py'] = 'yes'\n\ + \x20 f[o].attrs['x0'] = 9\n\ + \x20 del f[o].attrs['x1']\n" + )); + check_tools(&path, *dump); + let f = File::open(&path).unwrap(); + for attrs in [ + f.root().attrs().unwrap(), + f.dataset("d").unwrap().attrs().unwrap(), + ] { + assert!(matches!(attrs.get("from_h5py"), Some(AttrValue::String(s)) if s == "yes")); + assert!(matches!(attrs.get("x0"), Some(AttrValue::I64(9)))); + assert!(!attrs.contains_key("x1")); + assert!(attrs.contains_key("x2")); + } + } +} + +/// Files written by clawhdf5's own writer: appended to, overwritten and +/// given attributes, then read by h5py. +#[test] +fn edit_clawhdf5_written_files() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("ours.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("ext") + .with_i32_data(&(0..10).collect::>()) + .with_shape(&[10]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[4]) + .with_deflate(4); + b.create_dataset("plain").with_i32_data(&[1, 2, 3, 4, 5, 6]); + b.create_dataset("grid") + .with_i32_data(&(0..24).collect::>()) + .with_shape(&[4, 6]) + .with_maxshape(&[u64::MAX, 6]) + .with_chunks(&[2, 3]); + b.set_attr("version", AttrValue::I64(1)); + b.write(&path).unwrap(); + let mut ext = Model::new(&[10], |i| i as i32); + let mut plain = Model::new(&[6], |i| i as i32 + 1); + let mut grid = Model::new(&[4, 6], |i| i as i32); + let mut ed = FileEditor::open(&path).unwrap(); + let mut rng = Rng(9); + for _ in 0..200 { + let n = ext.shape[0]; + let add = 1 + rng.below(9); + ed.resize("ext", &[n + add]).unwrap(); + ext.resize(&[n + add], 0); + let vals: Vec = (0..add).map(|_| rng.next() as i32).collect(); + ed.write_values("ext", &block(&[n], &[add]), &vals).unwrap(); + ext.write_block(&[n], &[add], &vals); + } + for _ in 0..30 { + let rows = grid.shape[0] + rng.below(3); + ed.resize("grid", &[rows, 6]).unwrap(); + grid.resize(&[rows, 6], 0); + let sel = random_sel(&mut rng, &grid.shape.clone()); + let vals: Vec = (0..sel_coords(&sel, &grid.shape).len()) + .map(|_| rng.next() as i32) + .collect(); + ed.write_values("grid", &sel, &vals).unwrap(); + grid.write_sel(&sel, &vals); + } + ed.write_values( + "plain", + &Selection::Points(vec![vec![5], vec![0]]), + &[60, 10], + ) + .unwrap(); + plain.write_sel(&Selection::Points(vec![vec![5], vec![0]]), &[60, 10]); + ed.set_attr("/", "version", &AttrValue::I64(2)).unwrap(); + ed.set_attr("ext", "note", &AttrValue::String("appended".into())) + .unwrap(); + drop(ed); + verify(&path, "ext", &ext); + verify(&path, "plain", &plain); + verify(&path, "grid", &grid); + check_tools(&path, true); + py(&format!( + "import h5py\n\ + f = h5py.File({p:?}, 'r')\n\ + assert f.attrs['version'] == 2\n\ + assert f['ext'].attrs['note'] in ('appended', b'appended')\n", + p = path.to_str().unwrap() + )); +} + +/// One writer at a time: a second editor (or libhdf5 with file locking) +/// is refused until the first is dropped. +#[test] +fn editor_locks_the_file() { + let dir = tmpdir(); + let path = dir.path().join("lock.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("x").with_i32_data(&[1, 2, 3]); + b.write(&path).unwrap(); + let ed = FileEditor::open(&path).unwrap(); + assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); + if available(&python(), &["-c", "import h5py"]) { + let o = Command::new(python()) + .args([ + "-c", + &format!("import h5py; h5py.File({:?}, 'r+')", path.to_str().unwrap()), + ]) + .env_remove("HDF5_USE_FILE_LOCKING") + .output() + .unwrap(); + assert!(!o.status.success(), "h5py opened a locked file r+"); + } + drop(ed); + FileEditor::open(&path).unwrap(); +} + +/// Refused edits leave the file byte for byte as it was. +#[test] +fn refused_edits_change_nothing() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("refuse.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w') as f:\n\ + \x20 f.create_dataset('s', data=['a', 'bb'], dtype=h5py.string_dtype())\n\ + \x20 f.create_dataset('x', data=np.arange(6, dtype=' = (900..len).rev().collect(); + let mut rest: Vec = (0..900).collect(); + let mut rng = Rng(3); + for i in (1..rest.len()).rev() { + rest.swap(i, rng.below(i as u64 + 1) as usize); + } + order.extend(rest.iter().take(500)); + for (lv, tag) in [("'earliest'", "bt"), ("'v114'", "ea")] { + let dir = tmpdir(); + let a = dir.path().join(format!("ooo_{tag}_h5py.h5")); + let b = dir.path().join(format!("ooo_{tag}_edit.h5")); + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={lv}) as f:\n\ + \x20 d = f.create_dataset('x', shape=({len},), maxshape=(None,), chunks=(1,), dtype=' = (n..n + add) + .map(|i| ((i as f64 / 50.0).sin() * 1000.0).round() / 1000.0) + .collect(); + ed.write_values("x", &block(&[n], &[add]), &vals).unwrap(); + } + drop(ed); + let size = |p: &Path| std::fs::metadata(p).unwrap().len(); + let repacked = |p: &Path| { + let out = p.with_extension("repacked.h5"); + let _ = std::fs::remove_file(&out); + let o = Command::new("h5repack") + .args([p.to_str().unwrap(), out.to_str().unwrap()]) + .output() + .unwrap(); + assert!(o.status.success(), "{}", text(&o)); + size(&out) + }; + println!( + "chunks {chunk}{comp}, {rounds} appends of {add}: editor {} bytes \ + (repacked {}), libhdf5 {} bytes (repacked {})", + size(&b), + repacked(&b), + size(&a), + repacked(&a) + ); + } +} diff --git a/crates/clawhdf5/src/edit/btree1.rs b/crates/clawhdf5/src/edit/btree1.rs new file mode 100644 index 0000000..7289c3f --- /dev/null +++ b/crates/clawhdf5/src/edit/btree1.rs @@ -0,0 +1,403 @@ +//! Inserting into (and updating) a version-1 B-tree chunk index (node type +//! 1; layout versions 1-3), as libhdf5's `H5B_insert` does: +//! +//! - keys compare lexicographically over the chunk offsets *and* the +//! element-size coordinate (0 in a chunk's own key), so a node's final +//! ("right") key after an append is the last chunk's offsets with the +//! element-size coordinate set to the element size — the smallest key +//! greater than that chunk, which is what libhdf5 writes; +//! - a full node (2K children) splits before the insertion: the right-most +//! node of a level keeps 90% of its children, the left-most 10%, any other +//! half (libhdf5's default split ratios); siblings are relinked; +//! - a full root splits by moving its left half to a new node, so the root +//! keeps its address (the layout message never changes). +//! +//! Version-1 B-tree nodes carry no checksum. + +use std::cmp::Ordering; + +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Key { + pub(crate) size: u32, + pub(crate) mask: u32, + /// Offsets in every dimension, the element-size one last. + pub(crate) offs: Vec, +} + +fn cmp(a: &[u64], b: &[u64]) -> Ordering { + a.cmp(b) +} + +#[derive(Debug, Clone)] +struct Node { + addr: u64, + level: u8, + left: u64, + right: u64, + /// `children.len() + 1` keys. + keys: Vec, + children: Vec, +} + +pub(crate) struct BTree1 { + root: u64, + /// Children per node at most (2K). + two_k: usize, + ndims: usize, + elem_size: u64, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("chunk B-tree: {why}"), + )) +} + +enum Ins { + Done, + /// The node split; the new right sibling and its first key. + Split(Key, u64), +} + +impl BTree1 { + /// `k` is the file's chunk B-tree K (children per node are 2K); + /// `ndims` counts the element-size dimension. + pub(crate) fn new(root: u64, k: u16, ndims: usize, elem_size: u64) -> Result { + if k == 0 || ndims < 2 { + return Err(bad("bad parameters")); + } + Ok(Self { + root, + two_k: 2 * k as usize, + ndims, + elem_size, + }) + } + + fn key_size(&self) -> usize { + 8 + 8 * self.ndims + } + + fn node_size(&self, os: u8) -> usize { + let os = os as usize; + 8 + 2 * os + (self.two_k + 1) * self.key_size() + self.two_k * os + } + + fn read(&self, img: &Image<'_>, addr: u64) -> Result { + let os = img.os; + let osz = os as usize; + let d = img.read(addr, 8 + 2 * osz)?; + if &d[0..4] != b"TREE" || d[4] != 1 { + return Err(bad("not a chunk B-tree node")); + } + let level = d[5]; + let n = u16::from_le_bytes([d[6], d[7]]) as usize; + if n > self.two_k { + return Err(bad("node holds more children than 2K")); + } + let left = get_uint(&d[8..], os); + let right = get_uint(&d[8 + osz..], os); + let ks = self.key_size(); + let body = img.read(addr + 8 + 2 * osz as u64, (n + 1) * ks + n * osz)?; + let mut keys = Vec::with_capacity(n + 1); + let mut children = Vec::with_capacity(n); + let mut p = 0; + for i in 0..=n { + let k = &body[p..p + ks]; + keys.push(Key { + size: u32::from_le_bytes([k[0], k[1], k[2], k[3]]), + mask: u32::from_le_bytes([k[4], k[5], k[6], k[7]]), + offs: (0..self.ndims) + .map(|d| { + u64::from_le_bytes(k[8 + 8 * d..16 + 8 * d].try_into().unwrap_or([0; 8])) + }) + .collect(), + }); + p += ks; + if i < n { + children.push(get_uint(&body[p..], os)); + p += osz; + } + } + Ok(Node { + addr, + level, + left, + right, + keys, + children, + }) + } + + fn write(&self, img: &mut Image<'_>, node: &Node) -> Result<(), Error> { + let os = img.os; + let osz = os as usize; + let mut d = vec![0u8; self.node_size(os)]; + d[0..4].copy_from_slice(b"TREE"); + d[4] = 1; + d[5] = node.level; + d[6..8].copy_from_slice(&(node.children.len() as u16).to_le_bytes()); + put_uint(&mut d[8..], node.left, os); + put_uint(&mut d[8 + osz..], node.right, os); + let ks = self.key_size(); + let mut p = 8 + 2 * osz; + for (i, k) in node.keys.iter().enumerate() { + d[p..p + 4].copy_from_slice(&k.size.to_le_bytes()); + d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes()); + for (j, o) in k.offs.iter().enumerate() { + d[p + 8 + 8 * j..p + 16 + 8 * j].copy_from_slice(&o.to_le_bytes()); + } + p += ks; + if i < node.children.len() { + put_uint(&mut d[p..], node.children[i], os); + p += osz; + } + } + // Unused key/child slots stay zero, as libhdf5 leaves them. + img.write(node.addr, &d) + } + + /// Create a tree holding one chunk; returns it (its root is a new leaf). + pub(crate) fn create( + img: &mut Image<'_>, + k: u16, + ndims: usize, + elem_size: u64, + key: Key, + addr: u64, + ) -> Result { + let mut t = Self::new(0, k, ndims, elem_size)?; + let root = img.alloc(t.node_size(img.os) as u64)?; + t.root = root; + let right = t.right_key_after(&key); + let node = Node { + addr: root, + level: 0, + left: undef(img.os), + right: undef(img.os), + keys: vec![key, right], + children: vec![addr], + }; + t.write(img, &node)?; + Ok(t) + } + + pub(crate) fn root(&self) -> u64 { + self.root + } + + /// The smallest key above chunk `key`: its offsets with the element-size + /// coordinate one element in (what libhdf5 writes as a right key). + fn right_key_after(&self, key: &Key) -> Key { + let mut offs = key.offs.clone(); + if let Some(last) = offs.last_mut() { + *last = self.elem_size; + } + Key { + size: 0, + mask: 0, + offs, + } + } + + /// Insert chunk `key` at address `addr`, or update it when the tree + /// already has a chunk at those offsets. + pub(crate) fn insert(&mut self, img: &mut Image<'_>, key: Key, addr: u64) -> Result<(), Error> { + if key.offs.len() != self.ndims || key.offs[self.ndims - 1] != 0 { + return Err(bad("bad chunk key")); + } + let root = self.read(img, self.root)?; + if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? { + // The root split: move its (left) half to a new node so the root + // keeps its address, then make the root the parent of both. + let old = self.read(img, self.root)?; + let right = self.read(img, right_addr)?; + let new_left = img.alloc(self.node_size(img.os) as u64)?; + let mut moved = old.clone(); + moved.addr = new_left; + self.write(img, &moved)?; + let mut right = right; + right.left = new_left; + self.write(img, &right)?; + let first = old.keys[0].clone(); + let last = right + .keys + .last() + .cloned() + .ok_or_else(|| bad("empty node"))?; + let new_root = Node { + addr: self.root, + level: old.level + 1, + left: undef(img.os), + right: undef(img.os), + keys: vec![first, mid, last], + children: vec![new_left, right_addr], + }; + self.write(img, &new_root)?; + } + Ok(()) + } + + fn insert_at( + &self, + img: &mut Image<'_>, + mut node: Node, + key: &Key, + addr: u64, + depth: u8, + ) -> Result { + if depth == 0 { + return Err(bad("tree too deep")); + } + let n = node.children.len(); + if n == 0 { + return Err(bad("empty node")); + } + // The child whose range holds the key: the last i with + // keys[i] <= key (the first child when the key is below them all). + let mut i = node + .keys + .iter() + .take(n) + .rposition(|k| cmp(&k.offs, &key.offs) != Ordering::Greater) + .unwrap_or(0); + if node.level == 0 { + if node.keys[i].offs == key.offs { + node.keys[i].size = key.size; + node.keys[i].mask = key.mask; + node.children[i] = addr; + self.write(img, &node)?; + return Ok(Ins::Done); + } + // Insert after child i unless the key is below every child. + let pos = if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less { + 0 + } else { + i + 1 + }; + return self.add_child(img, node, pos, key.clone(), addr); + } + let child = self.read(img, node.children[i])?; + if child.level + 1 != node.level { + return Err(bad("inconsistent node levels")); + } + let ins = self.insert_at(img, child, key, addr, depth - 1)?; + let mut changed = false; + if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less && i == 0 { + node.keys[0] = key.clone(); + changed = true; + } + if cmp(&key.offs, &node.keys[n].offs) != Ordering::Less { + node.keys[n] = self.right_key_after(key); + changed = true; + } + match ins { + Ins::Done => { + if changed { + self.write(img, &node)?; + } + Ok(Ins::Done) + } + Ins::Split(mid, right) => { + i += 1; + self.add_child(img, node, i, mid, right) + } + } + } + + /// Insert child `addr` with left key `key` at position `pos` of `node` + /// (splitting it first when full), and write what changed. + fn add_child( + &self, + img: &mut Image<'_>, + mut node: Node, + pos: usize, + key: Key, + addr: u64, + ) -> Result { + let n = node.children.len(); + if n < self.two_k { + Self::insert_child(self, &mut node, pos, key, addr); + self.write(img, &node)?; + return Ok(Ins::Done); + } + // Split first (H5B__split): how many children stay left. + let undefined = undef(img.os); + let mut nleft = if node.right == undefined { + (self.two_k as f64 * 0.9) as usize + } else if node.left == undefined { + (self.two_k as f64 * 0.1) as usize + } else { + self.two_k / 2 + }; + if pos < nleft && nleft == self.two_k { + nleft -= 1; + } else if pos >= nleft && nleft == 0 { + nleft += 1; + } + let right_addr = img.alloc(self.node_size(img.os) as u64)?; + let mut right = Node { + addr: right_addr, + level: node.level, + left: node.addr, + right: node.right, + keys: node.keys[nleft..].to_vec(), + children: node.children[nleft..].to_vec(), + }; + if node.right != undefined { + let mut sib = self.read(img, node.right)?; + sib.left = right_addr; + self.write(img, &sib)?; + } + node.keys.truncate(nleft + 1); + node.children.truncate(nleft); + node.right = right_addr; + if pos <= nleft && !(pos == nleft && nleft < n && self.goes_right(&key, &right)) { + self.insert_child(&mut node, pos, key, addr); + } else { + self.insert_child(&mut right, pos - nleft, key, addr); + } + self.write(img, &node)?; + self.write(img, &right)?; + let mid = right.keys[0].clone(); + Ok(Ins::Split(mid, right_addr)) + } + + /// For an insertion exactly at the split point: whether the key belongs + /// to the right half (it is not below the right half's first key). + fn goes_right(&self, key: &Key, right: &Node) -> bool { + cmp(&key.offs, &right.keys[0].offs) != Ordering::Less + } + + fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) { + let n = node.children.len(); + if node.level == 0 { + // A leaf: the new chunk's key goes at `pos`. At the end, the + // node's right key moves up to stay above the new chunk. + if pos == n { + let right = self.right_key_after(&key); + let last = node.keys.len() - 1; + if cmp(&node.keys[last].offs, &right.offs) == Ordering::Less { + node.keys[last] = right; + } + node.keys.insert(n, key); + } else { + node.keys.insert(pos, key); + } + } else { + // An internal node: `key` is the new child's left key, taking + // position `pos` (the child's range starts there). + if pos == n { + // A child split off the last child: its right key is the + // parent's right key already. + node.keys.insert(n, key); + } else { + node.keys.insert(pos, key); + } + } + node.children.insert(pos, addr); + } +} diff --git a/crates/clawhdf5/src/edit/earray.rs b/crates/clawhdf5/src/edit/earray.rs new file mode 100644 index 0000000..49d9621 --- /dev/null +++ b/crates/clawhdf5/src/edit/earray.rs @@ -0,0 +1,512 @@ +//! Setting elements of an Extensible Array chunk index (layout v4, index +//! type 4), creating the index block, super blocks, data blocks and data +//! block pages the element needs, exactly as `H5EA__lookup_elmt` creates +//! them — including the header statistics libhdf5 keeps (blocks created, +//! their bytes, elements realised, one past the highest index set) and the +//! "block offset" each data block records. + +use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef}; +use crate::error::Error; + +/// A chunk index element. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Elem { + pub(crate) addr: u64, + pub(crate) size: u64, + pub(crate) mask: u32, +} + +/// Encode an element: the address, and for a filtered array the stored +/// size (in `elem_size - os - 4` bytes) and filter mask. `None` is the +/// fill element (undefined address, zero size and mask). +pub(crate) fn encode_elem( + e: Option, + filtered: bool, + elem_size: usize, + os: u8, +) -> Result, Error> { + let osz = os as usize; + let mut b = vec![0u8; if filtered { elem_size } else { osz }]; + let addr = e.map_or(undef(os), |e| e.addr); + put_uint(&mut b, addr, os); + if filtered { + let width = elem_size - osz - 4; + if let Some(e) = e { + if width < 8 && e.size >> (8 * width) != 0 { + return Err(Error::Unsupported(format!( + "filtered chunk of {} bytes does not fit the index's {width}-byte size field", + e.size + ))); + } + b[osz..osz + width].copy_from_slice(&e.size.to_le_bytes()[..width]); + b[osz + width..].copy_from_slice(&e.mask.to_le_bytes()); + } + } + Ok(b) +} + +/// The width libhdf5 gives the stored-size field of a filtered chunk index +/// element for chunks of `chunk_bytes` bytes (`H5D__earray_idx_create`, +/// `H5D__farray_idx_create`): one byte more than the nominal size needs — +/// except under layout message version 5 (HDF5 2.0's own format), which +/// always uses 8 bytes. +pub(crate) fn chunk_size_len(chunk_bytes: u64, layout_version: u8) -> usize { + if layout_version >= 5 { + return 8; + } + let log2 = if chunk_bytes <= 1 { + 0 + } else { + 63 - chunk_bytes.leading_zeros() + }; + (1 + ((log2 + 8) / 8) as usize).min(8) +} + +/// Creation parameters, in the layout message's order. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EaParams { + pub(crate) max_nelmts_bits: u8, + pub(crate) idx_blk_elmts: u8, + pub(crate) sup_blk_min_data_ptrs: u8, + pub(crate) data_blk_min_elmts: u8, + pub(crate) max_dblk_page_nelmts_bits: u8, +} + +#[derive(Debug, Clone, Copy)] +struct Level { + ndblks: u64, + dblk_nelmts: u64, + /// First element of the level, counted after the index block's own. + start_idx: u64, + /// Number of data blocks in the levels before this one. + start_dblk: u64, +} + +/// An open Extensible Array. +pub(crate) struct Ea { + hdr: u64, + filtered: bool, + elem_size: usize, + p: EaParams, + /// nsuper_blks, super_blk_size, ndata_blks, data_blk_size, + /// max_idx_set, nelmts. + stats: [u64; 6], + iblock: u64, + levels: Vec, + /// Levels whose data blocks the index block addresses directly. + direct_levels: usize, + ndblk_addrs: usize, + nsblk_addrs: usize, + dirty_hdr: bool, + /// Checksummed ranges changed by `set` (start -> checksum position), + /// recomputed once by `finish`. + dirty: std::collections::BTreeMap, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("Extensible Array: {why}"), + )) +} + +impl Ea { + fn layout(p: EaParams) -> Result<(Vec, usize, usize, usize), Error> { + let dmin = u64::from(p.data_blk_min_elmts); + if dmin == 0 || !dmin.is_power_of_two() || p.max_nelmts_bits > 64 { + return Err(bad("bad creation parameters")); + } + let nsblks = + 1 + (p.max_nelmts_bits as usize).saturating_sub(dmin.trailing_zeros() as usize); + let mut levels = Vec::with_capacity(nsblks); + let (mut start_idx, mut start_dblk) = (0u64, 0u64); + for u in 0..nsblks { + let ndblks = 1u64.checked_shl((u / 2) as u32).unwrap_or(u64::MAX); + let dblk_nelmts = dmin.checked_shl(u.div_ceil(2) as u32).unwrap_or(u64::MAX); + levels.push(Level { + ndblks, + dblk_nelmts, + start_idx, + start_dblk, + }); + start_idx = start_idx.saturating_add(ndblks.saturating_mul(dblk_nelmts)); + start_dblk = start_dblk.saturating_add(ndblks); + } + let ndblk_addrs = 2 * (p.sup_blk_min_data_ptrs as usize).saturating_sub(1); + let mut direct_levels = 0; + let mut n = 0u64; + while n < ndblk_addrs as u64 { + if direct_levels >= levels.len() { + return Err(bad("index block holds more data blocks than the array")); + } + n += levels[direct_levels].ndblks; + direct_levels += 1; + } + if n != ndblk_addrs as u64 { + return Err(bad("index block ends mid super block")); + } + Ok((levels, direct_levels, ndblk_addrs, nsblks - direct_levels)) + } + + fn arr_off_size(&self) -> usize { + (self.p.max_nelmts_bits as usize).div_ceil(8) + } + + fn page_nelmts(&self) -> u64 { + 1u64.checked_shl(u32::from(self.p.max_dblk_page_nelmts_bits)) + .unwrap_or(u64::MAX) + } + + fn slot_size(&self, os: u8) -> usize { + if self.filtered { + self.elem_size + } else { + os as usize + } + } + + /// Open the array whose header is at `hdr`. + pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result { + let os = img.os; + let ls = img.ls as usize; + let size = 12 + 6 * ls + os as usize + 4; + let d = img.read(hdr, size)?; + if &d[0..4] != b"EAHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let filtered = match d[5] { + 0 => false, + 1 => true, + _ => return Err(bad("unknown client")), + }; + let elem_size = d[6] as usize; + if filtered && elem_size < os as usize + 5 { + return Err(bad("element too small")); + } + let p = EaParams { + max_nelmts_bits: d[7], + idx_blk_elmts: d[8], + data_blk_min_elmts: d[9], + sup_blk_min_data_ptrs: d[10], + max_dblk_page_nelmts_bits: d[11], + }; + let mut stats = [0u64; 6]; + for (k, s) in stats.iter_mut().enumerate() { + *s = get_uint(&d[12 + k * ls..], img.ls); + } + let iblock = get_uint(&d[12 + 6 * ls..], os); + let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?; + Ok(Self { + hdr, + filtered, + elem_size, + p, + stats, + iblock, + levels, + direct_levels, + ndblk_addrs, + nsblk_addrs, + dirty_hdr: false, + dirty: Default::default(), + }) + } + + /// Create an empty array (header only; the index block comes with the + /// first element) and return it. + pub(crate) fn create( + img: &mut Image<'_>, + p: EaParams, + filtered: bool, + chunk_bytes: u64, + layout_version: u8, + ) -> Result { + let os = img.os; + let elem_size = if filtered { + os as usize + chunk_size_len(chunk_bytes, layout_version) + 4 + } else { + os as usize + }; + let size = 12 + 6 * img.ls as usize + os as usize + 4; + let hdr = img.alloc(size as u64)?; + let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?; + let mut ea = Self { + hdr, + filtered, + elem_size, + p, + stats: [0; 6], + iblock: undef(os), + levels, + direct_levels, + ndblk_addrs, + nsblk_addrs, + dirty_hdr: true, + dirty: Default::default(), + }; + ea.write_header(img)?; + Ok(ea) + } + + pub(crate) fn header_address(&self) -> u64 { + self.hdr + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + let ls = img.ls as usize; + let size = 12 + 6 * ls + os as usize + 4; + let mut d = vec![0u8; size]; + d[0..4].copy_from_slice(b"EAHD"); + d[4] = 0; + d[5] = u8::from(self.filtered); + d[6] = self.elem_size as u8; + d[7] = self.p.max_nelmts_bits; + d[8] = self.p.idx_blk_elmts; + d[9] = self.p.data_blk_min_elmts; + d[10] = self.p.sup_blk_min_data_ptrs; + d[11] = self.p.max_dblk_page_nelmts_bits; + for (k, s) in self.stats.iter().enumerate() { + put_uint(&mut d[12 + k * ls..], *s, img.ls); + } + put_uint(&mut d[12 + 6 * ls..], self.iblock, os); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]); + d[size - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(self.hdr, &d)?; + self.dirty_hdr = false; + Ok(()) + } + + /// Recompute the checksums of the blocks `set` changed; store changed + /// header statistics. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for (start, end) in std::mem::take(&mut self.dirty) { + rechecksum(img, start, end)?; + } + if self.dirty_hdr { + self.write_header(img)?; + } + Ok(()) + } + + fn fill_elems(&self, n: u64, os: u8) -> Result, Error> { + let one = encode_elem(None, self.filtered, self.elem_size, os)?; + let n = usize::try_from(n).map_err(|_| bad("block too large"))?; + Ok(one.repeat(n)) + } + + fn iblock_prefix(&self, os: u8) -> u64 { + 6 + u64::from(os) + } + + fn iblock_len(&self, os: u8) -> u64 { + let osz = os as u64; + self.iblock_prefix(os) + + u64::from(self.p.idx_blk_elmts) * self.slot_size(os) as u64 + + (self.ndblk_addrs + self.nsblk_addrs) as u64 * osz + } + + fn create_iblock(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + let len = self.iblock_len(os); + let addr = img.alloc(len + 4)?; + let mut d = Vec::with_capacity(len as usize + 4); + d.extend_from_slice(b"EAIB"); + d.push(0); + d.push(u8::from(self.filtered)); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.hdr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&self.fill_elems(u64::from(self.p.idx_blk_elmts), os)?); + let u = undef(os).to_le_bytes(); + for _ in 0..self.ndblk_addrs + self.nsblk_addrs { + d.extend_from_slice(&u[..os as usize]); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(addr, &d)?; + self.iblock = addr; + self.stats[5] += u64::from(self.p.idx_blk_elmts); + self.dirty_hdr = true; + Ok(()) + } + + fn block_prefix(&self, sig: &[u8; 4], off: u64, os: u8) -> Vec { + let mut d = Vec::new(); + d.extend_from_slice(sig); + d.push(0); + d.push(u8::from(self.filtered)); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.hdr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&off.to_le_bytes()[..self.arr_off_size()]); + d + } + + fn dblk_prefix_len(&self, os: u8) -> u64 { + 6 + u64::from(os) + self.arr_off_size() as u64 + } + + /// Create a data block of `nelmts` elements whose recorded block offset + /// is `off`; returns its address. + fn create_dblock(&mut self, img: &mut Image<'_>, nelmts: u64, off: u64) -> Result { + let os = img.os; + let es = self.slot_size(os) as u64; + let page = self.page_nelmts(); + let prefix = self.block_prefix(b"EADB", off, os); + let (size, body) = if nelmts > page { + // Paged: only the prefix (and its checksum) is written now; each + // page is written when an element in it is first set. + let npages = nelmts / page; + let size = prefix.len() as u64 + 4 + npages * (page * es + 4); + let mut d = prefix; + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + (size, d) + } else { + let mut d = prefix; + d.extend_from_slice(&self.fill_elems(nelmts, os)?); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + (d.len() as u64, d) + }; + let addr = img.alloc(size)?; + img.write(addr, &body)?; + self.stats[2] += 1; + self.stats[3] += size; + self.stats[5] += nelmts; + self.dirty_hdr = true; + Ok(addr) + } + + /// Set element `idx` to `e`. + pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + let os = img.os; + let osz = u64::from(os); + let es = self.slot_size(os) as u64; + let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + if self.iblock == undef(os) { + self.create_iblock(img)?; + } + let ib = self.iblock; + let ib_len = self.iblock_len(os); + let idx_blk = u64::from(self.p.idx_blk_elmts); + if idx < idx_blk { + img.write(ib + self.iblock_prefix(os) + idx * es, &enc)?; + self.dirty.insert(ib, ib + ib_len); + } else { + let rel = idx - idx_blk; + let u = self + .levels + .iter() + .position(|l| { + rel < l + .start_idx + .saturating_add(l.ndblks.saturating_mul(l.dblk_nelmts)) + }) + .ok_or_else(|| bad("index beyond the array's maximum"))?; + let l = self.levels[u]; + let dblks_at = ib + self.iblock_prefix(os) + idx_blk * es; + if u < self.direct_levels { + if l.dblk_nelmts > self.page_nelmts() { + return Err(Error::Unsupported( + "Extensible Array index block addressing a paged data block".into(), + )); + } + let local = (rel - l.start_idx) / l.dblk_nelmts; + let dblk_idx = l.start_dblk + local; + let slot = dblks_at + dblk_idx * osz; + let mut addr = get_uint(&img.read(slot, os as usize)?, os); + if addr == undef(os) { + // libhdf5 records start_idx + (global data block index) + // * nelmts here (H5EA__lookup_elmt), not the block's + // own first element; kept for byte-for-byte parity. + let off = l.start_idx + dblk_idx * l.dblk_nelmts; + addr = self.create_dblock(img, l.dblk_nelmts, off)?; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, addr, os); + img.write(slot, &a)?; + self.dirty.insert(ib, ib + ib_len); + } + let within = (rel - l.start_idx) % l.dblk_nelmts; + let at = addr + self.dblk_prefix_len(os) + within * es; + img.write(at, &enc)?; + self.dirty + .insert(addr, addr + self.dblk_prefix_len(os) + l.dblk_nelmts * es); + } else { + let s = (u - self.direct_levels) as u64; + let sslot = dblks_at + self.ndblk_addrs as u64 * osz + s * osz; + let page = self.page_nelmts(); + let npages = if l.dblk_nelmts > page { + l.dblk_nelmts / page + } else { + 0 + }; + let bitmap_len = npages.div_ceil(8) * l.ndblks; + let sb_prefix = self.dblk_prefix_len(os); + let sb_len = sb_prefix + bitmap_len + l.ndblks * osz; + let mut sb = get_uint(&img.read(sslot, os as usize)?, os); + if sb == undef(os) { + let mut d = self.block_prefix(b"EASB", l.start_idx, os); + d.resize(d.len() + bitmap_len as usize, 0); + let u8s = undef(os).to_le_bytes(); + for _ in 0..l.ndblks { + d.extend_from_slice(&u8s[..os as usize]); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + sb = img.alloc(d.len() as u64)?; + img.write(sb, &d)?; + self.stats[0] += 1; + self.stats[1] += d.len() as u64; + self.dirty_hdr = true; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, sb, os); + img.write(sslot, &a)?; + self.dirty.insert(ib, ib + ib_len); + } + let local = (rel - l.start_idx) / l.dblk_nelmts; + let dslot = sb + sb_prefix + bitmap_len + local * osz; + let mut addr = get_uint(&img.read(dslot, os as usize)?, os); + if addr == undef(os) { + let off = l.start_idx + local * l.dblk_nelmts; + addr = self.create_dblock(img, l.dblk_nelmts, off)?; + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, addr, os); + img.write(dslot, &a)?; + self.dirty.insert(sb, sb + sb_len); + } + let within = (rel - l.start_idx) % l.dblk_nelmts; + let dprefix = self.dblk_prefix_len(os); + if npages == 0 { + img.write(addr + dprefix + within * es, &enc)?; + self.dirty.insert(addr, addr + dprefix + l.dblk_nelmts * es); + } else { + let pg = within / page; + let page_at = addr + dprefix + 4 + pg * (page * es + 4); + let bit = local * npages + pg; + let bpos = sb + sb_prefix + bit / 8; + let mut byte = img.read(bpos, 1)?[0]; + let mask = 0x80u8 >> (bit % 8); + if byte & mask == 0 { + let fill = self.fill_elems(page, os)?; + img.write(page_at, &fill)?; + byte |= mask; + img.write(bpos, &[byte])?; + self.dirty.insert(sb, sb + sb_len); + } + img.write(page_at + (within % page) * es, &enc)?; + self.dirty.insert(page_at, page_at + page * es); + } + } + } + if idx + 1 > self.stats[4] { + self.stats[4] = idx + 1; + self.dirty_hdr = true; + } + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/farray.rs b/crates/clawhdf5/src/edit/farray.rs new file mode 100644 index 0000000..0a25fe7 --- /dev/null +++ b/crates/clawhdf5/src/edit/farray.rs @@ -0,0 +1,185 @@ +//! Setting elements of a Fixed Array chunk index (layout v4, index type 3), +//! creating the array (header and data block) when the dataset has none +//! yet, and a data block page when an element in it is first set. + +use crate::edit::earray::{Elem, chunk_size_len, encode_elem}; +use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef}; +use crate::error::Error; + +pub(crate) struct Fa { + filtered: bool, + elem_size: usize, + page_bits: u8, + nelmts: u64, + dblk: u64, + /// Checksummed ranges changed by `set`, recomputed by `finish`. + dirty: std::collections::BTreeMap, +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("Fixed Array: {why}"), + )) +} + +impl Fa { + fn slot(&self, os: u8) -> u64 { + if self.filtered { + self.elem_size as u64 + } else { + u64::from(os) + } + } + + fn page(&self) -> u64 { + 1u64.checked_shl(u32::from(self.page_bits)) + .unwrap_or(u64::MAX) + } + + /// Open the array whose header is at `hdr`. + pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result { + let os = img.os; + let size = 8 + img.ls as usize + os as usize + 4; + let d = img.read(hdr, size)?; + if &d[0..4] != b"FAHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let filtered = match d[5] { + 0 => false, + 1 => true, + _ => return Err(bad("unknown client")), + }; + let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let fa = Self { + filtered, + elem_size: d[6] as usize, + page_bits: d[7], + nelmts: get_uint(&d[8..], img.ls), + dblk: get_uint(&d[8 + img.ls as usize..], os), + dirty: Default::default(), + }; + if fa.filtered && fa.elem_size < os as usize + 5 { + return Err(bad("element too small")); + } + if fa.page_bits >= 64 || fa.dblk == undef(os) { + return Err(bad("bad header fields")); + } + Ok(fa) + } + + /// Create an array of `nelmts` fill elements; returns it and its + /// header address. + pub(crate) fn create( + img: &mut Image<'_>, + nelmts: u64, + page_bits: u8, + filtered: bool, + chunk_bytes: u64, + layout_version: u8, + ) -> Result<(Self, u64), Error> { + let os = img.os; + let osz = os as usize; + let elem_size = if filtered { + osz + chunk_size_len(chunk_bytes, layout_version) + 4 + } else { + osz + }; + let mut fa = Self { + filtered, + elem_size, + page_bits, + nelmts, + dblk: 0, + dirty: Default::default(), + }; + let hsize = 8 + img.ls as usize + osz + 4; + let hdr = img.alloc(hsize as u64)?; + // Data block. + let fill = encode_elem(None, filtered, elem_size, os)?; + let mut d = Vec::new(); + d.extend_from_slice(b"FADB"); + d.push(0); + d.push(u8::from(filtered)); + let mut a = vec![0u8; osz]; + put_uint(&mut a, hdr, os); + d.extend_from_slice(&a); + let page = fa.page(); + let n = usize::try_from(nelmts).map_err(|_| bad("too many elements"))?; + let total = if nelmts > page { + let npages = nelmts.div_ceil(page); + d.resize(d.len() + npages.div_ceil(8) as usize, 0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + // Pages are written when first used; their space is reserved. + d.len() as u64 + nelmts * fa.slot(os) + npages * 4 + } else { + d.extend_from_slice(&fill.repeat(n)); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + d.len() as u64 + }; + let dblk = img.alloc(total)?; + img.write(dblk, &d)?; + fa.dblk = dblk; + let mut h = vec![0u8; hsize]; + h[0..4].copy_from_slice(b"FAHD"); + h[5] = u8::from(filtered); + h[6] = elem_size as u8; + h[7] = page_bits; + put_uint(&mut h[8..], nelmts, img.ls); + put_uint(&mut h[8 + img.ls as usize..], dblk, os); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&h[..hsize - 4]); + h[hsize - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(hdr, &h)?; + Ok((fa, hdr)) + } + + /// Set element `idx` to `e`. + pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + let os = img.os; + if idx >= self.nelmts { + return Err(bad("index beyond the array")); + } + let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let es = self.slot(os); + let prefix = 6 + u64::from(os); + let page = self.page(); + if self.nelmts <= page { + img.write(self.dblk + prefix + idx * es, &enc)?; + self.dirty + .insert(self.dblk, self.dblk + prefix + self.nelmts * es); + return Ok(()); + } + let npages = self.nelmts.div_ceil(page); + let bitmap_len = npages.div_ceil(8); + let pages_at = self.dblk + prefix + bitmap_len + 4; + let p = idx / page; + let count = page.min(self.nelmts - p * page); + let page_at = pages_at + p * (page * es + 4); + let bpos = self.dblk + prefix + p / 8; + let mut byte = img.read(bpos, 1)?[0]; + let mask = 0x80u8 >> (p % 8); + if byte & mask == 0 { + let fill = encode_elem(None, self.filtered, self.elem_size, os)?; + img.write(page_at, &fill.repeat(count as usize))?; + byte |= mask; + img.write(bpos, &[byte])?; + self.dirty + .insert(self.dblk, self.dblk + prefix + bitmap_len); + } + img.write(page_at + (idx % page) * es, &enc)?; + self.dirty.insert(page_at, page_at + count * es); + Ok(()) + } + + /// Recompute the checksums of the blocks and pages `set` changed. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for (start, end) in std::mem::take(&mut self.dirty) { + rechecksum(img, start, end)?; + } + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs new file mode 100644 index 0000000..a100b66 --- /dev/null +++ b/crates/clawhdf5/src/edit/image.rs @@ -0,0 +1,317 @@ +//! The file as one edit sees it: the bytes on disk plus the edit's pending +//! writes, and an allocator that hands out space at the end of the file. +//! +//! An edit never writes to the file while it is being planned. Every change +//! is recorded here first (reads see them), so an edit that fails half-way — +//! a filter that cannot encode, a chunk index this code does not handle — +//! leaves the file exactly as it was. [`Image::commit`] then writes the +//! changes in an order that keeps the old metadata valid for as long as +//! possible (see there). + +use std::collections::BTreeMap; +use std::io::{Seek, SeekFrom, Write}; + +use crate::error::Error; + +/// Pending writes over the file's bytes, addressed as HDF5 addresses +/// (relative to the superblock). +pub(crate) struct Image<'a> { + /// The file from the superblock to its recorded end of allocation. + base: &'a [u8], + /// Pending writes: start address -> bytes. Never overlapping. + patches: BTreeMap>, + /// End of allocated space (grows with [`Self::alloc`]). + eoa: u64, + /// The end of allocated space when the edit started. + old_eoa: u64, + /// Width of addresses and lengths in the file. + pub(crate) os: u8, + pub(crate) ls: u8, +} + +impl<'a> Image<'a> { + pub(crate) fn new(base: &'a [u8], os: u8, ls: u8) -> Self { + let eoa = base.len() as u64; + Self { + base, + patches: BTreeMap::new(), + eoa, + old_eoa: eoa, + os, + ls, + } + } + + pub(crate) fn eoa(&self) -> u64 { + self.eoa + } + + pub(crate) fn old_eoa(&self) -> u64 { + self.old_eoa + } + + /// Whether the edit changes anything. + pub(crate) fn is_dirty(&self) -> bool { + !self.patches.is_empty() || self.eoa != self.old_eoa + } + + /// Allocate `size` bytes at the end of the file. The space reads as + /// zeros until written. Nothing is ever freed: space an edit stops + /// using (a relocated chunk, say) is leaked, as there is no free-space + /// manager. + pub(crate) fn alloc(&mut self, size: u64) -> Result { + let addr = self.eoa; + let end = addr + .checked_add(size) + .filter(|&e| self.os >= 8 || e < (1u64 << (8 * u32::from(self.os))) - 1) + .ok_or_else(|| Error::Unsupported("file would exceed its address size".into()))?; + self.eoa = end; + Ok(addr) + } + + /// If `[addr, addr + old_len)` is the last allocated space, grow it to + /// `new_len` bytes (a structure at the end of the file can grow where + /// it is) and return true. + pub(crate) fn grow_tail( + &mut self, + addr: u64, + old_len: u64, + new_len: u64, + ) -> Result { + if addr.checked_add(old_len) != Some(self.eoa) || new_len < old_len { + return Ok(false); + } + let old_end = self.eoa; + self.eoa = addr; + if let Err(e) = self.alloc(new_len) { + self.eoa = old_end; + return Err(e); + } + Ok(true) + } + + /// `len` bytes at `addr`, with the pending writes applied. + pub(crate) fn read(&self, addr: u64, len: usize) -> Result, Error> { + let end = addr + .checked_add(len as u64) + .filter(|&e| e <= self.eoa) + .ok_or_else(|| { + Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof { + expected: addr.saturating_add(len as u64) as usize, + available: self.eoa as usize, + }) + })?; + let mut out = vec![0u8; len]; + let base_len = self.base.len() as u64; + if addr < base_len { + let b_end = end.min(base_len); + out[..(b_end - addr) as usize] + .copy_from_slice(&self.base[addr as usize..b_end as usize]); + } + // Patches overlapping [addr, end): the last one starting before + // `end`, walking back while they still reach `addr`. + for (&p_start, bytes) in self.patches.range(..end).rev() { + let p_end = p_start + bytes.len() as u64; + if p_end <= addr { + break; + } + let lo = p_start.max(addr); + let hi = p_end.min(end); + out[(lo - addr) as usize..(hi - addr) as usize] + .copy_from_slice(&bytes[(lo - p_start) as usize..(hi - p_start) as usize]); + } + Ok(out) + } + + /// Record a write of `bytes` at `addr` (inside allocated space). + pub(crate) fn write(&mut self, addr: u64, bytes: &[u8]) -> Result<(), Error> { + if bytes.is_empty() { + return Ok(()); + } + let end = addr + .checked_add(bytes.len() as u64) + .filter(|&e| e <= self.eoa) + .ok_or_else(|| Error::Unsupported("write past the end of allocated space".into()))?; + // Fast path: inside, or extending, the one patch that starts at or + // before `addr` and reaches it (sequential writes into a block, and + // chunks allocated back to back, stay linear). + if let Some((&p_start, p)) = self.patches.range_mut(..=addr).next_back() + && p_start + p.len() as u64 >= addr + && self + .patches + .range(addr + 1..end.max(addr + 1)) + .next() + .is_none() + { + let p = self.patches.get_mut(&p_start).expect("found above"); + let off = (addr - p_start) as usize; + if off + bytes.len() > p.len() { + p.resize(off + bytes.len(), 0); + } + p[off..off + bytes.len()].copy_from_slice(bytes); + return Ok(()); + } + // Patches that overlap or touch [addr, end) merge into one. + let touching: Vec = self + .patches + .range(..=end) + .rev() + .take_while(|(s, b)| **s + b.len() as u64 >= addr) + .map(|(s, _)| *s) + .collect(); + if touching.is_empty() { + self.patches.insert(addr, bytes.to_vec()); + return Ok(()); + } + let lo = touching.iter().copied().min().map_or(addr, |s| s.min(addr)); + let hi = touching + .iter() + .map(|s| s + self.patches[s].len() as u64) + .max() + .map_or(end, |e| e.max(end)); + let mut merged = self.read(lo, (hi - lo) as usize)?; + merged[(addr - lo) as usize..(end - lo) as usize].copy_from_slice(bytes); + for s in touching { + self.patches.remove(&s); + } + self.patches.insert(lo, merged); + Ok(()) + } + + /// Write the edit to `file`, whose superblock is at `user_block`. + /// + /// Order: first everything in newly allocated space (new chunks, new + /// index blocks, relocated structures), which nothing on disk refers to + /// yet, then a sync; then the changes to existing bytes — raw data + /// overwritten in place and the metadata that links the new space in + /// (superblock end of file, chunk index entries, object header + /// messages) — then a sync. A crash during the first phase leaves the + /// file as it was (plus unreferenced bytes past its end of file); a + /// crash during the second can leave it inconsistent, as with libhdf5 + /// without SWMR: there is no journal. + pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> { + let old_eoa = self.old_eoa; + let mut in_place: Vec<(u64, &[u8])> = Vec::new(); + for (&addr, bytes) in &self.patches { + // A patch may run from existing bytes into new space (writes + // merge); its new part goes with the new space. + let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize; + let (old, new) = bytes.split_at(split); + if !new.is_empty() { + write_at(file, user_block + addr + split as u64, new)?; + } + if !old.is_empty() { + in_place.push((addr, old)); + } + } + if self.eoa > old_eoa { + let want = user_block + self.eoa; + if file.metadata()?.len() < want { + file.set_len(want)?; + } + } + file.sync_data()?; + for (addr, bytes) in in_place { + write_at(file, user_block + addr, bytes)?; + } + file.sync_all()?; + Ok(()) + } +} + +fn write_at(file: &mut std::fs::File, pos: u64, bytes: &[u8]) -> Result<(), Error> { + file.seek(SeekFrom::Start(pos))?; + file.write_all(bytes)?; + Ok(()) +} + +/// Little-endian encode of `v` in `width` bytes. +pub(crate) fn put_uint(buf: &mut [u8], v: u64, width: u8) { + let w = width as usize; + buf[..w].copy_from_slice(&v.to_le_bytes()[..w]); +} + +/// Little-endian decode of `width` bytes. +pub(crate) fn get_uint(buf: &[u8], width: u8) -> u64 { + let mut b = [0u8; 8]; + b[..width as usize].copy_from_slice(&buf[..width as usize]); + u64::from_le_bytes(b) +} + +/// The undefined address for `os`-byte addresses. +pub(crate) fn undef(os: u8) -> u64 { + if os >= 8 { + u64::MAX + } else { + (1u64 << (8 * u32::from(os))) - 1 + } +} + +/// Recompute the Jenkins checksum over `[start, end)` and store it at `end`. +pub(crate) fn rechecksum(img: &mut Image<'_>, start: u64, end: u64) -> Result<(), Error> { + let bytes = img.read(start, (end - start) as usize)?; + let sum = clawhdf5_format::checksum::jenkins_lookup3(&bytes); + img.write(end, &sum.to_le_bytes()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_see_writes_and_merges() { + let base = vec![1u8; 32]; + let mut img = Image::new(&base, 8, 8); + img.write(4, &[9, 9]).unwrap(); + img.write(8, &[7]).unwrap(); + img.write(5, &[3, 3, 3]).unwrap(); // extends the first up to the second + assert_eq!(img.read(3, 7).unwrap(), vec![1, 9, 3, 3, 3, 7, 1]); + img.write(2, &[4, 4, 4, 4, 4, 4, 4, 4]).unwrap(); // covers both: merged + assert_eq!(img.patches.len(), 1); + assert_eq!(img.read(1, 10).unwrap(), vec![1, 4, 4, 4, 4, 4, 4, 4, 4, 1]); + let a = img.alloc(10).unwrap(); + assert_eq!(a, 32); + assert_eq!(img.read(30, 4).unwrap(), vec![1, 1, 0, 0]); + img.write(40, &[5]).unwrap(); + assert_eq!(img.read(39, 3).unwrap(), vec![0, 5, 0]); + assert!(img.write(42, &[1]).is_err()); + } + + /// Random reads and writes against a flat copy of the bytes. + #[test] + fn matches_a_flat_model() { + let base: Vec = (0..200u32).map(|i| i as u8).collect(); + let mut img = Image::new(&base, 8, 8); + img.alloc(100).unwrap(); + let mut flat = base.clone(); + flat.resize(300, 0); + let mut x = 12345u64; + let mut next = |n: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x % n + }; + for step in 0..5000 { + let at = next(300); + let len = 1 + next(20).min(299 - at); + if step % 3 == 0 { + assert_eq!( + img.read(at, len as usize).unwrap(), + flat[at as usize..(at + len) as usize] + ); + } else { + let bytes: Vec = (0..len).map(|_| next(256) as u8).collect(); + img.write(at, &bytes).unwrap(); + flat[at as usize..(at + len) as usize].copy_from_slice(&bytes); + } + } + assert_eq!(img.read(0, 300).unwrap(), flat); + // Patches never overlap. + let mut end = 0; + for (s, b) in &img.patches { + assert!(*s >= end); + end = s + b.len() as u64; + } + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs new file mode 100644 index 0000000..b135a3f --- /dev/null +++ b/crates/clawhdf5/src/edit/mod.rs @@ -0,0 +1,1213 @@ +//! In-place modification of an existing HDF5 file: [`FileEditor`]. +//! +//! [`FileBuilder`](crate::FileBuilder) builds a whole file in memory; the +//! editor instead opens a file libhdf5 (or clawhdf5) wrote and changes it +//! where it lies: dataset values are overwritten in place, chunks are added +//! or relocated at the end of the file, the chunk index, dataspace and +//! object headers are patched, and every checksum of a structure it touched +//! is recomputed. +//! +//! Each operation is planned in memory first ([`image::Image`]): if any part +//! of it is unsupported, nothing is written. The plan is then committed with +//! the new space (new chunks, new index blocks) written and synced before +//! the existing bytes that link it in, then synced again. + +mod btree1; +mod earray; +mod farray; +mod image; +mod ohdr; +mod select; + +use std::collections::{BTreeMap, HashMap}; +use std::fs::{OpenOptions, TryLockError}; +use std::path::{Path, PathBuf}; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::data_read::NativeElement; +use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::filter_pipeline::FilterPipeline; +use clawhdf5_format::selection::Selection; + +use crate::error::Error; +use crate::reader::File; +use crate::types::AttrValue; +use btree1::{BTree1, Key}; +use earray::{Ea, EaParams, Elem}; +use farray::Fa; +use image::{Image, get_uint, put_uint, undef}; +use ohdr::{Header, MSG_ATTRIBUTE}; + +const MSG_DATASPACE: u16 = 0x01; +const MSG_LAYOUT: u16 = 0x08; +const MSG_EXTERNAL: u16 = 0x07; +const MSG_ATTR_INFO: u16 = 0x15; +/// Message flag: the message is shared (stored elsewhere). +const MSG_FLAG_SHARED: u8 = 0x02; + +/// An HDF5 file opened for in-place modification. +/// +/// Opening takes an exclusive advisory lock on the file (`flock`, the lock +/// libhdf5 itself takes when file locking is on), so a second editor, or +/// h5py opening the file for writing, fails until the editor is dropped. +/// Readers that do not lock ([`File`]) can still open it, but see a file +/// that may be mid-update. +/// +/// Every method is one self-contained edit: it re-reads the file's +/// metadata, applies the change, and syncs the file before returning. +/// +/// # What it can change +/// +/// - [`write_selection`](Self::write_selection) / +/// [`write_all`](Self::write_all): overwrite values of a compact, +/// contiguous or chunked dataset with values of the dataset's own +/// datatype, under any selection. Chunks are decoded, updated and +/// re-encoded; an unfiltered chunk is rewritten in place, a filtered one +/// in place when it still fits and otherwise at the end of the file. New +/// chunks are added to the chunk index: version-1 B-tree (layout v1-v3, +/// what h5py's default `libver` writes), Extensible Array, Fixed Array and +/// single-chunk indexes. A version-2 B-tree index (two or more unlimited +/// dimensions) can only have existing chunks overwritten in place, and an +/// implicit index only in place. +/// - [`resize`](Self::resize): grow a chunked dataset up to its maximum +/// dimensions (h5py's `Dataset.resize`); new chunks come with the writes. +/// - [`set_attr`](Self::set_attr): add or replace an attribute of any +/// object whose attributes are stored in its object header. +/// +/// Anything else is an [`Error::Unsupported`] and leaves the file untouched. +/// +/// # Crash safety and space +/// +/// There is no journal (libhdf5 has none either, outside SWMR). An edit +/// writes all of its new space — chunks and index blocks past the old end +/// of file — and syncs it before it changes any existing byte, so a crash +/// before that point leaves the file as it was; a crash while the existing +/// structures are being patched can leave the file inconsistent. +/// +/// Space is never reused: a filtered chunk that grows moves to the end of +/// the file and its old bytes are leaked, as are index blocks that are +/// replaced. `h5repack` reclaims such space. +#[derive(Debug)] +pub struct FileEditor { + path: PathBuf, + file: std::fs::File, +} + +/// Where a layout message keeps the fields an edit may change (offsets in +/// the message body). +#[derive(Debug, Default, Clone, Copy)] +struct LayoutPos { + version: u8, + /// Contiguous data address, or the chunk index address. + addr: Option, + /// Filtered single chunk: its stored size and filter mask. + single_size: Option, + single_mask: Option, + /// First chunk index creation parameter (Fixed/Extensible Array). + params: Option, + /// Compact raw data. + compact_data: Option, +} + +fn layout_pos(d: &[u8], os: u8, ls: u8) -> Result { + let short = || Error::Unsupported("layout message too short".into()); + let os = os as usize; + let ls = ls as usize; + let version = *d.first().ok_or_else(short)?; + let mut p = LayoutPos { + version, + ..LayoutPos::default() + }; + match version { + 1 | 2 => { + let nd = *d.get(1).ok_or_else(short)? as usize; + match *d.get(2).ok_or_else(short)? { + 0 => p.compact_data = Some(8 + nd * 4 + 4), + _ => p.addr = Some(8), + } + } + 3 => match *d.get(1).ok_or_else(short)? { + 0 => p.compact_data = Some(4), + 1 => p.addr = Some(2), + _ => p.addr = Some(3), + }, + 4 | 5 => match *d.get(1).ok_or_else(short)? { + 0 => p.compact_data = Some(4), + 1 => p.addr = Some(2), + 2 => { + let flags = *d.get(2).ok_or_else(short)?; + let nd = *d.get(3).ok_or_else(short)? as usize; + let enc = *d.get(4).ok_or_else(short)? as usize; + let mut q = 5 + nd * enc; + let itype = *d.get(q).ok_or_else(short)?; + q += 1; + match itype { + 1 if flags & 0x02 != 0 => { + p.single_size = Some(q); + p.single_mask = Some(q + ls); + p.addr = Some(q + ls + 4); + } + 1 | 2 => p.addr = Some(q), + 3 => { + p.params = Some(q); + p.addr = Some(q + 1); + } + 4 => { + p.params = Some(q); + p.addr = Some(q + 5); + } + 5 => { + p.params = Some(q); + p.addr = Some(q + 6); + } + _ => return Err(Error::Unsupported(format!("chunk index type {itype}"))), + } + } + c => return Err(Error::Unsupported(format!("layout class {c}"))), + }, + v => return Err(Error::Unsupported(format!("layout message version {v}"))), + } + let end = p + .addr + .map_or(0, |a| a + os) + .max(p.compact_data.unwrap_or(0)); + if end > d.len() { + return Err(short()); + } + Ok(p) +} + +/// A dataset as an edit sees it. +struct Target { + addr: u64, + dt: Datatype, + es: usize, + ds: Dataspace, + layout: DataLayout, + pipeline: Option, + /// One element of fill value. + fill: Vec, +} + +impl Target { + fn load(f: &File, path: &str) -> Result { + let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let d = f.dataset_at(addr)?; + let dt = d.datatype()?; + let es = dt.type_size() as usize; + if es == 0 { + return Err(Error::Unsupported("datatype of size 0".into())); + } + let ds = d.dataspace()?; + let layout = d.data_layout()?; + let pipeline = d.filter_pipeline()?.filter(|p| !p.filters.is_empty()); + let os = f.superblock().offset_size; + let ls = f.superblock().length_size; + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + f.as_bytes(), + &d.header().messages, + os, + ls, + )?; + let fill = match fill { + None => vec![0u8; es], + Some(v) if v.len() == es => v, + Some(_) => { + return Err(Error::Unsupported( + "fill value size differs from the element size".into(), + )); + } + }; + if d.header() + .messages + .iter() + .any(|m| m.msg_type.to_u16() == MSG_EXTERNAL) + { + return Err(Error::Unsupported( + "dataset stored in external files".into(), + )); + } + Ok(Self { + addr, + dt, + es, + ds, + layout, + pipeline, + fill, + }) + } + + fn dims(&self) -> &[u64] { + &self.ds.dimensions + } +} + +/// Datatypes whose stored bytes point elsewhere in the file (variable-length +/// data in a global heap, references) cannot be written as raw values. +fn check_plain(dt: &Datatype) -> Result<(), Error> { + match dt { + Datatype::VariableLength { .. } => { + Err(Error::Unsupported("writing variable-length data".into())) + } + Datatype::Reference { .. } => Err(Error::Unsupported("writing references".into())), + Datatype::Compound { members, .. } => { + members.iter().try_for_each(|m| check_plain(&m.datatype)) + } + Datatype::Array { base_type, .. } => check_plain(base_type), + Datatype::Enumeration { base_type, .. } => check_plain(base_type), + _ => Ok(()), + } +} + +fn row_major(coords: &[u64], dims: &[u64]) -> u64 { + coords + .iter() + .zip(dims) + .fold(0u64, |acc, (&c, &d)| acc * d + c) +} + +/// Linear index of a chunk in a Fixed Array (`swizzle` false) or Extensible +/// Array (`swizzle` true) index; see `clawhdf5_format`'s `chunk_grid`. +fn array_index( + scaled: &[u64], + dims: &[u64], + max: Option<&[u64]>, + cd: &[u64], + swizzle: bool, +) -> Result { + let rank = cd.len(); + let max_chunks: Vec = (0..rank) + .map(|d| { + let m = max.map_or(dims[d], |m| m[d]); + if m == u64::MAX { + u64::MAX + } else { + m.div_ceil(cd[d]).max(dims[d].div_ceil(cd[d])) + } + }) + .collect(); + let mut order: Vec = (0..rank).collect(); + if swizzle && let Some(u) = max_chunks.iter().position(|&m| m == u64::MAX) { + order.remove(u); + order.insert(0, u); + } + let mut idx = 0u64; + let mut down = 1u64; + for p in (0..rank).rev() { + let d = order[p]; + idx = scaled[d] + .checked_mul(down) + .and_then(|v| idx.checked_add(v)) + .ok_or_else(|| Error::Unsupported("chunk index overflows".into()))?; + if p > 0 { + if max_chunks[d] == u64::MAX { + return Err(Error::Unsupported( + "array chunk index with more than one unlimited dimension".into(), + )); + } + down = down + .checked_mul(max_chunks[d]) + .ok_or_else(|| Error::Unsupported("chunk index overflows".into()))?; + } + } + Ok(idx) +} + +/// The chunk index of one dataset, opened for changes. Index structures a +/// dataset does not have yet (no chunk was ever written) are created on the +/// first insertion, and the layout message is pointed at them. +enum IndexEdit { + BTree1(Option), + Single, + Implicit, + Fixed(Option), + Extensible(Option), + BTree2, +} + +struct ChunkedEdit<'t> { + t: &'t Target, + /// Spatial chunk dimensions. + cd: Vec, + chunk_bytes: usize, + index: IndexEdit, + /// The dataset's object header (for layout message changes). + hdr: Header, + layout_msg: usize, + lpos: LayoutPos, + btree_k: u16, +} + +impl<'t> ChunkedEdit<'t> { + fn new(f: &File, img: &Image<'_>, t: &'t Target) -> Result { + let DataLayout::Chunked { + chunk_dimensions, + btree_address, + version, + chunk_index_type, + dont_filter_partial_edge_chunks, + .. + } = &t.layout + else { + return Err(Error::Unsupported("not a chunked dataset".into())); + }; + if *dont_filter_partial_edge_chunks { + return Err(Error::Unsupported( + "datasets whose partial edge chunks are not filtered".into(), + )); + } + let rank = t.dims().len(); + if chunk_dimensions.len() < rank { + return Err(Error::Unsupported( + "chunk rank differs from the dataspace".into(), + )); + } + let cd: Vec = chunk_dimensions[..rank] + .iter() + .map(|&c| u64::from(c)) + .collect(); + let chunk_bytes = cd + .iter() + .try_fold(t.es as u64, |a, &c| a.checked_mul(c)) + .and_then(|b| usize::try_from(b).ok()) + .filter(|&b| b <= u32::MAX as usize) + .ok_or_else(|| Error::Unsupported("chunk larger than 4 GiB".into()))?; + let hdr = Header::load(img, t.addr)?; + let layout_msg = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, layout_msg)?, img.os, img.ls)?; + let index = match (*version, *chunk_index_type) { + (3, _) => IndexEdit::BTree1( + btree_address + .map(|a| BTree1::new(a, chunk_btree_k(f)?, rank + 1, t.es as u64)) + .transpose()?, + ), + (_, Some(1)) => IndexEdit::Single, + (_, Some(2)) => IndexEdit::Implicit, + (_, Some(3)) => IndexEdit::Fixed(btree_address.map(|a| Fa::open(img, a)).transpose()?), + (_, Some(4)) => { + IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?) + } + (_, Some(5)) => IndexEdit::BTree2, + (v, i) => { + return Err(Error::Unsupported(format!( + "chunked layout version {v}, index type {i:?}" + ))); + } + }; + Ok(Self { + t, + cd, + chunk_bytes, + index, + hdr, + layout_msg, + lpos, + btree_k: chunk_btree_k(f)?, + }) + } + + fn patch_layout_addr(&mut self, img: &mut Image<'_>, addr: u64) -> Result<(), Error> { + let at = self + .lpos + .addr + .ok_or_else(|| Error::Unsupported("layout message has no address".into()))?; + if self.lpos.version < 3 { + return Err(Error::Unsupported( + "creating a chunk index in a version 1/2 layout message".into(), + )); + } + let mut a = vec![0u8; img.os as usize]; + put_uint(&mut a, addr, img.os); + self.hdr.patch(img, self.layout_msg, at, &a) + } + + /// Record chunk `scaled` at `e` in the index (new, moved or resized). + fn set(&mut self, img: &mut Image<'_>, scaled: &[u64], e: Elem) -> Result<(), Error> { + let t = self.t; + let filtered = t.pipeline.is_some(); + match &mut self.index { + IndexEdit::BTree1(tree) => { + let size = u32::try_from(e.size) + .map_err(|_| Error::Unsupported("chunk larger than 4 GiB".into()))?; + let mut offs: Vec = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect(); + offs.push(0); + let key = Key { + size, + mask: e.mask, + offs, + }; + match tree { + Some(tree) => tree.insert(img, key, e.addr)?, + None => { + let new = BTree1::create( + img, + self.btree_k, + scaled.len() + 1, + t.es as u64, + key, + e.addr, + )?; + let root = new.root(); + *tree = Some(new); + self.patch_layout_addr(img, root)?; + } + } + } + IndexEdit::Single => { + if scaled.iter().any(|&s| s != 0) { + return Err(Error::Unsupported( + "single-chunk index with a second chunk".into(), + )); + } + match (self.lpos.single_size, self.lpos.single_mask) { + (Some(s), Some(m)) => { + let mut b = vec![0u8; img.ls as usize]; + put_uint(&mut b, e.size, img.ls); + self.hdr.patch(img, self.layout_msg, s, &b)?; + self.hdr + .patch(img, self.layout_msg, m, &e.mask.to_le_bytes())?; + } + _ if filtered => { + return Err(Error::Unsupported( + "filtered single chunk without a filtered-size field".into(), + )); + } + _ => {} + } + self.patch_layout_addr(img, e.addr)?; + } + IndexEdit::Implicit => { + return Err(Error::Unsupported( + "adding a chunk to an implicit chunk index".into(), + )); + } + IndexEdit::Fixed(fa) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, false)?; + if fa.is_none() { + let pbits_at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("Fixed Array parameters".into()))?; + let page_bits = self.hdr.data(img, self.layout_msg)?[pbits_at]; + let nelmts = (0..self.cd.len()) + .map(|d| { + let m = max.map_or(t.dims()[d], |m| m[d]); + m.div_ceil(self.cd[d]) + }) + .try_fold(1u64, |a, n| a.checked_mul(n)) + .ok_or_else(|| Error::Unsupported("Fixed Array too large".into()))?; + let (new, hdr_addr) = Fa::create( + img, + nelmts, + page_bits, + filtered, + self.chunk_bytes as u64, + self.lpos.version, + )?; + *fa = Some(new); + self.patch_layout_addr(img, hdr_addr)?; + } + if let IndexEdit::Fixed(Some(fa)) = &mut self.index { + fa.set(img, idx, e)?; + } + } + IndexEdit::Extensible(ea) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, true)?; + if ea.is_none() { + let at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("Extensible Array parameters".into()))?; + let d = self.hdr.data(img, self.layout_msg)?; + // Layout message order: max_nelmts_bits, idx_blk_elmts, + // sup_blk_min_data_ptrs, data_blk_min_elmts, + // max_dblk_page_nelmts_bits. + let p = EaParams { + max_nelmts_bits: d[at], + idx_blk_elmts: d[at + 1], + sup_blk_min_data_ptrs: d[at + 2], + data_blk_min_elmts: d[at + 3], + max_dblk_page_nelmts_bits: d[at + 4], + }; + let new = + Ea::create(img, p, filtered, self.chunk_bytes as u64, self.lpos.version)?; + let hdr_addr = new.header_address(); + *ea = Some(new); + self.patch_layout_addr(img, hdr_addr)?; + } + if let IndexEdit::Extensible(Some(ea)) = &mut self.index { + ea.set(img, idx, e)?; + } + } + IndexEdit::BTree2 => { + return Err(Error::Unsupported( + "adding, moving or resizing a chunk in a version-2 B-tree chunk index \ + (datasets with more than one unlimited dimension)" + .into(), + )); + } + } + Ok(()) + } + + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + match &mut self.index { + IndexEdit::Extensible(Some(ea)) => ea.finish(img)?, + IndexEdit::Fixed(Some(fa)) => fa.finish(img)?, + _ => {} + } + self.hdr.finish(img) + } +} + +/// The chunk B-tree's K: the superblock's (version 1), the superblock +/// extension's (versions 2 and 3), or libhdf5's default of 32. +fn chunk_btree_k(f: &File) -> Result { + let sb = f.superblock(); + if let Some(k) = sb.indexed_storage_internal_node_k { + return Ok(k); + } + let ext = clawhdf5_format::superblock_ext::read_superblock_extension(f.as_bytes(), sb)?; + Ok(ext + .and_then(|e| e.btree_k) + .map_or(32, |(chunk, _, _)| chunk)) +} + +impl FileEditor { + /// Open `path` for modification, locking it exclusively. + /// + /// Refused ([`Error::Unsupported`]) for files the editor cannot keep + /// consistent: a metadata cache image, paged or persistent free-space + /// management, a multi-file driver, a file another writer has marked + /// open (superblock version 3 consistency flags). + pub fn open>(path: P) -> Result { + let path = path.as_ref().to_path_buf(); + let file = OpenOptions::new().read(true).write(true).open(&path)?; + match file.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => { + return Err(Error::Locked(format!( + "{} is open for writing elsewhere", + path.display() + ))); + } + Err(TryLockError::Error(e)) => return Err(Error::Io(e)), + } + let ed = Self { path, file }; + let f = File::open(&ed.path)?; + check_editable(&f)?; + Ok(ed) + } + + /// The file's path. + pub fn path(&self) -> &Path { + &self.path + } + + fn edit( + &mut self, + op: impl FnOnce(&File, &mut Image<'_>) -> Result, + ) -> Result { + let f = File::open(&self.path)?; + check_editable(&f)?; + let sb = f.superblock().clone(); + let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); + let r = op(&f, &mut img)?; + if img.is_dirty() { + if img.eoa() != img.old_eoa() { + set_superblock_eof(&mut img, &sb)?; + } + img.commit(&mut self.file, f.user_block_size())?; + } + Ok(r) + } + + /// Overwrite the dataset's values under `selection` with `data`: the + /// selected elements' bytes in the dataset's own datatype and byte + /// order, in selection order (row-major over a hyperslab, the listed + /// order for points). + pub fn write_selection( + &mut self, + path: &str, + selection: &Selection, + data: &[u8], + ) -> Result<(), Error> { + self.edit(|f, img| write_selection(f, img, path, selection, data)) + } + + /// Overwrite every value of the dataset (see + /// [`write_selection`](Self::write_selection)). + pub fn write_all(&mut self, path: &str, data: &[u8]) -> Result<(), Error> { + self.write_selection(path, &Selection::All, data) + } + + /// [`write_selection`](Self::write_selection) with typed values; the + /// dataset's datatype must be `T`'s native representation. + pub fn write_values( + &mut self, + path: &str, + selection: &Selection, + values: &[T], + ) -> Result<(), Error> { + // SAFETY: `NativeElement` types have no padding and no invalid bit + // patterns, so their memory is plain bytes. + let bytes = unsafe { + std::slice::from_raw_parts(values.as_ptr().cast::(), std::mem::size_of_val(values)) + }; + self.edit(|f, img| { + let t = Target::load(f, path)?; + if !T::is_native(&t.dt) { + return Err(Error::InvalidArgument(format!( + "dataset {path} does not store {}", + std::any::type_name::() + ))); + } + write_selection(f, img, path, selection, bytes) + }) + } + + /// Change a chunked dataset's current dimensions to `shape`, which may + /// only grow each dimension, up to the dataset's maximum dimensions + /// (h5py's `Dataset.resize`). New elements read as the fill value until + /// written. Shrinking is [`Error::Unsupported`]. + pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> { + self.edit(|f, img| { + let t = Target::load(f, path)?; + let dims = t.dims().to_vec(); + if shape.len() != dims.len() { + return Err(Error::InvalidArgument(format!( + "rank {} for a dataset of rank {}", + shape.len(), + dims.len() + ))); + } + if shape == dims.as_slice() { + return Ok(()); + } + let max = t.ds.max_dimensions.clone().unwrap_or_else(|| dims.clone()); + for d in 0..dims.len() { + if shape[d] > max[d] { + return Err(Error::InvalidArgument(format!( + "dimension {d}: {} exceeds the maximum {}", + shape[d], max[d] + ))); + } + if shape[d] < dims[d] { + return Err(Error::Unsupported("shrinking a dataset".into())); + } + } + if !matches!(t.layout, DataLayout::Chunked { .. }) { + return Err(Error::Unsupported( + "resizing a dataset that is not chunked".into(), + )); + } + // Only the dataspace changes: every chunk index is keyed + // independently of the current extent (the array indexes by the + // maximum dimensions), and new chunks come with the writes. + let mut hdr = Header::load(img, t.addr)?; + let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::Dataspace, + ))?; + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared dataspace message".into())); + } + let body = hdr.data(img, i)?; + let first = match body.first() { + Some(1) => 8, + Some(2) => 4, + _ => return Err(Error::Unsupported("dataspace message version".into())), + }; + let ls = img.ls as usize; + let mut dims_bytes = vec![0u8; shape.len() * ls]; + for (d, &n) in shape.iter().enumerate() { + if ls < 8 && n >> (8 * ls) != 0 { + return Err(Error::InvalidArgument("dimension too large".into())); + } + put_uint(&mut dims_bytes[d * ls..], n, img.ls); + } + hdr.patch(img, i, first, &dims_bytes)?; + hdr.finish(img) + }) + } + + /// Set attribute `name` of the object at `path` (a group or dataset; + /// `"/"` is the root group) to `value`, replacing an attribute of that + /// name. The attribute goes into free space in the object header, or a + /// new header continuation chunk at the end of the file. + /// + /// [`Error::Unsupported`] for an object whose attributes are in dense + /// storage (or would have to move there: more than the object's + /// compact-attribute limit), one that tracks attribute creation order, + /// or one with shared attribute messages. + pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { + if name.is_empty() { + return Err(Error::InvalidArgument("empty attribute name".into())); + } + self.edit(|f, img| { + let addr = + clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let mut hdr = Header::load(img, addr)?; + if hdr.version == 2 && hdr.flags & 0x04 != 0 { + return Err(Error::Unsupported( + "object tracks attribute creation order".into(), + )); + } + if let Some(i) = hdr.find(MSG_ATTR_INFO) { + let d = hdr.data(img, i)?; + // version(1) flags(1) [max creation index(2)] fractal heap + // address, name index address, [order index address]. + let mut p = 2; + if d.get(1).is_some_and(|f| f & 0x01 != 0) { + p += 2; + } + let os = img.os as usize; + if d.len() < p + os { + return Err(Error::Unsupported("short attribute info message".into())); + } + if get_uint(&d[p..], img.os) != undef(img.os) { + return Err(Error::Unsupported( + "object with attributes in dense storage".into(), + )); + } + } + let mut existing = None; + let mut count = 0usize; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + count += 1; + if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { + existing = Some(i); + } + } + if existing.is_none() && hdr.version == 2 { + let max_compact = max_compact_attrs(img, &hdr)?; + if count + 1 > usize::from(max_compact) { + return Err(Error::Unsupported(format!( + "object already has {count} compact attributes (its limit is \ + {max_compact}); more need dense storage" + ))); + } + } + let msg = clawhdf5_format::type_builders::build_attr_message(name, value); + check_plain(&msg.datatype)?; + let body = if hdr.version == 1 { + encode_attr_v1(&msg, img.ls) + } else { + let mut b = msg.serialize_v3(img.ls); + if !name.is_ascii() { + b[8] = 1; // UTF-8 name + } + b + }; + if let Some(i) = existing { + hdr.delete(img, i)?; + } + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + hdr.finish(img) + }) + } +} + +/// libhdf5's checks before it opens a file for writing, and what this +/// editor cannot keep consistent. +fn check_editable(f: &File) -> Result<(), Error> { + let sb = f.superblock(); + if sb.version > 3 { + return Err(Error::Unsupported(format!( + "superblock version {}", + sb.version + ))); + } + if sb.version >= 3 && (sb.is_write_access() || sb.is_swmr_write()) { + return Err(Error::Unsupported( + "the file is marked as open for writing by another process".into(), + )); + } + if sb + .driver_info_address + .is_some_and(|a| a != undef(sb.offset_size)) + { + return Err(Error::Unsupported("files with a driver info block".into())); + } + if let Some(ext) = clawhdf5_format::superblock_ext::read_superblock_extension(f.as_bytes(), sb)? + { + if ext.cache_image.is_some() { + return Err(Error::Unsupported( + "files with a metadata cache image".into(), + )); + } + // Strategy 1 is H5F_FSPACE_STRATEGY_PAGE. + if let Some(fs) = ext.file_space_info + && (fs.persist || fs.strategy == 1) + { + return Err(Error::Unsupported( + "files with paged or persistent free-space management".into(), + )); + } + } + Ok(()) +} + +/// Store the new end of file in the superblock. +fn set_superblock_eof( + img: &mut Image<'_>, + sb: &clawhdf5_format::superblock::Superblock, +) -> Result<(), Error> { + let os = u64::from(sb.offset_size); + let (at, sum) = match sb.version { + 0 => (24 + 2 * os, None), + 1 => (28 + 2 * os, None), + 2 | 3 => (12 + 2 * os, Some(12 + 4 * os)), + v => return Err(Error::Unsupported(format!("superblock version {v}"))), + }; + let eof = sb + .base_address + .checked_add(img.eoa()) + .ok_or_else(|| Error::Unsupported("end of file overflows".into()))?; + let mut b = vec![0u8; os as usize]; + put_uint(&mut b, eof, sb.offset_size); + img.write(at, &b)?; + if let Some(end) = sum { + let bytes = img.read(0, end as usize)?; + let s = clawhdf5_format::checksum::jenkins_lookup3(&bytes); + img.write(end, &s.to_le_bytes())?; + } + Ok(()) +} + +/// The name bytes of an attribute message body (without the NUL). +fn attr_name(d: &[u8]) -> Result<&[u8], Error> { + let bad = || Error::Unsupported("malformed attribute message".into()); + let (len, at) = match d.first() { + Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), + Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), + _ => return Err(bad()), + }; + let name = d.get(at..at + len).ok_or_else(bad)?; + Ok(name.split(|&b| b == 0).next().unwrap_or(name)) +} + +/// A version-2 header's limit on compact attributes: stored when its flags +/// say so, else libhdf5's default of 8. +fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { + if hdr.flags & 0x10 == 0 { + return Ok(8); + } + let mut p = hdr.addr + 6; + if hdr.flags & 0x20 != 0 { + p += 16; + } + let b = img.read(p, 2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) +} + +/// A version-1 attribute message (what libhdf5 writes in a version-1 object +/// header): name, datatype and dataspace each padded to 8 bytes, the +/// dataspace as a version-1 dataspace message. +fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { + let mut name = a.name.as_bytes().to_vec(); + name.push(0); + let dt = a.datatype.serialize(); + let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; + if a.dataspace.space_type == DataspaceType::Simple { + for &d in &a.dataspace.dimensions { + let mut b = vec![0u8; ls as usize]; + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + } else { + ds[1] = 0; + } + let mut out = vec![1u8, 0]; + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); + out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); + for part in [&name, &dt, &ds] { + out.extend_from_slice(part); + out.resize(out.len().next_multiple_of(8), 0); + } + out.extend_from_slice(&a.raw_data); + out +} + +fn write_selection( + f: &File, + img: &mut Image<'_>, + path: &str, + sel: &Selection, + data: &[u8], +) -> Result<(), Error> { + let t = Target::load(f, path)?; + check_plain(&t.dt)?; + if t.ds.space_type == DataspaceType::Null { + return Err(Error::InvalidArgument( + "dataset has a null dataspace".into(), + )); + } + let dims = t.dims().to_vec(); + clawhdf5_format::partial_read::validate(sel, &dims) + .map_err(|e| Error::InvalidArgument(e.to_string()))?; + if let Selection::Hyperslab { + stride, + count, + block, + .. + } = sel + && (0..dims.len()).any(|d| count[d] > 1 && stride[d] < block[d]) + { + return Err(Error::InvalidArgument( + "overlapping hyperslab blocks".into(), + )); + } + let n = select::for_each_run(sel, &dims, |_, _, _| Ok(()))?; + let want = n + .checked_mul(t.es as u64) + .ok_or_else(|| Error::InvalidArgument("selection too large".into()))?; + if data.len() as u64 != want { + return Err(Error::InvalidArgument(format!( + "{} bytes for {n} elements of {} bytes", + data.len(), + t.es + ))); + } + if n == 0 { + return Ok(()); + } + let es = t.es; + match &t.layout { + DataLayout::Compact { data: stored } => { + let mut hdr = Header::load(img, t.addr)?; + let li = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, li)?, img.os, img.ls)?; + let at = lpos + .compact_data + .ok_or_else(|| Error::Unsupported("compact layout".into()))?; + let total = dims.iter().product::() as usize * es; + if stored.len() < total { + return Err(Error::Unsupported( + "compact data shorter than the dataset".into(), + )); + } + select::for_each_run(sel, &dims, |c, len, src| { + let off = at + row_major(c, &dims) as usize * es; + let s = src as usize * es; + hdr.patch(img, li, off, &data[s..s + len as usize * es]) + })?; + hdr.finish(img) + } + DataLayout::Contiguous { address, size } => { + let total = dims + .iter() + .try_fold(es as u64, |a, &d| a.checked_mul(d)) + .ok_or_else(|| Error::Unsupported("dataset too large".into()))?; + let base = match address { + Some(a) => { + if *size < total { + return Err(Error::Unsupported("contiguous storage too small".into())); + } + *a + } + None => { + // Never written (late allocation): allocate it now, as + // libhdf5 does on the first write, with the fill value. + let mut hdr = Header::load(img, t.addr)?; + let li = hdr.find(MSG_LAYOUT).ok_or(Error::MissingMessage( + clawhdf5_format::message_type::MessageType::DataLayout, + ))?; + let lpos = layout_pos(&hdr.data(img, li)?, img.os, img.ls)?; + if lpos.version < 3 { + return Err(Error::Unsupported( + "allocating storage for a version 1/2 layout message".into(), + )); + } + let at = lpos + .addr + .ok_or_else(|| Error::Unsupported("layout".into()))?; + let a = img.alloc(total)?; + if t.fill.iter().any(|&b| b != 0) { + let buf = t.fill.repeat(total as usize / es); + img.write(a, &buf)?; + } + let mut ab = vec![0u8; img.os as usize]; + put_uint(&mut ab, a, img.os); + hdr.patch(img, li, at, &ab)?; + // The size field follows the address. + let mut sb = vec![0u8; img.ls as usize]; + put_uint(&mut sb, total, img.ls); + hdr.patch(img, li, at + img.os as usize, &sb)?; + hdr.finish(img)?; + a + } + }; + select::for_each_run(sel, &dims, |c, len, src| { + let off = base + row_major(c, &dims) * es as u64; + let s = src as usize * es; + img.write(off, &data[s..s + len as usize * es]) + })?; + Ok(()) + } + DataLayout::Chunked { btree_address, .. } => { + let mut ce = ChunkedEdit::new(f, img, &t)?; + let existing: HashMap, ChunkInfo> = if btree_address.is_some() { + let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?; + chunks + .into_iter() + .map(|c| { + let s: Vec = + c.offsets.iter().zip(&ce.cd).map(|(o, cd)| o / cd).collect(); + (s, c) + }) + .collect() + } else { + HashMap::new() + }; + let cd = ce.cd.clone(); + let chunk_bytes = ce.chunk_bytes; + let rank = dims.len(); + let mut bufs: BTreeMap, Vec> = BTreeMap::new(); + let load = |scaled: &Vec| -> Result, Error> { + match existing.get(scaled) { + Some(info) => decode_chunk(img_read(f, info)?, &t, info, chunk_bytes), + None => Ok(t.fill.repeat(chunk_bytes / es)), + } + }; + select::for_each_run(sel, &dims, |coords, len, src| { + let mut c = coords.to_vec(); + let mut left = len; + let mut s = src as usize; + let l = rank.saturating_sub(1); + while left > 0 { + let scaled: Vec = c.iter().zip(&cd).map(|(x, d)| x / d).collect(); + let within: Vec = c.iter().zip(&cd).map(|(x, d)| x % d).collect(); + let n = if rank == 0 { + left + } else { + left.min(cd[l] - within[l]) + }; + if !bufs.contains_key(&scaled) { + let b = load(&scaled)?; + bufs.insert(scaled.clone(), b); + } + let buf = bufs.get_mut(&scaled).expect("inserted above"); + let at = row_major(&within, &cd) as usize * es; + let bytes = n as usize * es; + buf[at..at + bytes].copy_from_slice(&data[s * es..s * es + bytes]); + s += n as usize; + left -= n; + if rank > 0 { + c[l] += n; + } + } + Ok(()) + })?; + for (scaled, buf) in bufs { + let (bytes, mask) = match &t.pipeline { + Some(p) => ( + clawhdf5_format::filters::compress_chunk(&buf, p, es as u32)?, + 0u32, + ), + None => (buf, 0u32), + }; + let len = bytes.len() as u64; + let placed = match existing.get(&scaled) { + Some(info) if t.pipeline.is_none() => { + if u64::from(info.chunk_size) != len { + return Err(Error::Unsupported( + "unfiltered chunk stored at an unexpected size".into(), + )); + } + img.write(info.address, &bytes)?; + None + } + // Rewritten where it is when it still fits, or when it + // is the last thing in the file (the chunk an append + // keeps rewriting usually is) and can grow there. + Some(info) + if len <= u64::from(info.chunk_size) + || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => + { + img.write(info.address, &bytes)?; + (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some( + Elem { + addr: info.address, + size: len, + mask, + }, + ) + } + _ => { + let a = img.alloc(len)?; + img.write(a, &bytes)?; + Some(Elem { + addr: a, + size: len, + mask, + }) + } + }; + if let Some(e) = placed { + ce.set(img, &scaled, e)?; + } + } + ce.finish(img) + } + DataLayout::Virtual { .. } => Err(Error::Unsupported("writing a virtual dataset".into())), + } +} + +fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> { + let start = usize::try_from(info.address) + .map_err(|_| Error::Unsupported("chunk address out of range".into()))?; + f.as_bytes() + .get(start..start + info.chunk_size as usize) + .ok_or_else(|| { + Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof { + expected: start + info.chunk_size as usize, + available: f.as_bytes().len(), + }) + }) +} + +fn decode_chunk( + raw: &[u8], + t: &Target, + info: &ChunkInfo, + chunk_bytes: usize, +) -> Result, Error> { + let out = match &t.pipeline { + Some(p) if !clawhdf5_format::filters::all_filters_skipped(p, info.filter_mask) => { + clawhdf5_format::filters::decompress_chunk_masked( + raw, + p, + chunk_bytes, + t.es as u32, + info.filter_mask, + )? + } + _ => raw.to_vec(), + }; + if out.len() != chunk_bytes { + return Err(Error::Format( + clawhdf5_format::error::FormatError::ChunkedReadError(format!( + "chunk decodes to {} bytes, expected {chunk_bytes}", + out.len() + )), + )); + } + Ok(out) +} diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs new file mode 100644 index 0000000..5446702 --- /dev/null +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -0,0 +1,504 @@ +//! An object header as an edit sees it: every chunk and every message +//! (NIL and continuation messages included) with its position in the file, +//! so single messages can be changed in place, deleted (turned into NIL +//! messages) and added (into a NIL message big enough, or into a new +//! continuation chunk at the end of the file). +//! +//! Version-2 chunks carry a checksum, recomputed by [`Header::finish`] for +//! every chunk the edit touched; a version-1 header's message count is kept +//! up to date there too. + +use std::collections::BTreeSet; + +use crate::edit::image::{Image, get_uint, put_uint, rechecksum}; +use crate::error::Error; +use clawhdf5_format::error::FormatError; + +pub(crate) const MSG_NIL: u16 = 0x00; +pub(crate) const MSG_CONTINUATION: u16 = 0x10; +pub(crate) const MSG_ATTRIBUTE: u16 = 0x0C; + +/// One message: where its header and body are, and what it is. +#[derive(Debug, Clone)] +pub(crate) struct Msg { + pub(crate) chunk: usize, + pub(crate) hdr_pos: u64, + pub(crate) data_pos: u64, + pub(crate) size: usize, + pub(crate) mtype: u16, + pub(crate) flags: u8, + pub(crate) corder: Option, +} + +/// One chunk of the header. +#[derive(Debug, Clone)] +struct Chunk { + /// Where the checksummed bytes start (the `OHDR`/`OCHK` signature). + start: u64, + /// Where the checksum is (version 2 only). + checksum_at: Option, + /// Where the chunk's messages end. + end: u64, + /// Bytes at the end too few for a message header (version 2 only). + /// libhdf5 refuses a chunk with both a gap and a NIL message, so a + /// NIL message made in such a chunk must absorb the gap. + gap: u64, +} + +#[derive(Debug)] +pub(crate) struct Header { + pub(crate) addr: u64, + pub(crate) version: u8, + /// Version-2 header flags (0 for version 1). + pub(crate) flags: u8, + chunks: Vec, + pub(crate) msgs: Vec, + dirty: BTreeSet, + /// Messages added (a split NIL message, a new chunk's messages), for a + /// version-1 header's message count. + added: usize, +} + +const MAX_CHUNKS: usize = 1024; + +fn corrupt(why: &'static str) -> Error { + Error::Format(FormatError::InvalidObjectHeader(why)) +} + +impl Header { + /// Locate every chunk and message of the header at `addr`. + pub(crate) fn load(img: &Image<'_>, addr: u64) -> Result { + let sig = img.read(addr, 4)?; + let mut h = Header { + addr, + version: 0, + flags: 0, + chunks: Vec::new(), + msgs: Vec::new(), + dirty: BTreeSet::new(), + added: 0, + }; + let mut pending: Vec<(u64, u64)> = Vec::new(); + if sig == b"OHDR" { + let pre = img.read(addr, 6)?; + if pre[4] != 2 { + return Err(corrupt("bad object header version")); + } + h.version = 2; + h.flags = pre[5]; + let mut pos = addr + 6; + if h.flags & 0x20 != 0 { + pos += 16; + } + if h.flags & 0x10 != 0 { + pos += 4; + } + let w = 1u8 << (h.flags & 0x03); + let size = get_uint(&img.read(pos, w as usize)?, w); + pos += u64::from(w); + h.chunks.push(Chunk { + start: addr, + checksum_at: Some(pos + size), + end: pos + size, + gap: 0, + }); + h.scan(img, 0, pos, pos + size, &mut pending)?; + } else { + let pre = img.read(addr, 16)?; + if pre[0] != 1 { + return Err(corrupt("bad object header version")); + } + h.version = 1; + let size = u64::from(u32::from_le_bytes([pre[8], pre[9], pre[10], pre[11]])); + h.chunks.push(Chunk { + start: addr, + checksum_at: None, + end: addr + 16 + size, + gap: 0, + }); + h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; + } + while let Some((caddr, clen)) = pending.pop() { + if h.chunks.len() >= MAX_CHUNKS { + return Err(corrupt("too many object header chunks")); + } + let idx = h.chunks.len(); + if h.version == 2 { + if clen < 8 || img.read(caddr, 4)? != b"OCHK" { + return Err(corrupt("bad continuation chunk")); + } + h.chunks.push(Chunk { + start: caddr, + checksum_at: Some(caddr + clen - 4), + end: caddr + clen - 4, + gap: 0, + }); + h.scan(img, idx, caddr + 4, caddr + clen - 4, &mut pending)?; + } else { + h.chunks.push(Chunk { + start: caddr, + checksum_at: None, + end: caddr + clen, + gap: 0, + }); + h.scan(img, idx, caddr, caddr + clen, &mut pending)?; + } + } + Ok(h) + } + + /// Size of a message header in this object header. + pub(crate) fn hsize(&self) -> usize { + match (self.version, self.flags & 0x04 != 0) { + (1, _) => 8, + (_, true) => 6, + _ => 4, + } + } + + fn scan( + &mut self, + img: &Image<'_>, + chunk: usize, + start: u64, + end: u64, + pending: &mut Vec<(u64, u64)>, + ) -> Result<(), Error> { + let hs = self.hsize() as u64; + let bytes = img.read(start, (end - start) as usize)?; + let mut p = 0usize; + while (p as u64) + hs <= end - start { + let b = &bytes[p..]; + let (mtype, size, flags, corder) = if self.version == 1 { + ( + u16::from_le_bytes([b[0], b[1]]), + u16::from_le_bytes([b[2], b[3]]) as usize, + b[4], + None, + ) + } else { + ( + u16::from(b[0]), + u16::from_le_bytes([b[1], b[2]]) as usize, + b[3], + (hs == 6).then(|| u16::from_le_bytes([b[4], b[5]])), + ) + }; + let data_off = p + hs as usize; + if data_off + size > bytes.len() { + return Err(corrupt("message size exceeds buffer end")); + } + if mtype == MSG_CONTINUATION { + let d = &bytes[data_off..data_off + size]; + let os = img.os as usize; + let ls = img.ls as usize; + if d.len() < os + ls { + return Err(corrupt("short continuation message")); + } + pending.push((get_uint(d, img.os), get_uint(&d[os..], img.ls))); + } + self.msgs.push(Msg { + chunk, + hdr_pos: start + p as u64, + data_pos: start + data_off as u64, + size, + mtype, + flags, + corder, + }); + p = data_off + size; + } + self.chunks[chunk].gap = (end - start) - p as u64; + Ok(()) + } + + /// After message `i` became a NIL message: if its chunk ends in a gap, + /// grow the NIL message over it (it must be the chunk's last message). + fn absorb_gap(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> { + let m = self.msgs[i].clone(); + let c = &self.chunks[m.chunk]; + if c.gap == 0 { + return Ok(()); + } + if m.data_pos + m.size as u64 + c.gap != c.end { + return Err(Error::Unsupported( + "object header chunk ends in a gap that a free message cannot absorb".into(), + )); + } + let new_size = m.size + c.gap as usize; + if new_size > usize::from(u16::MAX) { + return Err(Error::Unsupported("object header message too large".into())); + } + self.write_msg_header(img, m.hdr_pos, MSG_NIL, new_size, 0, m.corder)?; + img.write(m.data_pos, &vec![0u8; new_size])?; + self.msgs[i].size = new_size; + self.chunks[m.chunk].gap = 0; + Ok(()) + } + + /// The first message of type `mtype`. + pub(crate) fn find(&self, mtype: u16) -> Option { + self.msgs.iter().position(|m| m.mtype == mtype) + } + + pub(crate) fn data(&self, img: &Image<'_>, i: usize) -> Result, Error> { + let m = &self.msgs[i]; + img.read(m.data_pos, m.size) + } + + /// Overwrite bytes of message `i`'s body, from `offset`. + pub(crate) fn patch( + &mut self, + img: &mut Image<'_>, + i: usize, + offset: usize, + bytes: &[u8], + ) -> Result<(), Error> { + let m = &self.msgs[i]; + if offset + bytes.len() > m.size { + return Err(Error::Unsupported( + "change does not fit the header message".into(), + )); + } + img.write(m.data_pos + offset as u64, bytes)?; + self.dirty.insert(m.chunk); + Ok(()) + } + + fn write_msg_header( + &mut self, + img: &mut Image<'_>, + hdr_pos: u64, + mtype: u16, + size: usize, + flags: u8, + corder: Option, + ) -> Result<(), Error> { + let mut h = vec![0u8; self.hsize()]; + if self.version == 1 { + h[0..2].copy_from_slice(&mtype.to_le_bytes()); + h[2..4].copy_from_slice(&(size as u16).to_le_bytes()); + h[4] = flags; + } else { + h[0] = mtype as u8; + h[1..3].copy_from_slice(&(size as u16).to_le_bytes()); + h[3] = flags; + if h.len() == 6 { + h[4..6].copy_from_slice(&corder.unwrap_or(0).to_le_bytes()); + } + } + img.write(hdr_pos, &h) + } + + /// Turn message `i` into a NIL message (its space becomes free). + pub(crate) fn delete(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> { + let m = self.msgs[i].clone(); + self.write_msg_header(img, m.hdr_pos, MSG_NIL, m.size, 0, m.corder)?; + img.write(m.data_pos, &vec![0u8; m.size])?; + self.msgs[i].mtype = MSG_NIL; + self.msgs[i].flags = 0; + self.dirty.insert(m.chunk); + self.absorb_gap(img, i) + } + + /// Body size a message of `len` bytes occupies (version 1 pads to 8). + fn padded(&self, len: usize) -> usize { + if self.version == 1 { + len.next_multiple_of(8) + } else { + len + } + } + + /// Whether a free slot of `slot` bytes can take a body of `need` bytes: + /// exactly, or with room left for a NIL message after it. + fn fits(&self, slot: usize, need: usize) -> bool { + slot == need || slot >= need + self.hsize() + } + + /// The smallest NIL message that can take `need` body bytes. + fn best_nil(&self, need: usize) -> Option { + self.msgs + .iter() + .enumerate() + .filter(|(_, m)| m.mtype == MSG_NIL && self.fits(m.size, need)) + .min_by_key(|(_, m)| m.size) + .map(|(i, _)| i) + } + + /// Put a message into slot `i` (a NIL message, or a message being + /// moved away), splitting off the rest as a NIL message. + fn place( + &mut self, + img: &mut Image<'_>, + i: usize, + mtype: u16, + flags: u8, + data: &[u8], + corder: Option, + ) -> Result<(), Error> { + let slot = self.msgs[i].clone(); + let need = self.padded(data.len()); + debug_assert!(self.fits(slot.size, need)); + let mut body = data.to_vec(); + body.resize(need, 0); + self.write_msg_header(img, slot.hdr_pos, mtype, need, flags, corder)?; + img.write(slot.data_pos, &body)?; + self.msgs[i] = Msg { + size: need, + mtype, + flags, + corder, + ..slot.clone() + }; + if slot.size > need { + let hs = self.hsize(); + let nil_hdr = slot.data_pos + need as u64; + let nil_size = slot.size - need - hs; + self.write_msg_header(img, nil_hdr, MSG_NIL, nil_size, 0, Some(0))?; + img.write(nil_hdr + hs as u64, &vec![0u8; nil_size])?; + self.msgs.push(Msg { + chunk: slot.chunk, + hdr_pos: nil_hdr, + data_pos: nil_hdr + hs as u64, + size: nil_size, + mtype: MSG_NIL, + flags: 0, + corder: (hs == 6).then_some(0), + }); + self.added += 1; + let nil = self.msgs.len() - 1; + self.absorb_gap(img, nil)?; + } + self.dirty.insert(slot.chunk); + Ok(()) + } + + /// Add a message: into free space in the header when there is some, + /// else into a new continuation chunk at the end of the file (whose + /// continuation message takes a NIL slot, or the slot of another + /// message — an attribute if possible — that moves into the new chunk + /// with it). + pub(crate) fn insert( + &mut self, + img: &mut Image<'_>, + mtype: u16, + flags: u8, + data: &[u8], + corder: Option, + ) -> Result<(), Error> { + if data.len() > usize::from(u16::MAX) { + return Err(Error::Unsupported( + "message larger than 64 KiB (would need dense storage)".into(), + )); + } + let need = self.padded(data.len()); + if let Some(i) = self.best_nil(need) { + return self.place(img, i, mtype, flags, data, corder); + } + let os = img.os as usize; + let ls = img.ls as usize; + let cont_need = self.padded(os + ls); + // Where the continuation message goes, and the message (if any) + // that moves out of that slot into the new chunk. + let (slot, moved) = match self.best_nil(cont_need) { + Some(i) => (i, None), + None => { + // Any message but a continuation can live in any chunk; + // prefer moving an attribute, then the smallest that fits. + let i = self + .msgs + .iter() + .enumerate() + .filter(|(_, m)| { + m.mtype != MSG_NIL + && m.mtype != MSG_CONTINUATION + && self.fits(m.size, cont_need) + }) + .min_by_key(|(_, m)| (m.mtype != MSG_ATTRIBUTE, m.size)) + .map(|(i, _)| i) + .ok_or_else(|| { + Error::Unsupported( + "no room in the object header for a continuation message".into(), + ) + })?; + let m = self.msgs[i].clone(); + let body = img.read(m.data_pos, m.size)?; + (i, Some((m, body))) + } + }; + + // The new chunk: [moved message] + new message + a NIL message + // holding spare room for later additions. + let hs = self.hsize(); + let spare = 64usize; + let mut payload = hs + need; + if let Some((m, _)) = &moved { + payload += hs + m.size; + } + let msgs_len = payload + hs + spare; + let (prefix, suffix) = if self.version == 2 { (4, 4) } else { (0, 0) }; + let chunk_len = prefix + msgs_len + suffix; + let caddr = img.alloc(chunk_len as u64)?; + if self.version == 2 { + img.write(caddr, b"OCHK")?; + } + let cidx = self.chunks.len(); + self.chunks.push(Chunk { + start: caddr, + checksum_at: (self.version == 2).then_some(caddr + (prefix + msgs_len) as u64), + end: caddr + (prefix + msgs_len) as u64, + gap: 0, + }); + let first = caddr + prefix as u64; + // Lay the chunk out as one NIL message, then place into it. + self.write_msg_header(img, first, MSG_NIL, msgs_len - hs, 0, Some(0))?; + self.msgs.push(Msg { + chunk: cidx, + hdr_pos: first, + data_pos: first + hs as u64, + size: msgs_len - hs, + mtype: MSG_NIL, + flags: 0, + corder: (hs == 6).then_some(0), + }); + self.added += 1; + if let Some((m, body)) = &moved { + let nil = self.msgs.len() - 1; + self.place(img, nil, m.mtype, m.flags, body, m.corder)?; + } + let nil = self.msgs.len() - 1; + self.place(img, nil, mtype, flags, data, corder)?; + + // Link it in. + let mut cont = vec![0u8; os + ls]; + put_uint(&mut cont, caddr, img.os); + put_uint(&mut cont[os..], chunk_len as u64, img.ls); + if moved.is_some() { + self.msgs[slot].mtype = MSG_NIL; // its content now lives in the new chunk + } + self.place(img, slot, MSG_CONTINUATION, 0, &cont, Some(0))?; + self.dirty.insert(cidx); + Ok(()) + } + + /// Recompute the checksum of every changed version-2 chunk; store a + /// version-1 header's new message count. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for &c in &self.dirty { + if let Some(at) = self.chunks[c].checksum_at { + rechecksum(img, self.chunks[c].start, at)?; + } + } + if self.version == 1 && self.added > 0 { + let old = u16::from_le_bytes(img.read(self.addr + 2, 2)?.try_into().unwrap_or([0; 2])); + let new = usize::from(old) + self.added; + let new = u16::try_from(new) + .map_err(|_| Error::Unsupported("too many object header messages".into()))?; + img.write(self.addr + 2, &new.to_le_bytes())?; + } + self.dirty.clear(); + self.added = 0; + Ok(()) + } +} diff --git a/crates/clawhdf5/src/edit/select.rs b/crates/clawhdf5/src/edit/select.rs new file mode 100644 index 0000000..4c34336 --- /dev/null +++ b/crates/clawhdf5/src/edit/select.rs @@ -0,0 +1,142 @@ +//! A selection as runs of consecutive elements along the last dimension, in +//! the order the selection's elements are numbered (row-major over a +//! hyperslab, as h5py and libhdf5 number them; a point list in its order). + +use clawhdf5_format::selection::Selection; + +use crate::error::Error; + +/// Call `f(coords, len, src)` for each run: `len` elements starting at +/// `coords` (consecutive in the last dimension), which are elements +/// `src..src + len` of the selection. Returns the number of elements. +/// The selection must already be validated against `dims`. +pub(crate) fn for_each_run( + sel: &Selection, + dims: &[u64], + mut f: impl FnMut(&[u64], u64, u64) -> Result<(), Error>, +) -> Result { + let rank = dims.len(); + let mut src = 0u64; + match sel { + Selection::None => {} + Selection::Points(pts) => { + for p in pts { + f(p, 1, src)?; + src += 1; + } + } + Selection::All => { + if rank == 0 { + f(&[], 1, 0)?; + return Ok(1); + } + if dims.contains(&0) { + return Ok(0); + } + let last = dims[rank - 1]; + let mut coords = vec![0u64; rank]; + loop { + f(&coords, last, src)?; + src += last; + if !advance(&mut coords[..rank - 1], &dims[..rank - 1]) { + break; + } + } + } + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + if rank == 0 { + return Err(Error::InvalidArgument( + "hyperslab selection on a scalar dataset".into(), + )); + } + if (0..rank).any(|d| count[d] == 0 || block[d] == 0) { + return Ok(0); + } + // Per-dimension extent of the selection: j in 0..count*block. + let ext: Vec = (0..rank).map(|d| count[d] * block[d]).collect(); + let coord = |d: usize, j: u64| start[d] + (j / block[d]) * stride[d] + j % block[d]; + let l = rank - 1; + // Along the last dimension, blocks merge when they touch. + let merged = stride[l] == block[l] || count[l] == 1; + let mut js = vec![0u64; rank - 1]; + let mut coords = vec![0u64; rank]; + loop { + for (d, &j) in js.iter().enumerate() { + coords[d] = coord(d, j); + } + if merged { + coords[l] = start[l]; + f(&coords, ext[l], src)?; + src += ext[l]; + } else { + for c in 0..count[l] { + coords[l] = start[l] + c * stride[l]; + f(&coords, block[l], src)?; + src += block[l]; + } + } + if !advance(&mut js, &ext[..l]) { + break; + } + } + } + } + Ok(src) +} + +/// Odometer step over `0..lim[d]`; false when it wraps around. +fn advance(v: &mut [u64], lim: &[u64]) -> bool { + for d in (0..v.len()).rev() { + v[d] += 1; + if v[d] < lim[d] { + return true; + } + v[d] = 0; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + fn collect(sel: &Selection, dims: &[u64]) -> Vec<(Vec, u64, u64)> { + let mut out = Vec::new(); + for_each_run(sel, dims, |c, n, s| { + out.push((c.to_vec(), n, s)); + Ok(()) + }) + .unwrap(); + out + } + + #[test] + fn runs() { + assert_eq!( + collect(&Selection::All, &[2, 3]), + vec![(vec![0, 0], 3, 0), (vec![1, 0], 3, 3)] + ); + let h = Selection::Hyperslab { + start: vec![1, 0], + stride: vec![2, 3], + count: vec![2, 2], + block: vec![1, 2], + }; + assert_eq!( + collect(&h, &[5, 6]), + vec![ + (vec![1, 0], 2, 0), + (vec![1, 3], 2, 2), + (vec![3, 0], 2, 4), + (vec![3, 3], 2, 6) + ] + ); + assert_eq!(collect(&Selection::All, &[]), vec![(vec![], 1, 0)]); + assert_eq!(collect(&Selection::All, &[0, 4]), vec![]); + } +} diff --git a/crates/clawhdf5/src/error.rs b/crates/clawhdf5/src/error.rs index de34355..110abe3 100644 --- a/crates/clawhdf5/src/error.rs +++ b/crates/clawhdf5/src/error.rs @@ -6,7 +6,10 @@ use clawhdf5_format::error::FormatError; use clawhdf5_format::message_type::MessageType; /// Errors that can occur when using the high-level API. +/// +/// Non-exhaustive: new kinds of failure may be added. #[derive(Debug)] +#[non_exhaustive] pub enum Error { /// I/O error from the filesystem. Io(std::io::Error), @@ -36,6 +39,16 @@ pub enum Error { /// Actual alignment of the data pointer. actual: usize, }, + /// The requested change is valid but not supported (by + /// [`FileEditor`](crate::FileEditor): a chunk index, filter or header + /// layout it cannot modify). Nothing was written. + Unsupported(String), + /// An argument does not fit the object (a selection outside the + /// dataset, a buffer of the wrong length, a shrinking resize, ...). + InvalidArgument(String), + /// The file is locked by another writer (another [`FileEditor`](crate::FileEditor), + /// or libhdf5 with file locking on). + Locked(String), } impl fmt::Display for Error { @@ -58,6 +71,9 @@ impl fmt::Display for Error { "zero-copy type mismatch: expected {expected}, got {actual}" ) } + Error::Unsupported(msg) => write!(f, "unsupported: {msg}"), + Error::InvalidArgument(msg) => write!(f, "invalid argument: {msg}"), + Error::Locked(msg) => write!(f, "file is locked: {msg}"), Error::ZeroCopyUnaligned { required, actual } => { write!( f, diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 222e432..280f64b 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -23,8 +23,25 @@ //! builder.set_attr("version", AttrValue::I64(1)); //! builder.write("output.h5").unwrap(); //! ``` +//! +//! # Modifying a file in place +//! +//! ```no_run +//! use clawhdf5::{FileEditor, Selection}; +//! +//! let mut ed = FileEditor::open("data.h5").unwrap(); +//! ed.resize("series", &[1100]).unwrap(); // a chunked dataset, maxshape (None,) +//! let tail = Selection::Hyperslab { +//! start: vec![1000], +//! stride: vec![1], +//! count: vec![100], +//! block: vec![1], +//! }; +//! ed.write_values("series", &tail, &[0.5f64; 100]).unwrap(); +//! ``` mod cache_image; +mod edit; pub mod error; pub mod lazy; #[cfg(feature = "mmap")] @@ -34,6 +51,7 @@ pub mod types; pub mod vlen; pub mod writer; +pub use edit::FileEditor; pub use error::Error; pub use lazy::{LazyDataset, LazyFile, LazyGroup}; #[cfg(feature = "mmap")] diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 1b20dd6..3a11e20 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -1104,13 +1104,18 @@ impl<'f> Dataset<'f> { .ok_or(Error::MissingMessage(msg_type)) } - fn datatype(&self) -> Result { + /// The dataset's object header as parsed. + pub(crate) fn header(&self) -> &ObjectHeader { + &self.header + } + + pub(crate) fn datatype(&self) -> Result { let data = self.required_payload(MessageType::Datatype)?; let (dt, _) = Datatype::parse_in_header(&data, self.header.version)?; Ok(dt) } - fn dataspace(&self) -> Result { + pub(crate) fn dataspace(&self) -> Result { let data = self.required_payload(MessageType::Dataspace)?; let mut ds = Dataspace::parse(&data, self.file.length_size())?; // libhdf5 reports a virtual dataset with unlimited or printf-style @@ -1130,7 +1135,7 @@ impl<'f> Dataset<'f> { Ok(ds) } - fn data_layout(&self) -> Result { + pub(crate) fn data_layout(&self) -> Result { let msg = find_message(&self.header, MessageType::DataLayout)?; Ok(DataLayout::parse( &msg.data, @@ -1143,7 +1148,7 @@ impl<'f> Dataset<'f> { /// that is present but unparseable is an error: treating it as "no /// filters" would hand the caller the still-compressed bytes as if they /// were the data. - fn filter_pipeline(&self) -> Result, Error> { + pub(crate) fn filter_pipeline(&self) -> Result, Error> { self.message_payload(MessageType::FilterPipeline)? .map(|data| FilterPipeline::parse(&data).map_err(Error::Format)) .transpose() diff --git a/crates/clawhdf5/tests/edit_tests.rs b/crates/clawhdf5/tests/edit_tests.rs new file mode 100644 index 0000000..2467256 --- /dev/null +++ b/crates/clawhdf5/tests/edit_tests.rs @@ -0,0 +1,138 @@ +//! `FileEditor` on files clawhdf5 writes, read back with our own reader +//! (libhdf5 interop is in `clawhdf5-tools/tests/edit_interop.rs`, which can +//! also run `h5rs check`). + +use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection}; + +fn block(start: u64, count: u64) -> Selection { + Selection::Hyperslab { + start: vec![start], + stride: vec![1], + count: vec![count], + block: vec![1], + } +} + +fn sample(dir: &std::path::Path) -> std::path::PathBuf { + let path = dir.join("f.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("ext") + .with_i32_data(&[0, 1, 2, 3, 4]) + .with_shape(&[5]) + .with_maxshape(&[u64::MAX]) + .with_chunks(&[4]) + .with_deflate(6); + b.create_dataset("raw") + .with_f64_data(&[0.5; 8]) + .with_shape(&[2, 4]) + .with_maxshape(&[u64::MAX, 4]) + .with_chunks(&[1, 4]); + b.create_dataset("flat").with_i64_data(&[1, 2, 3]); + b.set_attr("title", AttrValue::String("t".into())); + b.write(&path).unwrap(); + path +} + +#[test] +fn append_overwrite_and_attributes_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + let len_before = std::fs::metadata(&path).unwrap().len(); + let mut expect: Vec = (0..5).collect(); + let mut raw = vec![0.5f64; 8]; + { + let mut ed = FileEditor::open(&path).unwrap(); + for k in 0..300u64 { + let n = expect.len() as u64; + let add = 1 + k % 5; + ed.resize("ext", &[n + add]).unwrap(); + let vals: Vec = (0..add).map(|j| (n + j) as i32 * 2).collect(); + ed.write_values("ext", &block(n, add), &vals).unwrap(); + expect.extend(&vals); + } + // A filtered chunk rewritten with data that compresses worse moves. + let noisy: Vec = (0..4).map(|i| i * 7_919_993).collect(); + ed.write_values("ext", &block(0, 4), &noisy).unwrap(); + expect[..4].copy_from_slice(&noisy); + + ed.resize("raw", &[5, 4]).unwrap(); + raw.resize(20, 0.0); + let sel = Selection::Hyperslab { + start: vec![1, 1], + stride: vec![2, 2], + count: vec![2, 2], + block: vec![1, 1], + }; + ed.write_values("raw", &sel, &[1.0f64, 2.0, 3.0, 4.0]) + .unwrap(); + for (i, (r, c)) in [(1, 1), (1, 3), (3, 1), (3, 3)].iter().enumerate() { + raw[r * 4 + c] = i as f64 + 1.0; + } + ed.write_values("flat", &Selection::Points(vec![vec![2]]), &[30i64]) + .unwrap(); + ed.set_attr("/", "title", &AttrValue::String("a longer title".into())) + .unwrap(); + ed.set_attr("ext", "count", &AttrValue::I64(expect.len() as i64)) + .unwrap(); + } + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("ext").unwrap().read_i32().unwrap(), expect); + assert_eq!(f.dataset("raw").unwrap().shape().unwrap(), vec![5, 4]); + assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw); + assert_eq!( + f.dataset("flat").unwrap().read_i64().unwrap(), + vec![1, 2, 30] + ); + let root = f.root().attrs().unwrap(); + assert!(matches!(root.get("title"), Some(AttrValue::String(s)) if s == "a longer title")); + let ext = f.dataset("ext").unwrap().attrs().unwrap(); + assert!(matches!(ext.get("count"), Some(AttrValue::I64(n)) if *n == expect.len() as i64)); + assert!(std::fs::metadata(&path).unwrap().len() > len_before); +} + +#[test] +fn errors_leave_the_file_untouched() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + let before = std::fs::read(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); + assert!(ed.write_all("missing", &[0; 4]).is_err()); + // Wrong length, wrong type, outside the extent, beyond maxshape, + // shrinking, a rank change. + assert!(matches!( + ed.write_all("flat", &[0; 7]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.write_values("flat", &Selection::All, &[1i32, 2, 3]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.write_values("ext", &block(4, 2), &[1i32, 2]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.resize("raw", &[3, 5]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_)))); + assert!(matches!( + ed.resize("ext", &[4, 1]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.resize("flat", &[4]), + Err(Error::InvalidArgument(_)) + )); + assert!(matches!( + ed.set_attr("/", "", &AttrValue::I64(1)), + Err(Error::InvalidArgument(_)) + )); + // No-ops write nothing. + ed.resize("ext", &[5]).unwrap(); + ed.write_values("ext", &Selection::None, &[] as &[i32]) + .unwrap(); + drop(ed); + assert!(std::fs::read(&path).unwrap() == before); +}