From 17201e279dec2b4211a2ef809a96213c06f276ad Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:35:06 -0500 Subject: [PATCH 01/15] format: implicit chunk index addresses over the maximum chunk grid libhdf5 allocates an implicit index's chunks for the whole maximum extent and places chunk `scaled` at its row-major position in the maximum chunk grid (H5D__none_idx_get_addr, max_down_chunks). The reader used the current grid, so a dataset below its maximum shape with more chunk columns at its maximum read other chunks' values from the second chunk row on (h5py early allocation, fixed maxshape). generate_implicit_chunks_in_grid takes the maximum dimensions; generate_implicit_chunks keeps its signature (grid = current extent). Regression: implicit_chunks_use_the_maximum_grid here, and implicit_index_below_its_maximum_reads_like_libhdf5 (h5py file) with the editor tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/chunked_read.rs | 51 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index e249168..f428285 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -933,16 +933,40 @@ pub fn generate_implicit_chunks( dataset_dims: &[u64], chunk_dimensions: &[u32], element_size: u32, +) -> Vec { + generate_implicit_chunks_in_grid( + base_address, + dataset_dims, + dataset_dims, + chunk_dimensions, + element_size, + ) +} + +/// [`generate_implicit_chunks`] for a dataset whose maximum dimensions +/// (`max_dims`) exceed its current ones: libhdf5 allocates the chunks of +/// the whole maximum extent and places chunk `scaled` at its row-major +/// position in the *maximum* chunk grid (`H5D__none_idx_get_addr`, +/// `max_down_chunks`), so the current extent's chunks are not contiguous. +/// Only the chunks of the current extent are listed. +pub fn generate_implicit_chunks_in_grid( + base_address: u64, + dataset_dims: &[u64], + max_dims: &[u64], + chunk_dimensions: &[u32], + element_size: u32, ) -> Vec { let rank = chunk_dimensions.len(); let chunk_byte_size: u64 = chunk_dimensions.iter().map(|&d| d as u64).product::() * element_size as u64; let mut num_chunks_per_dim = Vec::with_capacity(rank); + let mut grid_per_dim = Vec::with_capacity(rank); for d in 0..rank { - let ds = dataset_dims[d]; let ch = chunk_dimensions[d] as u64; - num_chunks_per_dim.push(ds.div_ceil(ch)); + let n = dataset_dims[d].div_ceil(ch); + num_chunks_per_dim.push(n); + grid_per_dim.push(max_dims.get(d).map_or(n, |m| m.div_ceil(ch)).max(n)); } let total_chunks: u64 = num_chunks_per_dim.iter().product(); @@ -951,18 +975,22 @@ pub fn generate_implicit_chunks( for linear_idx in 0..total_chunks { let mut offsets = vec![0u64; rank]; let mut remaining = linear_idx; + let mut grid_idx = 0u64; + let mut down = 1u64; for d in (0..rank).rev() { let nchunks = num_chunks_per_dim[d]; let chunk_idx = remaining % nchunks; remaining /= nchunks; offsets[d] = chunk_idx * chunk_dimensions[d] as u64; + grid_idx = grid_idx.saturating_add(chunk_idx.saturating_mul(down)); + down = down.saturating_mul(grid_per_dim[d]); } chunks.push(ChunkInfo { chunk_size: chunk_byte_size as u32, filter_mask: 0, offsets, - address: base_address + linear_idx * chunk_byte_size, + address: base_address.saturating_add(grid_idx.saturating_mul(chunk_byte_size)), }); } @@ -1153,9 +1181,13 @@ pub fn list_chunks( (4, Some(2)) => { // Implicit index — use spatial chunk dims only let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank]; - generate_implicit_chunks( + generate_implicit_chunks_in_grid( addr, &dataspace.dimensions, + dataspace + .max_dimensions + .as_deref() + .unwrap_or(&dataspace.dimensions), spatial_chunk_dims, elem_size as u32, ) @@ -3015,6 +3047,17 @@ mod tests { } } + /// A dataset below its maximum extent: libhdf5 lays chunks out over the + /// maximum chunk grid, so row 1 starts after a whole maximum row (here + /// 4 chunks), not after the current row of 3. + #[test] + fn implicit_chunks_use_the_maximum_grid() { + let chunks = generate_implicit_chunks_in_grid(0x100, &[2, 3], &[4, 4], &[1, 1], 4); + let addrs: Vec = chunks.iter().map(|c| (c.address - 0x100) / 4).collect(); + assert_eq!(addrs, vec![0, 1, 2, 4, 5, 6]); + assert_eq!(chunks[3].offsets, vec![1, 0]); + } + #[test] fn implicit_chunks_partial_last() { // 25 elements, chunk size 10 => 3 chunks (last partial) From e9c71e5d2e7c237b7105caf9125d508be75d3aa2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:35:26 -0500 Subject: [PATCH 02/15] edit: version-2 B-tree chunk indexes, shrinking, early allocation FileEditor can now: - add, move and resize chunks of datasets with two or more unlimited dimensions (version-2 B-tree chunk index, record types 10/11). The new edit/btree2.rs follows libhdf5's H5B2 code: H5B2_update (modify, or insert into a leaf with room, or fall back to H5B2__insert), the preemptive split/redistribute loop with its two retries, split1, split_root (depth growth, node geometry per depth), redistribute2/3, cumulative record counts and the pointer widths H5B2__hdr_init derives, a checksum per node. Removal (H5B2_remove: merge2/3, redistribution, root collapse, the internal-record swap with a leaf) is there too. A dataset without an index yet gets one from the layout message's node size and split/merge percentages. - shrink a chunked dataset along any dimension (resize to a smaller shape), as H5D__set_extent / H5D__chunk_prune_by_extent do: the same chunks visited in the same order; chunks wholly outside the new extent are removed from the index (version-1 B-tree: H5B_remove with its sibling key and link fix-ups and the empty-root case; version-2 B-tree; Fixed/Extensible Array elements reset to the fill element; an implicit index keeps its chunks, as libhdf5 does) and their space noted as free; the part of each partial edge chunk outside the extent is overwritten with the fill value, so elements that come back after a later growth read as fill. - under early allocation, allocate and fill the chunks a growth brings in (H5D__chunk_allocate), which an implicit index needs: libhdf5 refills them, and they may hold the data of chunks pruned earlier. Tests (crates/clawhdf5-tools/tests/edit_coverage_interop.rs): growth in both dimensions of v110/latest files, unfiltered and deflated, gives node-for-node the version-2 B-tree libhdf5 builds (h5py with its chunk cache off, so chunks enter the index in the editor's order), through a depth increase; random chunk order; 60 random shrink/grow/write steps on Extensible Array, version-2 B-tree, Fixed Array, 1-D and implicit datasets (earliest/v110/latest, with and without gzip+shuffle) give the values h5py gets doing the same and the same index shape (version-1 and version-2 B-tree node shapes, Extensible Array statistics); h5py r+ continues on every result; h5dump and h5rs check accept them. edit_interop's version-2 B-tree case now appends instead of expecting a refusal; shrinking is no longer an error in edit_tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 832 ++++++++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 25 +- crates/clawhdf5/src/edit/btree1.rs | 167 ++- crates/clawhdf5/src/edit/btree2.rs | 1128 +++++++++++++++++ crates/clawhdf5/src/edit/earray.rs | 29 +- crates/clawhdf5/src/edit/farray.rs | 15 +- crates/clawhdf5/src/edit/image.rs | 17 + crates/clawhdf5/src/edit/mod.rs | 582 ++++++++- crates/clawhdf5/tests/edit_tests.rs | 32 +- 9 files changed, 2738 insertions(+), 89 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/edit_coverage_interop.rs create mode 100644 crates/clawhdf5/src/edit/btree2.rs diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs new file mode 100644 index 0000000..f2c27fd --- /dev/null +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -0,0 +1,832 @@ +//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends, +//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or +//! more unlimited dimensions), datasets shrunk, attributes in dense storage +//! (and moved there), and space reused within an editing session. Files are +//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by +//! clawhdf5; after each edit h5py must read exactly what a model says, +//! h5dump must read the file, `h5rs check --data` must find nothing, and our +//! reader must agree; h5py then opens the file `r+` and goes on. Where the +//! same operations can be done by libhdf5, the structures it builds (B-tree +//! shapes, heap and free-space bookkeeping) are compared with the editor's. +//! +//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips +//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::{Command, Output}; + +use clawhdf5::{Error, File, FileEditor, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn available(cmd: &str, args: &[&str]) -> bool { + Command::new(cmd) + .args(args) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Whether the tools are here; false (skip) or a panic when interop is +/// required. +fn tools_ok() -> bool { + let ok = + available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]); + if !ok { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available" + ); + eprintln!("SKIP: h5py/numpy or h5dump not available"); + } + ok +} + +fn py(script: &str) -> String { + let o = Command::new(python()) + .args(["-c", script]) + .output() + .expect("run python"); + assert!( + o.status.success(), + "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + String::from_utf8_lossy(&o.stdout).trim().to_string() +} + +fn text(o: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ) +} + +/// The file is valid for libhdf5's tools and for `h5rs check --data`. +/// HDF5 2.0 `latest` files are beyond h5dump 1.14. +fn check_tools(path: &Path, h5dump: bool) { + let p = path.to_str().unwrap(); + let o = Command::new(env!("CARGO_BIN_EXE_h5rs")) + .args(["check", "--data", "-q", p]) + .output() + .unwrap(); + assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o)); + if h5dump { + let o = Command::new("h5dump") + .args(["-o", "/dev/null", p]) + .output() + .unwrap(); + assert!( + o.status.success() && o.stderr.is_empty(), + "h5dump {p}:\n{}", + text(&o) + ); + } +} + +/// A dataset's expected contents. +#[derive(Clone, Debug)] +struct Model { + shape: Vec, + /// Row-major values. + data: Vec, +} + +impl Model { + fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self { + let n: u64 = shape.iter().product(); + Self { + shape: shape.to_vec(), + data: (0..n).map(f).collect(), + } + } + + fn index(&self, c: &[u64]) -> usize { + c.iter() + .zip(&self.shape) + .fold(0u64, |a, (&x, &d)| a * d + x) as usize + } + + /// Change the extent to `shape`: elements inside both keep their + /// values, new ones are `fill`. + fn resize(&mut self, shape: &[u64], fill: i32) { + let old = self.clone(); + *self = Self::new(shape, |_| fill); + let n: u64 = old.shape.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; old.shape.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = r % old.shape[d]; + r /= old.shape[d]; + } + if c.iter().zip(shape).all(|(x, s)| x < s) { + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } + } + } + + /// Apply a hyperslab write of `vals` (row-major over the block). + fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) { + let n: u64 = count.iter().product(); + for flat in 0..n { + let mut c = vec![0u64; count.len()]; + let mut r = flat; + for d in (0..c.len()).rev() { + c[d] = start[d] + r % count[d]; + r /= count[d]; + } + let i = self.index(&c); + self.data[i] = vals[flat as usize]; + } + } +} + +fn block(start: &[u64], count: &[u64]) -> Selection { + Selection::Hyperslab { + start: start.to_vec(), + stride: vec![1; start.len()], + count: count.to_vec(), + block: vec![1; start.len()], + } +} + +/// h5py and our reader both read `m` from dataset `name`. +fn verify(path: &Path, name: &str, m: &Model) { + let f = File::open(path).unwrap(); + let ds = f.dataset(name).unwrap(); + assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}"); + let got = ds.read_i32().unwrap(); + if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) { + panic!( + "our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}", + path.display(), + m.shape, + got[i], + m.data[i] + ); + } + let exp = path.with_extension("expect"); + let bytes: Vec = m.data.iter().flat_map(|v| v.to_le_bytes()).collect(); + std::fs::write(&exp, bytes).unwrap(); + let shape: Vec = m.shape.iter().map(|d| d.to_string()).collect(); + py(&format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + a = f[{name:?}][()]\n\ + e = np.fromfile({e:?}, dtype=' u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +fn tmpdir() -> tempfile::TempDir { + let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR")); + tempfile::TempDir::new_in(base).unwrap() +} + +#[allow(dead_code)] +fn unsupported(r: Result) { + match r { + Err(Error::Unsupported(_)) => {} + other => panic!("expected Error::Unsupported, got {other:?}"), + } +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} + +fn enc_size(v: u64) -> usize { + ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize +} + +/// The shape of the version-2 B-tree whose header is at `hdr` in `b` +/// (8-byte addresses and lengths): its depth, record count, and every +/// node's (depth, records) breadth-first. +fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) { + let h = &b[hdr..]; + assert_eq!(&h[0..4], b"BTHD"); + let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap())); + let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap())); + let depth = u16::from_le_bytes(h[12..14].try_into().unwrap()); + let root = le(&h[16..24]); + let root_n = le(&h[24..26]); + let total = le(&h[26..34]); + // Node geometry, as H5B2__hdr_init computes it. + let leaf_max = (node_size - 10) / rs; + let nrec_w = enc_size(leaf_max); + let mut cum = vec![(leaf_max, 0usize)]; + for d in 1..=u64::from(depth) { + let (below, below_w) = cum[d as usize - 1]; + let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 }; + let max = (node_size - 10 - ptr) / (rs + ptr); + let c = (max + 1) * below + max; + cum.push((c, enc_size(c))); + } + let mut out = Vec::new(); + let mut level = vec![(root, root_n)]; + let mut dep = depth; + loop { + let mut next = Vec::new(); + for &(addr, n) in &level { + out.push((dep, n)); + if dep == 0 { + assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF"); + continue; + } + let node = &b[addr as usize..]; + assert_eq!(&node[0..4], b"BTIN"); + let mut q = 6 + (n * rs) as usize; + let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 }; + for _ in 0..=n { + let a = le(&node[q..q + 8]); + let c = le(&node[q + 8..q + 8 + nrec_w]); + q += 8 + nrec_w + all_w; + next.push((a, c)); + } + } + if dep == 0 { + break; + } + dep -= 1; + level = next; + } + (depth, total, out) +} + +/// The shape of the file's only chunk B-tree (record type 10 or 11). +fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) { + let b = std::fs::read(path).unwrap(); + let hdrs: Vec = b + .windows(6) + .enumerate() + .filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11)) + .map(|(i, _)| i) + .collect(); + assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display()); + bt2_shape_at(&b, hdrs[0]) +} + +/// Grow a dataset with two unlimited dimensions in both directions many +/// times, writing the new parts, with libhdf5 (its chunk cache off, so each +/// chunk enters the index as it is written, in the same order as the +/// editor's) and with the editor: the same values, and the same B-tree — +/// depth, record count, and every node's record count — through leaf and +/// internal splits, redistributions and a depth increase. +fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) { + let dir = tmpdir(); + let a = dir.path().join(format!("bt2_{tag}_h5py.h5")); + let b = dir.path().join(format!("bt2_{tag}_edit.h5")); + // Extents: both dimensions grow; new columns fall between existing + // chunks in the index's key order (row first). + let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect(); + let val = |r: u64, c: u64| (r * 1000 + c) as i32; + let create = |p: &Path, write: bool| { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\ + \x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \ + dtype=' c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\ + \x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\ + \x20 r0, c0 = r, c\n", + p = p.to_str().unwrap(), + write = if write { "True" } else { "False" }, + )) + }; + create(&a, true); + create(&b, false); + let mut m = Model::new(&[0, 0], |_| 0); + let mut ed = FileEditor::open(&b).unwrap(); + let (mut r0, mut c0) = (0u64, 0u64); + for &(r, c) in &steps { + ed.resize("x", &[r, c]).unwrap(); + m.resize(&[r, c], 0); + if c > c0 && r0 > 0 { + let vals: Vec = (0..r0) + .flat_map(|i| (c0..c).map(move |j| val(i, j))) + .collect(); + ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals) + .unwrap(); + m.write_block(&[0, c0], &[r0, c - c0], &vals); + } + if r > r0 { + let vals: Vec = (r0..r) + .flat_map(|i| (0..c).map(move |j| val(i, j))) + .collect(); + ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals) + .unwrap(); + m.write_block(&[r0, 0], &[r - r0, c], &vals); + } + (r0, c0) = (r, c); + } + drop(ed); + verify(&b, "x", &m); + verify(&a, "x", &m); + check_tools(&b, h5dump); + let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b)); + assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}"); + assert_eq!(sb, sa, "B-tree shape differs from libhdf5's"); + // libhdf5 goes on growing what the editor built. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 r, c = d.shape\n\ + \x20 d.resize((r + 3, c + 2))\n\ + \x20 d[:, c:] = -1\n\ + \x20 d[r:, :] = -2\n", + p = b.to_str().unwrap() + )); + let (r, c) = (m.shape[0], m.shape[1]); + m.resize(&[r + 3, c + 2], 0); + m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]); + m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]); + verify(&b, "x", &m); + check_tools(&b, h5dump); +} + +/// Dataset `name`'s layout: its chunk index type (0 for a version-1 +/// B-tree), index address and chunk rank (element-size dimension +/// included). +fn chunk_index(path: &Path, name: &str) -> (u8, Option, usize) { + use clawhdf5_format::data_layout::DataLayout; + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + let f = File::open(path).unwrap(); + let sb = f.superblock(); + let (os, ls) = (sb.offset_size, sb.length_size); + let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap(); + let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap(); + let m = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + .unwrap(); + match DataLayout::parse(&m.data, os, ls).unwrap() { + DataLayout::Chunked { + chunk_dimensions, + btree_address, + chunk_index_type, + .. + } => ( + chunk_index_type.unwrap_or(0), + btree_address, + chunk_dimensions.len(), + ), + other => panic!("{other:?}"), + } +} + +/// A version-1 chunk B-tree's shape: every node's (level, entries), +/// breadth-first from the root. +fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> { + let ks = 8 + 8 * ndims; + let mut out = Vec::new(); + let mut level = vec![root]; + while !level.is_empty() { + let mut next = Vec::new(); + for a in level { + let n = &b[a as usize..]; + assert_eq!(&n[0..5], b"TREE\x01", "node at {a}"); + let lv = n[5]; + let e = le(&n[6..8]); + out.push((lv, e)); + if lv > 0 { + for i in 0..e as usize { + next.push(le( + &n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8] + )); + } + } + } + level = next; + } + out +} + +/// The shape of dataset `name`'s chunk index, for comparing with the one +/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or +/// nothing to compare (Fixed Array, implicit). +fn index_shape(path: &Path, name: &str) -> String { + let b = std::fs::read(path).unwrap(); + match chunk_index(path, name) { + (_, None, _) => "none".into(), + (0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)), + (5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)), + (4, Some(a), _) => { + let s: Vec = (0..6) + .map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k])) + .collect(); + format!("ea {s:?}") + } + (t, Some(_), _) => format!("type {t}"), + } +} + +/// One step of a resize workload. +#[derive(Debug, Clone)] +enum Op { + Resize(Vec), + /// A block write; values from `vals`. + Write(Vec, Vec, u64), +} + +fn op_vals(seed: u64, n: u64) -> Vec { + (0..n) + .map(|i| ((i * 31 + seed * 7919) % 100_000) as i32) + .collect() +} + +/// Random resizes (shrinking and growing any dimension within `max`) +/// and block writes. +fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec { + let mut shape = start.to_vec(); + let mut ops = Vec::new(); + for k in 0..n { + if k % 3 == 0 { + shape = (0..shape.len()) + .map(|d| { + let cap = max[d].min(shape[d] * 2 + 6); + rng.below(cap + 1) + }) + .collect(); + ops.push(Op::Resize(shape.clone())); + } else if shape.iter().all(|&s| s > 0) { + let st: Vec = shape.iter().map(|&s| rng.below(s)).collect(); + let cnt: Vec = (0..shape.len()) + .map(|d| 1 + rng.below((shape[d] - st[d]).min(6))) + .collect(); + ops.push(Op::Write(st, cnt, rng.next() % 1000)); + } + } + ops +} + +/// The python statements applying `ops` to dataset `d`. +fn py_ops(ops: &[Op]) -> String { + let mut s = String::new(); + for op in ops { + match op { + Op::Resize(shape) => { + s += &format!("\x20 d.resize({shape:?})\n") + .replace('[', "(") + .replace(']', ",)"); + } + Op::Write(st, cnt, seed) => { + let n: u64 = cnt.iter().product(); + let sl: Vec = st + .iter() + .zip(cnt) + .map(|(a, c)| format!("{a}:{}", a + c)) + .collect(); + s += &format!( + "\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\ + .astype(' Vec { + s.trim_matches(|c| c == '[' || c == ']') + .split(',') + .filter(|t| !t.trim().is_empty()) + .map(|t| t.trim().parse().unwrap()) + .collect() + }; + (p(x), p(y)) + }) + .unwrap(); + let initial = py(&format!( + "import h5py\n\ + d = h5py.File({p:?}, 'r')['x']\n\ + print(' '.join(str(v) for v in d[()].ravel()))\n", + p = b.to_str().unwrap() + )); + let mut m = Model { + shape: start.clone(), + data: initial + .split_whitespace() + .map(|v| v.parse().unwrap()) + .collect(), + }; + let mut rng = Rng(seed); + let ops = random_resize_ops(&mut rng, &start, &max, 60); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\ + \x20 d = f['x']\n{o}", + p = a.to_str().unwrap(), + o = py_ops(&ops) + )); + let mut ed = FileEditor::open(&b).unwrap(); + for (k, op) in ops.iter().enumerate() { + match op { + Op::Resize(s) => { + ed.resize("x", s) + .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}")); + m.resize(s, fill); + } + Op::Write(st, cnt, seed) => { + let vals = op_vals(*seed, cnt.iter().product()); + ed.write_values("x", &block(st, cnt), &vals) + .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}")); + m.write_block(st, cnt, &vals); + } + } + } + drop(ed); + if tag == "implicit" { + // An implicit index cannot drop chunks: libhdf5 leaves the data of + // chunks wholly outside a shrunk extent in place and does not + // refill those that come back unless they lie past the extent it + // grows from, so they can show old values. The editor must match + // libhdf5, not the model. + let got = py(&format!( + "import h5py\n\ + d = h5py.File({p:?}, 'r')['x']\n\ + print(' '.join(str(v) for v in d[()].ravel()))\n", + p = a.to_str().unwrap() + )); + m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect(); + } + verify(&a, "x", &m); + verify(&b, "x", &m); + check_tools(&b, h5dump); + assert_eq!( + index_shape(&b, "x"), + index_shape(&a, "x"), + "{tag}: chunk index differs from libhdf5's" + ); + // libhdf5 goes on: grow back to the start shape and write everything. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 d = f['x']\n\ + \x20 d.resize({s:?})\n\ + \x20 d[...] = 3\n", + p = b.to_str().unwrap(), + s = start + )); + let n: u64 = start.iter().product(); + verify( + &b, + "x", + &Model { + shape: start.clone(), + data: vec![3; n as usize], + }, + ); + check_tools(&b, h5dump); +} + +/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every +/// dimension, mixed with growth and writes, on every chunk index: the +/// version-1 B-tree (`earliest`), Extensible Array (one unlimited +/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape) +/// and implicit (early allocation), unfiltered and deflated. +#[test] +fn shrink_matches_libhdf5() { + if !tools_ok() { + return; + } + let kinds: &[(&str, &str, i32)] = &[ + ( + "ea", + "\x20 d = f.create_dataset('x', data=np.arange(40, dtype=' = (0..cnt[0] * cnt[1]) + .map(|_| (rng.next() % 1_000_000) as i32) + .collect(); + ed.write_values("x", &block(&[r0, c0], &cnt), &vals) + .unwrap(); + m.write_block(&[r0, c0], &cnt, &vals); + if step % 100 == 99 { + drop(ed); + verify(&path, "x", &m); + check_tools(&path, false); + ed = FileEditor::open(&path).unwrap(); + } + } + drop(ed); + verify(&path, "x", &m); + let shape = chunk_bt2_shape(&path); + assert!(shape.0 >= 1, "{shape:?}"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['x'][...] = 5\n", + p = path.to_str().unwrap() + )); + let n = m.data.len(); + m.data = vec![5; n]; + verify(&path, "x", &m); + check_tools(&path, false); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 85d4d0c..6a2dbf6 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -752,19 +752,20 @@ fn overwrite_every_layout() { } check_tools(&path, *dump); } - // A version-2 B-tree index can take new chunks only from libhdf5 for - // now: growing works, writing the new chunks is refused and changes - // nothing. + // A version-2 B-tree index takes new chunks too. if *lv != "'earliest'" { let mut ed = FileEditor::open(&path).unwrap(); - ed.resize("bt2", &[8, 6]).unwrap(); - let before = std::fs::read(&path).unwrap(); - unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12])); - assert!( - std::fs::read(&path).unwrap() == before, - "a refused edit changed the file" - ); - models[12].resize(&[8, 6], 0); + ed.resize("bt2", &[8, 7]).unwrap(); + models[12].resize(&[8, 7], 0); + let vals: Vec = (0..23).collect(); + ed.write_values("bt2", &block(&[6, 0], &[2, 7]), &vals[..14]) + .unwrap(); + models[12].write_block(&[6, 0], &[2, 7], &vals[..14]); + ed.write_values("bt2", &block(&[0, 6], &[6, 1]), &vals[14..20]) + .unwrap(); + models[12].write_block(&[0, 6], &[6, 1], &vals[14..20]); + drop(ed); + verify(&path, "bt2", &models[12]); } // libhdf5 goes on modifying what we wrote. py(&format!( @@ -1027,7 +1028,7 @@ fn refused_edits_change_nothing() { let before = std::fs::read(&path).unwrap(); let mut ed = FileEditor::open(&path).unwrap(); unsupported(ed.write_all("s", &[0u8; 32])); - unsupported(ed.resize("x", &[4])); + unsupported(ed.resize("c", &[4])); unsupported( ed.resize("c", &[6, 1]) .map_err(|_| Error::Unsupported(String::new())), diff --git a/crates/clawhdf5/src/edit/btree1.rs b/crates/clawhdf5/src/edit/btree1.rs index 7289c3f..25c626a 100644 --- a/crates/clawhdf5/src/edit/btree1.rs +++ b/crates/clawhdf5/src/edit/btree1.rs @@ -56,6 +56,15 @@ fn bad(why: &str) -> Error { )) } +/// What a removal did below a node (`H5B_ins_t`), with the removed chunk's +/// address and size. +enum Rm { + NotFound, + Noop((u64, u32)), + /// The child is gone: the parent must drop it. + Remove((u64, u32)), +} + enum Ins { Done, /// The node split; the new right sibling and its first key. @@ -144,7 +153,14 @@ impl BTree1 { put_uint(&mut d[8 + osz..], node.right, os); let ks = self.key_size(); let mut p = 8 + 2 * osz; - for (i, k) in node.keys.iter().enumerate() { + // An empty node (a root whose last chunk was removed) stores no + // keys, as libhdf5 writes it. + let nkeys = if node.children.is_empty() { + 0 + } else { + node.keys.len() + }; + for (i, k) in node.keys.iter().take(nkeys).enumerate() { d[p..p + 4].copy_from_slice(&k.size.to_le_bytes()); d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes()); for (j, o) in k.offs.iter().enumerate() { @@ -210,6 +226,18 @@ impl BTree1 { return Err(bad("bad chunk key")); } let root = self.read(img, self.root)?; + if root.children.is_empty() { + // Every chunk was removed (H5B__insert_helper's first + // insertion): the root, a leaf again, takes it. + let right = self.right_key_after(&key); + let node = Node { + level: 0, + keys: vec![key, right], + children: vec![addr], + ..root + }; + return self.write(img, &node); + } if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? { // The root split: move its (left) half to a new node so the root // keeps its address, then make the root the parent of both. @@ -372,6 +400,143 @@ impl BTree1 { cmp(&key.offs, &right.keys[0].offs) != Ordering::Less } + /// Remove the chunk at offsets `offs` (element-size coordinate 0), as + /// `H5B_remove` does for the chunk index (whose critical key is the + /// left one): no rebalancing; a node left without children is deleted + /// and its siblings relinked (the left one takes over its right key), + /// a root left empty becomes an empty leaf. Returns the chunk's address + /// and stored size, or `None` when the tree has no such chunk (nothing + /// changes then). Deleted nodes are freed in `img`. + pub(crate) fn remove( + &mut self, + img: &mut Image<'_>, + offs: &[u64], + ) -> Result, Error> { + if offs.len() != self.ndims { + return Err(bad("bad chunk key")); + } + let mut lt = None; + match self.remove_at(img, self.root, 0, offs, &mut lt, 64)? { + Rm::NotFound => Ok(None), + Rm::Noop(c) | Rm::Remove(c) => Ok(Some(c)), + } + } + + fn remove_at( + &self, + img: &mut Image<'_>, + addr: u64, + level: usize, + offs: &[u64], + lt_out: &mut Option, + depth: u8, + ) -> Result { + if depth == 0 { + return Err(bad("tree too deep")); + } + let mut node = self.read(img, addr)?; + let n = node.children.len(); + // H5D__btree_cmp3 over (keys[i], keys[i + 1]), binary search. + let (mut lo, mut hi, mut idx) = (0usize, n, 0usize); + let mut c = 1i32; + while lo < hi && c != 0 { + idx = (lo + hi) / 2; + c = if cmp(offs, &node.keys[idx + 1].offs) != Ordering::Less { + 1 + } else if cmp(offs, &node.keys[idx].offs) == Ordering::Less { + -1 + } else { + 0 + }; + if c < 0 { + hi = idx; + } else { + lo = idx + 1; + } + } + if c != 0 { + return Ok(Rm::NotFound); + } + let mut lt_changed = None; + let res = if node.level > 0 { + let child = self.read(img, node.children[idx])?; + if usize::from(child.level) + 1 != usize::from(node.level) { + return Err(bad("inconsistent node levels")); + } + self.remove_at( + img, + node.children[idx], + level + 1, + offs, + &mut lt_changed, + depth - 1, + )? + } else { + if node.keys[idx].offs != offs { + return Ok(Rm::NotFound); + } + Rm::Remove((node.children[idx], node.keys[idx].size)) + }; + let chunk = match res { + Rm::NotFound => return Ok(Rm::NotFound), + Rm::Noop(c) | Rm::Remove(c) => c, + }; + let mut dirty = false; + if let Some(k) = lt_changed { + node.keys[idx] = k; + dirty = true; + if idx == 0 { + *lt_out = Some(node.keys[0].clone()); + } + } + let out = Rm::Noop(chunk); + if let Rm::Remove(_) = res { + let undefined = undef(img.os); + if n == 1 { + if level > 0 { + if node.left != undefined { + let mut sib = self.read(img, node.left)?; + let last = sib.children.len(); + sib.keys[last] = node.keys[1].clone(); + sib.right = node.right; + self.write(img, &sib)?; + } + if node.right != undefined { + let mut sib = self.read(img, node.right)?; + sib.left = node.left; + self.write(img, &sib)?; + } + img.free(addr, self.node_size(img.os) as u64); + return Ok(Rm::Remove(chunk)); + } + node.children.clear(); + node.keys.truncate(1); + node.level = 0; + } else if idx == 0 { + node.keys.remove(0); + node.children.remove(0); + *lt_out = Some(node.keys[0].clone()); + } else { + // Right-most or middle child: its left key goes, the next + // key becomes the following child's left key. + node.keys.remove(idx); + node.children.remove(idx); + } + dirty = true; + } + if dirty { + self.write(img, &node)?; + } + // The left sibling's right key follows a changed left key. + if lt_out.is_some() && node.left != undef(img.os) && level > 0 { + let mut sib = self.read(img, node.left)?; + let last = sib.children.len(); + sib.keys[last] = node.keys[0].clone(); + self.write(img, &sib)?; + } + Ok(out) + } + fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) { let n = node.children.len(); if node.level == 0 { diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs new file mode 100644 index 0000000..8ff7205 --- /dev/null +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -0,0 +1,1128 @@ +//! Changing a version-2 B-tree (`BTHD`/`BTIN`/`BTLF`) in place, as libhdf5's +//! `H5B2` code does: the same insertion (`H5B2_update`, falling back to +//! `H5B2__insert` when the leaf is full), node splits and redistributions +//! (`H5B2__split1`, `H5B2__split_root`, `H5B2__redistribute2/3`), and removal +//! with merges and redistributions (`H5B2_remove`, `H5B2__merge2/3`, root +//! collapse, the internal-record swap with a leaf) — so a tree the editor +//! changes has the nodes libhdf5 would have made for the same operations. +//! +//! Split and merge thresholds come from the header's split and merge +//! percentages; each node pointer carries the child's record count and, +//! below the first internal level, the child subtree's total, in the widths +//! libhdf5 derives from the node size (`H5B2__hdr_init`). Every node written +//! gets its checksum; the header's depth, root and counts are updated by +//! [`Bt2::finish`]. +//! +//! Nodes are loaded on first use, changed in memory and written back by +//! [`Bt2::finish`]; nodes a merge empties are dropped and their space handed +//! to the image's free list ([`Image::free`]). + +use std::cmp::Ordering; +use std::collections::{BTreeSet, HashMap}; + +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +/// A comparison of the record being looked for against a stored record +/// (`Less`: the target sorts before the stored record). +pub(crate) type Cmp<'c> = dyn FnMut(&Image<'_>, &[u8]) -> Result + 'c; + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("version-2 B-tree: {why}"), + )) +} + +/// A pointer to a node, as its parent (or the header, for the root) holds +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Ptr { + addr: u64, + nrec: u16, + all: u64, +} + +#[derive(Debug, Clone)] +struct Node { + depth: u16, + recs: Vec>, + /// `recs.len() + 1` children (internal nodes only). + ptrs: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct Level { + max_nrec: u64, + split_nrec: u64, + merge_nrec: u64, + cum_max_nrec: u64, + cum_max_nrec_size: usize, +} + +/// `H5VM_limit_enc_size`: bytes to encode values up to `v`. +fn enc_size(v: u64) -> usize { + ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize +} + +pub(crate) struct Bt2 { + addr: u64, + tree_type: u8, + node_size: u32, + rec_size: usize, + split_pct: u8, + merge_pct: u8, + depth: u16, + root: Ptr, + levels: Vec, + max_nrec_size: usize, + nodes: HashMap, + dirty: BTreeSet, + hdr_dirty: bool, +} + +/// What `update` did below a node. +#[derive(Debug, PartialEq, Eq)] +enum Status { + Modified, + Inserted, + ChildFull, +} + +const MAX_DEPTH: u16 = 32; + +impl Bt2 { + fn header_len(os: u8, ls: u8) -> usize { + 16 + os as usize + 2 + ls as usize + 4 + } + + /// Open the tree whose header is at `addr`. + pub(crate) fn open(img: &Image<'_>, addr: u64) -> Result { + let len = Self::header_len(img.os, img.ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"BTHD" || d[4] != 0 { + return Err(bad("bad header")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let os = img.os as usize; + let mut t = Self { + addr, + tree_type: d[5], + node_size: u32::from_le_bytes([d[6], d[7], d[8], d[9]]), + rec_size: usize::from(u16::from_le_bytes([d[10], d[11]])), + depth: u16::from_le_bytes([d[12], d[13]]), + split_pct: d[14], + merge_pct: d[15], + root: Ptr { + addr: get_uint(&d[16..], img.os), + nrec: u16::from_le_bytes([d[16 + os], d[17 + os]]), + all: get_uint(&d[18 + os..], img.ls), + }, + levels: Vec::new(), + max_nrec_size: 0, + nodes: HashMap::new(), + dirty: BTreeSet::new(), + hdr_dirty: false, + }; + if t.depth > MAX_DEPTH || t.rec_size == 0 || t.split_pct == 0 || t.split_pct > 100 { + return Err(bad("unsupported header parameters")); + } + t.compute_levels(img.os)?; + Ok(t) + } + + /// Create an empty tree (header only, undefined root), as + /// `H5B2_create` does. + pub(crate) fn create( + img: &mut Image<'_>, + tree_type: u8, + node_size: u32, + rec_size: usize, + split_pct: u8, + merge_pct: u8, + ) -> Result { + let addr = img.alloc(Self::header_len(img.os, img.ls) as u64)?; + let mut t = Self { + addr, + tree_type, + node_size, + rec_size, + split_pct, + merge_pct, + depth: 0, + root: Ptr { + addr: undef(img.os), + nrec: 0, + all: 0, + }, + levels: Vec::new(), + max_nrec_size: 0, + nodes: HashMap::new(), + dirty: BTreeSet::new(), + hdr_dirty: true, + }; + t.compute_levels(img.os)?; + t.write_header(img)?; + Ok(t) + } + + pub(crate) fn address(&self) -> u64 { + self.addr + } + + pub(crate) fn record_size(&self) -> usize { + self.rec_size + } + + pub(crate) fn tree_type(&self) -> u8 { + self.tree_type + } + + /// Records in the whole tree. + pub(crate) fn len(&self) -> u64 { + self.root.all + } + + /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, + /// `H5B2__split_root`). + fn compute_levels(&mut self, os: u8) -> Result<(), Error> { + let ns = u64::from(self.node_size); + let rs = self.rec_size as u64; + let leaf_max = ns.saturating_sub(10) / rs; + if leaf_max < 2 || leaf_max > u64::from(u16::MAX) { + return Err(bad("node size holds too few or too many records")); + } + self.max_nrec_size = enc_size(leaf_max); + let pct = |n: u64, p: u8| n * u64::from(p) / 100; + let mut levels = vec![Level { + max_nrec: leaf_max, + split_nrec: pct(leaf_max, self.split_pct), + merge_nrec: pct(leaf_max, self.merge_pct), + cum_max_nrec: leaf_max, + cum_max_nrec_size: 0, + }]; + for d in 1..=self.depth { + let below = levels[usize::from(d) - 1]; + let ptr = u64::from(os) + + self.max_nrec_size as u64 + + if d > 1 { + below.cum_max_nrec_size as u64 + } else { + 0 + }; + let max = ns.saturating_sub(10 + ptr) / (rs + ptr); + if max < 2 { + return Err(bad("internal nodes hold too few records")); + } + let cum = max + .saturating_add(1) + .saturating_mul(below.cum_max_nrec) + .saturating_add(max); + levels.push(Level { + max_nrec: max, + split_nrec: pct(max, self.split_pct), + merge_nrec: pct(max, self.merge_pct), + cum_max_nrec: cum, + cum_max_nrec_size: enc_size(cum), + }); + } + if levels.iter().any(|l| l.split_nrec == 0) { + return Err(bad("split percentage too small")); + } + self.levels = levels; + Ok(()) + } + + fn ptr_size(&self, os: u8, depth: u16) -> usize { + os as usize + + self.max_nrec_size + + if depth > 1 { + self.levels[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + } + } + + /// Load the node `p` points at (a node of `depth`). + fn load(&mut self, img: &Image<'_>, p: Ptr, depth: u16) -> Result<(), Error> { + if let Some(n) = self.nodes.get(&p.addr) { + if n.depth != depth || n.recs.len() != usize::from(p.nrec) { + return Err(bad("node does not match its pointer")); + } + return Ok(()); + } + if p.addr == undef(img.os) { + return Err(bad("undefined node address")); + } + let nrec = usize::from(p.nrec); + if nrec as u64 > self.levels[usize::from(depth)].max_nrec { + return Err(bad("node holds more records than it can")); + } + let ps = if depth > 0 { + self.ptr_size(img.os, depth) + } else { + 0 + }; + let body = 6 + nrec * self.rec_size + if depth > 0 { (nrec + 1) * ps } else { 0 }; + if body + 4 > self.node_size as usize { + return Err(bad("node overflows its size")); + } + let d = img.read(p.addr, body + 4)?; + let sig: &[u8] = if depth == 0 { b"BTLF" } else { b"BTIN" }; + if &d[0..4] != sig || d[4] != 0 || d[5] != self.tree_type { + return Err(bad("bad node signature")); + } + let stored = u32::from_le_bytes(d[body..body + 4].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..body]) != stored { + return Err(bad("node checksum mismatch")); + } + let mut q = 6; + let mut recs = Vec::with_capacity(nrec); + for _ in 0..nrec { + recs.push(d[q..q + self.rec_size].to_vec()); + q += self.rec_size; + } + let mut ptrs = Vec::new(); + if depth > 0 { + let all_w = if depth > 1 { + self.levels[usize::from(depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for _ in 0..=nrec { + let addr = get_uint(&d[q..], img.os); + q += img.os as usize; + let n = le(&d[q..q + self.max_nrec_size]); + q += self.max_nrec_size; + let all = if depth > 1 { + let a = le(&d[q..q + all_w]); + q += all_w; + a + } else { + n + }; + let n = u16::try_from(n).map_err(|_| bad("record count"))?; + ptrs.push(Ptr { addr, nrec: n, all }); + } + } + self.nodes.insert(p.addr, Node { depth, recs, ptrs }); + Ok(()) + } + + fn node(&mut self, addr: u64) -> &mut Node { + self.dirty.insert(addr); + self.nodes.get_mut(&addr).expect("node loaded") + } + + fn peek(&self, addr: u64) -> &Node { + self.nodes.get(&addr).expect("node loaded") + } + + fn new_node(&mut self, img: &mut Image<'_>, depth: u16) -> Result { + let addr = img.alloc_reusing(u64::from(self.node_size))?; + self.nodes.insert( + addr, + Node { + depth, + recs: Vec::new(), + ptrs: Vec::new(), + }, + ); + self.dirty.insert(addr); + Ok(addr) + } + + fn drop_node(&mut self, img: &mut Image<'_>, addr: u64) { + self.nodes.remove(&addr); + self.dirty.remove(&addr); + img.free(addr, u64::from(self.node_size)); + } + + /// `H5B2__locate_record`: binary search; the index and the comparison + /// there. + fn locate( + img: &Image<'_>, + recs: &[Vec], + cmp: &mut Cmp<'_>, + ) -> Result<(usize, Ordering), Error> { + let (mut lo, mut hi) = (0usize, recs.len()); + let mut idx = 0; + let mut c = Ordering::Less; + while lo < hi && c != Ordering::Equal { + idx = (lo + hi) / 2; + c = cmp(img, &recs[idx])?; + if c == Ordering::Less { + hi = idx; + } else { + lo = idx + 1; + } + } + Ok((idx, c)) + } + + /// The record matching `cmp`, if any. + pub(crate) fn find( + &mut self, + img: &Image<'_>, + cmp: &mut Cmp<'_>, + ) -> Result>, Error> { + if self.root.addr == undef(img.os) || self.root.nrec == 0 { + return Ok(None); + } + let mut p = self.root; + let mut depth = self.depth; + loop { + self.load(img, p, depth)?; + let n = self.peek(p.addr); + let (idx, c) = Self::locate(img, &n.recs, cmp)?; + if c == Ordering::Equal { + return Ok(Some(n.recs[idx].clone())); + } + if depth == 0 { + return Ok(None); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + p = n.ptrs[i]; + depth -= 1; + } + } + + /// Every record, in key order. + pub(crate) fn records(&mut self, img: &Image<'_>) -> Result>, Error> { + let mut out = Vec::new(); + if self.root.addr != undef(img.os) && self.root.nrec > 0 { + self.walk(img, self.root, self.depth, &mut out)?; + } + Ok(out) + } + + fn walk( + &mut self, + img: &Image<'_>, + p: Ptr, + depth: u16, + out: &mut Vec>, + ) -> Result<(), Error> { + self.load(img, p, depth)?; + let n = self.peek(p.addr).clone(); + for i in 0..=n.recs.len() { + if depth > 0 { + self.walk(img, n.ptrs[i], depth - 1, out)?; + } + if i < n.recs.len() { + out.push(n.recs[i].clone()); + } + } + Ok(()) + } + + /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). + pub(crate) fn update( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + if rec.len() != self.rec_size { + return Err(bad("record of the wrong size")); + } + if self.root.addr == undef(img.os) { + let a = self.new_node(img, 0)?; + self.root = Ptr { + addr: a, + nrec: 0, + all: 0, + }; + self.hdr_dirty = true; + } + let mut root = self.root; + let st = self.update_at(img, &mut root, self.depth, cmp, rec)?; + self.root = root; + self.hdr_dirty = true; + if st == Status::ChildFull { + self.insert(img, cmp, rec)?; + } + Ok(()) + } + + fn update_at( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result { + self.load(img, *p, depth)?; + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c == Ordering::Equal { + self.node(p.addr).recs[idx] = rec.to_vec(); + return Ok(Status::Modified); + } + if depth == 0 { + if u64::from(p.nrec) == self.levels[0].split_nrec { + return Ok(Status::ChildFull); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + self.node(p.addr).recs.insert(i, rec.to_vec()); + p.nrec += 1; + p.all += 1; + return Ok(Status::Inserted); + } + let i = if c == Ordering::Greater { idx + 1 } else { idx }; + let mut child = self.peek(p.addr).ptrs[i]; + let st = self.update_at(img, &mut child, depth - 1, cmp, rec)?; + match st { + Status::Modified => Ok(st), + Status::Inserted => { + self.node(p.addr).ptrs[i] = child; + p.all += 1; + Ok(st) + } + Status::ChildFull => { + // H5B2__update_internal: a full child is split or + // redistributed here, unless this node is full too and the + // child and a neighbour are close to full (the split could + // reach this node): then the caller deals with it. + let d = usize::from(depth); + let (nrec, ptrs) = { + let n = self.peek(p.addr); + (n.recs.len(), n.ptrs.clone()) + }; + if nrec as u64 == self.levels[d].split_nrec { + let near = (self.levels[d - 1].split_nrec * 2).saturating_sub(1); + let pair = |a: usize, b: usize| { + u64::from(ptrs[a].nrec) + u64::from(ptrs[b].nrec) >= near + }; + let could_split = if i == 0 { + pair(0, 1) + } else if i == nrec { + pair(i - 1, i) + } else { + pair(i - 1, i) || pair(i, i + 1) + }; + if could_split { + return Ok(Status::ChildFull); + } + } + self.insert_internal(img, p, depth, cmp, rec)?; + Ok(Status::Inserted) + } + } + } + + /// `H5B2__insert`: insert a record that is not in the tree, splitting + /// or redistributing full nodes on the way down. + pub(crate) fn insert( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + if rec.len() != self.rec_size { + return Err(bad("record of the wrong size")); + } + if self.root.addr == undef(img.os) { + let a = self.new_node(img, 0)?; + self.root = Ptr { + addr: a, + nrec: 0, + all: 0, + }; + } else if u64::from(self.root.nrec) == self.levels[usize::from(self.depth)].split_nrec { + self.split_root(img)?; + } + self.hdr_dirty = true; + let mut root = self.root; + if self.depth > 0 { + self.insert_internal(img, &mut root, self.depth, cmp, rec)?; + } else { + self.insert_leaf(img, &mut root, cmp, rec)?; + } + self.root = root; + Ok(()) + } + + fn insert_leaf( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + self.load(img, *p, 0)?; + let mut i = 0; + if p.nrec > 0 { + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c == Ordering::Equal { + return Err(bad("record is already in the tree")); + } + i = if c == Ordering::Greater { idx + 1 } else { idx }; + } + if u64::from(p.nrec) >= self.levels[0].max_nrec { + return Err(bad("leaf full")); + } + self.node(p.addr).recs.insert(i, rec.to_vec()); + p.nrec += 1; + p.all += 1; + Ok(()) + } + + fn insert_internal( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + cmp: &mut Cmp<'_>, + rec: &[u8], + ) -> Result<(), Error> { + self.load(img, *p, depth)?; + let child_split = self.levels[usize::from(depth) - 1].split_nrec; + let mut i = self.child_index(img, p.addr, cmp)?; + // Preemptively split or redistribute the child to enter; a + // redistribution can move the insertion point into another full + // child, so after two tries it splits. + let mut retries = 2u32; + while u64::from(self.peek(p.addr).ptrs[i].nrec) == child_split { + let nrec = self.peek(p.addr).recs.len(); + let ptrs = self.peek(p.addr).ptrs.clone(); + let room = |j: usize| retries > 0 && u64::from(ptrs[j].nrec) < child_split; + if i == 0 { + if room(1) { + self.redistribute2(img, p.addr, depth, 0)?; + } else { + self.split1(img, p, depth, 0)?; + } + } else if i == nrec { + if room(i - 1) { + self.redistribute2(img, p.addr, depth, i - 1)?; + } else { + self.split1(img, p, depth, i)?; + } + } else if room(i + 1) || room(i - 1) { + self.redistribute3(img, p.addr, depth, i)?; + } else { + self.split1(img, p, depth, i)?; + } + i = self.child_index(img, p.addr, cmp)?; + retries = retries.saturating_sub(1); + } + let mut child = self.peek(p.addr).ptrs[i]; + if depth > 1 { + self.insert_internal(img, &mut child, depth - 1, cmp, rec)?; + } else { + self.insert_leaf(img, &mut child, cmp, rec)?; + } + self.node(p.addr).ptrs[i] = child; + p.all += 1; + Ok(()) + } + + /// The child of internal node `addr` to descend into for an insertion; + /// refuses a record that is already there. + fn child_index( + &mut self, + img: &Image<'_>, + addr: u64, + cmp: &mut Cmp<'_>, + ) -> Result { + let n = self.peek(addr); + if n.recs.is_empty() { + return Ok(0); + } + let (idx, c) = Self::locate(img, &n.recs, cmp)?; + match c { + Ordering::Equal => Err(bad("record is already in the tree")), + Ordering::Greater => Ok(idx + 1), + Ordering::Less => Ok(idx), + } + } + + /// Load the children `lo..=hi` of internal node `parent` (at `depth`). + fn load_children( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + lo: usize, + hi: usize, + ) -> Result<(), Error> { + for i in lo..=hi { + let c = self.peek(parent).ptrs[i]; + self.load(img, c, depth - 1)?; + } + Ok(()) + } + + /// Total records under a node with these records and children. + fn subtree_total(nrec: usize, ptrs: &[Ptr]) -> u64 { + nrec as u64 + ptrs.iter().map(|p| p.all).sum::() + } + + /// Lay `recs`/`ptrs` (the concatenation of several siblings and the + /// parent records between them) out over the children `lo..` of + /// `parent`, with `counts[k]` records in the k-th; the records between + /// them go back into the parent. + fn relayout( + &mut self, + parent: u64, + depth: u16, + lo: usize, + addrs: &[u64], + mut recs: Vec>, + mut ptrs: Vec, + counts: &[usize], + ) { + let internal = depth > 1; + let mut new_ptrs = Vec::with_capacity(addrs.len()); + let mut seps = Vec::with_capacity(addrs.len().saturating_sub(1)); + for (k, &a) in addrs.iter().enumerate() { + let n = counts[k]; + let rest_r = recs.split_off(n); + let mine_r = std::mem::replace(&mut recs, rest_r); + let mine_p = if internal { + let rest_p = ptrs.split_off(n + 1); + std::mem::replace(&mut ptrs, rest_p) + } else { + Vec::new() + }; + if k + 1 < addrs.len() { + seps.push(recs.remove(0)); + } + let all = if internal { + Self::subtree_total(n, &mine_p) + } else { + n as u64 + }; + new_ptrs.push(Ptr { + addr: a, + nrec: n as u16, + all, + }); + let node = self.node(a); + node.recs = mine_r; + node.ptrs = mine_p; + } + let pn = self.node(parent); + for (k, s) in seps.into_iter().enumerate() { + pn.recs[lo + k] = s; + } + for (k, p) in new_ptrs.into_iter().enumerate() { + pn.ptrs[lo + k] = p; + } + } + + /// The children `lo..=hi` of `parent` and the parent records between + /// them, concatenated. + fn gather(&self, parent: u64, lo: usize, hi: usize) -> (Vec, Vec>, Vec) { + let pn = self.peek(parent); + let mut addrs = Vec::new(); + let mut recs = Vec::new(); + let mut ptrs = Vec::new(); + for i in lo..=hi { + let c = self.peek(pn.ptrs[i].addr); + addrs.push(pn.ptrs[i].addr); + recs.extend(c.recs.iter().cloned()); + ptrs.extend(c.ptrs.iter().copied()); + if i < hi { + recs.push(pn.recs[i].clone()); + } + } + (addrs, recs, ptrs) + } + + /// `H5B2__redistribute2`: even out children `idx` and `idx + 1`. + fn redistribute2( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + self.load_children(img, parent, depth, idx, idx + 1)?; + let (l, r) = { + let pn = self.peek(parent); + ( + usize::from(pn.ptrs[idx].nrec), + usize::from(pn.ptrs[idx + 1].nrec), + ) + }; + let (nl, nr) = if l < r { + let nr = (l + r) / 2; + (l + r - nr, nr) + } else { + let nl = (l + r) / 2; + (nl, l + r - nl) + }; + let (addrs, recs, ptrs) = self.gather(parent, idx, idx + 1); + self.relayout(parent, depth, idx, &addrs, recs, ptrs, &[nl, nr]); + Ok(()) + } + + /// `H5B2__redistribute3`: even out children `idx - 1`, `idx`, `idx + 1`. + fn redistribute3( + &mut self, + img: &Image<'_>, + parent: u64, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + self.load_children(img, parent, depth, idx - 1, idx + 1)?; + let total = { + let pn = self.peek(parent); + (idx - 1..=idx + 1) + .map(|i| usize::from(pn.ptrs[i].nrec)) + .sum::() + + 2 + }; + let nm = (total - 2) / 3; + let nl = ((total - 2) - nm) / 2; + let nr = (total - 2) - (nl + nm); + let (addrs, recs, ptrs) = self.gather(parent, idx - 1, idx + 1); + self.relayout(parent, depth, idx - 1, &addrs, recs, ptrs, &[nl, nm, nr]); + Ok(()) + } + + /// `H5B2__split1`: split child `idx` of the node `p` points at, which + /// gains a record and a child. + fn split1( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx, idx)?; + let right = self.new_node(img, depth - 1)?; + let old = usize::from(self.peek(parent).ptrs[idx].nrec); + let mid = old / 2; + { + let pn = self.node(parent); + pn.recs.insert(idx, Vec::new()); + pn.ptrs.insert( + idx + 1, + Ptr { + addr: right, + nrec: 0, + all: 0, + }, + ); + } + let left = self.peek(parent).ptrs[idx].addr; + let (recs, ptrs) = { + let l = self.peek(left); + (l.recs.clone(), l.ptrs.clone()) + }; + self.relayout( + parent, + depth, + idx, + &[left, right], + recs, + ptrs, + &[mid, old - mid - 1], + ); + p.nrec += 1; + Ok(()) + } + + /// `H5B2__split_root`: the tree grows a level. + fn split_root(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + if self.depth >= MAX_DEPTH { + return Err(bad("tree too deep")); + } + self.depth += 1; + self.compute_levels(img.os)?; + let old = self.root; + let new_root = self.new_node(img, self.depth)?; + self.node(new_root).ptrs.push(old); + let mut root = Ptr { + addr: new_root, + nrec: 0, + all: old.all, + }; + let depth = self.depth; + self.split1(img, &mut root, depth, 0)?; + self.root = root; + self.hdr_dirty = true; + Ok(()) + } + + /// Remove the record `cmp` matches (`H5B2_remove`); returns it, or + /// `None` when the tree has no such record (nothing changes then). + pub(crate) fn remove( + &mut self, + img: &mut Image<'_>, + cmp: &mut Cmp<'_>, + ) -> Result>, Error> { + let Some(found) = self.find(img, cmp)? else { + return Ok(None); + }; + let mut root = self.root; + if self.depth > 0 { + let mut decreased = false; + self.remove_internal(img, &mut root, self.depth, true, None, cmp, &mut decreased)?; + if decreased { + self.depth -= 1; + self.compute_levels(img.os)?; + } + } else { + self.remove_leaf(img, &mut root, cmp)?; + } + root.all -= 1; + self.root = root; + self.hdr_dirty = true; + Ok(Some(found)) + } + + fn remove_leaf( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + cmp: &mut Cmp<'_>, + ) -> Result<(), Error> { + self.load(img, *p, 0)?; + let (idx, c) = Self::locate(img, &self.peek(p.addr).recs, cmp)?; + if c != Ordering::Equal { + return Err(bad("record to remove not found")); + } + self.node(p.addr).recs.remove(idx); + if self.peek(p.addr).recs.is_empty() { + self.drop_node(img, p.addr); + p.addr = undef(img.os); + } + p.nrec -= 1; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn remove_internal( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + is_root: bool, + mut swap: Option<(u64, usize)>, + cmp: &mut Cmp<'_>, + decreased: &mut bool, + ) -> Result<(), Error> { + self.load(img, *p, depth)?; + let merge = self.levels[usize::from(depth) - 1].merge_nrec; + let addr = p.addr; + let pn = self.peek(addr); + // libhdf5 checks every level; only a root can hold a single record. + if is_root + && pn.recs.len() == 1 + && u64::from(pn.ptrs[0].nrec) + u64::from(pn.ptrs[1].nrec) <= merge * 2 + 1 + { + // Collapse the root: merge its two children into the left one, + // which becomes the root. + let mut dummy = *p; + self.merge2(img, &mut dummy, depth, 0)?; + let child = self.peek(addr).ptrs[0]; + self.drop_node(img, addr); + p.addr = child.addr; + p.nrec = child.nrec; + *decreased = true; + if depth > 1 { + return self.remove_internal(img, p, depth - 1, true, swap, cmp, decreased); + } + return self.remove_leaf(img, p, cmp); + } + let mut idx; + let mut c = Ordering::Less; + if swap.is_some() { + idx = 0; + } else { + let (i, cc) = Self::locate(img, &pn.recs, cmp)?; + c = cc; + idx = if cc != Ordering::Less { i + 1 } else { i }; + } + let mut retries = 2u32; + while u64::from(self.peek(addr).ptrs[idx].nrec) == merge { + let n = self.peek(addr).recs.len(); + if n == 0 { + return Err(bad("internal node without records")); + } + let ptrs = self.peek(addr).ptrs.clone(); + let spare = |j: usize| retries > 0 && u64::from(ptrs[j].nrec) > merge; + if idx == 0 { + if spare(1) { + self.redistribute2(img, addr, depth, 0)?; + } else { + self.merge2(img, p, depth, 0)?; + } + } else if idx == n { + if spare(idx - 1) { + self.redistribute2(img, addr, depth, idx - 1)?; + } else { + self.merge2(img, p, depth, idx - 1)?; + } + } else if spare(idx + 1) || spare(idx - 1) { + self.redistribute3(img, addr, depth, idx)?; + } else { + self.merge3(img, p, depth, idx)?; + } + if swap.is_some() { + idx = 0; + } else { + let (i, cc) = Self::locate(img, &self.peek(addr).recs, cmp)?; + c = cc; + idx = if cc != Ordering::Less { i + 1 } else { i }; + } + retries = retries.saturating_sub(1); + } + if swap.is_none() && c == Ordering::Equal { + swap = Some((addr, idx - 1)); + } + if let (Some((sa, si)), 1) = (swap, depth) { + // H5B2__swap_leaf: the record to delete trades places with the + // leaf's first record. + let child = self.peek(addr).ptrs[idx]; + self.load(img, child, 0)?; + let leaf_first = self.peek(child.addr).recs[0].clone(); + let deleted = std::mem::replace(&mut self.node(sa).recs[si], leaf_first); + self.node(child.addr).recs[0] = deleted; + } + let mut child = self.peek(addr).ptrs[idx]; + if depth > 1 { + self.remove_internal(img, &mut child, depth - 1, false, swap, cmp, decreased)?; + } else { + self.remove_leaf(img, &mut child, cmp)?; + } + child.all -= 1; + self.node(addr).ptrs[idx] = child; + Ok(()) + } + + /// `H5B2__merge2`: merge child `idx + 1` of the node `p` points at into + /// child `idx`. + fn merge2( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx, idx + 1)?; + let (addrs, recs, ptrs) = self.gather(parent, idx, idx + 1); + let n = recs.len(); + { + let pn = self.node(parent); + pn.recs.remove(idx); + pn.ptrs.remove(idx + 1); + } + self.relayout(parent, depth, idx, &addrs[..1], recs, ptrs, &[n]); + self.drop_node(img, addrs[1]); + p.nrec -= 1; + Ok(()) + } + + /// `H5B2__merge3`: merge children `idx - 1`, `idx`, `idx + 1` of the + /// node `p` points at into two. + fn merge3( + &mut self, + img: &mut Image<'_>, + p: &mut Ptr, + depth: u16, + idx: usize, + ) -> Result<(), Error> { + let parent = p.addr; + self.load_children(img, parent, depth, idx - 1, idx + 1)?; + let (addrs, recs, ptrs) = self.gather(parent, idx - 1, idx + 1); + let total = recs.len(); + let nl = (total - 1) / 2; + { + let pn = self.node(parent); + pn.recs.remove(idx); + pn.ptrs.remove(idx + 1); + } + self.relayout( + parent, + depth, + idx - 1, + &addrs[..2], + recs, + ptrs, + &[nl, total - 1 - nl], + ); + self.drop_node(img, addrs[2]); + p.nrec -= 1; + Ok(()) + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let len = Self::header_len(img.os, img.ls); + let os = img.os as usize; + let mut d = vec![0u8; len]; + d[0..4].copy_from_slice(b"BTHD"); + d[5] = self.tree_type; + d[6..10].copy_from_slice(&self.node_size.to_le_bytes()); + d[10..12].copy_from_slice(&(self.rec_size as u16).to_le_bytes()); + d[12..14].copy_from_slice(&self.depth.to_le_bytes()); + d[14] = self.split_pct; + d[15] = self.merge_pct; + put_uint(&mut d[16..], self.root.addr, img.os); + d[16 + os..18 + os].copy_from_slice(&self.root.nrec.to_le_bytes()); + put_uint(&mut d[18 + os..], self.root.all, img.ls); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]); + d[len - 4..].copy_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d)?; + self.hdr_dirty = false; + Ok(()) + } + + /// Write every changed node and the header. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + for addr in std::mem::take(&mut self.dirty) { + let Some(n) = self.nodes.get(&addr) else { + continue; + }; + let mut d = Vec::with_capacity(self.node_size as usize); + d.extend_from_slice(if n.depth == 0 { b"BTLF" } else { b"BTIN" }); + d.push(0); + d.push(self.tree_type); + for r in &n.recs { + d.extend_from_slice(r); + } + if n.depth > 0 { + let all_w = if n.depth > 1 { + self.levels[usize::from(n.depth) - 1].cum_max_nrec_size + } else { + 0 + }; + for p in &n.ptrs { + let mut a = vec![0u8; img.os as usize]; + put_uint(&mut a, p.addr, img.os); + d.extend_from_slice(&a); + d.extend_from_slice(&u64::from(p.nrec).to_le_bytes()[..self.max_nrec_size]); + if n.depth > 1 { + d.extend_from_slice(&p.all.to_le_bytes()[..all_w]); + } + } + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + if d.len() > self.node_size as usize { + return Err(bad("node overflows its size")); + } + d.resize(self.node_size as usize, 0); + img.write(addr, &d)?; + } + if self.hdr_dirty { + self.write_header(img)?; + } + Ok(()) + } +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} diff --git a/crates/clawhdf5/src/edit/earray.rs b/crates/clawhdf5/src/edit/earray.rs index 49d9621..bae196f 100644 --- a/crates/clawhdf5/src/edit/earray.rs +++ b/crates/clawhdf5/src/edit/earray.rs @@ -383,12 +383,23 @@ impl Ea { } /// Set element `idx` to `e`. - pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + /// Set element `idx` to `e`, or back to the fill element (`None`: a + /// removed chunk, `H5D__earray_idx_remove`), which creates no block. + pub(crate) fn set( + &mut self, + img: &mut Image<'_>, + idx: u64, + e: Option, + ) -> Result<(), Error> { let os = img.os; let osz = u64::from(os); let es = self.slot_size(os) as u64; - let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let enc = encode_elem(e, self.filtered, self.elem_size, os)?; + let clear = e.is_none(); if self.iblock == undef(os) { + if clear { + return Ok(()); + } self.create_iblock(img)?; } let ib = self.iblock; @@ -420,6 +431,9 @@ impl Ea { let dblk_idx = l.start_dblk + local; let slot = dblks_at + dblk_idx * osz; let mut addr = get_uint(&img.read(slot, os as usize)?, os); + if addr == undef(os) && clear { + return Ok(()); + } if addr == undef(os) { // libhdf5 records start_idx + (global data block index) // * nelmts here (H5EA__lookup_elmt), not the block's @@ -449,6 +463,9 @@ impl Ea { let sb_prefix = self.dblk_prefix_len(os); let sb_len = sb_prefix + bitmap_len + l.ndblks * osz; let mut sb = get_uint(&img.read(sslot, os as usize)?, os); + if sb == undef(os) && clear { + return Ok(()); + } if sb == undef(os) { let mut d = self.block_prefix(b"EASB", l.start_idx, os); d.resize(d.len() + bitmap_len as usize, 0); @@ -471,6 +488,9 @@ impl Ea { let local = (rel - l.start_idx) / l.dblk_nelmts; let dslot = sb + sb_prefix + bitmap_len + local * osz; let mut addr = get_uint(&img.read(dslot, os as usize)?, os); + if addr == undef(os) && clear { + return Ok(()); + } if addr == undef(os) { let off = l.start_idx + local * l.dblk_nelmts; addr = self.create_dblock(img, l.dblk_nelmts, off)?; @@ -491,6 +511,9 @@ impl Ea { let bpos = sb + sb_prefix + bit / 8; let mut byte = img.read(bpos, 1)?[0]; let mask = 0x80u8 >> (bit % 8); + if byte & mask == 0 && clear { + return Ok(()); + } if byte & mask == 0 { let fill = self.fill_elems(page, os)?; img.write(page_at, &fill)?; @@ -503,7 +526,7 @@ impl Ea { } } } - if idx + 1 > self.stats[4] { + if !clear && idx + 1 > self.stats[4] { self.stats[4] = idx + 1; self.dirty_hdr = true; } diff --git a/crates/clawhdf5/src/edit/farray.rs b/crates/clawhdf5/src/edit/farray.rs index 0a25fe7..e7a1696 100644 --- a/crates/clawhdf5/src/edit/farray.rs +++ b/crates/clawhdf5/src/edit/farray.rs @@ -137,13 +137,19 @@ impl Fa { Ok((fa, hdr)) } - /// Set element `idx` to `e`. - pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> { + /// Set element `idx` to `e`, or back to the fill element (`None`: a + /// removed chunk, `H5D__farray_idx_remove`), which creates no page. + pub(crate) fn set( + &mut self, + img: &mut Image<'_>, + idx: u64, + e: Option, + ) -> Result<(), Error> { let os = img.os; if idx >= self.nelmts { return Err(bad("index beyond the array")); } - let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?; + let enc = encode_elem(e, self.filtered, self.elem_size, os)?; let es = self.slot(os); let prefix = 6 + u64::from(os); let page = self.page(); @@ -162,6 +168,9 @@ impl Fa { let bpos = self.dblk + prefix + p / 8; let mut byte = img.read(bpos, 1)?[0]; let mask = 0x80u8 >> (p % 8); + if byte & mask == 0 && e.is_none() { + return Ok(()); + } if byte & mask == 0 { let fill = encode_elem(None, self.filtered, self.elem_size, os)?; img.write(page_at, &fill.repeat(count as usize))?; diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index 254abcd..722534d 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -35,6 +35,8 @@ pub(crate) struct Image<'a> { /// Width of addresses and lengths in the file. pub(crate) os: u8, pub(crate) ls: u8, + /// Space the edit stopped using. + freed: Vec<(u64, u64)>, } impl<'a> Image<'a> { @@ -47,6 +49,7 @@ impl<'a> Image<'a> { old_eoa: eoa, os, ls, + freed: Vec::new(), } } @@ -77,6 +80,20 @@ impl<'a> Image<'a> { Ok(addr) } + /// Allocate `size` bytes for metadata or data, from space an earlier + /// edit of this session freed when there is a block that fits, + /// otherwise at the end of the file. + pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result { + self.alloc(size) + } + + /// Note that the edit no longer uses `[addr, addr + len)`. + pub(crate) fn free(&mut self, addr: u64, len: u64) { + if len > 0 { + self.freed.push((addr, len)); + } + } + /// If `[addr, addr + old_len)` is the last allocated space, grow it to /// `new_len` bytes (a structure at the end of the file can grow where /// it is) and return true. diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 4903672..b14244a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -16,6 +16,7 @@ //! is alive while the file changes (see `image`). mod btree1; +mod btree2; mod earray; mod farray; mod image; @@ -39,6 +40,7 @@ use crate::error::Error; use crate::reader::File; use crate::types::AttrValue; use btree1::{BTree1, Key}; +use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; use image::{Image, get_uint, put_uint, undef}; @@ -249,6 +251,14 @@ impl Target { fn dims(&self) -> &[u64] { &self.ds.dimensions } + + /// The chunk index (or implicit chunks') address. + fn layout_address(&self) -> Option { + match &self.layout { + DataLayout::Chunked { btree_address, .. } => *btree_address, + _ => None, + } + } } /// Datatypes whose stored bytes point elsewhere in the file (variable-length @@ -331,7 +341,7 @@ enum IndexEdit { Implicit, Fixed(Option), Extensible(Option), - BTree2, + BTree2(Option), } struct ChunkedEdit<'t> { @@ -398,7 +408,9 @@ impl<'t> ChunkedEdit<'t> { (_, Some(4)) => { IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?) } - (_, Some(5)) => IndexEdit::BTree2, + (_, Some(5)) => { + IndexEdit::BTree2(btree_address.map(|a| Bt2::open(img, a)).transpose()?) + } (v, i) => { return Err(Error::Unsupported(format!( "chunked layout version {v}, index type {i:?}" @@ -520,7 +532,7 @@ impl<'t> ChunkedEdit<'t> { self.patch_layout_addr(img, hdr_addr)?; } if let IndexEdit::Fixed(Some(fa)) = &mut self.index { - fa.set(img, idx, e)?; + fa.set(img, idx, Some(e))?; } } IndexEdit::Extensible(ea) => { @@ -549,30 +561,165 @@ impl<'t> ChunkedEdit<'t> { self.patch_layout_addr(img, hdr_addr)?; } if let IndexEdit::Extensible(Some(ea)) = &mut self.index { - ea.set(img, idx, e)?; + ea.set(img, idx, Some(e))?; } } - IndexEdit::BTree2 => { - return Err(Error::Unsupported( - "adding, moving or resizing a chunk in a version-2 B-tree chunk index \ - (datasets with more than one unlimited dimension)" - .into(), - )); + IndexEdit::BTree2(tree) => { + if tree.is_none() { + let at = self + .lpos + .params + .ok_or_else(|| Error::Unsupported("B-tree v2 parameters".into()))?; + let d = self.hdr.data(img, self.layout_msg)?; + let node_size = u32::from_le_bytes([d[at], d[at + 1], d[at + 2], d[at + 3]]); + let size_len = if filtered { + earray::chunk_size_len(self.chunk_bytes as u64, self.lpos.version) + 4 + } else { + 0 + }; + let rec_size = img.os as usize + size_len + 8 * self.cd.len(); + let new = Bt2::create( + img, + if filtered { 11 } else { 10 }, + node_size, + rec_size, + d[at + 4], + d[at + 5], + )?; + let a = new.address(); + *tree = Some(new); + self.patch_layout_addr(img, a)?; + } + let IndexEdit::BTree2(Some(tree)) = &mut self.index else { + unreachable!("created above") + }; + let rec = bt2_chunk_record(tree, scaled, e, filtered, img.os)?; + let mut cmp = bt2_chunk_cmp(scaled, tree.record_size()); + tree.update(img, &mut cmp, &rec)?; } } Ok(()) } + /// Remove chunk `scaled` (stored at `info`) from the index, as the + /// index's `remove` operation does when libhdf5 prunes a dataset, and + /// free its space. An implicit index cannot drop chunks: libhdf5 leaves + /// them (and their data) where they are, and so does this. + fn remove( + &mut self, + img: &mut Image<'_>, + scaled: &[u64], + info: &ChunkInfo, + ) -> Result<(), Error> { + let t = self.t; + match &mut self.index { + IndexEdit::Implicit => return Ok(()), + IndexEdit::Single => { + return Err(Error::Unsupported( + "removing the chunk of a single-chunk index".into(), + )); + } + IndexEdit::BTree1(Some(tree)) => { + let mut offs: Vec = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect(); + offs.push(0); + if tree.remove(img, &offs)?.is_none() { + return Err(Error::Unsupported("chunk missing from its B-tree".into())); + } + } + IndexEdit::BTree2(Some(tree)) => { + let mut cmp = bt2_chunk_cmp(scaled, tree.record_size()); + if tree.remove(img, &mut cmp)?.is_none() { + return Err(Error::Unsupported("chunk missing from its B-tree".into())); + } + } + IndexEdit::Fixed(Some(fa)) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, false)?; + fa.set(img, idx, None)?; + } + IndexEdit::Extensible(Some(ea)) => { + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, t.dims(), max, &self.cd, true)?; + ea.set(img, idx, None)?; + } + _ => return Err(Error::Unsupported("chunk index missing".into())), + } + img.free(info.address, u64::from(info.chunk_size)); + Ok(()) + } + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { match &mut self.index { IndexEdit::Extensible(Some(ea)) => ea.finish(img)?, IndexEdit::Fixed(Some(fa)) => fa.finish(img)?, + IndexEdit::BTree2(Some(t)) => t.finish(img)?, _ => {} } self.hdr.finish(img) } } +/// A version-2 B-tree chunk record (type 10, or 11 when filtered): the +/// chunk's address, [its stored size and filter mask,] and its scaled +/// offsets. +fn bt2_chunk_record( + tree: &Bt2, + scaled: &[u64], + e: Elem, + filtered: bool, + os: u8, +) -> Result, Error> { + let rs = tree.record_size(); + let want_type = if filtered { 11 } else { 10 }; + let osz = os as usize; + let size_len = rs + .checked_sub(osz + 8 * scaled.len()) + .and_then(|n| { + if filtered { + n.checked_sub(4).filter(|&w| (1..=8).contains(&w)) + } else { + (n == 0).then_some(0) + } + }) + .filter(|_| tree.tree_type() == want_type) + .ok_or_else(|| Error::Unsupported("version-2 B-tree chunk record layout".into()))?; + let mut r = vec![0u8; rs]; + put_uint(&mut r, e.addr, os); + let mut q = osz; + if filtered { + if size_len < 8 && e.size >> (8 * size_len) != 0 { + return Err(Error::Unsupported(format!( + "filtered chunk of {} bytes does not fit the index's {size_len}-byte size field", + e.size + ))); + } + r[q..q + size_len].copy_from_slice(&e.size.to_le_bytes()[..size_len]); + q += size_len; + r[q..q + 4].copy_from_slice(&e.mask.to_le_bytes()); + q += 4; + } + for &s in scaled { + r[q..q + 8].copy_from_slice(&s.to_le_bytes()); + q += 8; + } + Ok(r) +} + +/// Compare chunk `scaled` with a chunk record (`H5D__bt2_compare`: the +/// scaled offsets, the first dimension most significant). +fn bt2_chunk_cmp( + scaled: &[u64], + rec_size: usize, +) -> impl FnMut(&Image<'_>, &[u8]) -> Result + '_ { + move |_, r| { + let at = rec_size - 8 * scaled.len(); + let theirs = (0..scaled.len()).map(|d| { + u64::from_le_bytes(r[at + 8 * d..at + 8 * d + 8].try_into().unwrap_or([0; 8])) + }); + Ok(scaled.iter().copied().cmp(theirs)) + } +} + /// The chunk B-tree's K: the superblock's (version 1), the superblock /// extension's (versions 2 and 3), or libhdf5's default of 32. fn chunk_btree_k(f: &File) -> Result { @@ -694,10 +841,15 @@ impl FileEditor { }) } - /// Change a chunked dataset's current dimensions to `shape`, which may - /// only grow each dimension, up to the dataset's maximum dimensions - /// (h5py's `Dataset.resize`). New elements read as the fill value until - /// written. Shrinking is [`Error::Unsupported`]. + /// Change a chunked dataset's current dimensions to `shape`: each + /// dimension may grow up to the dataset's maximum or shrink (h5py's + /// `Dataset.resize`), as `H5Dset_extent` does. New elements read as the + /// fill value until written (with early allocation their chunks are + /// allocated and filled now). Shrinking removes the chunks wholly + /// outside the new extent from the chunk index and frees them, and + /// overwrites the part of each partial edge chunk outside it with the + /// fill value, so elements that come back after a later growth read as + /// the fill value (`H5D__chunk_prune_by_extent`). pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> { self.edit(|f, img| { let t = Target::load(f, path)?; @@ -720,18 +872,16 @@ impl FileEditor { shape[d], max[d] ))); } - if shape[d] < dims[d] { - return Err(Error::Unsupported("shrinking a dataset".into())); - } } if !matches!(t.layout, DataLayout::Chunked { .. }) { return Err(Error::Unsupported( "resizing a dataset that is not chunked".into(), )); } - // Only the dataspace changes: every chunk index is keyed - // independently of the current extent (the array indexes by the - // maximum dimensions), and new chunks come with the writes. + // The dataspace changes; every chunk index is keyed + // independently of the current extent (the arrays index by the + // maximum dimensions). Growth allocates chunks only under early + // allocation, shrinking prunes. let mut hdr = Header::load(img, t.addr)?; let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage( clawhdf5_format::message_type::MessageType::Dataspace, @@ -754,7 +904,15 @@ impl FileEditor { put_uint(&mut dims_bytes[d * ls..], n, img.ls); } hdr.patch(img, i, first, &dims_bytes)?; - hdr.finish(img) + let fill = fill_info(img, &hdr)?; + hdr.finish(img)?; + let expand = shape.iter().zip(&dims).any(|(n, o)| n > o); + let shrink = shape.iter().zip(&dims).any(|(n, o)| n < o); + let early = expand && fill.alloc_time == ALLOC_EARLY; + if early || shrink { + resize_chunks(f, img, &t, &dims, shape, &fill, early, shrink)?; + } + Ok(()) }) } @@ -1187,52 +1345,7 @@ fn write_selection( Ok(()) })?; for (scaled, buf) in bufs { - // As libhdf5 does: an optional filter that fails (LZF that - // does not shrink the chunk) is skipped and its mask bit set. - let (bytes, mask) = match &t.pipeline { - Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?, - None => (buf, 0u32), - }; - let len = bytes.len() as u64; - let placed = match existing.get(&scaled) { - Some(info) if t.pipeline.is_none() => { - if u64::from(info.chunk_size) != len { - return Err(Error::Unsupported( - "unfiltered chunk stored at an unexpected size".into(), - )); - } - img.write(info.address, &bytes)?; - None - } - // Rewritten where it is when it still fits, or when it - // is the last thing in the file (the chunk an append - // keeps rewriting usually is) and can grow there. - Some(info) - if len <= u64::from(info.chunk_size) - || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => - { - img.write(info.address, &bytes)?; - (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some( - Elem { - addr: info.address, - size: len, - mask, - }, - ) - } - _ => { - let a = img.alloc(len)?; - img.write(a, &bytes)?; - Some(Elem { - addr: a, - size: len, - mask, - }) - } - }; - if let Some(e) = placed { - ce.set(img, &scaled, e)?; - } + store_chunk(img, &mut ce, existing.get(&scaled), &scaled, buf)?; } ce.finish(img) } @@ -1240,6 +1353,341 @@ fn write_selection( } } +/// `H5D_ALLOC_TIME_EARLY`. +const ALLOC_EARLY: u8 = 1; + +/// When a dataset's storage is allocated and filled, from its fill value +/// message (`H5O__fill_new_decode`); a dataset without one uses the chunked +/// defaults (incremental allocation, fill if set). +struct FillInfo { + alloc_time: u8, + /// 0 on allocation, 1 never, 2 if set. + fill_time: u8, + /// The fill value is undefined (`H5D_FILL_VALUE_UNDEFINED`). + undefined: bool, +} + +fn fill_info(img: &Image<'_>, hdr: &Header) -> Result { + let mut fi = FillInfo { + alloc_time: 3, + fill_time: 2, + undefined: false, + }; + let Some(i) = hdr.find(0x05) else { + return Ok(fi); + }; + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared fill value message".into())); + } + let d = hdr.data(img, i)?; + let short = || Error::Unsupported("short fill value message".into()); + match d.first() { + Some(1) | Some(2) => { + fi.alloc_time = *d.get(1).ok_or_else(short)?; + fi.fill_time = *d.get(2).ok_or_else(short)?; + fi.undefined = d[0] == 2 && *d.get(3).ok_or_else(short)? == 0; + } + Some(3) => { + let flags = *d.get(1).ok_or_else(short)?; + fi.alloc_time = flags & 0x03; + fi.fill_time = (flags >> 2) & 0x03; + fi.undefined = flags & 0x10 != 0; + } + _ => return Err(Error::Unsupported("fill value message version".into())), + } + Ok(fi) +} + +/// Visit the chunks libhdf5 visits, in its order, when the extent changes +/// from `old` to `new`: for allocation (`H5D__chunk_allocate`), every +/// chunk of the new extent outside the old one; `f` gets each scaled +/// offset. +fn for_each_new_chunk( + old: &[u64], + new: &[u64], + cd: &[u64], + mut f: impl FnMut(&[u64]) -> Result<(), Error>, +) -> Result<(), Error> { + let rank = new.len(); + if rank == 0 || new.contains(&0) { + return Ok(()); + } + let min: Vec = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect(); + let mut max: Vec = (0..rank).map(|d| (new[d] - 1) / cd[d]).collect(); + for op in 0..rank { + if min[op] > max[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min[op]; + loop { + f(&scaled)?; + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max[i] { + scaled[i] = if i == op { min[i] } else { 0 }; + } else { + carry = false; + break; + } + } + if carry { + break; + } + } + if min[op] == 0 { + break; + } + max[op] = min[op] - 1; + } + Ok(()) +} + +/// What pruning does to one chunk. +enum Prune { + /// Overwrite the part outside the new extent with the fill value. + Fill, + /// Drop it from the index. + Remove, +} + +/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks +/// from `old` to `new`, in its order. +fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { + let rank = new.len(); + let mut out = Vec::new(); + if old.contains(&0) { + return out; + } + let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); + let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_fill: Vec = (0..rank) + .map(|d| { + if new[d] == 0 { + -1 + } else { + ((new[d].min(old[d]) - 1) / cd[d]) as i64 + } + }) + .collect(); + let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); + let fill_dim: Vec = (0..rank) + .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) + .collect(); + for op in 0..rank { + if !shrunk[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min_mod[op]; + let mut outside: Vec = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); + let mut n_out = outside.iter().filter(|&&o| o).count(); + loop { + if n_out == 0 { + out.push((scaled.clone(), Prune::Fill)); + } else { + out.push((scaled.clone(), Prune::Remove)); + } + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max_mod[i] { + if i == op { + scaled[i] = min_mod[i]; + if outside[i] && fill_dim[i] { + outside[i] = false; + n_out -= 1; + } + } else { + scaled[i] = 0; + if outside[i] && max_fill[i] >= 0 { + outside[i] = false; + n_out -= 1; + } + } + } else { + if !outside[i] && scaled[i] as i64 > max_fill[i] { + outside[i] = true; + n_out += 1; + } + carry = false; + break; + } + } + if carry { + break; + } + } + if min_mod[op] == 0 { + // Every chunk was visited (the dimension shrank to nothing). + break; + } + max_mod[op] = min_mod[op] - 1; + } + out +} + +/// The chunk work of a resize: under early allocation, allocate and fill +/// the chunks the growth brings in (`H5D__chunk_allocate`); when a +/// dimension shrank, prune (`H5D__chunk_prune_by_extent`). +#[allow(clippy::too_many_arguments)] +fn resize_chunks( + f: &File, + img: &mut Image<'_>, + t: &Target, + old: &[u64], + new: &[u64], + fill: &FillInfo, + early: bool, + shrink: bool, +) -> Result<(), Error> { + let mut ce = ChunkedEdit::new(f, img, t)?; + let cd = ce.cd.clone(); + let es = t.es; + let chunk_bytes = ce.chunk_bytes; + let existing: HashMap, ChunkInfo> = match &t.layout { + DataLayout::Chunked { + btree_address: Some(_), + .. + } => { + let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?; + chunks + .into_iter() + .map(|c| { + let s: Vec = c.offsets.iter().zip(&cd).map(|(o, c)| o / c).collect(); + (s, c) + }) + .collect() + } + _ => HashMap::new(), + }; + let fill_chunk = t.fill.repeat(chunk_bytes / es); + if early { + let should_fill = + fill.fill_time == 0 || (fill.fill_time == 2 && !fill.undefined) || t.pipeline.is_some(); + let implicit = matches!(ce.index, IndexEdit::Implicit); + let base = t.layout_address(); + for_each_new_chunk(old, new, &cd, |scaled| { + if implicit { + if should_fill { + let base = + base.ok_or_else(|| Error::Unsupported("implicit index address".into()))?; + let max = t.ds.max_dimensions.as_deref(); + let idx = array_index(scaled, new, max, &cd, false)?; + let at = idx + .checked_mul(chunk_bytes as u64) + .and_then(|o| o.checked_add(base)) + .ok_or_else(|| Error::Unsupported("chunk address overflows".into()))?; + img.write(at, &fill_chunk)?; + } + Ok(()) + } else if existing.contains_key(scaled) { + Ok(()) + } else { + store_chunk(img, &mut ce, None, scaled, fill_chunk.clone()) + } + })?; + } + if shrink && !existing.is_empty() { + for (scaled, what) in prune_plan(old, new, &cd) { + let Some(info) = existing.get(&scaled) else { + continue; + }; + match what { + Prune::Remove => ce.remove(img, &scaled, info)?, + Prune::Fill => { + let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?; + // Keep [0, count) in each dimension; fill the rest. + let count: Vec = (0..cd.len()) + .map(|d| cd[d].min(new[d] - scaled[d] * cd[d])) + .collect(); + let n: u64 = cd.iter().product(); + for flat in 0..n { + let mut r = flat; + let mut inside = true; + for d in (0..cd.len()).rev() { + if r % cd[d] >= count[d] { + inside = false; + } + r /= cd[d]; + } + if !inside { + let at = flat as usize * es; + buf[at..at + es].copy_from_slice(&t.fill); + } + } + store_chunk(img, &mut ce, Some(info), &scaled, buf)?; + } + } + } + } + ce.finish(img) +} + +/// Encode chunk `scaled` (decoded bytes `buf`) and store it: an existing +/// unfiltered chunk in place; a filtered one in place when it still fits +/// (or can grow at the end of the file), else in new space, the old space +/// freed; a new chunk in new space. The index is updated when the chunk's +/// address, size or filter mask changed. +fn store_chunk( + img: &mut Image<'_>, + ce: &mut ChunkedEdit<'_>, + existing: Option<&ChunkInfo>, + scaled: &[u64], + buf: Vec, +) -> Result<(), Error> { + let t = ce.t; + // As libhdf5 does: an optional filter that fails (LZF that does not + // shrink the chunk) is skipped and its mask bit set. + let (bytes, mask) = match &t.pipeline { + Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, t.es as u32)?, + None => (buf, 0u32), + }; + let len = bytes.len() as u64; + let placed = match existing { + Some(info) if t.pipeline.is_none() => { + if u64::from(info.chunk_size) != len { + return Err(Error::Unsupported( + "unfiltered chunk stored at an unexpected size".into(), + )); + } + img.write(info.address, &bytes)?; + None + } + // Rewritten where it is when it still fits, or when it is the last + // thing in the file (the chunk an append keeps rewriting usually is) + // and can grow there. + Some(info) + if len <= u64::from(info.chunk_size) + || img.grow_tail(info.address, u64::from(info.chunk_size), len)? => + { + img.write(info.address, &bytes)?; + (len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(Elem { + addr: info.address, + size: len, + mask, + }) + } + _ => { + let a = img.alloc_reusing(len)?; + img.write(a, &bytes)?; + if let Some(info) = existing { + img.free(info.address, u64::from(info.chunk_size)); + } + Some(Elem { + addr: a, + size: len, + mask, + }) + } + }; + if let Some(e) = placed { + ce.set(img, scaled, e)?; + } + Ok(()) +} + fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> { let start = usize::try_from(info.address) .map_err(|_| Error::Unsupported("chunk address out of range".into()))?; diff --git a/crates/clawhdf5/tests/edit_tests.rs b/crates/clawhdf5/tests/edit_tests.rs index 2467256..9190a2c 100644 --- a/crates/clawhdf5/tests/edit_tests.rs +++ b/crates/clawhdf5/tests/edit_tests.rs @@ -98,8 +98,8 @@ fn errors_leave_the_file_untouched() { let mut ed = FileEditor::open(&path).unwrap(); assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_)))); assert!(ed.write_all("missing", &[0; 4]).is_err()); - // Wrong length, wrong type, outside the extent, beyond maxshape, - // shrinking, a rank change. + // Wrong length, wrong type, outside the extent, beyond maxshape, a + // rank change, resizing a dataset that is not chunked. assert!(matches!( ed.write_all("flat", &[0; 7]), Err(Error::InvalidArgument(_)) @@ -116,7 +116,10 @@ fn errors_leave_the_file_untouched() { ed.resize("raw", &[3, 5]), Err(Error::InvalidArgument(_)) )); - assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_)))); + assert!(matches!( + ed.resize("flat", &[2]), + Err(Error::Unsupported(_)) + )); assert!(matches!( ed.resize("ext", &[4, 1]), Err(Error::InvalidArgument(_)) @@ -136,3 +139,26 @@ fn errors_leave_the_file_untouched() { drop(ed); assert!(std::fs::read(&path).unwrap() == before); } + +/// Shrinking and growing again on a file clawhdf5 wrote: elements that come +/// back read as the fill value, the ones kept keep their values. +#[test] +fn shrink_then_grow_reads_fill() { + let dir = tempfile::tempdir().unwrap(); + let path = sample(dir.path()); + { + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("ext", &[2]).unwrap(); + ed.resize("ext", &[9]).unwrap(); + ed.resize("raw", &[1, 4]).unwrap(); + ed.resize("raw", &[3, 4]).unwrap(); + } + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("ext").unwrap().read_i32().unwrap(), + [0, 1, 0, 0, 0, 0, 0, 0, 0] + ); + let mut raw = vec![0.0f64; 12]; + raw[..4].fill(0.5); + assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw); +} From 7e5e920c727009ee2b8ec1e77c530d384dc2240e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:56:09 -0500 Subject: [PATCH 03/15] format: read object headers with long continuation chains libhdf5 follows any number of continuation chunks, and a header that is full gains one per message added (each new chunk holding the next continuation message), so a version-1 header with a few dozen attributes added one at a time is a chain dozens of chunks long. The reader recursed once per chunk and refused a chain deeper than 32 (NestingDepthExceeded): h5py read such files, we did not. Version-2 headers stopped at 256 continuation chunks. Version-1 chunks are now followed with an explicit stack (the same depth-first message order as before), version-2 ones as before; both refuse a chunk address seen twice (a cycle, what the limits guarded against) and more than 65 536 chunks. Regression: long_v1_continuation_chains_are_read (a 200-chunk chain), v1_continuation_cycles_are_refused; the dense-attribute interop test's 'earliest' case produces such a chain. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/object_header.rs | 164 +++++++++++++++----- 1 file changed, 122 insertions(+), 42 deletions(-) diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 0d031ed..383d553 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -1,7 +1,9 @@ //! HDF5 Object Header parsing (v1 and v2). #[cfg(not(feature = "std"))] -use alloc::vec::Vec; +use alloc::{collections::BTreeSet, vec, vec::Vec}; +#[cfg(feature = "std")] +use std::collections::BTreeSet; use byteorder::{ByteOrder, LittleEndian}; @@ -196,7 +198,6 @@ impl ObjectHeader { header_data_size, offset_size, length_size, - MAX_V1_CONTINUATION_DEPTH, &mut messages, )?; // libhdf5 reads every message in the first chunk and refuses a header @@ -230,49 +231,55 @@ impl ObjectHeader { /// 8; libhdf5 refuses a message that is not aligned, that runs past the /// end of the chunk, or leftover bytes too few for a message header (a /// "gap", which only version 2 allows). - #[allow(clippy::too_many_arguments)] + /// + /// Continuation chunks are followed depth-first, as met, with an + /// explicit stack: a chain of continuation chunks as long as libhdf5 + /// writes (each new chunk holding the next continuation message, one + /// per attribute added to a full header) is read without recursion. + /// A chunk address seen twice is a cycle and refused, as is a header of + /// more than [`MAX_V1_CHUNKS`] chunks. fn parse_v1_chunk( file: &S, offset: u64, length: usize, offset_size: u8, length_size: u8, - depth_remaining: u16, messages: &mut Vec, ) -> Result { - if depth_remaining == 0 { - return Err(FormatError::NestingDepthExceeded); - } - let chunk = read_exact_at(file, offset, length)?; - let data: &[u8] = &chunk; - let end = length; - let mut pos = 0usize; + let mut stack = vec![(read_exact_at(file, offset, length)?, 0usize)]; + let mut seen = BTreeSet::new(); + seen.insert(offset); let mut count = 0usize; - while pos < end { - if end - pos < V1_MSG_HEADER_SIZE { + while let Some((chunk, pos)) = stack.last_mut() { + let data: &[u8] = chunk; + let end = data.len(); + if *pos >= end { + stack.pop(); + continue; + } + if end - *pos < V1_MSG_HEADER_SIZE { return Err(FormatError::InvalidObjectHeader( "gap found in early version of file format", )); } - let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); - let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; - let msg_flags = data[pos + 4]; - // reserved(3) at pos+5..pos+8 - pos += V1_MSG_HEADER_SIZE; + let p = *pos; + let msg_type_raw = LittleEndian::read_u16(&data[p..p + 2]); + let msg_data_size = LittleEndian::read_u16(&data[p + 2..p + 4]) as usize; + let msg_flags = data[p + 4]; + // reserved(3) at p+5..p+8 + let p = p + V1_MSG_HEADER_SIZE; if !msg_data_size.is_multiple_of(8) { return Err(FormatError::InvalidObjectHeader("message not aligned")); } - if msg_data_size > end - pos { + if msg_data_size > end - p { return Err(FormatError::InvalidObjectHeader( "message size exceeds buffer end", )); } - let body = &data[pos..pos + msg_data_size]; + let body = &data[p..p + msg_data_size]; check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; - count += 1; - let msg_type = MessageType::from_u16(msg_type_raw); if msg_type != MessageType::Nil { messages.push(HeaderMessage { @@ -283,22 +290,26 @@ impl ObjectHeader { data: body.to_vec(), }); } - pos += msg_data_size; - // Follow continuations (v1 continuation chunks are just raw // messages, no signature); check_message has checked the body. - if msg_type == MessageType::ObjectHeaderContinuation { - let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?; - let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?; - Self::parse_v1_chunk( - file, - cont_offset as u64, - cont_length, - offset_size, - length_size, - depth_remaining - 1, - messages, - )?; + let cont = if msg_type == MessageType::ObjectHeaderContinuation { + Some(( + read_offset(body, 0, offset_size)?, + to_usize(read_offset(body, offset_size as usize, length_size)?)?, + )) + } else { + None + }; + *pos = p + msg_data_size; + // Only the first chunk's messages are held to the prefix count. + if stack.len() == 1 { + count += 1; + } + if let Some((cont_offset, cont_length)) = cont { + if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { + return Err(FormatError::NestingDepthExceeded); + } + stack.push((read_exact_at(file, cont_offset, cont_length)?, 0)); } } @@ -425,13 +436,15 @@ impl ObjectHeader { &mut continuations, )?; - // Follow continuations (limit to prevent cycles in malformed data) - let mut cont_remaining = 256u16; + // Follow continuations. A chunk address seen twice is a cycle in + // malformed data; a valid header can have many chunks (libhdf5 adds + // one whenever a message no longer fits), up to the same bound as a + // version-1 header. + let mut seen = BTreeSet::new(); while let Some((cont_offset, cont_length)) = continuations.pop() { - if cont_remaining == 0 { + if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { return Err(FormatError::NestingDepthExceeded); } - cont_remaining -= 1; Self::parse_v2_continuation( file, cont_offset as u64, @@ -594,8 +607,10 @@ const V2_PREFIX_MAX: usize = 34; /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3). const V1_MSG_HEADER_SIZE: usize = 8; -/// How deep version-1 continuation chunks may chain (malformed-data guard). -const MAX_V1_CONTINUATION_DEPTH: u16 = 32; +/// Most chunks a version-1 object header may have (malformed-data guard; +/// libhdf5 has no limit, and a header that gains one continuation chunk per +/// attribute added can have many). +const MAX_V1_CHUNKS: usize = 1 << 16; /// Every defined version-2 object header status flag (libhdf5 /// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation @@ -901,6 +916,71 @@ mod tests { assert_eq!(hdr.messages[1].data[..2], [5, 6]); } + /// A version-1 header whose continuation chunks form a chain: chunk k + /// holds a Dataspace message `[k]` and the continuation to chunk k + 1. + /// With `cycle`, the last chunk points back at the first continuation + /// chunk. + fn v1_chain(n: usize, cycle: bool) -> Vec { + // Each continuation chunk: dataspace (8 + 8) + continuation (8 + 16). + let chunk_len = 40u64; + let first = 64u64; + let cont = |addr: u64| { + let mut b = addr.to_le_bytes().to_vec(); + b.extend_from_slice(&chunk_len.to_le_bytes()); + b + }; + let mut data = build_v1_header(&[(0x0010, &cont(first)[..], 0)], 8, 8); + data.resize(first as usize, 0); + for k in 0..n { + let mut c = Vec::new(); + c.extend_from_slice(&1u16.to_le_bytes()); + c.extend_from_slice(&8u16.to_le_bytes()); + c.extend_from_slice(&[0; 4]); + c.extend_from_slice(&(k as u64).to_le_bytes()); + let next = if k + 1 < n { + first + (k as u64 + 1) * chunk_len + } else if cycle { + first + } else { + // The last chunk ends in a NIL message instead. + c.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]); + c.extend_from_slice(&[0; 16]); + data.extend_from_slice(&c); + continue; + }; + c.extend_from_slice(&0x10u16.to_le_bytes()); + c.extend_from_slice(&16u16.to_le_bytes()); + c.extend_from_slice(&[0; 4]); + c.extend_from_slice(&cont(next)); + data.extend_from_slice(&c); + } + data + } + + /// libhdf5 reads any chain of continuation chunks (a header grows one + /// per attribute added when full); the reader used to stop at 32. + #[test] + fn long_v1_continuation_chains_are_read() { + let data = v1_chain(200, false); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + let spaces: Vec = hdr + .messages + .iter() + .filter(|m| m.msg_type == MessageType::Dataspace) + .map(|m| m.data[0]) + .collect(); + assert_eq!(spaces, (0..200).map(|k| k as u8).collect::>()); + } + + #[test] + fn v1_continuation_cycles_are_refused() { + let data = v1_chain(5, true); + assert!(matches!( + ObjectHeader::parse(&data, 0, 8, 8), + Err(FormatError::NestingDepthExceeded) + )); + } + #[test] fn parse_v1_unknown_message_ok() { let messages = [(0x00FFu16, &[0xAA, 0xBB][..], 0u8)]; From 773f427f1685cd882e5f82fcd0e9d9d62e1a298b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 16:58:08 -0500 Subject: [PATCH 04/15] edit: dense attributes, compact-to-dense transition, creation order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FileEditor::set_attr now handles every attribute storage libhdf5 uses for version-2 object headers: - objects that track (and index) attribute creation order: compact attributes carry their creation index in the message header, the Attribute Info message its maximum; - the move to dense storage when an object reaches its compact limit (or an attribute is too large for a header message), as H5O__attr_create does it: a new fractal heap, name index (v2 B-tree type 8) and, when creation order is indexed, creation-order index (type 9); the compact attributes moved over in header message order, their messages freed; - objects already in dense storage (h5py- or clawhdf5-written): new attributes inserted (H5A__dense_insert), an attribute replaced by one of the same encoded size rewritten in its heap object (H5A__dense_write), otherwise removed from both indexes and the heap and inserted anew. edit/fheap.rs follows H5HF: managed objects go to the best-fitting free section of the heap's free-space manager (FSHD/FSSE, kept as libhdf5 keeps it — sorted sections, counts, section info reallocated when its size changes, the manager deleted when empty); otherwise to a new direct block: the root direct block of an empty heap, else the block at the allocation iterator in the root indirect block (created from the root direct block, doubled as needed), with libhdf5's managed-space, allocated space, free space and iterator bookkeeping. Objects above the managed limit are huge objects in their own space, indexed by the huge-object B-tree (type 1). Removed objects return their space merged with adjacent free space. Refused before anything is written: heaps with I/O filters, child indirect blocks, an object larger than the next heap block (libhdf5 skips blocks and records them as free space), free sections other than those inside direct blocks, removing a direct block's last object (libhdf5 frees the block), directly addressed huge objects. Attributes are encoded as libhdf5 does when h5py opens a file r+ (low bound "earliest"): message version 1 (3 for non-ASCII names), simple dataspaces with their maximum dimensions. Header chunks are now visited in libhdf5's order (FIFO), which is also the order attributes move to dense storage in. Tests (edit_coverage_interop): 40 attributes on each of a plain group, a group tracking and indexing creation order, and a dataset (earliest, v110, latest), some above the 4 KiB managed limit, then same-size rewrites: the heap statistics, free-space sections and both index B-trees node for node equal libhdf5's doing the same through h5py; then replacements of other sizes, h5py adds/deletes/rewrites; h5py, h5dump, h5rs check and our reader agree throughout, h5py's attribute count included. clawhdf5-written dense storage (tracked and untracked) is extended the same way; refusals leave the file byte for byte as it was. edit_interop's attribute test now expects dense storage and tracked creation order to work. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 495 +++++++- crates/clawhdf5-tools/tests/edit_interop.rs | 19 +- crates/clawhdf5/src/edit/attrs.rs | 513 ++++++++ crates/clawhdf5/src/edit/btree2.rs | 35 +- crates/clawhdf5/src/edit/fheap.rs | 1121 +++++++++++++++++ crates/clawhdf5/src/edit/mod.rs | 190 +-- crates/clawhdf5/src/edit/ohdr.rs | 8 +- 7 files changed, 2165 insertions(+), 216 deletions(-) create mode 100644 crates/clawhdf5/src/edit/attrs.rs create mode 100644 crates/clawhdf5/src/edit/fheap.rs diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index f2c27fd..036839f 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -213,7 +213,6 @@ fn tmpdir() -> tempfile::TempDir { tempfile::TempDir::new_in(base).unwrap() } -#[allow(dead_code)] fn unsupported(r: Result) { match r { Err(Error::Unsupported(_)) => {} @@ -830,3 +829,497 @@ fn btree2_random_chunk_order() { verify(&path, "x", &m); check_tools(&path, false); } + +// ---- dense attributes ---- + +/// An attribute value both sides can write with the same encoded size: a +/// 1-D int64 array or a fixed-length string. +#[derive(Clone, Debug, PartialEq)] +enum AV { + Ints(Vec), + Str(String), +} + +impl AV { + fn value(&self) -> clawhdf5::AttrValue { + match self { + AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()), + AV::Str(s) => clawhdf5::AttrValue::String(s.clone()), + } + } + + fn py(&self) -> String { + match self { + AV::Ints(v) => format!("np.array({v:?}, dtype=' format!("np.bytes_({s:?})"), + } + } + + fn py_expect(&self) -> String { + match self { + AV::Ints(v) => format!("{v:?}"), + AV::Str(s) => format!("{s:?}"), + } + } + + fn make(i: u64, salt: u64) -> Self { + match i % 11 { + 5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)), + 1 | 4 | 8 => AV::Str( + (0..10 + (i * 13 + salt) % 300) + .map(|k| (b'a' + ((k + salt) % 26) as u8) as char) + .collect(), + ), + _ => AV::Ints( + (0..1 + (i + salt) % 9) + .map(|k| (k * 31 + salt) as i64) + .collect(), + ), + } + } +} + +/// An object's dense attribute storage as libhdf5 would compare it: the +/// heap's statistics and root shape, its free sections (heap offsets and +/// sizes), and the shapes of the name and creation-order index B-trees. +fn dense_info(path: &Path, obj: &str) -> String { + use clawhdf5_format::message_type::MessageType; + use clawhdf5_format::object_header::ObjectHeader; + let f = File::open(path).unwrap(); + let sb = f.superblock(); + let (os, ls) = (sb.offset_size, sb.length_size); + assert_eq!((os, ls), (8, 8)); + let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap(); + let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap(); + let Some(m) = oh + .messages + .iter() + .find(|m| m.msg_type == MessageType::AttributeInfo) + else { + return "no attribute info".into(); + }; + let d = &m.data; + let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 }; + let heap = le(&d[p..p + 8]); + p += 8; + let name = le(&d[p..p + 8]); + let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16])); + if heap == u64::MAX { + return "compact".into(); + } + let b = std::fs::read(path).unwrap(); + let h = &b[heap as usize..]; + assert_eq!(&h[0..4], b"FRHP"); + // next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge + // size, huge objs (8-byte fields from offset 14). + let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]); + let fs = stat(3); + // Root rows: after 12 8-byte fields, width, start, max direct, max + // index, start rows, root address. + let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8; + let mut out = format!( + "heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}", + stat(0), + stat(2), + stat(4), + stat(5), + stat(6), + stat(7), + stat(8), + stat(9), + le(&h[rows_at..rows_at + 2]) + ); + if fs != u64::MAX { + let s = &b[fs as usize..]; + assert_eq!(&s[0..4], b"FSHD"); + let tot = le(&s[6..14]); + let n = le(&s[14..22]); + // counts (4 x 8), 4 x u16, max section size, section info address. + let at = 6 + 32 + 8 + 8; + let sect_addr = le(&s[at..at + 8]); + let sect_size = le(&s[at + 8..at + 16]); + let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize]; + // Sections after the prefix (signature, version, header address); + // trailing zero padding and the checksum are left out. + let body = &ss[13..ss.len() - 4]; + let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count(); + out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used])); + } else { + out += " no-fs"; + } + out += &format!(" names={:?}", bt2_shape_at(&b, name as usize)); + if let Some(o) = order { + out += &format!(" order={:?}", bt2_shape_at(&b, o as usize)); + } + out +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// h5py and our reader both see `want` on each object (other attributes +/// may exist; these must have these values), and h5py's attribute count +/// agrees with libhdf5's object info. +fn check_attr_values(path: &Path, want: &[(String, String, AV)]) { + let f = File::open(path).unwrap(); + for (o, n, v) in want { + let attrs = if o == "d" { + f.dataset(o).unwrap().attrs().unwrap() + } else { + f.group(o).unwrap().attrs().unwrap() + }; + let got = attrs + .get(n.as_str()) + .unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len())); + match (got, v) { + (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"), + (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"), + (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"), + other => panic!("{o}/{n}: {other:?}"), + } + } + let exp: Vec = want + .iter() + .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect())) + .collect(); + let script = format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + want = [{w}]\n\ + for o, n, v in want:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 a = [a] if isinstance(a, int) else a\n\ + \x20 assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\ + for o in set(x[0] for x in want):\n\ + \x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\ + \x20 assert len(list(f[o].attrs)) == len(f[o].attrs)\n", + p = path.to_str().unwrap(), + w = exp.join(", ") + ); + let sp = path.with_extension("check.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); +} + +/// Run python statements (inside `with h5py.File(path, 'r+') as f:`). +fn py_r_plus(path: &Path, lines: &[String]) { + let script = format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n{l}", + p = path.to_str().unwrap(), + l = lines.concat() + ); + let sp = path.with_extension("ops.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "h5py workload failed:\n{}", text(&o)); +} + +type AttrOp = (String, String, AV); + +fn set_want(want: &mut Vec, o: &str, n: &str, v: AV) { + want.retain(|(wo, wn, _)| !(wo == o && wn == n)); + want.push((o.into(), n.into(), v)); +} + +/// The same attribute workload by libhdf5 and by the editor on objects +/// with compact attributes that move to dense storage (a plain group, a +/// group tracking and indexing creation order, a dataset): 40 new +/// attributes each (some larger than the heap's 4 KiB managed limit), then +/// same-size rewrites — the heaps, their free space and both index B-trees +/// must come out as libhdf5 makes them; then replacements of another size, +/// then h5py adds, deletes and rewrites attributes. +fn dense_workload(libver: &str, h5dump: bool, tag: &str) { + let dir = tmpdir(); + let a = dir.path().join(format!("dense_{tag}_h5py.h5")); + let b = dir.path().join(format!("dense_{tag}_edit.h5")); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \ + f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + for o in objs { + for i in 0..3i64 { + want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i]))); + } + } + // Phase 1: new attributes. + let mut ops: Vec = Vec::new(); + for i in 0..40u64 { + for (k, o) in objs.iter().enumerate() { + ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64))); + } + } + let lines: Vec = ops + .iter() + .map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.create({n:?}, {})\n", v.py())) + .collect(); + py_r_plus(&a, &lines); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, n, v) in &ops { + ed.set_attr(o, n, &v.value()) + .unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}")); + set_want(&mut want, o, n, v.clone()); + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + check_attr_values(&a, &want); + for o in objs { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{tag}: dense storage of {o} differs from libhdf5's after insertions" + ); + } + // Phase 2: same-size rewrites (H5Awrite in place). + let ops2: Vec = ops + .iter() + .step_by(4) + .map(|(o, n, v)| { + let nv = match v { + AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()), + AV::Str(s) => AV::Str(s.chars().rev().collect()), + }; + (o.clone(), n.clone(), nv) + }) + .collect(); + let lines: Vec = ops2 + .iter() + .map(|(o, n, v)| format!("\x20 f[{o:?}].attrs.modify({n:?}, {})\n", v.py())) + .collect(); + py_r_plus(&a, &lines); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, n, v) in &ops2 { + ed.set_attr(o, n, &v.value()).unwrap(); + set_want(&mut want, o, n, v.clone()); + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + for o in objs { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{tag}: dense storage of {o} differs from libhdf5's after rewrites" + ); + } + // Phase 3: replacements of another size (values only: h5py replaces + // through a temporary attribute and a rename). + let mut ed = FileEditor::open(&b).unwrap(); + for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) { + let nv = match v { + AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()), + AV::Str(s) => AV::Str(format!("{s}-{k}")), + }; + let before = std::fs::read(&b).unwrap(); + match ed.set_attr(o, n, &nv.value()) { + Ok(()) => set_want(&mut want, o, n, nv), + Err(Error::Unsupported(msg)) => { + assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}"); + assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote"); + } + Err(e) => panic!("{tag}: replace {o}/{n}: {e}"), + } + } + drop(ed); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + // libhdf5 goes on: adds, deletes and rewrites. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['g', 't', 'd']:\n\ + \x20 for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\ + \x20 del f[o].attrs['n3']\n\ + \x20 del f[o].attrs['c1']\n\ + \x20 f[o].attrs['n7'] = 'rewritten by h5py'\n", + p = b.to_str().unwrap() + )); + want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7"); + check_tools(&b, h5dump); + check_attr_values(&b, &want); +} + +#[test] +fn dense_attributes_match_libhdf5() { + if !tools_ok() { + return; + } + for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] + .iter() + .enumerate() + { + dense_workload(lv, *dump, &format!("{i}")); + } +} + +/// Dense attributes in files clawhdf5 writes (its own heap and index +/// layout, no free-space manager), with and without tracked creation +/// order: attributes added (moving a dataset's attributes to dense storage +/// too), rewritten and replaced, then h5py goes on. +#[test] +fn dense_attributes_on_clawhdf5_files() { + if !tools_ok() { + return; + } + for track in [false, true] { + let dir = tmpdir(); + let path = dir.path().join(format!("ours_dense_{track}.h5")); + let mut b = clawhdf5::FileBuilder::new(); + b.track_order(track); + for i in 0..12i64 { + b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3])); + } + b.create_dataset("d") + .with_i32_data(&[1, 2, 3]) + .set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5])); + b.write(&path).unwrap(); + let mut want: Vec = (0..12i64) + .map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3]))) + .collect(); + want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5]))); + let mut ed = FileEditor::open(&path).unwrap(); + for i in 0..30u64 { + for (k, o) in ["/", "d"].iter().enumerate() { + // Small attributes: a larger one needs a heap block bigger than + // the next (see dense_attribute_refusals_change_nothing). + let v = match AV::make(i, k as u64 + 7) { + AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()), + v => v, + }; + ed.set_attr(o, &format!("n{i}"), &v.value()) + .unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}")); + set_want(&mut want, o, &format!("n{i}"), v); + } + } + // Rewrites in place, then replacements of another size. + for i in (0..12i64).step_by(3) { + let v = AV::Ints(vec![-i; 3]); + ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap(); + set_want(&mut want, "/", &format!("r{i}"), v); + } + for i in (1..30u64).step_by(7) { + let v = AV::Str(format!("replaced {i}")); + match ed.set_attr("d", &format!("n{i}"), &v.value()) { + Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v), + Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"), + Err(e) => panic!("{e}"), + } + } + drop(ed); + check_tools(&path, true); + let want_py: Vec = want + .iter() + .map(|(o, n, v)| { + let o = if o == "/" { "/".to_string() } else { o.clone() }; + (o, n.clone(), v.clone()) + }) + .collect(); + check_root_and_attrs(&path, &want_py); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['/', 'd']:\n\ + \x20 f[o].attrs['from_h5py'] = np.arange(3)\n\ + \x20 del f[o].attrs['n2']\n", + p = path.to_str().unwrap() + )); + want.retain(|(_, n, _)| n != "n2"); + check_tools(&path, true); + check_root_and_attrs(&path, &want); + } +} + +/// `check_attr_values`, with the root group as "/". +fn check_root_and_attrs(path: &Path, want: &[AttrOp]) { + let f = File::open(path).unwrap(); + for (o, n, v) in want { + let attrs = match o.as_str() { + "/" => f.root().attrs().unwrap(), + "d" => f.dataset(o).unwrap().attrs().unwrap(), + _ => f.group(o).unwrap().attrs().unwrap(), + }; + let got = attrs + .get(n.as_str()) + .unwrap_or_else(|| panic!("{o}/{n} missing")); + match (got, v) { + (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"), + (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"), + (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"), + other => panic!("{o}/{n}: {other:?}"), + } + } + let exp: Vec = want + .iter() + .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect())) + .collect(); + let script = format!( + "import h5py, numpy as np\n\ + f = h5py.File({p:?}, 'r')\n\ + want = [{w}]\n\ + for o, n, v in want:\n\ + \x20 a = f[o].attrs[n]\n\ + \x20 a = a.decode() if isinstance(a, bytes) else a\n\ + \x20 a = a.tolist() if hasattr(a, 'tolist') else a\n\ + \x20 a = [a] if isinstance(a, int) else a\n\ + \x20 assert a == v, (o, n)\n\ + for o in set(x[0] for x in want):\n\ + \x20 assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n", + p = path.to_str().unwrap(), + w = exp.join(", ") + ); + let sp = path.with_extension("check.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); +} + +/// What the editor refuses in dense storage — an object larger than the +/// next heap block (libhdf5 would skip blocks and record their space as +/// free, which this editor does not do) — is `Error::Unsupported`, and the +/// file is left byte for byte as it was. +#[test] +fn dense_attribute_refusals_change_nothing() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("dense_refuse.h5"); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", + p = path.to_str().unwrap() + )); + let before = std::fs::read(&path).unwrap(); + let mut ed = FileEditor::open(&path).unwrap(); + unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000)))); + drop(ed); + assert!( + std::fs::read(&path).unwrap() == before, + "a refused edit wrote" + ); + // What libhdf5 does with it instead works on the untouched file. + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['g'].attrs['big'] = 'x' * 2000\n\ + \x20 assert len(f['g'].attrs) == 13\n", + p = path.to_str().unwrap() + )); + check_tools(&path, true); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index 6a2dbf6..bc4c9a6 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -836,19 +836,16 @@ fn attributes_in_place() { .unwrap(); want.push(("d", "units".into(), AttrValue::String("km".into()))); if *lv != "'earliest'" { - // Up to the compact limit (8) and no further; attributes in - // dense storage and tracked creation order are refused, and a - // refused edit writes nothing. + // Up to the compact limit (8), then into dense storage; objects + // already in dense storage and ones tracking creation order. ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap(); want.push(("g", "eighth".into(), AttrValue::I64(8))); - let before = std::fs::read(&path).unwrap(); - unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9))); - unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1))); - unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1))); - assert!( - std::fs::read(&path).unwrap() == before, - "a refused edit changed the file" - ); + ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap(); + want.push(("g", "ninth".into(), AttrValue::I64(9))); + ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap(); + want.push(("dense", "k0".into(), AttrValue::I64(1))); + ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap(); + want.push(("tracked", "b".into(), AttrValue::I64(1))); } drop(ed); check_tools(&path, *dump); diff --git a/crates/clawhdf5/src/edit/attrs.rs b/crates/clawhdf5/src/edit/attrs.rs new file mode 100644 index 0000000..5772db3 --- /dev/null +++ b/crates/clawhdf5/src/edit/attrs.rs @@ -0,0 +1,513 @@ +//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do: +//! compact attributes are object header messages (with their creation +//! index in the message header when the object tracks creation order); +//! when an object reaches its compact limit (or an attribute is too large +//! for a header message) its attributes move to dense storage — a fractal +//! heap for the encoded messages, a version-2 B-tree indexing them by name +//! hash (record type 8) and, when creation order is indexed, a second one +//! by creation index (type 9) — and the Attribute Info message points at +//! them. + +use std::cmp::Ordering; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::dataspace::DataspaceType; + +use crate::edit::btree2::Bt2; +use crate::edit::fheap::Heap; +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::edit::ohdr::{Header, MSG_ATTRIBUTE}; +use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain}; +use crate::error::Error; +use crate::reader::File; +use crate::types::AttrValue; + +/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage. +const MESG_MAX_SIZE: usize = 65536; +/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object +/// that does not track creation order. +const NO_CRT_IDX: u16 = u16::MAX; +/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`, +/// `H5A_CORDER_BT2_*`). +const NAME_BT2_TYPE: u8 = 8; +const CORDER_BT2_TYPE: u8 = 9; +const ATTR_BT2_NODE: u32 = 512; +/// Heap IDs in attribute records. +const ID_LEN: usize = 8; + +/// An object's Attribute Info message. +#[derive(Debug, Clone)] +struct AInfo { + /// Its message index in the header. + idx: usize, + track: bool, + index: bool, + max_crt: u16, + fheap: u64, + name_bt2: u64, + corder_bt2: u64, +} + +impl AInfo { + fn load(img: &Image<'_>, hdr: &Header) -> Result, Error> { + let Some(idx) = hdr.find(MSG_ATTR_INFO) else { + return Ok(None); + }; + if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute info message".into())); + } + let d = hdr.data(img, idx)?; + let os = img.os as usize; + let short = || Error::Unsupported("short attribute info message".into()); + if d.first() != Some(&0) { + return Err(Error::Unsupported("attribute info message version".into())); + } + let flags = *d.get(1).ok_or_else(short)?; + let track = flags & 0x01 != 0; + let index = flags & 0x02 != 0; + let mut p = 2; + let mut max_crt = 0; + if track { + let b = d.get(p..p + 2).ok_or_else(short)?; + max_crt = u16::from_le_bytes([b[0], b[1]]); + p += 2; + } + let n = if index { 3 } else { 2 }; + if d.len() < p + n * os { + return Err(short()); + } + Ok(Some(Self { + idx, + track, + index, + max_crt, + fheap: get_uint(&d[p..], img.os), + name_bt2: get_uint(&d[p + os..], img.os), + corder_bt2: if index { + get_uint(&d[p + 2 * os..], img.os) + } else { + undef(img.os) + }, + })) + } + + fn dense(&self, os: u8) -> bool { + self.fheap != undef(os) + } + + /// Store the changeable fields back into the message. + fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> { + let os = img.os as usize; + let mut p = 2; + if self.track { + hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?; + p += 2; + } + let mut a = vec![0u8; os]; + for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2] + .into_iter() + .enumerate() + .take(if self.index { 3 } else { 2 }) + { + put_uint(&mut a, v, img.os); + hdr.patch(img, self.idx, p + k * os, &a)?; + } + Ok(()) + } + + /// The next creation index (`H5O__attr_create`), or libhdf5's "none". + fn next_crt(&mut self) -> Result { + if !self.track { + return Ok(NO_CRT_IDX); + } + if self.max_crt == NO_CRT_IDX { + return Err(Error::Unsupported( + "object's attribute creation index is exhausted".into(), + )); + } + self.max_crt += 1; + Ok(self.max_crt - 1) + } +} + +/// The name bytes of an attribute message body (without the NUL). +pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> { + let bad = || Error::Unsupported("malformed attribute message".into()); + let (len, at) = match d.first() { + Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), + Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), + _ => return Err(bad()), + }; + let name = d.get(at..at + len).ok_or_else(bad)?; + Ok(name.split(|&b| b == 0).next().unwrap_or(name)) +} + +/// A new Attribute Info message for a version-2 header with flags +/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order +/// tracked / indexed as the header's flags say, the maximum creation index, +/// and no dense storage (undefined fractal heap and B-tree addresses). +fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec { + let track = hdr_flags & 0x04 != 0; + let index = hdr_flags & 0x08 != 0; + let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; + if track { + b.extend_from_slice(&max_crt.to_le_bytes()); + } + let undef_addr = vec![0xffu8; os as usize]; + b.extend_from_slice(&undef_addr); + b.extend_from_slice(&undef_addr); + if index { + b.extend_from_slice(&undef_addr); + } + b +} + +/// A version-2 header's limit on compact attributes: stored when its flags +/// say so, else libhdf5's default of 8. +fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { + if hdr.flags & 0x10 == 0 { + return Ok(8); + } + let mut p = hdr.addr + 6; + if hdr.flags & 0x20 != 0 { + p += 16; + } + let b = img.read(p, 2)?; + Ok(u16::from_le_bytes([b[0], b[1]])) +} + +/// A version-1 attribute message (what libhdf5 writes in a version-1 object +/// header): name, datatype and dataspace each padded to 8 bytes, the +/// dataspace as a version-1 dataspace message. +fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { + let mut name = a.name.as_bytes().to_vec(); + name.push(0); + let dt = a.datatype.serialize(); + let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; + if a.dataspace.space_type == DataspaceType::Simple { + let mut b = vec![0u8; ls as usize]; + for &d in &a.dataspace.dimensions { + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + if let Some(max) = &a.dataspace.max_dimensions { + ds[2] = 0x01; + for &d in max { + put_uint(&mut b, d, ls); + ds.extend_from_slice(&b); + } + } + } else { + ds[1] = 0; + } + let mut out = vec![1u8, 0]; + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); + out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); + for part in [&name, &dt, &ds] { + out.extend_from_slice(part); + out.resize(out.len().next_multiple_of(8), 0); + } + out.extend_from_slice(&a.raw_data); + out +} + +/// Dense storage opened for changes. +struct Dense { + heap: Heap, + names: Bt2, + order: Option, +} + +/// `H5_checksum_lookup3` of a name, as the name index keys it. +fn name_hash(name: &[u8]) -> u32 { + clawhdf5_format::checksum::jenkins_lookup3(name) +} + +/// Compare attribute `name` (hash `hash`) with a name-index record +/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name). +fn cmp_name( + heap: &Heap, + img: &Image<'_>, + hash: u32, + name: &[u8], + rec: &[u8], +) -> Result { + let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]); + match hash.cmp(&theirs) { + Ordering::Equal => { + if rec[ID_LEN] & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported( + "shared attribute in dense storage".into(), + )); + } + let obj = heap.read(img, &rec[..ID_LEN])?; + Ok(name.cmp(attr_name(&obj)?)) + } + o => Ok(o), + } +} + +fn corder_of(rec: &[u8]) -> u32 { + u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]]) +} + +impl Dense { + fn open(img: &Image<'_>, ai: &AInfo) -> Result { + let heap = Heap::open(img, ai.fheap)?; + let names = Bt2::open(img, ai.name_bt2)?; + if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 { + return Err(Error::Unsupported("attribute name index layout".into())); + } + let order = if ai.index { + let t = Bt2::open(img, ai.corder_bt2)?; + if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 { + return Err(Error::Unsupported( + "attribute creation-order index layout".into(), + )); + } + Some(t) + } else { + None + }; + Ok(Self { heap, names, order }) + } + + /// `H5A__dense_create`: heap, name index, [creation-order index]. + fn create(img: &mut Image<'_>, index: bool) -> Result { + let heap = Heap::create_attribute_heap(img)?; + let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?; + let order = if index { + Some(Bt2::create( + img, + CORDER_BT2_TYPE, + ATTR_BT2_NODE, + ID_LEN + 5, + 100, + 40, + )?) + } else { + None + }; + Ok(Self { heap, names, order }) + } + + /// `H5A__dense_insert` of an encoded attribute message. + fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> { + let name = attr_name(body)?.to_vec(); + let id = self.heap.insert(img, body)?; + if id.len() != ID_LEN { + return Err(Error::Unsupported("attribute heap ID length".into())); + } + let hash = name_hash(&name); + let mut rec = id.clone(); + rec.push(0); + rec.extend_from_slice(&u32::from(crt).to_le_bytes()); + rec.extend_from_slice(&hash.to_le_bytes()); + let heap = &self.heap; + self.names + .insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?; + if let Some(t) = &mut self.order { + let key = u32::from(crt); + t.insert( + img, + &mut |_, r| Ok(key.cmp(&corder_of(r))), + &rec[..ID_LEN + 5], + )?; + } + Ok(()) + } + + fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + self.heap.finish(img)?; + self.names.finish(img)?; + if let Some(t) = &mut self.order { + t.finish(img)?; + } + Ok(()) + } +} + +/// Set attribute `name` of the object at `path` to `value`. +pub(super) fn set_attr( + f: &File, + img: &mut Image<'_>, + path: &str, + name: &str, + value: &AttrValue, +) -> Result<(), Error> { + let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; + let mut hdr = Header::load(img, addr)?; + let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value); + check_plain(&msg.datatype)?; + // libhdf5 encodes a simple dataspace with its maximum dimensions (the + // current ones when none were given), so an attribute takes the same + // space in a header or heap as when libhdf5 writes it. + if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() { + msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone()); + } + // H5A__set_version: version 1 unless the name is not ASCII (then 3), + // raised to the file's low bound — which is the earliest for a file + // libhdf5 opens without a libver setting (h5py's `r+`). + let body = if hdr.version == 1 || name.is_ascii() { + encode_attr_v1(&msg, img.ls) + } else { + let mut b = msg.serialize_v3(img.ls); + if !name.is_ascii() { + b[8] = 1; // UTF-8 name + } + b + }; + let mut ainfo = if hdr.version == 2 { + AInfo::load(img, &hdr)? + } else { + None + }; + if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) { + let mut ai = ai.clone(); + set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?; + return hdr.finish(img); + } + + let mut existing = None; + let mut count = 0usize; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + count += 1; + if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { + existing = Some(i); + } + } + if let Some(i) = existing { + hdr.delete(img, i)?; + count -= 1; + } + if hdr.version == 1 { + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; + return hdr.finish(img); + } + let tracked = hdr.flags & 0x04 != 0; + // H5O__attr_create: a missing Attribute Info message starts from + // nothing (and is added below, holding the new maximum creation index). + let new_ainfo = ainfo.is_none(); + let mut ai = ainfo.take().unwrap_or(AInfo { + idx: usize::MAX, + track: tracked, + index: hdr.flags & 0x08 != 0, + max_crt: 0, + fheap: undef(img.os), + name_bt2: undef(img.os), + corder_bt2: undef(img.os), + }); + let max_compact = usize::from(max_compact_attrs(img, &hdr)?); + if count == max_compact || body.len() >= MESG_MAX_SIZE { + if new_ainfo { + return Err(Error::Unsupported( + "dense attribute storage for an object without an Attribute Info message".into(), + )); + } + to_dense(img, &mut hdr, &mut ai)?; + set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?; + return hdr.finish(img); + } + let crt = ai.next_crt()?; + let corder = tracked.then_some(crt); + if new_ainfo { + // libhdf5 appends the Attribute Info message before the attribute + // when free space holds both, else after it, so that a new + // continuation chunk made for the attribute has room for it too. + let a = attr_info_message(hdr.flags, ai.max_crt, img.os); + let first = hdr.has_free(a.len() + hdr.hsize() + body.len()); + if first { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?; + } + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?; + if !first { + hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?; + } + } else { + hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?; + ai.store(img, &mut hdr)?; + } + hdr.finish(img) +} + +/// Move every compact attribute of the object into new dense storage, in +/// header message order (`H5O__attr_to_dense_cb`), leaving free space where +/// the messages were. +fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> { + let mut dense = Dense::create(img, ai.index)?; + for i in 0..hdr.msgs.len() { + if hdr.msgs[i].mtype != MSG_ATTRIBUTE { + continue; + } + if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { + return Err(Error::Unsupported("shared attribute message".into())); + } + let body = hdr.data(img, i)?; + let crt = if ai.track { + hdr.msgs[i].corder.unwrap_or(0) + } else { + NO_CRT_IDX + }; + dense.insert(img, &body, crt)?; + hdr.delete(img, i)?; + } + dense.finish(img)?; + ai.fheap = dense.heap.address(); + ai.name_bt2 = dense.names.address(); + if let Some(t) = &dense.order { + ai.corder_bt2 = t.address(); + } + ai.store(img, hdr) +} + +/// Set an attribute of an object whose attributes are in dense storage: an +/// attribute of that name whose new encoding has the old one's size is +/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one +/// is removed (`H5A__dense_remove`: name index, creation-order index, heap +/// object) and the new one inserted with the next creation index. +fn set_dense( + img: &mut Image<'_>, + hdr: &mut Header, + ai: &mut AInfo, + name: &[u8], + body: &[u8], +) -> Result<(), Error> { + let mut dense = Dense::open(img, ai)?; + let hash = name_hash(name); + let found = { + let heap = &dense.heap; + dense + .names + .find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))? + }; + if let Some(rec) = found { + if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? { + return dense.finish(img); + } + { + let heap = &dense.heap; + dense + .names + .remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?; + } + if let Some(t) = &mut dense.order { + let key = corder_of(&rec); + t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))? + .ok_or_else(|| { + Error::Unsupported("attribute missing from its creation-order index".into()) + })?; + } + dense.heap.remove(img, &rec[..ID_LEN])?; + } + let crt = ai.next_crt()?; + dense.insert(img, body, crt)?; + dense.finish(img)?; + ai.store(img, hdr) +} diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 8ff7205..80be9f0 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -180,11 +180,6 @@ impl Bt2 { self.tree_type } - /// Records in the whole tree. - pub(crate) fn len(&self) -> u64 { - self.root.all - } - /// Node geometry for depths `0..=self.depth` (`H5B2__hdr_init`, /// `H5B2__split_root`). fn compute_levels(&mut self, os: u8) -> Result<(), Error> { @@ -389,35 +384,6 @@ impl Bt2 { } } - /// Every record, in key order. - pub(crate) fn records(&mut self, img: &Image<'_>) -> Result>, Error> { - let mut out = Vec::new(); - if self.root.addr != undef(img.os) && self.root.nrec > 0 { - self.walk(img, self.root, self.depth, &mut out)?; - } - Ok(out) - } - - fn walk( - &mut self, - img: &Image<'_>, - p: Ptr, - depth: u16, - out: &mut Vec>, - ) -> Result<(), Error> { - self.load(img, p, depth)?; - let n = self.peek(p.addr).clone(); - for i in 0..=n.recs.len() { - if depth > 0 { - self.walk(img, n.ptrs[i], depth - 1, out)?; - } - if i < n.recs.len() { - out.push(n.recs[i].clone()); - } - } - Ok(()) - } - /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). pub(crate) fn update( &mut self, @@ -665,6 +631,7 @@ impl Bt2 { /// parent records between them) out over the children `lo..` of /// `parent`, with `counts[k]` records in the k-th; the records between /// them go back into the parent. + #[allow(clippy::too_many_arguments)] fn relayout( &mut self, parent: u64, diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs new file mode 100644 index 0000000..7c7b72a --- /dev/null +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -0,0 +1,1121 @@ +//! Changing a fractal heap (`FRHP`) in place, as libhdf5's `H5HF` code does +//! for the heaps that hold dense attributes: managed objects go into the +//! best-fitting free section the heap's free-space manager records (a new +//! direct block when none fits: the root direct block of an empty heap, the +//! next block of the root indirect block — created from the root direct +//! block, and doubled, as needed), huge objects (larger than the heap's +//! managed maximum) into their own file space tracked by the huge-object +//! version-2 B-tree; removed objects return their space to the free-space +//! manager, merged with adjacent free space. +//! +//! The heap's free-space manager (`FSHD` header, `FSSE` section info) is +//! kept exactly as libhdf5 keeps it: sections sorted by size then offset, +//! the header's counts and sizes, section info moved when its size changes, +//! the manager deleted when it tracks nothing. Header statistics (managed +//! space, allocated space, free space, allocation iterator, object counts) +//! follow libhdf5's arithmetic. +//! +//! What libhdf5 would do differently is refused ([`Error::Unsupported`], +//! before anything is written): I/O filters on the heap, child indirect +//! blocks, skipped blocks (a first object too large for the next block), +//! free sections other than "single" ones, freeing a whole direct block, +//! directly addressed huge objects, tiny objects. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::edit::btree2::Bt2; +use crate::edit::image::{Image, get_uint, put_uint, undef}; +use crate::error::Error; + +fn unsupported(why: &str) -> Error { + Error::Unsupported(format!("fractal heap: {why}")) +} + +fn bad(why: &str) -> Error { + Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError( + format!("fractal heap: {why}"), + )) +} + +/// `H5VM_log2_gen`: floor(log2(v)). +fn log2(v: u64) -> u32 { + 63 - v.max(1).leading_zeros() +} + +/// `H5VM_limit_enc_size`. +fn enc_size(v: u64) -> usize { + (log2(v) / 8 + 1) as usize +} + +fn le(b: &[u8]) -> u64 { + b.iter() + .take(8) + .enumerate() + .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) +} + +/// Free-space client ID for fractal heaps. +const FS_CLIENT_FHEAP: u8 = 0; +/// `H5HF_FSPACE_SHRINK` / `H5HF_FSPACE_EXPAND`. +const FS_SHRINK: u16 = 80; +const FS_EXPAND: u16 = 120; +/// Section classes the heap registers (single, first row, normal row, +/// indirect). +const FS_NCLASSES: u16 = 4; +/// Huge-object B-tree (`H5HF_HUGE_BT2_*`): record type 1 (indirectly +/// accessed, unfiltered), node size and split/merge percentages. +const HUGE_BT2_TYPE: u8 = 1; +const HUGE_BT2_NODE: u32 = 512; + +/// The heap's free-space manager: only "single" sections (free space inside +/// a direct block), keyed by heap offset. +struct FreeSpace { + addr: u64, + max_sect_addr: u16, + max_sect_size: u64, + shrink: u16, + expand: u16, + nclasses: u16, + sect_addr: u64, + sect_size: u64, + alloc_sect_size: u64, + sects: BTreeMap, +} + +impl FreeSpace { + fn hdr_len(os: u8, ls: u8) -> usize { + 6 + 4 * ls as usize + 8 + ls as usize + os as usize + 2 * ls as usize + 4 + } + + fn open(img: &Image<'_>, addr: u64) -> Result { + let (os, ls) = (img.os, img.ls); + let len = Self::hdr_len(os, ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"FSHD" || d[4] != 0 { + return Err(bad("bad free-space header")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("free-space header checksum mismatch")); + } + if d[5] != FS_CLIENT_FHEAP { + return Err(bad("free-space manager of another client")); + } + let l = ls as usize; + let mut p = 6; + let mut next = |w: usize| { + let v = le(&d[p..p + w]); + p += w; + v + }; + let tot_space = next(l); + let tot_count = next(l); + let serial = next(l); + let ghost = next(l); + let nclasses = next(2) as u16; + let shrink = next(2) as u16; + let expand = next(2) as u16; + let max_sect_addr = next(2) as u16; + let max_sect_size = next(l); + let sect_addr = next(os as usize); + let sect_size = next(l); + let alloc_sect_size = next(l); + if ghost != 0 || tot_count != serial { + return Err(unsupported("free space in row or indirect sections")); + } + let mut fs = Self { + addr, + max_sect_addr, + max_sect_size, + shrink, + expand, + nclasses, + sect_addr, + sect_size, + alloc_sect_size, + sects: BTreeMap::new(), + }; + if serial == 0 { + return Ok(fs); + } + if sect_addr == undef(os) || sect_size < 9 + os as u64 || sect_size > alloc_sect_size { + return Err(bad("bad free-space section info")); + } + let n = usize::try_from(sect_size).map_err(|_| bad("section info too large"))?; + let s = img.read(sect_addr, n)?; + let stored = u32::from_le_bytes(s[n - 4..].try_into().unwrap_or([0; 4])); + if &s[0..4] != b"FSSE" + || s[4] != 0 + || get_uint(&s[5..], os) != addr + || clawhdf5_format::checksum::jenkins_lookup3(&s[..n - 4]) != stored + { + return Err(bad("bad free-space section info")); + } + let cnt_w = enc_size(serial); + let len_w = enc_size(max_sect_size); + let off_w = (usize::from(max_sect_addr)).div_ceil(8); + let mut q = 5 + os as usize; + let mut seen = 0u64; + let mut total = 0u64; + while seen < serial { + if q + cnt_w + len_w > n - 4 { + return Err(bad("section info ends early")); + } + let count = le(&s[q..q + cnt_w]); + q += cnt_w; + let size = le(&s[q..q + len_w]); + q += len_w; + if count == 0 || size == 0 { + return Err(bad("empty section size node")); + } + for _ in 0..count { + if q + off_w + 1 > n - 4 { + return Err(bad("section info ends early")); + } + let off = le(&s[q..q + off_w]); + let class = s[q + off_w]; + q += off_w + 1; + if class != 0 { + return Err(unsupported("free space in row or indirect sections")); + } + if fs.sects.insert(off, size).is_some() { + return Err(bad("duplicate free section")); + } + seen += 1; + total += size; + } + } + if total != tot_space { + return Err(bad("free-space total disagrees with its sections")); + } + Ok(fs) + } + + /// Best fit (`H5FS__sect_find_node`): the smallest section of at least + /// `size` bytes, the lowest offset among equals. + fn find(&self, size: u64) -> Option<(u64, u64)> { + self.sects + .iter() + .filter(|&(_, &s)| s >= size) + .min_by_key(|&(&o, &s)| (s, o)) + .map(|(&o, &s)| (o, s)) + } + + /// The serialized section info's size (`H5FS__sect_serialize_size`). + fn needed(&self, os: u8) -> u64 { + let n = self.sects.len() as u64; + let prefix = 4 + 1 + u64::from(os) + 4; + if n == 0 { + return prefix; + } + let sizes: BTreeSet = self.sects.values().copied().collect(); + prefix + + sizes.len() as u64 * (enc_size(n) + enc_size(self.max_sect_size)) as u64 + + n * (u64::from(self.max_sect_addr).div_ceil(8) + 1) + } + + fn serialize(&self, os: u8) -> Vec { + let n = self.sects.len() as u64; + let mut by_size: BTreeMap> = BTreeMap::new(); + for (&o, &s) in &self.sects { + by_size.entry(s).or_default().push(o); + } + let cnt_w = enc_size(n); + let len_w = enc_size(self.max_sect_size); + let off_w = usize::from(self.max_sect_addr).div_ceil(8); + let mut d = Vec::new(); + d.extend_from_slice(b"FSSE"); + d.push(0); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.addr, os); + d.extend_from_slice(&a); + for (size, offs) in by_size { + d.extend_from_slice(&(offs.len() as u64).to_le_bytes()[..cnt_w]); + d.extend_from_slice(&size.to_le_bytes()[..len_w]); + for o in offs { + d.extend_from_slice(&o.to_le_bytes()[..off_w]); + d.push(0); + } + } + d + } + + fn write_header(&self, img: &mut Image<'_>) -> Result<(), Error> { + let (os, ls) = (img.os, img.ls); + let len = Self::hdr_len(os, ls); + let l = ls as usize; + let n = self.sects.len() as u64; + let tot: u64 = self.sects.values().sum(); + let mut d = Vec::with_capacity(len); + d.extend_from_slice(b"FSHD"); + d.push(0); + d.push(FS_CLIENT_FHEAP); + for v in [tot, n, n, 0] { + d.extend_from_slice(&v.to_le_bytes()[..l]); + } + for v in [self.nclasses, self.shrink, self.expand, self.max_sect_addr] { + d.extend_from_slice(&v.to_le_bytes()); + } + d.extend_from_slice(&self.max_sect_size.to_le_bytes()[..l]); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.sect_addr, os); + d.extend_from_slice(&a); + d.extend_from_slice(&self.sect_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.alloc_sect_size.to_le_bytes()[..l]); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d) + } +} + +/// An open fractal heap. +pub(crate) struct Heap { + addr: u64, + id_len: u16, + flags: u8, + max_man_size: u32, + next_huge_id: u64, + huge_bt2: u64, + total_man_free: u64, + fs_addr: u64, + man_size: u64, + man_alloc_size: u64, + man_iter_off: u64, + man_nobjs: u64, + huge_size: u64, + huge_nobjs: u64, + tiny_size: u64, + tiny_nobjs: u64, + width: u16, + start_block_size: u64, + max_direct_size: u64, + max_index: u16, + start_root_rows: u16, + root: u64, + root_rows: u16, + heap_off_size: usize, + heap_len_size: usize, + max_direct_rows: usize, + max_root_rows: usize, + /// Root indirect block children (all rows), when the root is one. + ents: Vec, + iblock_dirty: bool, + fs: Option, + fs_dirty: bool, + /// Direct blocks changed (address -> size), rechecksummed by `finish`. + dirty_dblocks: BTreeMap, +} + +impl Heap { + /// Header size (unfiltered heaps): the fixed fields, ten lengths and + /// two addresses of statistics, the doubling table, the checksum. + fn header_len(os: u8, ls: u8) -> usize { + 26 + 12 * ls as usize + 3 * os as usize + } + + /// Open the heap whose header is at `addr`. + pub(crate) fn open(img: &Image<'_>, addr: u64) -> Result { + let (os, ls) = (img.os, img.ls); + let o = os as usize; + let l = ls as usize; + let len = Self::header_len(os, ls); + let d = img.read(addr, len)?; + if &d[0..4] != b"FRHP" || d[4] != 0 { + return Err(bad("bad header")); + } + let filter_len = u16::from_le_bytes([d[7], d[8]]); + if filter_len != 0 { + return Err(unsupported("heaps with I/O filters")); + } + let stored = u32::from_le_bytes(d[len - 4..].try_into().unwrap_or([0; 4])); + if clawhdf5_format::checksum::jenkins_lookup3(&d[..len - 4]) != stored { + return Err(bad("header checksum mismatch")); + } + let mut p = 14usize; + let mut next = |w: usize| { + let v = le(&d[p..p + w]); + p += w; + v + }; + let next_huge_id = next(l); + let huge_bt2 = next(o); + let total_man_free = next(l); + let fs_addr = next(o); + let man_size = next(l); + let man_alloc_size = next(l); + let man_iter_off = next(l); + let man_nobjs = next(l); + let huge_size = next(l); + let huge_nobjs = next(l); + let tiny_size = next(l); + let tiny_nobjs = next(l); + let width = next(2) as u16; + let start_block_size = next(l); + let max_direct_size = next(l); + let max_index = next(2) as u16; + let start_root_rows = next(2) as u16; + let root = next(o); + let root_rows = next(2) as u16; + let pow2 = |v: u64| v != 0 && v.is_power_of_two(); + if !pow2(u64::from(width)) + || !pow2(start_block_size) + || !pow2(max_direct_size) + || max_direct_size < start_block_size + || !(1..=64).contains(&max_index) + { + return Err(bad("bad doubling table")); + } + let start_bits = log2(start_block_size) as usize; + let first_row_bits = start_bits + log2(u64::from(width)) as usize; + if usize::from(max_index) < first_row_bits { + return Err(bad("bad doubling table")); + } + let max_direct_rows = log2(max_direct_size) as usize - start_bits + 2; + let max_root_rows = usize::from(max_index) - first_row_bits + 1; + let heap_off_size = usize::from(max_index).div_ceil(8); + // H5HF__hdr_finish_init_phase1: object lengths are encoded in the + // bytes a direct block offset, or a managed object's size, needs. + let max_man_size = u32::from_le_bytes([d[10], d[11], d[12], d[13]]); + let len_bits = |v: u64| (log2(v) as usize).div_ceil(8); + let heap_len_size = len_bits(max_direct_size).min(len_bits(u64::from(max_man_size))); + let id_len = u16::from_le_bytes([d[5], d[6]]); + let mut h = Self { + addr, + id_len, + flags: d[9], + max_man_size, + next_huge_id, + huge_bt2, + total_man_free, + fs_addr, + man_size, + man_alloc_size, + man_iter_off, + man_nobjs, + huge_size, + huge_nobjs, + tiny_size, + tiny_nobjs, + width, + start_block_size, + max_direct_size, + max_index, + start_root_rows, + root, + root_rows, + heap_off_size, + heap_len_size, + max_direct_rows, + max_root_rows, + ents: Vec::new(), + iblock_dirty: false, + fs: None, + fs_dirty: false, + dirty_dblocks: BTreeMap::new(), + }; + if usize::from(id_len) != 1 + heap_off_size + heap_len_size && id_len < 8 { + return Err(unsupported("unusual heap ID length")); + } + if root_rows > 0 { + if root == undef(os) || usize::from(root_rows) > max_root_rows { + return Err(bad("bad root indirect block")); + } + let n = usize::from(root_rows) * usize::from(width); + let blen = h.iblock_len(root_rows, os); + let b = img.read(root, blen)?; + let stored = u32::from_le_bytes(b[blen - 4..].try_into().unwrap_or([0; 4])); + if &b[0..4] != b"FHIB" + || b[4] != 0 + || get_uint(&b[5..], os) != addr + || le(&b[5 + o..5 + o + heap_off_size]) != 0 + || clawhdf5_format::checksum::jenkins_lookup3(&b[..blen - 4]) != stored + { + return Err(bad("bad root indirect block")); + } + let at = 5 + o + heap_off_size; + h.ents = (0..n).map(|i| get_uint(&b[at + i * o..], os)).collect(); + let direct = h.max_direct_rows * usize::from(width); + if h.ents.iter().skip(direct).any(|&a| a != undef(os)) { + return Err(unsupported("child indirect blocks")); + } + } + if fs_addr != undef(os) { + let fs = FreeSpace::open(img, fs_addr)?; + if fs.max_sect_addr != max_index { + return Err(bad("free-space manager does not match the heap")); + } + h.fs = Some(fs); + } + Ok(h) + } + + /// Create an empty heap with libhdf5's attribute-heap parameters + /// (`H5A__dense_create`: width 4, 1 KiB starting blocks, 64 KiB direct + /// blocks, 40-bit offsets, 1 starting root row, checksummed direct + /// blocks, 4 KiB managed objects, 8-byte IDs). + pub(crate) fn create_attribute_heap(img: &mut Image<'_>) -> Result { + let (os, ls) = (img.os, img.ls); + let addr = img.alloc_reusing(Self::header_len(os, ls) as u64)?; + let mut h = Self { + addr, + id_len: 8, + flags: 0x02, + max_man_size: 4096, + next_huge_id: 0, + huge_bt2: undef(os), + total_man_free: 0, + fs_addr: undef(os), + man_size: 0, + man_alloc_size: 0, + man_iter_off: 0, + man_nobjs: 0, + huge_size: 0, + huge_nobjs: 0, + tiny_size: 0, + tiny_nobjs: 0, + width: 4, + start_block_size: 1024, + max_direct_size: 65536, + max_index: 40, + start_root_rows: 1, + root: undef(os), + root_rows: 0, + heap_off_size: 5, + heap_len_size: 2, + max_direct_rows: 8, + max_root_rows: 29, + ents: Vec::new(), + iblock_dirty: false, + fs: None, + fs_dirty: false, + dirty_dblocks: BTreeMap::new(), + }; + h.write_header(img)?; + Ok(h) + } + + pub(crate) fn address(&self) -> u64 { + self.addr + } + + fn overhead(&self, os: u8) -> u64 { + 5 + u64::from(os) + self.heap_off_size as u64 + if self.flags & 0x02 != 0 { 4 } else { 0 } + } + + fn row_size(&self, row: usize) -> u64 { + if row == 0 { + self.start_block_size + } else { + self.start_block_size << (row - 1) + } + } + + fn row_off(&self, row: usize) -> u64 { + if row == 0 { + 0 + } else { + (self.start_block_size * u64::from(self.width)) << (row - 1) + } + } + + fn iblock_len(&self, rows: u16, os: u8) -> usize { + 5 + os as usize + + self.heap_off_size + + usize::from(rows) * usize::from(self.width) * os as usize + + 4 + } + + fn first_row_bits(&self) -> u32 { + log2(self.start_block_size) + log2(u64::from(self.width)) + } + + /// Row and column of heap offset `off` in the root indirect block + /// (`H5HF__dtable_lookup`). + fn lookup(&self, off: u64) -> (usize, usize) { + let w = u64::from(self.width); + if off < self.start_block_size * w { + (0, (off / self.start_block_size) as usize) + } else { + let hb = log2(off); + let row = (hb - self.first_row_bits() + 1) as usize; + (row, ((off - (1u64 << hb)) / self.row_size(row)) as usize) + } + } + + /// The direct block holding heap offset `off`: (address, size, heap + /// offset of the block). + fn dblock_of(&self, os: u8, off: u64) -> Result<(u64, u64, u64), Error> { + if self.root == undef(os) { + return Err(bad("empty heap")); + } + if self.root_rows == 0 { + if off >= self.start_block_size { + return Err(bad("offset outside the heap")); + } + return Ok((self.root, self.start_block_size, 0)); + } + let (row, col) = self.lookup(off); + if row >= self.max_direct_rows || row >= usize::from(self.root_rows) { + return Err(unsupported("objects below child indirect blocks")); + } + let a = self.ents[row * usize::from(self.width) + col]; + if a == undef(os) { + return Err(bad("object in an unallocated block")); + } + Ok(( + a, + self.row_size(row), + self.row_off(row) + self.row_size(row) * col as u64, + )) + } + + /// Decode a managed heap ID: (offset, length). + fn man_id(&self, id: &[u8]) -> (u64, u64) { + let o = &id[1..]; + ( + le(&o[..self.heap_off_size]), + le(&o[self.heap_off_size..self.heap_off_size + self.heap_len_size]), + ) + } + + fn huge_id(&self, id: &[u8], os: u8, ls: u8) -> Result { + let w = usize::from(self.id_len - 1).min(8); + if self.huge_ids_direct(os, ls) { + return Err(unsupported("directly addressed huge objects")); + } + Ok(le(&id[1..1 + w])) + } + + fn huge_ids_direct(&self, os: u8, ls: u8) -> bool { + usize::from(os) + usize::from(ls) < usize::from(self.id_len) + } + + /// Where an object lives: (file address, length). + fn locate(&self, img: &Image<'_>, id: &[u8]) -> Result<(u64, u64), Error> { + if id.len() != usize::from(self.id_len) || id[0] >> 6 != 0 { + return Err(bad("bad heap ID")); + } + match (id[0] >> 4) & 0x03 { + 0 => { + let (off, len) = self.man_id(id); + let (a, size, boff) = self.dblock_of(img.os, off)?; + let within = off - boff; + if within < self.overhead(img.os) || within + len > size { + return Err(bad("object outside its block")); + } + Ok((a + within, len)) + } + 1 => { + if self.huge_ids_direct(img.os, img.ls) { + return Err(unsupported("directly addressed huge objects")); + } + let key = self.huge_id(id, img.os, img.ls)?; + let mut t = Bt2::open(img, self.huge_bt2)?; + let rec = t + .find( + img, + &mut |_, r| Ok(key.cmp(&huge_rec_id(r, img.os, img.ls))), + )? + .ok_or_else(|| bad("huge object missing from its B-tree"))?; + Ok(( + get_uint(&rec, img.os), + get_uint(&rec[img.os as usize..], img.ls), + )) + } + _ => Err(unsupported("tiny objects")), + } + } + + /// The object `id` names. + pub(crate) fn read(&self, img: &Image<'_>, id: &[u8]) -> Result, Error> { + let (a, len) = self.locate(img, id)?; + let n = usize::try_from(len).map_err(|_| bad("object too large"))?; + img.read(a, n) + } + + /// Overwrite managed object `id` with `obj` of the same length + /// (`H5HF_write`). + pub(crate) fn write_in_place( + &mut self, + img: &mut Image<'_>, + id: &[u8], + obj: &[u8], + ) -> Result { + if (id[0] >> 4) & 0x03 == 1 { + // H5HF__huge_write: an unfiltered huge object in place. + let (a, len) = self.locate(img, id)?; + if len != obj.len() as u64 { + return Ok(false); + } + img.write(a, obj)?; + return Ok(true); + } + if (id[0] >> 4) & 0x03 != 0 { + return Ok(false); + } + let (off, len) = self.man_id(id); + if len != obj.len() as u64 { + return Ok(false); + } + let (a, size, boff) = self.dblock_of(img.os, off)?; + img.write(a + (off - boff), obj)?; + self.dirty_dblocks.insert(a, size); + Ok(true) + } + + /// Insert `obj`; returns its heap ID (`H5HF_insert`). + pub(crate) fn insert(&mut self, img: &mut Image<'_>, obj: &[u8]) -> Result, Error> { + let os = img.os; + let n = obj.len() as u64; + if n == 0 { + return Err(bad("empty object")); + } + if n > u64::from(self.max_man_size) { + return self.huge_insert(img, obj); + } + let tiny_max = u64::from(self.id_len) - 1; + if n <= tiny_max { + return Err(unsupported("tiny objects")); + } + let (off, size) = match self.fs.as_ref().and_then(|fs| fs.find(n)) { + Some((o, s)) => { + self.fs.as_mut().expect("found above").sects.remove(&o); + (o, s) + } + None => self.dblock_new(img, n)?, + }; + // H5HF__sect_single_reduce: the object goes at the section's start. + if size > n { + self.fs_add(img, off + n, size - n)?; + } + self.fs_dirty = true; + let (a, bsize, boff) = self.dblock_of(os, off)?; + img.write(a + (off - boff), obj)?; + self.dirty_dblocks.insert(a, bsize); + self.man_nobjs += 1; + self.total_man_free = self.total_man_free.saturating_sub(n); + let mut id = vec![0u8; usize::from(self.id_len)]; + id[1..1 + self.heap_off_size].copy_from_slice(&off.to_le_bytes()[..self.heap_off_size]); + id[1 + self.heap_off_size..1 + self.heap_off_size + self.heap_len_size] + .copy_from_slice(&n.to_le_bytes()[..self.heap_len_size]); + Ok(id) + } + + /// Add a single section (no merging: `H5FS_sect_add` without + /// `H5FS_ADD_RETURNED_SPACE`), creating the free-space manager if the + /// heap has none. + fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { + if self.fs.is_none() { + let addr = img.alloc_reusing(FreeSpace::hdr_len(img.os, img.ls) as u64)?; + self.fs = Some(FreeSpace { + addr, + max_sect_addr: self.max_index, + max_sect_size: self.max_direct_size, + shrink: FS_SHRINK, + expand: FS_EXPAND, + nclasses: FS_NCLASSES, + sect_addr: undef(img.os), + sect_size: 0, + alloc_sect_size: 0, + sects: BTreeMap::new(), + }); + self.fs_addr = addr; + } + self.fs + .as_mut() + .expect("created above") + .sects + .insert(off, size); + self.fs_dirty = true; + Ok(()) + } + + /// `H5HF__man_dblock_new`: a direct block for an object of `request` + /// bytes; returns its free section (not in the free-space manager). + fn dblock_new(&mut self, img: &mut Image<'_>, request: u64) -> Result<(u64, u64), Error> { + let os = img.os; + let mut min = if request < self.start_block_size { + self.start_block_size + } else { + 1u64 << (1 + log2(request)) + }; + if min < self.overhead(os) + request { + min *= 2; + } + if min > self.max_direct_size { + return Err(bad("object larger than a direct block")); + } + if self.root == undef(os) && min == self.start_block_size { + let a = self.dblock_create(img, 0, self.start_block_size)?; + self.root = a; + self.root_rows = 0; + self.man_size = self.start_block_size; + self.total_man_free += self.start_block_size - self.overhead(os); + return Ok((self.overhead(os), self.start_block_size - self.overhead(os))); + } + if self.root == undef(os) { + return Err(unsupported( + "a first object too large for the starting block", + )); + } + // H5HF__hdr_update_iter. + if self.root_rows == 0 { + self.root_create(img, min)?; + } + let (mut row, mut col) = self.iter_pos(); + let min_row = self.size_to_row(min); + if min_row > row && row < usize::from(self.root_rows) { + return Err(unsupported("skipping blocks too small for an object")); + } + while row >= usize::from(self.root_rows) { + self.root_double(img, min)?; + (row, col) = self.iter_pos(); + } + if row >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + let size = self.row_size(row); + if min > size { + return Err(unsupported("skipping blocks too small for an object")); + } + self.man_iter_off += size; + let entry = row * usize::from(self.width) + col; + let block_off = self.row_off(row) + size * col as u64; + let a = self.dblock_create(img, block_off, size)?; + self.ents[entry] = a; + self.iblock_dirty = true; + Ok((block_off + self.overhead(os), size - self.overhead(os))) + } + + /// The allocation iterator's (row, column) in the root indirect block. + fn iter_pos(&self) -> (usize, usize) { + if self.man_iter_off >= self.man_size { + (usize::from(self.root_rows), 0) + } else { + self.lookup(self.man_iter_off) + } + } + + /// `H5HF__dtable_size_to_row`. + fn size_to_row(&self, size: u64) -> usize { + if size == self.start_block_size { + 0 + } else { + (log2(size) - log2(self.start_block_size) + 1) as usize + } + } + + /// Allocate and write an empty direct block at heap offset `block_off`. + fn dblock_create( + &mut self, + img: &mut Image<'_>, + block_off: u64, + size: u64, + ) -> Result { + let os = img.os; + let a = img.alloc_reusing(size)?; + let n = usize::try_from(size).map_err(|_| bad("block too large"))?; + let mut d = vec![0u8; n]; + d[0..4].copy_from_slice(b"FHDB"); + put_uint(&mut d[5..], self.addr, os); + let at = 5 + os as usize; + d[at..at + self.heap_off_size] + .copy_from_slice(&block_off.to_le_bytes()[..self.heap_off_size]); + img.write(a, &d)?; + self.dirty_dblocks.insert(a, size); + self.man_alloc_size += size; + Ok(a) + } + + /// `H5HF__man_iblock_root_create`: the root direct block becomes entry + /// 0 of a new root indirect block. + fn root_create(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + let os = img.os; + let mut nrows = if self.start_root_rows == 0 { + self.max_root_rows + } else { + usize::from(self.start_root_rows) + }; + if self.start_root_rows != 0 { + let mut block_row_off = (log2(min) - log2(self.start_block_size)) as usize; + if block_row_off > 0 { + block_row_off += 1; + } + nrows = nrows.max(1 + block_row_off); + } + if nrows > self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + if min > self.start_block_size { + return Err(unsupported("skipping blocks too small for an object")); + } + let have_direct = self.root != undef(os); + let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; + let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + self.ents = vec![undef(os); nrows * usize::from(self.width)]; + if have_direct { + self.ents[0] = self.root; + self.man_iter_off = self.start_block_size; + } else { + self.man_iter_off = 0; + } + self.iblock_dirty = true; + self.root_rows = rows; + self.root = a; + let w = u64::from(self.width); + let ov = self.overhead(os); + let mut acc: u64 = (0..nrows).map(|u| (self.row_size(u) - ov) * w).sum(); + if have_direct { + acc -= self.row_size(0) - ov; + } + self.man_size = self.row_off(nrows); + self.total_man_free += acc; + Ok(()) + } + + /// `H5HF__man_iblock_root_double`: the root indirect block gets twice + /// its rows, at a new address. + fn root_double(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + let os = img.os; + let old = usize::from(self.root_rows); + let (row, _) = self.iter_pos(); + let next_size = self.row_size(row.min(self.max_root_rows - 1)); + if old < self.max_direct_rows && min > next_size { + return Err(unsupported("skipping blocks too small for an object")); + } + let new = (2 * old).min(self.max_root_rows); + if new > self.max_direct_rows || new == old { + return Err(unsupported("child indirect blocks")); + } + img.free(self.root, self.iblock_len(self.root_rows, os) as u64); + let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; + let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let w = usize::from(self.width); + self.ents.resize(new * w, undef(os)); + let ov = self.overhead(os); + let acc: u64 = (old * w..new * w).map(|u| self.row_size(u / w) - ov).sum(); + self.root_rows = rows; + self.root = a; + self.iblock_dirty = true; + self.man_size = 2 * self.row_off(new - 1); + self.total_man_free += acc; + Ok(()) + } + + /// Remove object `id` (`H5HF_remove`): a managed object's space goes + /// back to the free-space manager, merged with free space next to it; a + /// huge object's file space is freed. + pub(crate) fn remove(&mut self, img: &mut Image<'_>, id: &[u8]) -> Result<(), Error> { + match (id[0] >> 4) & 0x03 { + 0 => { + let os = img.os; + let (off, len) = self.man_id(id); + let (_, size, boff) = self.dblock_of(os, off)?; + let (mut lo, mut hi) = (off, off + len); + if let Some(fs) = &self.fs { + if let Some((&o, &s)) = fs.sects.range(..off).next_back() + && o + s == off + { + lo = o; + } + if let Some(&s) = fs.sects.get(&hi) { + hi += s; + } + } + if hi - lo == size - self.overhead(os) && boff <= lo { + return Err(unsupported( + "removing the last object of a direct block (libhdf5 frees the block)", + )); + } + if let Some(fs) = &mut self.fs { + fs.sects.remove(&lo); + fs.sects.remove(&(off + len)); + } + self.fs_add(img, lo, hi - lo)?; + self.total_man_free += len; + self.man_nobjs -= 1; + Ok(()) + } + 1 => { + let key = self.huge_id(id, img.os, img.ls)?; + let (os, ls) = (img.os, img.ls); + let mut t = Bt2::open(img, self.huge_bt2)?; + let rec = t + .remove(img, &mut |_, r| Ok(key.cmp(&huge_rec_id(r, os, ls))))? + .ok_or_else(|| bad("huge object missing from its B-tree"))?; + t.finish(img)?; + let len = get_uint(&rec[os as usize..], ls); + img.free(get_uint(&rec, os), len); + self.huge_size = self.huge_size.saturating_sub(len); + self.huge_nobjs = self.huge_nobjs.saturating_sub(1); + Ok(()) + } + _ => Err(unsupported("tiny objects")), + } + } + + /// `H5HF__huge_insert` (unfiltered heap, IDs that index the huge-object + /// B-tree). + fn huge_insert(&mut self, img: &mut Image<'_>, obj: &[u8]) -> Result, Error> { + let (os, ls) = (img.os, img.ls); + if self.huge_ids_direct(os, ls) { + return Err(unsupported("directly addressed huge objects")); + } + let rs = os as usize + 2 * ls as usize; + let mut t = if self.huge_bt2 == undef(os) { + let t = Bt2::create(img, HUGE_BT2_TYPE, HUGE_BT2_NODE, rs, 100, 40)?; + self.huge_bt2 = t.address(); + t + } else { + Bt2::open(img, self.huge_bt2)? + }; + if t.record_size() != rs || t.tree_type() != HUGE_BT2_TYPE { + return Err(unsupported("huge-object B-tree layout")); + } + let n = obj.len() as u64; + let a = img.alloc_reusing(n)?; + img.write(a, obj)?; + let w = usize::from(self.id_len - 1).min(8); + let max_id = if w >= 8 { + u64::MAX + } else { + (1u64 << (8 * w)) - 1 + }; + if self.flags & 0x01 != 0 || self.next_huge_id >= max_id { + return Err(unsupported("huge object IDs wrapped")); + } + self.next_huge_id += 1; + let key = self.next_huge_id; + if key == max_id { + self.flags |= 0x01; + } + let mut rec = vec![0u8; rs]; + put_uint(&mut rec, a, os); + put_uint(&mut rec[os as usize..], n, ls); + put_uint(&mut rec[os as usize + ls as usize..], key, ls); + t.insert(img, &mut |_, r| Ok(key.cmp(&huge_rec_id(r, os, ls))), &rec)?; + t.finish(img)?; + self.huge_size += n; + self.huge_nobjs += 1; + let mut id = vec![0u8; usize::from(self.id_len)]; + id[0] = 0x10; + id[1..1 + w].copy_from_slice(&key.to_le_bytes()[..w]); + Ok(id) + } + + fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let (os, ls) = (img.os, img.ls); + let l = ls as usize; + let mut d = Vec::with_capacity(Self::header_len(os, ls)); + d.extend_from_slice(b"FRHP"); + d.push(0); + d.extend_from_slice(&self.id_len.to_le_bytes()); + d.extend_from_slice(&0u16.to_le_bytes()); + d.push(self.flags); + d.extend_from_slice(&self.max_man_size.to_le_bytes()); + let addr = |d: &mut Vec, v: u64| { + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, v, os); + d.extend_from_slice(&a); + }; + d.extend_from_slice(&self.next_huge_id.to_le_bytes()[..l]); + addr(&mut d, self.huge_bt2); + d.extend_from_slice(&self.total_man_free.to_le_bytes()[..l]); + addr(&mut d, self.fs_addr); + for v in [ + self.man_size, + self.man_alloc_size, + self.man_iter_off, + self.man_nobjs, + self.huge_size, + self.huge_nobjs, + self.tiny_size, + self.tiny_nobjs, + ] { + d.extend_from_slice(&v.to_le_bytes()[..l]); + } + d.extend_from_slice(&self.width.to_le_bytes()); + d.extend_from_slice(&self.start_block_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.max_direct_size.to_le_bytes()[..l]); + d.extend_from_slice(&self.max_index.to_le_bytes()); + d.extend_from_slice(&self.start_root_rows.to_le_bytes()); + addr(&mut d, self.root); + d.extend_from_slice(&self.root_rows.to_le_bytes()); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&d); + d.extend_from_slice(&sum.to_le_bytes()); + img.write(self.addr, &d) + } + + /// Write back everything that changed: direct block checksums, the + /// root indirect block, the free-space manager, the header. + pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> { + let os = img.os; + if self.flags & 0x02 != 0 { + let at = 5 + os as usize + self.heap_off_size; + for (a, size) in std::mem::take(&mut self.dirty_dblocks) { + let n = usize::try_from(size).map_err(|_| bad("block too large"))?; + let mut b = img.read(a, n)?; + b[at..at + 4].fill(0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&b); + img.write(a + at as u64, &sum.to_le_bytes())?; + } + } + if self.iblock_dirty && self.root_rows > 0 { + let len = self.iblock_len(self.root_rows, os); + let mut b = Vec::with_capacity(len); + b.extend_from_slice(b"FHIB"); + b.push(0); + let mut a = vec![0u8; os as usize]; + put_uint(&mut a, self.addr, os); + b.extend_from_slice(&a); + b.extend_from_slice(&vec![0u8; self.heap_off_size]); + for &e in &self.ents { + put_uint(&mut a, e, os); + b.extend_from_slice(&a); + } + let sum = clawhdf5_format::checksum::jenkins_lookup3(&b); + b.extend_from_slice(&sum.to_le_bytes()); + img.write(self.root, &b)?; + self.iblock_dirty = false; + } + if self.fs_dirty { + self.fs_dirty = false; + if let Some(mut fs) = self.fs.take() { + let hdr_len = FreeSpace::hdr_len(os, img.ls) as u64; + if fs.sects.is_empty() { + // H5HF__space_close: a manager that tracks nothing is + // deleted. + img.free(fs.addr, hdr_len); + if fs.sect_addr != undef(os) { + img.free(fs.sect_addr, fs.alloc_sect_size); + } + self.fs_addr = undef(os); + } else { + let need = fs.needed(os); + if fs.sect_addr == undef(os) || need != fs.alloc_sect_size { + if fs.sect_addr != undef(os) { + img.free(fs.sect_addr, fs.alloc_sect_size); + } + fs.sect_addr = img.alloc_reusing(need)?; + fs.alloc_sect_size = need; + } + fs.sect_size = fs.alloc_sect_size; + let mut s = fs.serialize(os); + let n = usize::try_from(fs.sect_size).map_err(|_| bad("section info"))?; + s.resize(n - 4, 0); + let sum = clawhdf5_format::checksum::jenkins_lookup3(&s); + s.extend_from_slice(&sum.to_le_bytes()); + img.write(fs.sect_addr, &s)?; + fs.write_header(img)?; + self.fs = Some(fs); + } + } + } + self.write_header(img) + } +} + +/// The ID in a huge-object B-tree record (type 1: address, length, ID). +fn huge_rec_id(r: &[u8], os: u8, ls: u8) -> u64 { + get_uint(&r[os as usize + ls as usize..], ls) +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index b14244a..c32431a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -15,10 +15,12 @@ //! file) is dropped before the first write, so no `&[u8]` over the mapping //! is alive while the file changes (see `image`). +mod attrs; mod btree1; mod btree2; mod earray; mod farray; +mod fheap; mod image; mod ohdr; mod select; @@ -27,7 +29,6 @@ use std::collections::{BTreeMap, HashMap}; use std::fs::{OpenOptions, TryLockError}; use std::path::{Path, PathBuf}; -use clawhdf5_format::attribute::AttributeMessage; use clawhdf5_format::chunked_read::{ChunkInfo, list_chunks}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_read::NativeElement; @@ -43,8 +44,8 @@ use btree1::{BTree1, Key}; use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; -use image::{Image, get_uint, put_uint, undef}; -use ohdr::{Header, MSG_ATTRIBUTE}; +use image::{Image, put_uint, undef}; +use ohdr::Header; const MSG_DATASPACE: u16 = 0x01; const MSG_LAYOUT: u16 = 0x08; @@ -918,101 +919,29 @@ impl FileEditor { /// Set attribute `name` of the object at `path` (a group or dataset; /// `"/"` is the root group) to `value`, replacing an attribute of that - /// name. The attribute goes into free space in the object header, or a - /// new header continuation chunk at the end of the file. + /// name, as libhdf5 creates attributes (`H5O__attr_create`). /// - /// [`Error::Unsupported`] for an object whose attributes are in dense - /// storage (or would have to move there: more than the object's - /// compact-attribute limit), one that tracks attribute creation order, - /// or one with shared attribute messages. + /// A compact attribute goes into free space in the object header, or a + /// new header continuation chunk; an object that tracks creation order + /// gives it the next creation index. When the object reaches its + /// compact-attribute limit (or the attribute is too large for a header + /// message) its attributes move to dense storage, and objects already + /// using dense storage get the attribute there: a fractal heap object + /// (in free heap space, a new heap block, or its own file space when + /// larger than the heap's managed limit) indexed by name and, when the + /// object indexes creation order, by creation index. An attribute + /// replaced by one of the same encoded size is rewritten where it is. + /// + /// [`Error::Unsupported`] for shared attribute messages, heaps this + /// editor cannot extend the way libhdf5 would (child indirect blocks, + /// skipped blocks, free space other than within direct blocks, freeing + /// a whole block), and version-1 object headers asked for an attribute + /// larger than a header message holds. pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { if name.is_empty() { return Err(Error::InvalidArgument("empty attribute name".into())); } - self.edit(|f, img| { - let addr = - clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?; - let mut hdr = Header::load(img, addr)?; - if hdr.version == 2 && hdr.flags & 0x04 != 0 { - return Err(Error::Unsupported( - "object tracks attribute creation order".into(), - )); - } - if let Some(i) = hdr.find(MSG_ATTR_INFO) { - let d = hdr.data(img, i)?; - // version(1) flags(1) [max creation index(2)] fractal heap - // address, name index address, [order index address]. - let mut p = 2; - if d.get(1).is_some_and(|f| f & 0x01 != 0) { - p += 2; - } - let os = img.os as usize; - if d.len() < p + os { - return Err(Error::Unsupported("short attribute info message".into())); - } - if get_uint(&d[p..], img.os) != undef(img.os) { - return Err(Error::Unsupported( - "object with attributes in dense storage".into(), - )); - } - } - let mut existing = None; - let mut count = 0usize; - for i in 0..hdr.msgs.len() { - if hdr.msgs[i].mtype != MSG_ATTRIBUTE { - continue; - } - if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 { - return Err(Error::Unsupported("shared attribute message".into())); - } - count += 1; - if attr_name(&hdr.data(img, i)?)? == name.as_bytes() { - existing = Some(i); - } - } - if existing.is_none() && hdr.version == 2 { - let max_compact = max_compact_attrs(img, &hdr)?; - if count + 1 > usize::from(max_compact) { - return Err(Error::Unsupported(format!( - "object already has {count} compact attributes (its limit is \ - {max_compact}); more need dense storage" - ))); - } - } - let msg = clawhdf5_format::type_builders::build_attr_message(name, value); - check_plain(&msg.datatype)?; - let body = if hdr.version == 1 { - encode_attr_v1(&msg, img.ls) - } else { - let mut b = msg.serialize_v3(img.ls); - if !name.is_ascii() { - b[8] = 1; // UTF-8 name - } - b - }; - if let Some(i) = existing { - hdr.delete(img, i)?; - } - // libhdf5 counts a version-2 header's attributes through its - // Attribute Info message and reports none without one; like - // H5O__attr_create, add it when missing: before the attribute - // when free space holds both (libhdf5's order), else after it, - // so that a new continuation chunk made for the attribute has - // room for it too. - let ainfo = (hdr.version == 2 && hdr.find(MSG_ATTR_INFO).is_none()) - .then(|| attr_info_message(hdr.flags, img.os)); - let ainfo_first = ainfo - .as_ref() - .is_some_and(|a| hdr.has_free(a.len() + hdr.hsize() + body.len())); - if let Some(a) = ainfo.as_ref().filter(|_| ainfo_first) { - hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; - } - hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?; - if let Some(a) = ainfo.as_ref().filter(|_| !ainfo_first) { - hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, a, None)?; - } - hdr.finish(img) - }) + self.edit(|f, img| attrs::set_attr(f, img, path, name, value)) } } @@ -1097,81 +1026,6 @@ fn set_superblock_eof( Ok(()) } -/// The name bytes of an attribute message body (without the NUL). -fn attr_name(d: &[u8]) -> Result<&[u8], Error> { - let bad = || Error::Unsupported("malformed attribute message".into()); - let (len, at) = match d.first() { - Some(1) | Some(2) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8), - Some(3) => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9), - _ => return Err(bad()), - }; - let name = d.get(at..at + len).ok_or_else(bad)?; - Ok(name.split(|&b| b == 0).next().unwrap_or(name)) -} - -/// A new Attribute Info message for a version-2 header with flags -/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order -/// tracked / indexed as the header's flags say, maximum creation index 0, -/// and no dense storage (undefined fractal heap and B-tree addresses). -fn attr_info_message(hdr_flags: u8, os: u8) -> Vec { - let track = hdr_flags & 0x04 != 0; - let index = hdr_flags & 0x08 != 0; - let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)]; - if track { - b.extend_from_slice(&0u16.to_le_bytes()); - } - let undef_addr = vec![0xffu8; os as usize]; - b.extend_from_slice(&undef_addr); - b.extend_from_slice(&undef_addr); - if index { - b.extend_from_slice(&undef_addr); - } - b -} - -/// A version-2 header's limit on compact attributes: stored when its flags -/// say so, else libhdf5's default of 8. -fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result { - if hdr.flags & 0x10 == 0 { - return Ok(8); - } - let mut p = hdr.addr + 6; - if hdr.flags & 0x20 != 0 { - p += 16; - } - let b = img.read(p, 2)?; - Ok(u16::from_le_bytes([b[0], b[1]])) -} - -/// A version-1 attribute message (what libhdf5 writes in a version-1 object -/// header): name, datatype and dataspace each padded to 8 bytes, the -/// dataspace as a version-1 dataspace message. -fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec { - let mut name = a.name.as_bytes().to_vec(); - name.push(0); - let dt = a.datatype.serialize(); - let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0]; - if a.dataspace.space_type == DataspaceType::Simple { - for &d in &a.dataspace.dimensions { - let mut b = vec![0u8; ls as usize]; - put_uint(&mut b, d, ls); - ds.extend_from_slice(&b); - } - } else { - ds[1] = 0; - } - let mut out = vec![1u8, 0]; - out.extend_from_slice(&(name.len() as u16).to_le_bytes()); - out.extend_from_slice(&(dt.len() as u16).to_le_bytes()); - out.extend_from_slice(&(ds.len() as u16).to_le_bytes()); - for part in [&name, &dt, &ds] { - out.extend_from_slice(part); - out.resize(out.len().next_multiple_of(8), 0); - } - out.extend_from_slice(&a.raw_data); - out -} - fn write_selection( f: &File, img: &mut Image<'_>, diff --git a/crates/clawhdf5/src/edit/ohdr.rs b/crates/clawhdf5/src/edit/ohdr.rs index f3abfe0..8aa2a15 100644 --- a/crates/clawhdf5/src/edit/ohdr.rs +++ b/crates/clawhdf5/src/edit/ohdr.rs @@ -59,7 +59,7 @@ pub(crate) struct Header { added: usize, } -const MAX_CHUNKS: usize = 1024; +const MAX_CHUNKS: usize = 1 << 16; fn corrupt(why: &'static str) -> Error { Error::Format(FormatError::InvalidObjectHeader(why)) @@ -118,7 +118,11 @@ impl Header { }); h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?; } - while let Some((caddr, clen)) = pending.pop() { + // Continuation chunks in the order their messages are found, as + // H5O_protect loads them (so messages keep libhdf5's order). + let mut next = 0; + while let Some(&(caddr, clen)) = pending.get(next) { + next += 1; if h.chunks.len() >= MAX_CHUNKS { return Err(corrupt("too many object header chunks")); } From dc9cfba6bb39b1b4481ca86d577d7e6c26aacf7e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:04:15 -0500 Subject: [PATCH 05/15] edit: reuse space freed earlier in the editing session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A FileEditor now keeps the space its edits free — a filtered chunk that moved, chunks a shrink removed, B-tree nodes merged away, a heap's replaced root indirect block or free-space section info, huge objects replaced — and later edits allocate from it (best fit, lowest address among equals, zeroed) before growing the file. An edit never reuses what it frees itself: until it is committed the file still refers to that space. Reused blocks are written in the commit's first phase with the space past the old end of file (nothing on disk refers to them yet), before any existing byte changes, so the crash-safety ordering holds. Space still free when the editor is dropped is leaked, as libhdf5 leaks it without a persistent free-space manager (files that have one, or use paged aggregation, are still refused at open). FileEditor::reusable_bytes reports what is left to reuse. Tests: FreeList merging and best fit, the edit-local rule and the commit split (image unit tests); a shrink followed by regrowth writing the same data reuses every removed chunk and leaves the file size unchanged, while one editor per edit grows the file, h5py/h5dump/h5rs check read both and h5py continues (freed_space_is_reused_within_a_session). measure_append_waste (edit_interop, ignored), same workload, one editor, before -> after (bytes; libhdf5 in brackets), on tank 2026-09-26: gzip chunks 1024, 1000 appends of 100: 307210 -> 306780 (306058); gzip chunks 4096, 2000 appends of 10: 119684 -> 79829 (50292); unfiltered unchanged (no space is freed). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 73 ++++++ crates/clawhdf5-tools/tests/edit_interop.rs | 7 +- crates/clawhdf5/src/edit/btree2.rs | 2 +- crates/clawhdf5/src/edit/fheap.rs | 14 +- crates/clawhdf5/src/edit/image.rs | 226 +++++++++++++++--- crates/clawhdf5/src/edit/mod.rs | 58 +++-- 6 files changed, 317 insertions(+), 63 deletions(-) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 036839f..332dffa 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1323,3 +1323,76 @@ fn dense_attribute_refusals_change_nothing() { )); check_tools(&path, true); } + +/// Space one edit frees is reused by later edits of the same editor: the +/// chunks a shrink removes are where the chunks of the following growth +/// go, so the file does not grow; with a new editor per edit (nothing to +/// reuse) it does. h5py, h5dump and `h5rs check` read the result, and +/// h5py goes on. +#[test] +fn freed_space_is_reused_within_a_session() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let mut sizes = Vec::new(); + for session in [true, false] { + let path = dir.path().join(format!("reuse_{session}.h5")); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 f.create_dataset('x', data=np.zeros(2000, dtype=' = (0..2000).map(|_| rng.next() as i32).collect(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.write_values("x", &Selection::All, &vals).unwrap(); + drop(ed); + let len0 = std::fs::metadata(&path).unwrap().len(); + let mut ed = FileEditor::open(&path).unwrap(); + ed.resize("x", &[1000]).unwrap(); + if session { + assert!(ed.reusable_bytes() > 0); + } else { + ed = { + drop(ed); + FileEditor::open(&path).unwrap() + }; + } + ed.resize("x", &[2000]).unwrap(); + ed.write_values("x", &block(&[1000], &[1000]), &vals[1000..]) + .unwrap(); + if session { + assert_eq!(ed.reusable_bytes(), 0, "every freed chunk is reused"); + } + drop(ed); + let len1 = std::fs::metadata(&path).unwrap().len(); + sizes.push((len0, len1)); + let m = Model { + shape: vec![2000], + data: vals.clone(), + }; + verify(&path, "x", &m); + check_tools(&path, true); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 f['x'].resize((2100,))\n\ + \x20 f['x'][2000:] = 9\n", + p = path.to_str().unwrap() + )); + let mut m = m; + m.resize(&[2100], 0); + m.write_block(&[2000], &[100], &[9; 100]); + verify(&path, "x", &m); + check_tools(&path, true); + } + let (reuse, fresh) = (sizes[0], sizes[1]); + assert_eq!( + reuse.1, reuse.0, + "a session reusing freed chunks does not grow the file" + ); + assert!(fresh.1 > fresh.0, "without reuse the file grows"); +} diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index bc4c9a6..cb345f8 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -1122,9 +1122,10 @@ fn out_of_order_chunk_creation_matches_libhdf5() { } } -/// Not a check: prints how much space an append workload leaks (the editor -/// never reuses space), against libhdf5 doing the same appends and against -/// `h5repack` of each. Run with `--ignored --nocapture`. +/// Not a check: prints how much space an append workload leaks (one +/// editor for the whole workload, which reuses the space it frees but not +/// space it cannot fit a grown chunk into), against libhdf5 doing the same +/// appends and against `h5repack` of each. Run with `--ignored --nocapture`. #[test] #[ignore] fn measure_append_waste() { diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 80be9f0..5761e1e 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -316,7 +316,7 @@ impl Bt2 { } fn new_node(&mut self, img: &mut Image<'_>, depth: u16) -> Result { - let addr = img.alloc_reusing(u64::from(self.node_size))?; + let addr = img.alloc(u64::from(self.node_size))?; self.nodes.insert( addr, Node { diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs index 7c7b72a..2fab657 100644 --- a/crates/clawhdf5/src/edit/fheap.rs +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -455,7 +455,7 @@ impl Heap { /// blocks, 4 KiB managed objects, 8-byte IDs). pub(crate) fn create_attribute_heap(img: &mut Image<'_>) -> Result { let (os, ls) = (img.os, img.ls); - let addr = img.alloc_reusing(Self::header_len(os, ls) as u64)?; + let addr = img.alloc(Self::header_len(os, ls) as u64)?; let mut h = Self { addr, id_len: 8, @@ -706,7 +706,7 @@ impl Heap { /// heap has none. fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { if self.fs.is_none() { - let addr = img.alloc_reusing(FreeSpace::hdr_len(img.os, img.ls) as u64)?; + let addr = img.alloc(FreeSpace::hdr_len(img.os, img.ls) as u64)?; self.fs = Some(FreeSpace { addr, max_sect_addr: self.max_index, @@ -813,7 +813,7 @@ impl Heap { size: u64, ) -> Result { let os = img.os; - let a = img.alloc_reusing(size)?; + let a = img.alloc(size)?; let n = usize::try_from(size).map_err(|_| bad("block too large"))?; let mut d = vec![0u8; n]; d[0..4].copy_from_slice(b"FHDB"); @@ -851,7 +851,7 @@ impl Heap { } let have_direct = self.root != undef(os); let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; - let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let a = img.alloc(self.iblock_len(rows, os) as u64)?; self.ents = vec![undef(os); nrows * usize::from(self.width)]; if have_direct { self.ents[0] = self.root; @@ -889,7 +889,7 @@ impl Heap { } img.free(self.root, self.iblock_len(self.root_rows, os) as u64); let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; - let a = img.alloc_reusing(self.iblock_len(rows, os) as u64)?; + let a = img.alloc(self.iblock_len(rows, os) as u64)?; let w = usize::from(self.width); self.ents.resize(new * w, undef(os)); let ov = self.overhead(os); @@ -973,7 +973,7 @@ impl Heap { return Err(unsupported("huge-object B-tree layout")); } let n = obj.len() as u64; - let a = img.alloc_reusing(n)?; + let a = img.alloc(n)?; img.write(a, obj)?; let w = usize::from(self.id_len - 1).min(8); let max_id = if w >= 8 { @@ -1096,7 +1096,7 @@ impl Heap { if fs.sect_addr != undef(os) { img.free(fs.sect_addr, fs.alloc_sect_size); } - fs.sect_addr = img.alloc_reusing(need)?; + fs.sect_addr = img.alloc(need)?; fs.alloc_sect_size = need; } fs.sect_size = fs.alloc_sect_size; diff --git a/crates/clawhdf5/src/edit/image.rs b/crates/clawhdf5/src/edit/image.rs index 722534d..08d389c 100644 --- a/crates/clawhdf5/src/edit/image.rs +++ b/crates/clawhdf5/src/edit/image.rs @@ -35,8 +35,61 @@ pub(crate) struct Image<'a> { /// Width of addresses and lengths in the file. pub(crate) os: u8, pub(crate) ls: u8, - /// Space the edit stopped using. + /// Space the edit stopped using. Not reused by this edit: until the + /// edit is committed, the file's metadata still points at it. freed: Vec<(u64, u64)>, + /// Space earlier edits of the session freed, available to this one. + reusable: FreeList, + /// Blocks this edit took from `reusable`: nothing on disk refers to + /// them, so they are written with the new space, before the changes + /// that link them in (see [`Plan::commit`]). + fresh: Vec<(u64, u64)>, +} + +/// Free space, address -> length, adjacent blocks merged. +#[derive(Debug, Clone, Default)] +pub(crate) struct FreeList(BTreeMap); + +impl FreeList { + /// Add `[addr, addr + len)`, merged with neighbours it touches. + pub(crate) fn add(&mut self, addr: u64, len: u64) { + if len == 0 { + return; + } + let (mut lo, mut hi) = (addr, addr.saturating_add(len)); + if let Some((&a, &l)) = self.0.range(..=lo).next_back() + && a + l >= lo + { + lo = a; + hi = hi.max(a + l); + self.0.remove(&a); + } + while let Some((&a, &l)) = self.0.range(lo..=hi).next() { + hi = hi.max(a + l); + self.0.remove(&a); + } + self.0.insert(lo, hi - lo); + } + + /// Take `size` bytes from the smallest block that holds them (the + /// lowest address among equals), from its start. + fn take(&mut self, size: u64) -> Option { + let (&a, &l) = self + .0 + .iter() + .filter(|&(_, &l)| l >= size) + .min_by_key(|&(&a, &l)| (l, a))?; + self.0.remove(&a); + if l > size { + self.0.insert(a + size, l - size); + } + Some(a) + } + + /// Total bytes. + pub(crate) fn total(&self) -> u64 { + self.0.values().sum() + } } impl<'a> Image<'a> { @@ -50,9 +103,23 @@ impl<'a> Image<'a> { os, ls, freed: Vec::new(), + reusable: FreeList::default(), + fresh: Vec::new(), } } + /// Let the edit allocate from `free` (space earlier edits freed). + pub(crate) fn with_reusable(mut self, free: FreeList) -> Self { + // Only space inside the file as it is now. + self.reusable = FreeList( + free.0 + .into_iter() + .filter(|&(a, l)| a.saturating_add(l) <= self.old_eoa) + .collect(), + ); + self + } + pub(crate) fn eoa(&self) -> u64 { self.eoa } @@ -66,11 +133,24 @@ impl<'a> Image<'a> { !self.patches.is_empty() || self.eoa != self.old_eoa } - /// Allocate `size` bytes at the end of the file. The space reads as - /// zeros until written. Nothing is ever freed: space an edit stops - /// using (a relocated chunk, say) is leaked, as there is no free-space - /// manager. + /// Allocate `size` bytes: from space an earlier edit of this session + /// freed when a block holds them (best fit), else at the end of the + /// file. The space reads as zeros until written. pub(crate) fn alloc(&mut self, size: u64) -> Result { + if size > 0 + && let Some(a) = self.reusable.take(size) + { + self.fresh.push((a, size)); + let n = usize::try_from(size) + .map_err(|_| Error::Unsupported("allocation too large".into()))?; + self.write(a, &vec![0u8; n])?; + return Ok(a); + } + self.alloc_end(size) + } + + /// Allocate `size` bytes at the end of the file. + fn alloc_end(&mut self, size: u64) -> Result { let addr = self.eoa; let end = addr .checked_add(size) @@ -80,14 +160,8 @@ impl<'a> Image<'a> { Ok(addr) } - /// Allocate `size` bytes for metadata or data, from space an earlier - /// edit of this session freed when there is a block that fits, - /// otherwise at the end of the file. - pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result { - self.alloc(size) - } - - /// Note that the edit no longer uses `[addr, addr + len)`. + /// Note that the edit no longer uses `[addr, addr + len)`; later edits + /// of the session may reuse it. pub(crate) fn free(&mut self, addr: u64, len: u64) { if len > 0 { self.freed.push((addr, len)); @@ -108,7 +182,7 @@ impl<'a> Image<'a> { } let old_end = self.eoa; self.eoa = addr; - if let Err(e) = self.alloc(new_len) { + if let Err(e) = self.alloc_end(new_len) { self.eoa = old_end; return Err(e); } @@ -206,12 +280,24 @@ impl<'a> Image<'a> { /// The edit's writes, detached from the base bytes (see the module's /// invariant: the reader that owns them can then be dropped before /// anything is written). - pub(crate) fn into_plan(self) -> Plan { - Plan { - patches: self.patches, - eoa: self.eoa, - old_eoa: self.old_eoa, + pub(crate) fn into_plan(self) -> (Plan, FreeList) { + // What the session may reuse once this edit is committed: what it + // did not take, and what it freed. + let mut free = self.reusable; + for (a, l) in self.freed { + free.add(a, l); } + let mut fresh = self.fresh; + fresh.sort_unstable(); + ( + Plan { + patches: self.patches, + eoa: self.eoa, + old_eoa: self.old_eoa, + fresh, + }, + free, + ) } } @@ -220,33 +306,57 @@ pub(crate) struct Plan { patches: BTreeMap>, eoa: u64, old_eoa: u64, + /// Reused blocks (sorted): written with the new space. + fresh: Vec<(u64, u64)>, } impl Plan { + /// Whether `addr` is in space nothing on disk refers to yet (past the + /// old end of file, or in a reused block), and up to where (before + /// `end`) that stays so. + fn new_space(&self, addr: u64, end: u64) -> (bool, u64) { + if addr >= self.old_eoa { + return (true, end); + } + let limit = end.min(self.old_eoa); + // The reused block holding `addr`, or the next one after it. + let i = self.fresh.partition_point(|&(a, l)| a + l <= addr); + match self.fresh.get(i) { + Some(&(a, l)) if a <= addr => (true, limit.min(a + l)), + Some(&(a, _)) => (false, limit.min(a)), + None => (false, limit), + } + } + /// Write the edit to `file`, whose superblock is at `user_block`. /// /// Order: first everything in newly allocated space (new chunks, new - /// index blocks, relocated structures), which nothing on disk refers to - /// yet, then a sync; then the changes to existing bytes — raw data - /// overwritten in place and the metadata that links the new space in - /// (superblock end of file, chunk index entries, object header + /// index blocks, relocated structures — past the old end of file, or in + /// space an earlier edit of the session freed), which nothing on disk + /// refers to yet, then a sync; then the changes to existing bytes — raw + /// data overwritten in place and the metadata that links the new space + /// in (superblock end of file, chunk index entries, object header /// messages) — then a sync. A crash during the first phase leaves the - /// file as it was (plus unreferenced bytes past its end of file); a - /// crash during the second can leave it inconsistent, as with libhdf5 - /// without SWMR: there is no journal. + /// file as it was (plus unreferenced bytes); a crash during the second + /// can leave it inconsistent, as with libhdf5 without SWMR: there is no + /// journal. pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> { let old_eoa = self.old_eoa; let mut in_place: Vec<(u64, &[u8])> = Vec::new(); for (&addr, bytes) in &self.patches { - // A patch may run from existing bytes into new space (writes - // merge); its new part goes with the new space. - let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize; - let (old, new) = bytes.split_at(split); - if !new.is_empty() { - write_at(file, user_block + addr + split as u64, new)?; - } - if !old.is_empty() { - in_place.push((addr, old)); + // A patch may run across new and existing space (writes + // merge): split it where that changes. + let end = addr + bytes.len() as u64; + let mut at = addr; + while at < end { + let (new, upto) = self.new_space(at, end); + let part = &bytes[(at - addr) as usize..(upto - addr) as usize]; + if new { + write_at(file, user_block + at, part)?; + } else { + in_place.push((at, part)); + } + at = upto; } } if self.eoa > old_eoa { @@ -322,6 +432,52 @@ mod tests { assert!(img.write(42, &[1]).is_err()); } + #[test] + fn free_list_merges_and_takes_best_fit() { + let mut f = FreeList::default(); + f.add(100, 10); + f.add(120, 5); + f.add(110, 10); // joins both neighbours + assert_eq!( + f.0.iter().map(|(&a, &l)| (a, l)).collect::>(), + [(100, 25)] + ); + f.add(300, 8); + f.add(200, 40); + // Best fit: the 8-byte block for 6 bytes, from its start. + assert_eq!(f.take(6), Some(300)); + assert_eq!(f.take(30), Some(200)); + assert_eq!(f.take(26), None); + assert_eq!(f.total(), 25 + 2 + 10); + } + + /// An edit allocates from space earlier edits freed (zeroed), never + /// from what it frees itself; the plan writes reused blocks with the + /// new space. + #[test] + fn reuse_across_edits_only() { + let base = vec![7u8; 64]; + let mut free = FreeList::default(); + free.add(8, 16); + let mut img = Image::new(&base, 8, 8).with_reusable(free); + img.free(32, 16); // freed by this edit: not reusable yet + let a = img.alloc(16).unwrap(); + assert_eq!(a, 8); + assert_eq!(img.read(8, 16).unwrap(), vec![0u8; 16]); + let b = img.alloc(8).unwrap(); + assert_eq!(b, 64, "the edit's own freed space is not reused"); + img.write(4, &[1; 8]).unwrap(); // existing bytes 4..8, reused 8..12 + let (plan, next) = img.into_plan(); + assert_eq!(plan.new_space(4, 12), (false, 8)); + assert_eq!(plan.new_space(8, 12), (true, 12)); + assert_eq!(plan.new_space(30, 40), (false, 40)); + assert_eq!(plan.new_space(64, 72), (true, 72)); + assert_eq!( + next.0.iter().map(|(&a, &l)| (a, l)).collect::>(), + [(32, 16)] + ); + } + /// Random reads and writes against a flat copy of the bytes. #[test] fn matches_a_flat_model() { diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index c32431a..336fbb8 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -44,7 +44,7 @@ use btree1::{BTree1, Key}; use btree2::Bt2; use earray::{Ea, EaParams, Elem}; use farray::Fa; -use image::{Image, put_uint, undef}; +use image::{FreeList, Image, put_uint, undef}; use ohdr::Header; const MSG_DATASPACE: u16 = 0x01; @@ -74,16 +74,18 @@ const MSG_FLAG_DONTSHARE: u8 = 0x04; /// contiguous or chunked dataset with values of the dataset's own /// datatype, under any selection. Chunks are decoded, updated and /// re-encoded; an unfiltered chunk is rewritten in place, a filtered one -/// in place when it still fits and otherwise at the end of the file. New -/// chunks are added to the chunk index: version-1 B-tree (layout v1-v3, -/// what h5py's default `libver` writes), Extensible Array, Fixed Array and -/// single-chunk indexes. A version-2 B-tree index (two or more unlimited -/// dimensions) can only have existing chunks overwritten in place, and an -/// implicit index only in place. -/// - [`resize`](Self::resize): grow a chunked dataset up to its maximum -/// dimensions (h5py's `Dataset.resize`); new chunks come with the writes. +/// in place when it still fits and otherwise in new space. New chunks are +/// added to the chunk index: version-1 B-tree (layout v1-v3, what h5py's +/// default `libver` writes), Extensible Array, Fixed Array, version-2 +/// B-tree (two or more unlimited dimensions) and single-chunk indexes, as +/// libhdf5 adds them (the same splits and blocks). An implicit index can +/// only be written in place. +/// - [`resize`](Self::resize): grow or shrink a chunked dataset within its +/// maximum dimensions (h5py's `Dataset.resize`), pruning the chunks a +/// shrink leaves outside the extent as libhdf5 does. /// - [`set_attr`](Self::set_attr): add or replace an attribute of any -/// object whose attributes are stored in its object header. +/// object, in its object header or in dense storage (moving attributes +/// there when the object reaches its compact limit). /// /// Anything else is an [`Error::Unsupported`] and leaves the file untouched. /// @@ -95,13 +97,20 @@ const MSG_FLAG_DONTSHARE: u8 = 0x04; /// before that point leaves the file as it was; a crash while the existing /// structures are being patched can leave the file inconsistent. /// -/// Space is never reused: a filtered chunk that grows moves to the end of -/// the file and its old bytes are leaked, as are index blocks that are -/// replaced. `h5repack` reclaims such space. +/// Space an edit stops using — a filtered chunk that moved, chunks a +/// shrink removed, index nodes a B-tree merged away, a heap's replaced +/// blocks — is reused by later edits of the same editor (best fit, at the +/// lowest address), never by the edit that freed it: until that edit is +/// committed the file still refers to it. Space reused this way is written +/// with the new space, before any existing byte changes. Freed space still +/// unused when the editor is dropped is lost, as it is when libhdf5 closes +/// a file without a persistent free-space manager; `h5repack` reclaims it. #[derive(Debug)] pub struct FileEditor { path: PathBuf, file: std::fs::File, + /// Space edits of this session freed, which later ones reuse. + free: FreeList, } /// Where a layout message keeps the fields an edit may change (offsets in @@ -754,7 +763,11 @@ impl FileEditor { } Err(TryLockError::Error(e)) => return Err(Error::Io(e)), } - let ed = Self { path, file }; + let ed = Self { + path, + file, + free: FreeList::default(), + }; let f = File::open(&ed.path)?; check_editable(&f)?; Ok(ed) @@ -765,6 +778,12 @@ impl FileEditor { &self.path } + /// Bytes earlier edits of this editor freed that later ones can still + /// reuse. + pub fn reusable_bytes(&self) -> u64 { + self.free.total() + } + /// Plan an edit over the file's current bytes, then commit it. /// /// The reader (a memory map of the file, with the `mmap` feature) is @@ -779,7 +798,8 @@ impl FileEditor { check_editable(&f)?; let sb = f.superblock().clone(); let user_block = f.user_block_size(); - let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size); + let mut img = Image::new(f.as_bytes(), sb.offset_size, sb.length_size) + .with_reusable(self.free.clone()); let r = op(&f, &mut img).map_err(unsupported_filter)?; let plan = if img.is_dirty() { if img.eoa() != img.old_eoa() { @@ -790,10 +810,14 @@ impl FileEditor { None }; drop(f); - if let Some(plan) = plan { + if let Some((plan, free)) = plan { #[cfg(test)] tests::note_commit(&self.path); + // A commit that fails part-way leaves the file in an unknown + // state: reuse nothing after it. + self.free = FreeList::default(); plan.commit(&mut self.file, user_block)?; + self.free = free; } Ok(r) } @@ -1524,7 +1548,7 @@ fn store_chunk( }) } _ => { - let a = img.alloc_reusing(len)?; + let a = img.alloc(len)?; img.write(a, &bytes)?; if let Some(info) = existing { img.free(info.address, u64::from(info.chunk_size)); From 1ffd013de910012444d89b648fb082154551ee84 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:11:36 -0500 Subject: [PATCH 06/15] edit: delete a heap's huge-object B-tree with its last huge object libhdf5 deletes a fractal heap's huge-object B-tree when the heap is closed with no huge object left (H5HF__huge_term) and starts huge IDs over. The editor left the empty tree, and a read-only libhdf5 then failed to list the object's attributes: closing the heap tried to delete the tree ("no write intent on file"), and h5dump failed the same way. Replacing an object's last huge attribute (one above the heap's 4 KiB managed limit) now deletes the tree (header and nodes freed), resets the next huge ID and the wrapped flag, as libhdf5 does; a later huge attribute creates a new tree. Bt2::delete frees a whole tree. Found by the extended random-operation test; regression: last_huge_attribute_replaced (fails before: h5dump, h5py listing), which also compares the heap with libhdf5's after the same replacement. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 54 +++++++++++++++++++ crates/clawhdf5/src/edit/btree2.rs | 26 +++++++++ crates/clawhdf5/src/edit/fheap.rs | 9 ++++ 3 files changed, 89 insertions(+) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 332dffa..839e32c 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1396,3 +1396,57 @@ fn freed_space_is_reused_within_a_session() { ); assert!(fresh.1 > fresh.0, "without reuse the file grows"); } + +/// Replacing the last huge attribute (larger than the heap's managed +/// limit) of an object with a small one deletes the heap's huge-object +/// B-tree, as libhdf5 does when it closes the heap (`H5HF__huge_term`). A +/// heap left with an empty huge-object B-tree made read-only libhdf5 fail +/// to list the attributes ("no write intent on file"). +#[test] +fn last_huge_attribute_replaced() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let a = dir.path().join("huge_h5py.h5"); + let b = dir.path().join("huge_edit.h5"); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(10): g.attrs.create(f'k{{i}}', np.array([i], dtype=' = (0..10i64) + .map(|i| ("g".to_string(), format!("k{i}"), AV::Ints(vec![i]))) + .collect(); + want.push(("g".into(), "big".into(), AV::Ints(vec![1, 2]))); + check_attr_values(&b, &want); + let info = |p: &Path| { + let s = dense_info(p, "g"); + s[..s.find(" fs ").or(s.find(" no-fs")).unwrap()].to_string() + }; + assert_eq!(info(&b), info(&a), "heap after the replacement"); + // A huge attribute again starts the huge-object B-tree over. + let mut ed = FileEditor::open(&b).unwrap(); + ed.set_attr("g", "big2", &clawhdf5::AttrValue::String("x".repeat(7000))) + .unwrap(); + drop(ed); + want.push(("g".into(), "big2".into(), AV::Str("x".repeat(7000)))); + check_tools(&b, true); + check_attr_values(&b, &want); +} diff --git a/crates/clawhdf5/src/edit/btree2.rs b/crates/clawhdf5/src/edit/btree2.rs index 5761e1e..71bd9b2 100644 --- a/crates/clawhdf5/src/edit/btree2.rs +++ b/crates/clawhdf5/src/edit/btree2.rs @@ -384,6 +384,32 @@ impl Bt2 { } } + /// Delete the whole tree (`H5B2_delete`): every node and the header go + /// to the image's free list. + pub(crate) fn delete(mut self, img: &mut Image<'_>) -> Result<(), Error> { + if self.root.addr != undef(img.os) && self.root.nrec > 0 { + let mut level = vec![self.root]; + let mut depth = self.depth; + loop { + let mut next = Vec::new(); + for p in level { + self.load(img, p, depth)?; + if depth > 0 { + next.extend(self.peek(p.addr).ptrs.iter().copied()); + } + img.free(p.addr, u64::from(self.node_size)); + } + if depth == 0 { + break; + } + depth -= 1; + level = next; + } + } + img.free(self.addr, Self::header_len(img.os, img.ls) as u64); + Ok(()) + } + /// Insert `rec`, or replace the record `cmp` matches (`H5B2_update`). pub(crate) fn update( &mut self, diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs index 2fab657..03e6b1e 100644 --- a/crates/clawhdf5/src/edit/fheap.rs +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -948,6 +948,15 @@ impl Heap { img.free(get_uint(&rec, os), len); self.huge_size = self.huge_size.saturating_sub(len); self.huge_nobjs = self.huge_nobjs.saturating_sub(1); + // H5HF__huge_term: with no huge object left, the huge-object + // B-tree is deleted and IDs start over (libhdf5 does it when + // it closes the heap, and a read-only libhdf5 fails to). + if self.huge_nobjs == 0 { + t.delete(img)?; + self.huge_bt2 = undef(os); + self.next_huge_id = 0; + self.flags &= !0x01; + } Ok(()) } _ => Err(unsupported("tiny objects")), From 304aed581309fccce336088c7238389baae2a342 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:11:44 -0500 Subject: [PATCH 07/15] tests: random editor operations include shrinking, 2-D growth, dense attributes random_operations_match_a_model (edit_interop) now drives, on every libver (earliest, v114, latest) and filter set (none, gzip + shuffle + fletcher32, LZF): - a dataset with one unlimited dimension and one with two (a version-2 B-tree chunk index under v114/latest), resized to random shapes that shrink and grow any resizable dimension, with block and point writes; - attributes on the first dataset under 20 names with values of random types and sizes (scalars, int64 arrays, short strings, strings above the heap's managed limit), so they move to dense storage on version-2 headers and are replaced by values of other sizes; against a model where shrunk-away elements that come back read as the fill value, compared with our reader and with h5py/numpy every 40 steps, with h5dump and h5rs check; h5py then grows both datasets and adds an attribute. The attribute check also compares h5py's attribute count with libhdf5's object info. CLAWHDF5_EDIT_SEED reruns the workloads with other random choices (seeds 1000-4000 pass). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-tools/tests/edit_interop.rs | 183 ++++++++++++++------ 1 file changed, 129 insertions(+), 54 deletions(-) diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index cb345f8..3a37e83 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -113,7 +113,8 @@ impl Model { .fold(0u64, |a, (&x, &d)| a * d + x) as usize } - /// Grow to `shape`, new elements `fill`. + /// Change the extent to `shape`: elements inside both keep their + /// values, new ones are `fill`. fn resize(&mut self, shape: &[u64], fill: i32) { let old = self.clone(); *self = Self::new(shape, |_| fill); @@ -125,8 +126,10 @@ impl Model { c[d] = r % old.shape[d]; r /= old.shape[d]; } - let i = self.index(&c); - self.data[i] = old.data[flat as usize]; + if c.iter().zip(shape).all(|(x, s)| x < s) { + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } } } @@ -293,9 +296,24 @@ fn append_many_gzip() { } } -/// Random operations — grow, hyperslab writes, point writes, attributes — -/// on a 2-D dataset with one unlimited dimension, checked against a model -/// after every few operations. +/// A random attribute value: a scalar, an int64 array, a short string or +/// one larger than a heap's managed-object limit (a huge heap object once +/// the attributes are in dense storage). +fn random_attr(rng: &mut Rng) -> AttrValue { + match rng.below(8) { + 0..=2 => AttrValue::I64(rng.next() as i64 >> 3), + 3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()), + 5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)), + _ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)), + } +} + +/// Random operations — growth and shrinking along any dimension, hyperslab +/// and point writes, attributes (enough names to move them to dense storage +/// on version-2 object headers, replaced with values of any size) — on a +/// 2-D dataset with one unlimited dimension and one with two (a version-2 +/// B-tree chunk index under `v114`/`latest`), checked against a model (and +/// through h5py, numpy) after every few operations. fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { let dir = tmpdir(); let path = dir.path().join(format!("rand_{tag}.h5")); @@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ \x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \ dtype=' = Vec::new(); + let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)]; + models[0].write_block(&[1, 2], &[2, 4], &[5; 8]); + models[1].write_block(&[0, 1], &[4, 4], &[8; 16]); + let fills = [-9, 3]; + let names = ["m", "b"]; + let mut attrs: Vec<(String, AttrValue)> = Vec::new(); let mut rng = Rng(seed); let mut ed = FileEditor::open(&path).unwrap(); - for step in 0..120 { - match rng.below(10) { - 0..=1 => { - let rows = m.shape[0] + 1 + rng.below(5); - ed.resize("m", &[rows, 7]).unwrap(); - m.resize(&[rows, 7], -9); + for step in 0..160 { + let d = rng.below(2) as usize; + let name = names[d]; + let m = &mut models[d]; + match rng.below(12) { + 0..=2 => { + // Grow or shrink: dimension 1 of "m" is fixed at 7. + let rows = rng.below(m.shape[0] + 6); + let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) }; + ed.resize(name, &[rows, cols]).unwrap(); + m.resize(&[rows, cols], fills[d]); } - 2..=6 => { + 3..=7 if m.shape.iter().all(|&s| s > 0) => { let r0 = rng.below(m.shape[0]); - let c0 = rng.below(7); + let c0 = rng.below(m.shape[1]); let cnt = [ 1 + rng.below((m.shape[0] - r0).min(6)), - 1 + rng.below(7 - c0), + 1 + rng.below((m.shape[1] - c0).min(6)), ]; let n = cnt[0] * cnt[1]; let vals: Vec = (0..n).map(|_| (rng.next() % 100_000) as i32).collect(); - ed.write_values("m", &block(&[r0, c0], &cnt), &vals) + ed.write_values(name, &block(&[r0, c0], &cnt), &vals) .unwrap(); m.write_block(&[r0, c0], &cnt, &vals); } - 7 => { + 8 if m.shape.iter().all(|&s| s > 0) => { let pts: Vec> = (0..1 + rng.below(4)) - .map(|_| vec![rng.below(m.shape[0]), rng.below(7)]) + .map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])]) .collect(); let vals: Vec = pts.iter().map(|_| rng.next() as i32).collect(); - ed.write_values("m", &Selection::Points(pts.clone()), &vals) + ed.write_values(name, &Selection::Points(pts.clone()), &vals) .unwrap(); for (p, v) in pts.iter().zip(&vals) { let i = m.index(p); m.data[i] = *v; } } - _ => { - let k = rng.below(6); - let name = format!("a{k}"); - let v = rng.next() as i64; - match ed.set_attr("m", &name, &AttrValue::I64(v)) { + 9..=11 => { + let k = rng.below(20); + let aname = format!("a{k}"); + let v = random_attr(&mut rng); + match ed.set_attr("m", &aname, &v) { Ok(()) => { - attrs.retain(|(n, _)| *n != name); - attrs.push((name, v)); + attrs.retain(|(n, _)| *n != aname); + attrs.push((aname, v)); } - Err(e) => panic!("set_attr {name}: {e}"), + // Replacing the only attribute in a heap block with one + // of another size would have libhdf5 free the block. + Err(Error::Unsupported(msg)) if msg.contains("last object") => {} + Err(e) => panic!("set_attr {aname}: {e}"), } } + _ => {} } - if step % 30 == 29 { + if step % 40 == 39 { drop(ed); - verify(&path, "m", &m); + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_tools(&path, h5dump); check_attrs(&path, "m", &attrs); ed = FileEditor::open(&path).unwrap(); } } drop(ed); - verify(&path, "m", &m); + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_attrs(&path, "m", &attrs); py(&format!( "import h5py, numpy as np\n\ with h5py.File({p:?}, 'r+') as f:\n\ - \x20 d = f['m']\n\ - \x20 n = d.shape[0]\n\ - \x20 d.resize((n + 3, 7))\n\ - \x20 d[n:, :] = 42\n\ - \x20 d.attrs['from_h5py'] = 1.5\n", + \x20 for name, cols in (('m', 7), ('b', None)):\n\ + \x20 d = f[name]\n\ + \x20 n, c = d.shape\n\ + \x20 d.resize((n + 3, cols or c + 2))\n\ + \x20 d[n:, :] = 42\n\ + \x20 f['m'].attrs['from_h5py'] = 1.5\n", p = path.to_str().unwrap() )); - let n = m.shape[0]; - m.resize(&[n + 3, 7], -9); - m.write_block(&[n, 0], &[3, 7], &[42; 21]); - verify(&path, "m", &m); + for (d, m) in models.iter_mut().enumerate() { + let (n, c) = (m.shape[0], m.shape[1]); + let c2 = if d == 0 { 7 } else { c + 2 }; + m.resize(&[n + 3, c2], fills[d]); + m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]); + } + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_tools(&path, h5dump); + check_attrs(&path, "m", &attrs); } -fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) { +/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of +/// its attributes agrees with libhdf5's object info). +fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) { let f = File::open(path).unwrap(); let got = f.dataset(obj).unwrap().attrs().unwrap(); for (n, v) in attrs { - match got.get(n) { - Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"), - other => panic!("attribute {n}: {other:?}"), - } + let g = got + .get(n) + .unwrap_or_else(|| panic!("attribute {n} missing")); + // Our reader reports a one-element array as a scalar. + let v = match v { + AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]), + v => v, + }; + assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}"); } - let want: Vec = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect(); - py(&format!( + let want: Vec = attrs + .iter() + .map(|(n, v)| { + let pv = match v { + AttrValue::I64(x) => format!("{x}"), + AttrValue::I64Array(a) => format!("{a:?}"), + AttrValue::String(s) => format!("{s:?}"), + other => panic!("{other:?}"), + }; + format!("{n:?}: {pv}") + }) + .collect(); + let script = format!( "import h5py\n\ f = h5py.File({p:?}, 'r')\n\ want = {{{w}}}\n\ - got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\ - assert got == want, (got, want)\n", + a = f[{obj:?}].attrs\n\ + def norm(v):\n\ + \x20 v = v.decode() if isinstance(v, bytes) else v\n\ + \x20 return v.tolist() if hasattr(v, 'tolist') else v\n\ + got = {{k: norm(v) for k, v in a.items() if k in want}}\n\ + assert got == want, sorted(set(want) ^ set(got))\n\ + assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n", p = path.to_str().unwrap(), w = want.join(", ") - )); + ); + let sp = path.with_extension("attrs.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); } #[test] @@ -411,7 +482,11 @@ fn random_operations_match_a_model() { if !tools_ok() { return; } - let mut seed = 1; + // CLAWHDF5_EDIT_SEED runs the same workloads with other random choices. + let mut seed = std::env::var("CLAWHDF5_EDIT_SEED") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1); for (i, (lv, dump)) in LIBVERS.iter().enumerate() { // h5dump has no LZF decoder (h5py's own filter). for (j, (extra, lzf)) in [ From 955fdb660d5121211d06192101d443c12d06fa11 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:12:14 -0500 Subject: [PATCH 08/15] tests: resize and write clawhdf5-written datasets on every chunk index Version-2 B-tree (its writer's own node size and single-leaf layout), Extensible Array and Fixed Array datasets written by FileBuilder, with and without deflate, resized up and down along both dimensions and written at random against a model; h5py, h5dump and h5rs check read the result, and h5py resizes and rewrites every dataset afterwards. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 839e32c..ffe13d9 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1450,3 +1450,83 @@ fn last_huge_attribute_replaced() { check_tools(&b, true); check_attr_values(&b, &want); } + +/// Datasets clawhdf5 writes — a version-2 B-tree index (two unlimited +/// dimensions, its own node size and a single leaf sized to its records), +/// an Extensible Array, a Fixed Array (fixed maximum shape), deflated and +/// not — resized up and down and written at random, against a model; h5py, +/// h5dump and `h5rs check` read the result and h5py goes on. +#[test] +fn resize_clawhdf5_written_datasets() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + let path = dir.path().join("ours_resize.h5"); + let mut b = clawhdf5::FileBuilder::new(); + let grid: Vec = (0..48).collect(); + let specs: [(&str, [u64; 2], [u64; 2], bool); 4] = [ + ("bt2", [6, 8], [u64::MAX, u64::MAX], false), + ("bt2_gz", [6, 8], [u64::MAX, u64::MAX], true), + ("ea", [6, 8], [u64::MAX, 8], true), + ("fa", [6, 8], [20, 12], false), + ]; + for (name, shape, max, gz) in specs { + let d = b + .create_dataset(name) + .with_i32_data(&grid) + .with_shape(&shape) + .with_maxshape(&max) + .with_chunks(&[2, 3]); + if gz { + d.with_deflate(4); + } + } + b.write(&path).unwrap(); + assert_eq!(chunk_index(&path, "bt2").0, 5, "version-2 B-tree index"); + assert_eq!(chunk_index(&path, "ea").0, 4, "Extensible Array index"); + let mut models: Vec = specs + .iter() + .map(|_| Model::new(&[6, 8], |i| i as i32)) + .collect(); + let mut rng = Rng(77); + let mut ed = FileEditor::open(&path).unwrap(); + for step in 0..120 { + let k = rng.below(4) as usize; + let (name, _, max, _) = specs[k]; + let m = &mut models[k]; + if step % 4 == 0 { + let s: Vec = (0..2) + .map(|d| rng.below((m.shape[d] * 2 + 4).min(max[d]) + 1)) + .collect(); + ed.resize(name, &s).unwrap(); + m.resize(&s, 0); + } else if m.shape.iter().all(|&s| s > 0) { + let st: Vec = m.shape.iter().map(|&s| rng.below(s)).collect(); + let cnt: Vec = (0..2) + .map(|d| 1 + rng.below((m.shape[d] - st[d]).min(5))) + .collect(); + let vals: Vec = (0..cnt[0] * cnt[1]).map(|_| rng.next() as i32).collect(); + ed.write_values(name, &block(&st, &cnt), &vals).unwrap(); + m.write_block(&st, &cnt, &vals); + } + } + drop(ed); + for ((name, ..), m) in specs.iter().zip(&models) { + verify(&path, name, m); + } + check_tools(&path, true); + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for n in ['bt2', 'bt2_gz', 'ea', 'fa']:\n\ + \x20 d = f[n]\n\ + \x20 d.resize((6, 8))\n\ + \x20 d[...] = np.arange(48, dtype=' Date: Sat, 26 Sep 2026 17:13:31 -0500 Subject: [PATCH 09/15] =?UTF-8?q?docs:=20editor=20coverage=20=E2=80=94=20v?= =?UTF-8?q?ersion-2=20B-trees,=20shrinking,=20dense=20attributes,=20reuse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHANGELOG (Unreleased): the new FileEditor operations, space reuse, and the two reader fixes (implicit index grid, object-header continuation chains). known-issues: the editor's remaining refusals (skipped heap blocks, heaps with filters or child indirect blocks, freeing a heap block, implicit-index insertions, ...) and the append-waste sizes before and after reuse (measure_append_waste, tank 2026-09-26; file sizes are deterministic). range-reads design: status note on the reader changes. README and CLAUDE.md: what the editor covers and how to test it. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 69 +++++++++++++++++++++++++++++++++++++- CLAUDE.md | 16 ++++++--- README.md | 6 +++- docs/design/range-reads.md | 8 ++++- docs/known-issues.md | 57 ++++++++++++++++++------------- 5 files changed, 124 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa0a30f..551b079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,72 @@ ## Unreleased +### In-place editing: version-2 B-tree indexes, shrinking, dense attributes (2026-09-26) +- **`FileEditor` adds, moves and resizes chunks of datasets with two or + more unlimited dimensions** (version-2 B-tree chunk index, record types + 10/11), as libhdf5's `H5B2` code does: `H5B2_update`'s insert-or-modify, + the preemptive split/redistribute loop, `split1`/`split_root` (depth + growth), `redistribute2/3`, and removal with `merge2/3`, root collapse + and the internal-record swap; node pointer widths and cumulative record + counts per depth; a missing index is created from the layout message's + parameters. After the same growth libhdf5's and the editor's trees are + node for node the same (tested through a depth increase). +- **`FileEditor::resize` shrinks** along any dimension (h5py's + `Dataset.resize` to a smaller shape), as `H5D__chunk_prune_by_extent` + does, visiting the same chunks in the same order: chunks wholly outside + the new extent leave the index (version-1 B-tree removal with libhdf5's + sibling key and link fix-ups and empty-root case, version-2 B-tree + removal, Fixed/Extensible Array elements reset; an implicit index keeps + its chunks, as in libhdf5) and their space is freed; the part of a + partial edge chunk outside the extent is overwritten with the fill value, + so it reads as fill after a later growth. Growth under early allocation + now allocates and fills the new chunks (`H5D__chunk_allocate`), which an + implicit index needs. Shrinking was `Error::Unsupported`. +- **`FileEditor::set_attr` handles dense attribute storage and creation + order**: objects that track (and index) attribute creation order; the + move to dense storage when an object reaches its compact limit (or an + attribute is too large for a header message), as `H5O__attr_create` + does it (new fractal heap, name index, creation-order index when + indexed, compact attributes moved over in header order); objects + already in dense storage (h5py- or clawhdf5-written): insertion, + same-size rewrites in place, other replacements by removal and + insertion. The heap is changed as `H5HF` changes it — best-fit free + sections from its free-space manager (kept as libhdf5 keeps `FSHD`/ + `FSSE`), new direct blocks through the root indirect block (created, + doubled), huge objects through the huge-object B-tree (deleted with the + last huge object), removed objects' space merged back — with libhdf5's + statistics: after the same attribute workload the heap, its free space + and both index B-trees equal libhdf5's. Attributes are encoded as libhdf5 + encodes them for a file h5py opens `r+` (message version 1, 3 for + non-ASCII names; simple dataspaces with their maximum dimensions). + Still refused: see `docs/known-issues.md`. +- **Freed space is reused within an editing session.** A `FileEditor` + reuses (best fit, zeroed) what its earlier edits freed — moved filtered + chunks, pruned chunks, merged B-tree nodes, replaced heap blocks — never + what the current edit frees, and writes reused blocks with the new space + before any existing byte changes. `FileEditor::reusable_bytes`. The + append workload of `measure_append_waste` leaks less (sizes in + `docs/known-issues.md`). +- **Reader: implicit chunk indexes below their maximum shape.** libhdf5 + places an implicit index's chunks by their position in the *maximum* + chunk grid; the reader used the current grid and returned other chunks' + values from the second chunk row on (h5py early allocation with a fixed + `maxshape` larger than the shape). + `chunked_read::generate_implicit_chunks_in_grid` takes the maximum. +- **Reader: object headers with long continuation chains.** A version-1 + header whose continuation chunks chain more than 32 deep (a header that + gains a chunk per attribute added when full, as libhdf5 and the editor + grow it) was refused with `NestingDepthExceeded`; version-2 headers + stopped at 256 chunks. Chunks are now followed without recursion, in the + same order; a chunk address seen twice (a cycle) or more than 65 536 + chunks are refused. +- Tests: `crates/clawhdf5-tools/tests/edit_coverage_interop.rs` (h5py + `earliest`/`v110`/`latest` and clawhdf5-written files; structure + comparisons with libhdf5 for version-2 B-trees, shrink on every index, + and dense attribute heaps); the random-operation property test in + `edit_interop.rs` now shrinks, grows two unlimited dimensions and moves + attributes to dense storage (`CLAWHDF5_EDIT_SEED` for other seeds). + ### Name lookups through the name index (2026-09-26) - **Finding one link or attribute by name reads the name index, not every entry.** In a dense group (links in a fractal heap) the v2 B-tree name @@ -237,7 +303,8 @@ before any existing byte changes, then the metadata that links it in, then a second sync. There is no journal: a crash during the second phase can leave the file inconsistent (as with libhdf5 without SWMR). - Freed space is not reused (see `docs/known-issues.md`). + Freed space is not reused (see `docs/known-issues.md`; since reused + within an editing session, above). - Tests: `crates/clawhdf5-tools/tests/edit_interop.rs` (h5py `earliest`, `v114` and `latest` files and clawhdf5 files; after every round h5py reads the expected values, h5dump and `h5rs check --data` accept the diff --git a/CLAUDE.md b/CLAUDE.md index f493066..14463ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,12 +151,18 @@ Cargo workspace with 18 crates under `crates/` (plus `libaec-sys`, an internal F `MemorySource` for this bookkeeping is inferred from the caller-supplied `source_channel` string (a heuristic, not an authenticated trust boundary). - In-place modification: `clawhdf5::FileEditor` (`crates/clawhdf5/src/edit/`) - overwrites values, grows chunked datasets and sets attributes in existing - files (h5py- or clawhdf5-written) without rewriting them; anything it - cannot do safely is `Error::Unsupported` before any write (limits in + overwrites values, grows and shrinks chunked datasets (every chunk index, + version-2 B-trees included) and sets attributes (compact and dense + storage) in existing files (h5py- or clawhdf5-written) without rewriting + them, changing indexes and heaps as libhdf5 does (index shapes and heap + bookkeeping are compared with libhdf5's in the tests); space an edit + frees is reused by later edits of the same editor. Anything it cannot do + safely is `Error::Unsupported` before any write (limits in `docs/known-issues.md`). Test changes with - `cargo test -p clawhdf5-tools --test edit_interop` (h5py, h5dump, - `h5rs check`). + `cargo test -p clawhdf5-tools --test edit_interop --test + edit_coverage_interop` (h5py, h5dump, `h5rs check`, structure comparisons + with libhdf5; libhdf5 sources for the algorithms are at + github.com/HDFGroup/hdf5, tag `hdf5_1_14_6`). - GPU-accelerated vector distance computation (`clawhdf5-gpu`, wgpu); HDF5 I/O itself is CPU-only - Browser: `clawhdf5-wasm` (wasm-bindgen, read-only, file held in memory; no Zstd/SZIP since they link C) and the `examples/wasm-viewer/` page. diff --git a/README.md b/README.md index 88ad61b..a87a43b 100644 --- a/README.md +++ b/README.md @@ -444,9 +444,13 @@ ed.resize("x", &[1100])?; // h5py: ds.resize((1100,)) let sel = Selection::Hyperslab { start: vec![1000], stride: vec![1], count: vec![100], block: vec![1] }; ed.write_values("x", &sel, &[0.5f64; 100])?; // ds[1000:1100] = 0.5 ed.set_attr("x", "units", &AttrValue::String("m/s".into()))?; +ed.resize("x", &[900])?; // shrinking prunes chunks, like h5py ``` -Each call changes the file in place (no rewrite) and syncs it. What it +Each call changes the file in place (no rewrite) and syncs it. Any chunk +index (version-2 B-trees for several unlimited dimensions included) and +attributes in compact or dense storage are handled as libhdf5 handles +them; space an edit frees is reused by later edits of the same editor. What it cannot change safely is refused before anything is written; see [known issues](docs/known-issues.md) for the limits. diff --git a/docs/design/range-reads.md b/docs/design/range-reads.md index 1c7fa55..7490c7a 100644 --- a/docs/design/range-reads.md +++ b/docs/design/range-reads.md @@ -3,7 +3,13 @@ Status: proposal, 2026-09-26; the plan for Phase 3's largest architectural change. Progress: M1, first part (the `Storage` trait and the metadata parsers listed in `CHANGELOG.md` under "Range reads, milestone M1") is done; -group B-tree v2 lookups, dense groups and the facade are not converted yet. Every count below was +group B-tree v2 lookups, dense groups and the facade are not converted yet. +Later the same day (branch `feat/p3-editor-coverage`) two reader fixes touched +converted code without changing the plan: object-header continuation chunks +are followed without recursion (still one bounded `read_at` per chunk), and +implicit chunk indexes are addressed over the maximum chunk grid (in +`chunked_read`, an M2 module). The in-place editor (`FileEditor`) keeps +working on the whole file in memory; it is not part of this design. Every count below was taken on `tank` on 2026-09-26 at commit `de2a53f`, with the commands given next to it. No timing numbers appear here on purpose: the machine was shared with other build jobs when this was written. diff --git a/docs/known-issues.md b/docs/known-issues.md index dcb365f..1b05553 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -26,40 +26,49 @@ libhdf5 modify them. ## In-place modification (`FileEditor`) limits -**Status:** open (documented 2026-09-26). `clawhdf5::FileEditor` refuses, -with `Error::Unsupported` and without writing anything: -- new, moved or resized chunks in a **version-2 B-tree** chunk index (what - libhdf5 uses for two or more unlimited dimensions) — existing unfiltered - chunks, and filtered ones that re-encode to the same size and filter - mask, are - overwritten in place; `resize` works — and new chunks in an **implicit** - index (it has all of its chunks from the start); -- **shrinking** a dataset; +**Status:** open (documented 2026-09-26, updated the same day when +version-2 B-tree chunk indexes, shrinking, dense attributes and space +reuse were added). `clawhdf5::FileEditor` refuses, with +`Error::Unsupported` and without writing anything: +- new chunks in an **implicit** index (it has all of its chunks from the + start; they are written in place, and allocated/filled on growth under + early allocation as libhdf5 does); - variable-length and reference data; - chunks through a filter this build cannot encode (scale-offset, N-Bit, SZIP, or a plugin filter it lacks), even an optional one: libhdf5 skips an optional filter only when its own build lacks it, which none does for these; -- attributes of an object in **dense storage**, past its compact limit (8 - by default) or with tracked **creation order**; +- attributes in dense storage when the heap cannot take them the way + libhdf5 would: an attribute that needs a heap block larger than the next + one (libhdf5 skips blocks and records them as free space — in practice an + attribute of roughly 1 to 4 KiB going into a young heap), a heap with I/O + filters or child indirect blocks (more than about 512 KiB of attributes), + free space the heap tracks outside direct blocks, replacing the last + attribute left in a heap block by one of another size (libhdf5 frees the + block), directly addressed huge objects; and shared attribute messages; +- version-1 object headers asked for an attribute larger than a header + message (they have no dense storage); - partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external raw data files, virtual datasets; - files with a metadata cache image, paged or persistent free-space management, a driver info block, or version-3 consistency flags set. -**Space is never reused.** There is no free-space manager: the old bytes of -a filtered chunk that grows and has to move, and of an attribute that is -replaced by a larger one, are leaked (`h5repack` reclaims them). A chunk -that is the last thing in the file grows in place instead, which covers the -usual append. Measured 2026-09-26 on tank with -`cargo test --release -p clawhdf5-tools --test edit_interop -- --ignored ---nocapture measure_append_waste` (file sizes are deterministic): 1000 -appends of 100 `f8` values to a 1-D dataset with 1024-element chunks give -810 504 bytes unfiltered, as libhdf5's file, and 307 210 bytes with gzip -(libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values with -4096-element gzip chunks give 119 684 bytes against libhdf5's 50 292 -(`h5repack`: 49 930), because the chunk being appended to is followed by -new index blocks and moves each time it grows. +**Space is reused only within one editor.** Space an edit frees (a filtered +chunk that moves, chunks a shrink removes, B-tree nodes merged away, a +heap's replaced blocks) is reused by later edits of the same `FileEditor`; +what is left when it is dropped is leaked, as libhdf5 leaks it without a +persistent free-space manager (`h5repack` reclaims it). A chunk that is the +last thing in the file grows in place, which covers the usual append. +Measured 2026-09-26 on tank with `cargo test -p clawhdf5-tools --test +edit_interop -- --ignored --nocapture measure_append_waste` (one editor for +the whole workload; file sizes are deterministic): 1000 appends of 100 `f8` +values to a 1-D dataset with 1024-element chunks give 810 504 bytes +unfiltered, as libhdf5's file, and 306 780 bytes with gzip (307 210 before +reuse; libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values +with 4096-element gzip chunks give 79 829 bytes (119 684 before reuse) +against libhdf5's 50 292 (`h5repack`: 49 930): the chunk being appended to +is followed by new index blocks and moves each time it grows, and the +space it leaves is too small for its next, larger version. **No journal.** A crash while an edit patches existing structures can leave the file inconsistent; see the `FileEditor` documentation. From 4e8109770d41ac22f6f8cc19c7d9bc1d5571d350 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 17:25:28 -0500 Subject: [PATCH 10/15] tests: shrink and regrow datasets allocated early Datasets with early allocation and unlimited dimensions (Extensible Array, version-2 B-tree, version-1 B-tree under earliest), unfiltered and deflated: the random resize workload gives the values h5py gets and the same chunk index shape, with every chunk a growth brings in allocated and filled as H5D__chunk_allocate does. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/edit_coverage_interop.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index ffe13d9..7fc309a 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -717,6 +717,42 @@ fn shrink_matches_libhdf5() { false, 99, ); + // Early allocation with unlimited dimensions (Extensible Array, + // version-2 B-tree; version-1 B-tree under `earliest`), filtered and + // not: growth allocates and fills every new chunk, as libhdf5 does. + for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] + .iter() + .enumerate() + { + for (j, (max, z)) in [ + ("(None, 9)", ""), + ("(None, None)", ""), + ("(None, 9)", ", compression='gzip'"), + ("(None, None)", ", compression='gzip'"), + ] + .iter() + .enumerate() + { + shrink_workload( + &format!("early_{i}_{j}"), + lv, + &format!( + "\x20 f.create_dataset('x', data=np.arange(72, dtype=' Date: Sat, 26 Sep 2026 18:23:59 -0500 Subject: [PATCH 11/15] filters: Fletcher-32 as libhdf5 computes it Our checksum reduced its sums with `% 65535`; libhdf5's H5_checksum_fletcher32 folds them with `(s & 0xffff) + (s >> 16)`, which leaves 0xffff where the modulo leaves 0. On about one chunk in 32768 libhdf5 refused the chunks we wrote and we refused the chunks it wrote. Every release since v2.1.0 is affected. clawhdf5_format::checksum::fletcher32 is a port of H5_checksum_fletcher32 and the filter's only implementation. Verification also accepts the byte-swapped form libhdf5 accepts (1.6.2 and earlier) and the `% 65535` form earlier releases wrote, so their files stay readable. The new interop test compares the checksum with libhdf5's own function (ctypes) on every 1- and 2-byte input and 40 000 random and fold-heavy inputs, and moves fold-case chunks between h5py and FileBuilder/FileEditor in both directions; with the old filters.rs the three file tests fail. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 26 ++ crates/clawhdf5-accel/src/lib.rs | 5 +- crates/clawhdf5-format/src/checksum.rs | 52 ++++ crates/clawhdf5-format/src/filters.rs | 66 +--- crates/clawhdf5/tests/fletcher32_interop.rs | 321 ++++++++++++++++++++ docs/known-issues.md | 35 +++ 6 files changed, 451 insertions(+), 54 deletions(-) create mode 100644 crates/clawhdf5/tests/fletcher32_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 551b079..964390d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ ## Unreleased +### Correctness: Fletcher-32 (2026-09-26) +- **Fletcher-32 checksums disagreed with libhdf5's on about one chunk in + 32768** (fixed 2026-09-26). **Every release is affected, v2.1.0 through + v2.7.0**, both directions: `FileBuilder`/`FileWriter` (`with_fletcher32`) + and, before release, `FileEditor` wrote chunks that h5py and libhdf5 + refuse ("filter returned failure during read"), and every reader + rejected valid libhdf5-written chunks with `Fletcher32Mismatch`. Our + checksum reduced its sums with `% 65535`; libhdf5's + `H5_checksum_fletcher32` uses the ones'-complement fold + `(s & 0xffff) + (s >> 16)`, which leaves 0xffff where the modulo leaves + 0, so the two differ whenever a sum is a non-zero multiple of 65535. + `clawhdf5_format::checksum::fletcher32` (new, public) is a port of + `H5_checksum_fletcher32` and the only implementation; the filter writes + and verifies with it, and, as libhdf5 does, also accepts a stored + checksum with the bytes of each 16-bit half swapped (libhdf5 1.6.2 and + earlier) and the `% 65535` form v2.7.0 and earlier wrote, so their files + stay readable. Tests: `crates/clawhdf5/tests/fletcher32_interop.rs` compares + it with libhdf5's own function (through ctypes) on every 1- and 2-byte + input and 40 000 random and fold-heavy ones, and has h5py read + fold-case chunks written by `FileBuilder` and `FileEditor` and us read + h5py's. Files written by earlier releases read with a fixed build; to + make one readable by libhdf5, rewrite its Fletcher-32 datasets with a + fixed build (see `docs/known-issues.md`). `clawhdf5_accel::checksum_fletcher32` is a + different, textbook Fletcher-32 (sums start at 0xffff) and is not used + for HDF5. + ### In-place editing: version-2 B-tree indexes, shrinking, dense attributes (2026-09-26) - **`FileEditor` adds, moves and resizes chunks of datasets with two or more unlimited dimensions** (version-2 B-tree chunk index, record types diff --git a/crates/clawhdf5-accel/src/lib.rs b/crates/clawhdf5-accel/src/lib.rs index 81a2f03..20d52a3 100644 --- a/crates/clawhdf5-accel/src/lib.rs +++ b/crates/clawhdf5-accel/src/lib.rs @@ -237,7 +237,10 @@ pub fn f16_to_f32_batch(input: &[u16], output: &mut [f32]) { convert::f16_to_f32_batch(input, output); } -/// Compute Fletcher-32 checksum. +/// Compute a textbook Fletcher-32 checksum (both sums start at 0xffff). +/// +/// This is not HDF5's checksum; the Fletcher-32 I/O filter uses +/// `clawhdf5_format::checksum::fletcher32`. pub fn checksum_fletcher32(data: &[u8]) -> u32 { checksum::checksum_fletcher32(data) } diff --git a/crates/clawhdf5-format/src/checksum.rs b/crates/clawhdf5-format/src/checksum.rs index 525689f..363d97f 100644 --- a/crates/clawhdf5-format/src/checksum.rs +++ b/crates/clawhdf5-format/src/checksum.rs @@ -14,6 +14,45 @@ pub fn jenkins_lookup3(data: &[u8]) -> u32 { hashlittle(data, 0) } +/// HDF5's Fletcher-32 checksum, as the Fletcher-32 I/O filter (filter id 3) +/// stores it after each chunk. +/// +/// A line-for-line port of `H5_checksum_fletcher32` (H5checksum.c, libhdf5 +/// 1.8 through 1.14): big-endian 16-bit words summed in blocks of 360, each +/// sum reduced after a block by the ones'-complement fold +/// `(s & 0xffff) + (s >> 16)` rather than `% 65535`, an odd trailing byte +/// taken as the high byte of a last word, and a final fold of both sums. +/// The fold and `% 65535` differ whenever a sum is a non-zero multiple of +/// 65535: the fold leaves 0xffff where the modulo gives 0, so the two +/// disagree on about one chunk in 32768 and libhdf5 rejects the other's +/// checksum. This must stay the only implementation. +pub fn fletcher32(data: &[u8]) -> u32 { + let mut sum1: u32 = 0; + let mut sum2: u32 = 0; + // 360 words keep both sums inside 32 bits between folds (the bound + // libhdf5 uses: after a fold sum1 < 0x10200, so sum2 stays below + // 360 * 361 / 2 * 0xffff + 360 * 0x10200 + 0x1fffe < 2^32). The adds wrap + // like the C unsigned arithmetic all the same. + let (words, odd) = data.as_chunks::<2>(); + for block in words.chunks(360) { + for w in block { + sum1 = sum1.wrapping_add((u32::from(w[0]) << 8) | u32::from(w[1])); + sum2 = sum2.wrapping_add(sum1); + } + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + } + if let [last] = odd { + sum1 = sum1.wrapping_add(u32::from(*last) << 8); + sum2 = sum2.wrapping_add(sum1); + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + } + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + (sum2 << 16) | sum1 +} + /// Compute CRC32 (IEEE / ISO 3309) over data. /// /// When the `fast-checksum` feature is enabled, this uses hardware CRC32 @@ -207,6 +246,19 @@ fn hashlittle(data: &[u8], initval: u32) -> u32 { mod tests { use super::*; + /// Values of libhdf5's `H5_checksum_fletcher32` (h5py 3.x's bundled + /// libhdf5, called through ctypes). The first three are sums that are + /// multiples of 65535, where `% 65535` gave 0 instead of 0xffff. + #[test] + fn fletcher32_matches_libhdf5() { + assert_eq!(fletcher32(&[0x00, 0x01, 0xff, 0xfe]), 0x0001_ffff); + assert_eq!(fletcher32(&[0xff; 720]), 0xffff_ffff); + assert_eq!(fletcher32(&[0xff; 721]), 0xff00_ff00); + assert_eq!(fletcher32(&[0xff; 1441]), 0xff00_ff00); + assert_eq!(fletcher32(&[]), 0); + assert_eq!(fletcher32(&[7]), 0x0700_0700); + } + #[test] fn empty_input() { // Empty input should return the initial state after no mixing diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 6390aee..47402a6 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -1678,56 +1678,6 @@ fn shuffle_compress_general(data: &[u8], n: usize, element_size: usize, result: } } -/// Compute HDF5 Fletcher32 checksum over data. -/// HDF5 uses a modified Fletcher32 that operates on 16-bit words. -/// -/// Optimized with wider accumulators: processes blocks of 360 words before -/// taking the modulo, reducing the number of expensive modulo operations. -/// (360 is the maximum block size that avoids u32 overflow for sum2.) -fn fletcher32_compute(data: &[u8]) -> u32 { - let mut sum1: u32 = 0; - let mut sum2: u32 = 0; - - // Process in blocks of 360 16-bit words (720 bytes) to delay modulo. - // Max sum1 before mod: 360 * 65535 = 23_592_600 < u32::MAX - // Max sum2 before mod: 360 * 23_592_600 ~ 8.5B > u32::MAX, but actual - // sum2 accumulates incrementally, so worst case is 360*360*65535/2 which - // fits in u64. We use u32 with block size 360 which is safe. - const BLOCK_WORDS: usize = 360; - const BLOCK_BYTES: usize = BLOCK_WORDS * 2; - - let mut offset = 0; - let len = data.len(); - - while offset + BLOCK_BYTES <= len { - let end = offset + BLOCK_BYTES; - let mut i = offset; - while i < end { - let val = ((data[i] as u32) << 8) | (data[i + 1] as u32); - sum1 += val; - sum2 += sum1; - i += 2; - } - sum1 %= 65535; - sum2 %= 65535; - offset = end; - } - - // Handle remaining bytes - while offset < len { - let val = if offset + 1 < len { - ((data[offset] as u32) << 8) | (data[offset + 1] as u32) - } else { - (data[offset] as u32) << 8 - }; - sum1 = (sum1 + val) % 65535; - sum2 = (sum2 + sum1) % 65535; - offset += 2; - } - - (sum2 << 16) | sum1 -} - /// Verify Fletcher32 checksum and strip it from the data. /// The last 4 bytes are the stored checksum. fn fletcher32_verify(data: &[u8]) -> Result, FormatError> { @@ -1749,8 +1699,18 @@ fn fletcher32_payload(data: &[u8]) -> Result { data[data.len() - 2], data[data.len() - 1], ]); - let computed = fletcher32_compute(payload); - if stored != computed { + let computed = crate::checksum::fletcher32(payload); + // libhdf5 also accepts the checksum with the bytes of each 16-bit half + // swapped, which is how 1.6.2 and earlier stored it + // (H5Z__filter_fletcher32's `reversed_fletcher`). + let reversed = ((computed & 0x00ff_00ff) << 8) | ((computed >> 8) & 0x00ff_00ff); + // clawhdf5 v2.7.0 and earlier reduced the sums `% 65535`, which gives 0 + // where libhdf5's fold gives 0xffff; accept that form too, so that files + // those releases wrote can still be read (and rewritten for libhdf5). + // It differs from `computed` only in a half that is 0xffff. + let half = |h: u32| if h == 0xffff { 0 } else { h }; + let legacy = (half(computed >> 16) << 16) | half(computed & 0xffff); + if stored != computed && stored != reversed && stored != legacy { return Err(FormatError::Fletcher32Mismatch { expected: stored, computed, @@ -1761,7 +1721,7 @@ fn fletcher32_payload(data: &[u8]) -> Result { /// Append Fletcher32 checksum to data. fn fletcher32_append(data: &[u8]) -> Result, FormatError> { - let checksum = fletcher32_compute(data); + let checksum = crate::checksum::fletcher32(data); let mut result = data.to_vec(); result.extend_from_slice(&checksum.to_le_bytes()); Ok(result) diff --git a/crates/clawhdf5/tests/fletcher32_interop.rs b/crates/clawhdf5/tests/fletcher32_interop.rs new file mode 100644 index 0000000..b6bf6b0 --- /dev/null +++ b/crates/clawhdf5/tests/fletcher32_interop.rs @@ -0,0 +1,321 @@ +//! Fletcher-32 against libhdf5. +//! +//! libhdf5's `H5_checksum_fletcher32` reduces its sums with the +//! ones'-complement fold `(s & 0xffff) + (s >> 16)`, which leaves 0xffff +//! where `% 65535` leaves 0. Our checksum once used `% 65535`, so on about +//! one chunk in 32768 (a sum that is a non-zero multiple of 65535) libhdf5 +//! rejected the chunks we wrote and we rejected the chunks it wrote. +//! +//! - The checksum is compared with libhdf5's own `H5_checksum_fletcher32`, +//! called through ctypes from the library h5py loads, over every one-byte +//! and two-byte input and a large corpus of random and fold-heavy inputs. +//! - Chunks engineered to hit the fold are written by `FileBuilder` and by +//! `FileEditor` and read by h5py, and written by h5py and read by us. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{File, FileBuilder, FileEditor}; +use clawhdf5_format::checksum::fletcher32; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn have_h5py() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .is_ok_and(|o| o.status.success()); + if !ok { + assert!( + std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + } + ok +} + +fn run_python(script: &str, args: &[&str]) -> String { + let out = Command::new(python()) + .arg("-c") + .arg(script) + .args(args) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "python failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn tmp(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("clawhdf5_fletcher32_{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + dir.join(name) +} + +/// The checksum our code computed before it was fixed: each sum reduced +/// `% 65535`. Only used to show that the test data hits the disagreement. +fn fletcher32_mod(data: &[u8]) -> u32 { + let (mut s1, mut s2) = (0u64, 0u64); + for w in data.chunks(2) { + let v = (u64::from(w[0]) << 8) | w.get(1).map_or(0, |&b| u64::from(b)); + s1 = (s1 + v) % 65535; + s2 = (s2 + s1) % 65535; + } + ((s2 as u32) << 16) | s1 as u32 +} + +/// Splitmix64, so the data is the same on every run. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } +} + +/// Writes every case of the file `argv[1]` (u32 LE length + bytes) back to +/// `argv[2]` as libhdf5's checksum of each, u32 LE. +const LIBHDF5_CHECKSUMS: &str = r#" +import ctypes, glob, os, struct, sys +import h5py +cands = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, 'h5py.libs', 'libhdf5-*.so*')) +cands += glob.glob(os.path.join(os.path.dirname(h5py.__file__), '.dylibs', 'libhdf5*.dylib')) +if cands: + lib = ctypes.CDLL(cands[0]) +else: + # A system h5py links the system libhdf5, already loaded. + import h5py.h5 + lib = ctypes.CDLL(h5py.h5.__file__) +f = lib.H5_checksum_fletcher32 +f.restype = ctypes.c_uint32 +f.argtypes = [ctypes.c_char_p, ctypes.c_size_t] +data = open(sys.argv[1], 'rb').read() +out = bytearray() +i = 0 +while i < len(data): + (n,) = struct.unpack_from('> = Vec::new(); + // Every one-byte input (the odd-length path alone) and every one-word + // input (65535 = 0xffff is the smallest fold). + cases.extend((0..=255u8).map(|b| vec![b])); + cases.extend((0..=u16::MAX).map(|w| w.to_be_bytes().to_vec())); + let mut rng = Rng(0x5eed_f1e7); + // Words drawn from values that make multiples of 65535 frequent, at + // lengths around the 360-word block boundaries, odd and even. + const FOLDY: [u16; 6] = [0, 1, 0xfffe, 0xffff, 0x8000, 0x7fff]; + for _ in 0..40_000 { + let len = match rng.next() % 4 { + 0 => (rng.next() % 16) as usize, + 1 => 718 + (rng.next() % 6) as usize, + 2 => 1438 + (rng.next() % 6) as usize, + _ => (rng.next() % 3000) as usize, + }; + let foldy = rng.next().is_multiple_of(2); + let mut v = Vec::with_capacity(len + 1); + while v.len() < len { + let w = if foldy { + FOLDY[(rng.next() % 6) as usize] + } else { + rng.next() as u16 + }; + v.extend_from_slice(&w.to_be_bytes()); + } + v.truncate(len); + cases.push(v); + } + // Long runs of 0xff: sums are multiples of 65535 at every block. + for len in [720, 721, 1440, 1441, 7200, 65536, 65537] { + cases.push(vec![0xff; len]); + } + let mut blob = Vec::new(); + for c in &cases { + blob.extend_from_slice(&(c.len() as u32).to_le_bytes()); + blob.extend_from_slice(c); + } + let input = tmp("cases.bin"); + let output = tmp("sums.bin"); + std::fs::write(&input, &blob).unwrap(); + run_python( + LIBHDF5_CHECKSUMS, + &[input.to_str().unwrap(), output.to_str().unwrap()], + ); + let sums = std::fs::read(&output).unwrap(); + assert_eq!(sums.len(), cases.len() * 4); + let mut folds = 0; + for (c, s) in cases.iter().zip(sums.as_chunks::<4>().0) { + let want = u32::from_le_bytes(*s); + assert_eq!( + fletcher32(c), + want, + "checksum of {} bytes {:02x?}...", + c.len(), + &c[..c.len().min(16)] + ); + if fletcher32_mod(c) != want { + folds += 1; + } + } + // The corpus must exercise the case `% 65535` got wrong. + assert!(folds > 500, "only {folds} fold cases"); +} + +const CHUNK: usize = 8; + +/// `n` chunks of `CHUNK` bytes, each one a chunk on which the old +/// `% 65535` checksum and libhdf5's differ (sum1, sum2 or both a non-zero +/// multiple of 65535), with an ordinary chunk between them. +fn fold_chunks(n: usize) -> Vec { + let mut rng = Rng(42); + let mut out = Vec::new(); + let mut found = 0; + while found < n { + // Build a chunk whose sum1 is a multiple of 65535 half the time, + // otherwise search at random for a sum2 fold. + let mut c: Vec = (0..CHUNK).map(|_| rng.next() as u8).collect(); + if found % 2 == 0 { + let words: u64 = c[..CHUNK - 2] + .chunks(2) + .map(|w| (u64::from(w[0]) << 8) | u64::from(w[1])) + .sum(); + let last = ((65535 - words % 65535) % 65535) as u16; + c[CHUNK - 2..].copy_from_slice(&last.to_be_bytes()); + } + if fletcher32(&c) != fletcher32_mod(&c) { + out.extend_from_slice(&c); + out.extend((0..CHUNK).map(|i| i as u8 + 1)); + found += 1; + } + } + out +} + +#[test] +fn h5py_reads_fold_case_chunks_we_write() { + if !have_h5py() { + return; + } + let data = fold_chunks(32); + // FileBuilder. + let built = tmp("built.h5"); + let mut b = FileBuilder::new(); + b.create_dataset("d") + .with_u8_data(&data) + .with_chunks(&[CHUNK as u64]) + .with_fletcher32(); + b.write(&built).unwrap(); + // FileEditor, into a dataset h5py created. + let edited = tmp("edited.h5"); + run_python( + "import sys, h5py, numpy as np\n\ + with h5py.File(sys.argv[1], 'w') as f:\n\ + \x20 f.create_dataset('d', data=np.zeros(int(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)", + &[edited.to_str().unwrap(), &data.len().to_string()], + ); + FileEditor::open(&edited) + .unwrap() + .write_all("d", &data) + .unwrap(); + for path in [&built, &edited] { + let got = run_python( + "import sys, h5py\n\ + with h5py.File(sys.argv[1], 'r') as f:\n\ + \x20 assert f['d'].fletcher32\n\ + \x20 print(f['d'][:].tobytes().hex())", + &[path.to_str().unwrap()], + ); + assert_eq!(got, hex(&data), "{}", path.display()); + } +} + +#[test] +fn we_read_fold_case_chunks_h5py_writes() { + if !have_h5py() { + return; + } + let data = fold_chunks(32); + let path = tmp("h5py.h5"); + run_python( + "import sys, h5py, numpy as np\n\ + with h5py.File(sys.argv[1], 'w') as f:\n\ + \x20 f.create_dataset('d', data=np.frombuffer(bytes.fromhex(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)", + &[path.to_str().unwrap(), &hex(&data)], + ); + let file = File::open(&path).unwrap(); + let ds = file.dataset("d").unwrap(); + assert_eq!( + ds.read_selection(&clawhdf5_format::selection::Selection::All) + .unwrap(), + data + ); +} + +/// A checksum stored with the bytes of each 16-bit half swapped, as +/// libhdf5 1.6.2 and earlier wrote it, is accepted as libhdf5 accepts it; +/// so is the `% 65535` form clawhdf5 v2.7.0 and earlier wrote, so that +/// their files stay readable. +#[test] +fn legacy_checksums_are_accepted() { + use clawhdf5_format::filter_pipeline::{FILTER_FLETCHER32, FilterDescription, FilterPipeline}; + let payload = [1u8, 2, 3, 4, 5]; + let sum = fletcher32(&payload); + let swapped = ((sum & 0x00ff_00ff) << 8) | ((sum >> 8) & 0x00ff_00ff); + assert_ne!(sum, swapped); + let pipeline = FilterPipeline { + version: 2, + filters: vec![FilterDescription { + filter_id: FILTER_FLETCHER32, + name: None, + client_data: vec![], + flags: 0, + }], + }; + for stored in [sum, swapped] { + let mut chunk = payload.to_vec(); + chunk.extend_from_slice(&stored.to_le_bytes()); + let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1) + .unwrap(); + assert_eq!(out, payload); + } + // Our old checksum of a fold-case chunk. + let fold = fold_chunks(1); + let fold = &fold[..CHUNK]; + let old = fletcher32_mod(fold); + assert_ne!(old, fletcher32(fold)); + let mut chunk = fold.to_vec(); + chunk.extend_from_slice(&old.to_le_bytes()); + let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, CHUNK, 1).unwrap(); + assert_eq!(out, fold); + let mut chunk = payload.to_vec(); + chunk.extend_from_slice(&(sum ^ 1).to_le_bytes()); + assert!( + clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1).is_err() + ); +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 1b05553..25d0960 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,41 @@ deleting it. --- +## Fletcher-32 checksums disagreed with libhdf5 on about 1 chunk in 32768 + +**Status:** fixed 2026-09-26, after v2.7.0. **Every release (v2.1.0 to +v2.7.0) is affected**, in both directions. + +Our Fletcher-32 reduced its two running sums with `% 65535`; libhdf5's +`H5_checksum_fletcher32` (H5checksum.c) folds them with +`(s & 0xffff) + (s >> 16)`. Both are arithmetic mod 65535, but where a sum +is a non-zero multiple of 65535 the fold leaves 0xffff and the modulo 0, so +the checksums differ — for random data about one chunk in 32768 (each of +the two sums hits it with probability about 1/65535). Found by the review +of the editor work: a random-edit fuzzer with gzip + Fletcher-32 hit it on +13 of about 100 seeds. + +- Chunks we wrote (`FileBuilder`/`FileWriter` `with_fletcher32`, and the + unreleased `FileEditor`) with such a sum are refused by h5py and libhdf5: + "filter returned failure during read". h5py writing `[1, 0xfffe]` as + big-endian `u2` stores checksum `0x0001ffff`; we computed `0x00010000`. +- Chunks libhdf5 wrote with such a sum were refused by every reader here + with `Fletcher32Mismatch`; the data itself was never wrong. + +**Fix:** `clawhdf5_format::checksum::fletcher32`, a port of +`H5_checksum_fletcher32`, used by the filter for writing and verifying. It +also accepts a checksum whose 16-bit halves are byte-swapped, as libhdf5 +does for files from 1.6.2 and earlier, and the `% 65535` form clawhdf5 +v2.7.0 and earlier wrote (the two differ only in a half that is 0xffff). +**Test:** +`crates/clawhdf5/tests/fletcher32_interop.rs` (libhdf5's own function +through ctypes on every 1- and 2-byte input plus 40 000 random and +fold-heavy inputs; h5py reads fold-case chunks from `FileBuilder` and +`FileEditor`; we read h5py's). **Existing data:** a Fletcher-32 dataset +written by v2.7.0 or earlier may hold chunks libhdf5 cannot read; a fixed +build reads them. Rewrite such datasets with a fixed build (read, then +write them again) before handing the file to libhdf5 or h5py. + ## LZF/Blosc chunks written with a stale filter mask **Status:** fixed 2026-09-26, before any release (the LZF and Blosc writers From 930921e8cb992263825b3e9572d9820c0bc76067 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:27:00 -0500 Subject: [PATCH 12/15] edit: shrink by visiting the chunks that exist prune_plan stored one Vec for every chunk coordinate of the region a shrink cuts off, existing or not, so a sparse dataset exhausted memory (about 62 bytes per coordinate; (4, 2e7) with chunks (1, 1) took 2.5 GB, larger extents never finished). It now places each existing chunk in H5D__chunk_prune_by_extent's walk (its pass, then its coordinates) and sorts, which gives the same chunks, order and actions in memory and time proportional to the chunks that exist. A unit test checks the plan against the full walk (kept as the test's reference) for 3000 random extents and chunk subsets. The interop test shrinks a (4, 10^12) dataset with chunks (1, 1) and 9 chunks (v1 and v2 B-tree): 0.56 s and 43 MB peak; the old code aborted on allocation under an 8 GB limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +- .../tests/edit_coverage_interop.rs | 62 +++++ crates/clawhdf5/src/edit/mod.rs | 246 +++++++++++++----- 3 files changed, 245 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 964390d..6ae2b02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,10 @@ partial edge chunk outside the extent is overwritten with the fill value, so it reads as fill after a later growth. Growth under early allocation now allocates and fills the new chunks (`H5D__chunk_allocate`), which an - implicit index needs. Shrinking was `Error::Unsupported`. + implicit index needs. Shrinking was `Error::Unsupported`. Only the + chunks that exist are visited (placed in libhdf5's order), so shrinking + a sparse dataset costs memory and time in its chunks, not in the + coordinates cut off (a 2 x 10^12-coordinate shrink takes 0.6 s). - **`FileEditor::set_attr` handles dense attribute storage and creation order**: objects that track (and index) attribute creation order; the move to dense storage when an object reaches its compact limit (or an diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 7fc309a..fff256c 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1566,3 +1566,65 @@ fn resize_clawhdf5_written_datasets() { } check_tools(&path, true); } + +/// Shrinking a huge, sparse dataset costs time and memory in the chunks +/// that exist, not in the chunk coordinates cut off. The editor once stored +/// every coordinate of the cut-off region (about 62 bytes each), so this +/// 2 x 10^12-coordinate shrink ran out of memory. +#[test] +fn shrinking_a_huge_sparse_dataset_is_bounded() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + const N: u64 = 1_000_000_000_000; + for libver in ["earliest", "v110"] { + let path = dir.path().join(format!("sparse_{libver}.h5")); + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'w', libver=({libver:?}, 'latest')) as f:\n\ + \x20 d = f.create_dataset('b', shape=(4, {N}), maxshape=(None, None), chunks=(1, 1), dtype=' = got + .as_chunks::<4>() + .0 + .iter() + .map(|c| i32::from_le_bytes(*c)) + .collect(); + assert_eq!(got, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0], "{libver}"); + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 336fbb8..e1d301a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -1330,16 +1330,31 @@ enum Prune { Remove, } -/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks -/// from `old` to `new`, in its order. -fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { +/// The chunks of `existing` (keyed by scaled coordinates) that +/// `H5D__chunk_prune_by_extent` visits when the extent shrinks from `old` +/// to `new`, in its order, with what it does to each. +/// +/// libhdf5 walks every chunk coordinate of the region cut off, one pass per +/// shrunk dimension `op` in order: the coordinates with `scaled[op]` at or +/// past the new extent's chunk and every earlier shrunk dimension below its +/// own, row-major. It looks each one up in the index and stores nothing, +/// so a sparse dataset costs it time, not memory. Only chunks that exist +/// can be acted on, so this places each existing chunk in that walk (its +/// pass, then its coordinates) and sorts: the same chunks in the same +/// order, in memory and time proportional to the chunks that exist. +fn prune_plan<'c>( + old: &[u64], + new: &[u64], + cd: &[u64], + existing: &'c HashMap, ChunkInfo>, +) -> Vec<(&'c [u64], &'c ChunkInfo, Prune)> { let rank = new.len(); - let mut out = Vec::new(); - if old.contains(&0) { - return out; + if old.contains(&0) || cd.contains(&0) { + return Vec::new(); } - let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); - let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + // The last chunk index still (partly) inside the new extent; -1 when + // the dimension shrank to nothing. let max_fill: Vec = (0..rank) .map(|d| { if new[d] == 0 { @@ -1350,60 +1365,25 @@ fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { }) .collect(); let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); - let fill_dim: Vec = (0..rank) - .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) - .collect(); - for op in 0..rank { - if !shrunk[op] { - continue; - } - let mut scaled = vec![0u64; rank]; - scaled[op] = min_mod[op]; - let mut outside: Vec = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); - let mut n_out = outside.iter().filter(|&&o| o).count(); - loop { - if n_out == 0 { - out.push((scaled.clone(), Prune::Fill)); + let mut out: Vec<(usize, &[u64], &ChunkInfo, Prune)> = existing + .iter() + .filter(|(s, _)| s.len() == rank && s.iter().zip(&max_mod).all(|(c, m)| c <= m)) + .filter_map(|(s, info)| { + // The first shrunk dimension whose pass reaches this chunk; the + // passes before it covered only coordinates below their minimum. + let op = (0..rank).find(|&d| new[d] < old[d] && s[d] >= min_mod[d])?; + // Chunks with any coordinate past the last partly kept chunk go; + // the others lose the part outside the new extent. + let what = if (0..rank).all(|u| s[u] as i64 <= max_fill[u]) { + Prune::Fill } else { - out.push((scaled.clone(), Prune::Remove)); - } - let mut carry = true; - for i in (0..rank).rev() { - scaled[i] += 1; - if scaled[i] > max_mod[i] { - if i == op { - scaled[i] = min_mod[i]; - if outside[i] && fill_dim[i] { - outside[i] = false; - n_out -= 1; - } - } else { - scaled[i] = 0; - if outside[i] && max_fill[i] >= 0 { - outside[i] = false; - n_out -= 1; - } - } - } else { - if !outside[i] && scaled[i] as i64 > max_fill[i] { - outside[i] = true; - n_out += 1; - } - carry = false; - break; - } - } - if carry { - break; - } - } - if min_mod[op] == 0 { - // Every chunk was visited (the dimension shrank to nothing). - break; - } - max_mod[op] = min_mod[op] - 1; - } - out + Prune::Remove + }; + Some((op, s.as_slice(), info, what)) + }) + .collect(); + out.sort_unstable_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))); + out.into_iter().map(|(_, s, i, w)| (s, i, w)).collect() } /// The chunk work of a resize: under early allocation, allocate and fill @@ -1468,12 +1448,9 @@ fn resize_chunks( })?; } if shrink && !existing.is_empty() { - for (scaled, what) in prune_plan(old, new, &cd) { - let Some(info) = existing.get(&scaled) else { - continue; - }; + for (scaled, info, what) in prune_plan(old, new, &cd, &existing) { match what { - Prune::Remove => ce.remove(img, &scaled, info)?, + Prune::Remove => ce.remove(img, scaled, info)?, Prune::Fill => { let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?; // Keep [0, count) in each dimension; fill the rest. @@ -1495,7 +1472,7 @@ fn resize_chunks( buf[at..at + es].copy_from_slice(&t.fill); } } - store_chunk(img, &mut ce, Some(info), &scaled, buf)?; + store_chunk(img, &mut ce, Some(info), scaled, buf)?; } } } @@ -1610,6 +1587,141 @@ fn decode_chunk( #[cfg(test)] mod tests { + use super::{Prune, prune_plan}; + use clawhdf5_format::chunked_read::ChunkInfo; + use std::collections::HashMap; + + /// `H5D__chunk_prune_by_extent`'s walk over every chunk coordinate of the + /// region cut off, stored (the implementation before `prune_plan` + /// visited only existing chunks). + fn prune_walk(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { + let rank = new.len(); + let mut out = Vec::new(); + if old.contains(&0) { + return out; + } + let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); + let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_fill: Vec = (0..rank) + .map(|d| { + if new[d] == 0 { + -1 + } else { + ((new[d].min(old[d]) - 1) / cd[d]) as i64 + } + }) + .collect(); + let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); + let fill_dim: Vec = (0..rank) + .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) + .collect(); + for op in 0..rank { + if !shrunk[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min_mod[op]; + let mut outside: Vec = + (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); + let mut n_out = outside.iter().filter(|&&o| o).count(); + loop { + if n_out == 0 { + out.push((scaled.clone(), Prune::Fill)); + } else { + out.push((scaled.clone(), Prune::Remove)); + } + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max_mod[i] { + if i == op { + scaled[i] = min_mod[i]; + if outside[i] && fill_dim[i] { + outside[i] = false; + n_out -= 1; + } + } else { + scaled[i] = 0; + if outside[i] && max_fill[i] >= 0 { + outside[i] = false; + n_out -= 1; + } + } + } else { + if !outside[i] && scaled[i] as i64 > max_fill[i] { + outside[i] = true; + n_out += 1; + } + carry = false; + break; + } + } + if carry { + break; + } + } + if min_mod[op] == 0 { + // Every chunk was visited (the dimension shrank to nothing). + break; + } + max_mod[op] = min_mod[op] - 1; + } + out + } + + /// `prune_plan` over a random subset of chunks gives the chunks of the + /// full walk that exist, in its order and with its actions. + #[test] + fn prune_plan_follows_the_full_walk() { + let mut x: u64 = 0x1234_5678; + let mut rnd = |n: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x % n + }; + for _ in 0..3000 { + let rank = 1 + rnd(3) as usize; + let cd: Vec = (0..rank).map(|_| 1 + rnd(4)).collect(); + let old: Vec = (0..rank).map(|_| 1 + rnd(14)).collect(); + let new: Vec = old + .iter() + .map(|&o| if rnd(2) == 0 { o } else { rnd(o + 1) }) + .collect(); + let grid: Vec = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect(); + let mut existing = HashMap::new(); + let total: u64 = grid.iter().product(); + for flat in 0..total { + if rnd(3) == 0 { + continue; + } + let mut r = flat; + let mut s = vec![0; rank]; + for d in (0..rank).rev() { + s[d] = r % grid[d]; + r /= grid[d]; + } + let info = ChunkInfo { + chunk_size: 1, + filter_mask: 0, + offsets: s.iter().zip(&cd).map(|(a, b)| a * b).collect(), + address: flat, + }; + existing.insert(s, info); + } + let want: Vec<(Vec, bool)> = prune_walk(&old, &new, &cd) + .into_iter() + .filter(|(s, _)| existing.contains_key(s)) + .map(|(s, w)| (s, matches!(w, Prune::Fill))) + .collect(); + let got: Vec<(Vec, bool)> = prune_plan(&old, &new, &cd, &existing) + .into_iter() + .map(|(s, _, w)| (s.to_vec(), matches!(w, Prune::Fill))) + .collect(); + assert_eq!(got, want, "old {old:?} new {new:?} chunks {cd:?}"); + } + } + use std::cell::Cell; use std::path::Path; From a69c5be8b2bd6d55ffbce4c33e37bd5907edaf44 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:29:31 -0500 Subject: [PATCH 13/15] format: read object header chunks from a queue, one buffer at a time The version-1 chunk walk nested continuation chunks depth-first and kept every enclosing chunk's buffer alive, up to 65 536 chunks. With storage that hands out owned buffers (CountingStorage, the Storage trait, remote storage) a crafted chain of chunks nested in each other read and held the square of the file's size (a 192 KB file read 768 MB). Chunks are now read from a FIFO queue of (address, length) pairs in the order their continuation messages are found, as H5O_protect does and as the editor's header walker already did, each buffer released before the next read. In both header versions a chunk starting at an address seen before (cycle) is refused, and so are chunks adding up to more than the file, which bounds a header's reads by the file's size. Overlap itself is allowed: libhdf5 reads cve-2025-7067.h5, whose continuation chunk overlaps chunk 0 (refusing overlap cost that conformance file). Tests: the nested chain is refused having read at most the file (it read n^2 bytes before); a 3000-chunk chain reads each chunk once; a chunk's messages follow the whole previous chunk (they were inserted at the continuation message); an overlapping continuation chunk is read. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +- crates/clawhdf5-format/src/object_header.rs | 287 ++++++++++++++------ 2 files changed, 218 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae2b02..1949400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,9 +87,15 @@ header whose continuation chunks chain more than 32 deep (a header that gains a chunk per attribute added when full, as libhdf5 and the editor grow it) was refused with `NestingDepthExceeded`; version-2 headers - stopped at 256 chunks. Chunks are now followed without recursion, in the - same order; a chunk address seen twice (a cycle) or more than 65 536 - chunks are refused. + stopped at 256 chunks. Chunks are now read one at a time from a queue, + in the order their continuation messages are found (libhdf5's + `H5O_protect` order, which the editor already used; a version-1 + chunk's messages used to be inserted at its continuation message), each + buffer released before the next is read; a chunk address seen twice (a + cycle), chunks adding up to more than the file (a crafted chain of + chunks nested in each other made storage with owned buffers read and + hold the square of the file's size), or more than 65 536 chunks are + refused, so a header's chunks read at most the file's size. - Tests: `crates/clawhdf5-tools/tests/edit_coverage_interop.rs` (h5py `earliest`/`v110`/`latest` and clawhdf5-written files; structure comparisons with libhdf5 for version-2 B-trees, shrink on every index, diff --git a/crates/clawhdf5-format/src/object_header.rs b/crates/clawhdf5-format/src/object_header.rs index 383d553..ab652cb 100644 --- a/crates/clawhdf5-format/src/object_header.rs +++ b/crates/clawhdf5-format/src/object_header.rs @@ -232,12 +232,14 @@ impl ObjectHeader { /// end of the chunk, or leftover bytes too few for a message header (a /// "gap", which only version 2 allows). /// - /// Continuation chunks are followed depth-first, as met, with an - /// explicit stack: a chain of continuation chunks as long as libhdf5 - /// writes (each new chunk holding the next continuation message, one - /// per attribute added to a full header) is read without recursion. - /// A chunk address seen twice is a cycle and refused, as is a header of - /// more than [`MAX_V1_CHUNKS`] chunks. + /// Continuation chunks are read in the order their messages are found, + /// as `H5O_protect` loads them (so the messages keep libhdf5's order): + /// a queue of (address, length) pairs, each chunk read, parsed and + /// released before the next, so only one chunk buffer is alive at a + /// time whatever the storage. Every chunk must start at a new address + /// (else a cycle), and the chunks together may be no larger than the + /// file, so the bytes read stay within the file's size; a header of + /// more than [`MAX_V1_CHUNKS`] chunks is refused. fn parse_v1_chunk( file: &S, offset: u64, @@ -246,74 +248,67 @@ impl ObjectHeader { length_size: u8, messages: &mut Vec, ) -> Result { - let mut stack = vec![(read_exact_at(file, offset, length)?, 0usize)]; - let mut seen = BTreeSet::new(); - seen.insert(offset); - let mut count = 0usize; - - while let Some((chunk, pos)) = stack.last_mut() { - let data: &[u8] = chunk; + let mut spans = ChunkSpans::new(file.len(), offset, length)?; + let mut queue: Vec<(u64, usize)> = vec![(offset, length)]; + let mut chunk0_count = 0usize; + let mut next = 0usize; + while let Some(&(chunk_offset, chunk_length)) = queue.get(next) { + let chunk = read_exact_at(file, chunk_offset, chunk_length)?; + let data: &[u8] = &chunk; let end = data.len(); - if *pos >= end { - stack.pop(); - continue; - } - if end - *pos < V1_MSG_HEADER_SIZE { - return Err(FormatError::InvalidObjectHeader( - "gap found in early version of file format", - )); - } - let p = *pos; - let msg_type_raw = LittleEndian::read_u16(&data[p..p + 2]); - let msg_data_size = LittleEndian::read_u16(&data[p + 2..p + 4]) as usize; - let msg_flags = data[p + 4]; - // reserved(3) at p+5..p+8 - let p = p + V1_MSG_HEADER_SIZE; - - if !msg_data_size.is_multiple_of(8) { - return Err(FormatError::InvalidObjectHeader("message not aligned")); - } - if msg_data_size > end - p { - return Err(FormatError::InvalidObjectHeader( - "message size exceeds buffer end", - )); - } - let body = &data[p..p + msg_data_size]; - check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; - let msg_type = MessageType::from_u16(msg_type_raw); - if msg_type != MessageType::Nil { - messages.push(HeaderMessage { - msg_type, - size: msg_data_size, - flags: msg_flags, - creation_order: None, - data: body.to_vec(), - }); - } - // Follow continuations (v1 continuation chunks are just raw - // messages, no signature); check_message has checked the body. - let cont = if msg_type == MessageType::ObjectHeaderContinuation { - Some(( - read_offset(body, 0, offset_size)?, - to_usize(read_offset(body, offset_size as usize, length_size)?)?, - )) - } else { - None - }; - *pos = p + msg_data_size; - // Only the first chunk's messages are held to the prefix count. - if stack.len() == 1 { - count += 1; - } - if let Some((cont_offset, cont_length)) = cont { - if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { - return Err(FormatError::NestingDepthExceeded); + let mut pos = 0usize; + let mut count = 0usize; + while pos < end { + if end - pos < V1_MSG_HEADER_SIZE { + return Err(FormatError::InvalidObjectHeader( + "gap found in early version of file format", + )); } - stack.push((read_exact_at(file, cont_offset, cont_length)?, 0)); - } - } + let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]); + let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize; + let msg_flags = data[pos + 4]; + // reserved(3) at pos+5..pos+8 + pos += V1_MSG_HEADER_SIZE; - Ok(count) + if !msg_data_size.is_multiple_of(8) { + return Err(FormatError::InvalidObjectHeader("message not aligned")); + } + if msg_data_size > end - pos { + return Err(FormatError::InvalidObjectHeader( + "message size exceeds buffer end", + )); + } + let body = &data[pos..pos + msg_data_size]; + check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?; + count += 1; + let msg_type = MessageType::from_u16(msg_type_raw); + if msg_type != MessageType::Nil { + messages.push(HeaderMessage { + msg_type, + size: msg_data_size, + flags: msg_flags, + creation_order: None, + data: body.to_vec(), + }); + } + // Queue continuations (v1 continuation chunks are just raw + // messages, no signature); check_message has checked the body. + if msg_type == MessageType::ObjectHeaderContinuation { + let cont_offset = read_offset(body, 0, offset_size)?; + let cont_length = + to_usize(read_offset(body, offset_size as usize, length_size)?)?; + spans.add(cont_offset, cont_length)?; + queue.push((cont_offset, cont_length)); + } + pos += msg_data_size; + } + // Only the first chunk's messages are held to the prefix count. + if next == 0 { + chunk0_count = count; + } + next += 1; + } + Ok(chunk0_count) } fn parse_v2( @@ -436,15 +431,14 @@ impl ObjectHeader { &mut continuations, )?; - // Follow continuations. A chunk address seen twice is a cycle in - // malformed data; a valid header can have many chunks (libhdf5 adds - // one whenever a message no longer fits), up to the same bound as a + // Follow continuations, one chunk buffer at a time. A chunk address + // seen twice is a cycle in malformed data, and the chunks may add up + // to no more than the file; a valid header can have many chunks (libhdf5 adds one + // whenever a message no longer fits), up to the same bound as a // version-1 header. - let mut seen = BTreeSet::new(); + let mut spans = ChunkSpans::new(file.len(), base as u64, chunk0_msg_end.saturating_add(4))?; while let Some((cont_offset, cont_length)) = continuations.pop() { - if !seen.insert(cont_offset) || seen.len() > MAX_V1_CHUNKS { - return Err(FormatError::NestingDepthExceeded); - } + spans.add(cont_offset as u64, cont_length)?; Self::parse_v2_continuation( file, cont_offset as u64, @@ -607,6 +601,45 @@ const V2_PREFIX_MAX: usize = 34; /// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3). const V1_MSG_HEADER_SIZE: usize = 8; +/// The chunks of one object header read so far. A chunk starting where +/// another did is a cycle. Chunks of a valid header do not overlap, so +/// together they are no larger than the file; a header whose chunks add up +/// to more is refused, which bounds what its chunks can make a reader read +/// (a crafted chain of chunks each nested in the last would otherwise read +/// the file over and over). Overlap itself is not refused: libhdf5 reads +/// such headers (`cve-2025-7067.h5` has one). +struct ChunkSpans { + starts: BTreeSet, + /// Bytes of the chunks so far, and the most they may add up to. + total: u64, + budget: u64, +} + +impl ChunkSpans { + fn new(file_len: u64, start: u64, len: usize) -> Result { + let mut s = Self { + starts: BTreeSet::new(), + total: 0, + budget: file_len, + }; + s.add(start, len)?; + Ok(s) + } + + fn add(&mut self, start: u64, len: usize) -> Result<(), FormatError> { + if !self.starts.insert(start) || self.starts.len() > MAX_V1_CHUNKS { + return Err(FormatError::NestingDepthExceeded); + } + self.total = self.total.saturating_add(len as u64); + if self.total > self.budget { + return Err(FormatError::InvalidObjectHeader( + "object header chunks larger than the file", + )); + } + Ok(()) + } +} + /// Most chunks a version-1 object header may have (malformed-data guard; /// libhdf5 has no limit, and a header that gains one continuation chunk per /// attribute added can have many). @@ -972,6 +1005,104 @@ mod tests { assert_eq!(spaces, (0..200).map(|k| k as u8).collect::>()); } + /// A crafted version-1 header whose continuation chunks nest: each + /// chunk's continuation message points at the rest of that chunk. Read + /// depth-first with every enclosing chunk kept alive, from storage that + /// hands out owned buffers, it read n^2 bytes and held them all at once + /// (a 192 KB file read 768 MB). Chunks adding up to more than the file + /// are refused, and the bytes read stay within the file's size. + #[test] + fn nested_v1_continuation_chunks_are_bounded() { + use crate::storage::CountingStorage; + let n = 2000u64; + let a = 64u64; + let cont = |addr: u64, len: u64| { + let mut m = vec![0x10, 0, 16, 0, 0, 0, 0, 0]; + m.extend_from_slice(&addr.to_le_bytes()); + m.extend_from_slice(&len.to_le_bytes()); + m + }; + // Prefix: version 1, one message, reference count 1, 24 bytes. + let mut buf = vec![1, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0]; + buf.extend_from_slice(&cont(a, 24 * n)); + buf.resize(a as usize, 0); + for k in 0..n { + if k + 1 < n { + buf.extend_from_slice(&cont(a + 24 * (k + 1), 24 * (n - k - 1))); + } else { + buf.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]); + buf.extend_from_slice(&[0; 16]); + } + } + let len = buf.len() as u64; + let s = CountingStorage::new(buf); + assert!(matches!( + ObjectHeader::parse_in(&s, 0, 8, 8), + Err(FormatError::InvalidObjectHeader( + "object header chunks larger than the file" + )) + )); + assert!( + s.bytes_read() <= 2 * len, + "read {} of {len}", + s.bytes_read() + ); + } + + /// libhdf5 reads a continuation chunk that overlaps the chunk holding + /// its message (`cve-2025-7067.h5` has one), and so does this reader. + #[test] + fn overlapping_v1_continuation_chunk_is_read() { + // Chunk 0 (at 16): continuation (24 bytes), then a NIL message at + // 40; the continuation chunk is that NIL message's 8-byte header. + let mut cont = 40u64.to_le_bytes().to_vec(); + cont.extend_from_slice(&8u64.to_le_bytes()); + let data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0000, &[][..], 0)], 8, 8); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + assert_eq!(hdr.messages.len(), 1); + } + + /// A valid chain over owned-buffer storage reads each chunk once. + #[test] + fn long_v1_chain_reads_each_chunk_once() { + use crate::storage::CountingStorage; + let data = v1_chain(3000, false); + let len = data.len() as u64; + let s = CountingStorage::new(data); + let hdr = ObjectHeader::parse_in(&s, 0, 8, 8).unwrap(); + assert_eq!( + hdr.messages + .iter() + .filter(|m| m.msg_type == MessageType::Dataspace) + .count(), + 3000 + ); + assert!(s.bytes_read() <= len, "read {} of {len}", s.bytes_read()); + } + + /// Continuation chunks are read in the order their messages are found + /// (libhdf5's `H5O_protect`), so a chunk's messages follow every + /// message of the chunk before, not the continuation message. + #[test] + fn v1_continuation_messages_keep_libhdf5_order() { + // Chunk 0: continuation to A, dataspace [1]; A: dataspace [2]. + let a = 64u64; + let mut cont = a.to_le_bytes().to_vec(); + cont.extend_from_slice(&16u64.to_le_bytes()); + let mut data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0001, &[1; 8][..], 0)], 8, 8); + data.resize(a as usize, 0); + data.extend_from_slice(&[1, 0, 8, 0, 0, 0, 0, 0]); + data.extend_from_slice(&[2; 8]); + let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap(); + let spaces: Vec = hdr + .messages + .iter() + .filter(|m| m.msg_type == MessageType::Dataspace) + .map(|m| m.data[0]) + .collect(); + assert_eq!(spaces, [1, 2]); + } + #[test] fn v1_continuation_cycles_are_refused() { let data = v1_chain(5, true); From c2ae7846c93581a9698934a994a64b96daeca9a7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:30:33 -0500 Subject: [PATCH 14/15] docs: label the h5repack sizes in the append-waste figures 306 104 and 49 930 are h5repack of the editor's file; the text read as if they were h5repack of libhdf5's, which measures 305 954 and 50 188. Both are now given, from measure_append_waste rerun on 2026-09-26 (file sizes unchanged). Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/known-issues.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/known-issues.md b/docs/known-issues.md index 25d0960..a178d47 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -98,10 +98,12 @@ Measured 2026-09-26 on tank with `cargo test -p clawhdf5-tools --test edit_interop -- --ignored --nocapture measure_append_waste` (one editor for the whole workload; file sizes are deterministic): 1000 appends of 100 `f8` values to a 1-D dataset with 1024-element chunks give 810 504 bytes -unfiltered, as libhdf5's file, and 306 780 bytes with gzip (307 210 before -reuse; libhdf5: 306 058; `h5repack`: 306 104); 2000 appends of 10 values -with 4096-element gzip chunks give 79 829 bytes (119 684 before reuse) -against libhdf5's 50 292 (`h5repack`: 49 930): the chunk being appended to +unfiltered, as libhdf5's file (`h5repack` of either: 810 360), and 306 780 +bytes with gzip (307 210 before reuse; libhdf5's file: 306 058; `h5repack` +of the editor's file: 306 104, of libhdf5's: 305 954); 2000 appends of 10 +values with 4096-element gzip chunks give 79 829 bytes (119 684 before +reuse) against libhdf5's 50 292 (`h5repack` of the editor's file: 49 930, +of libhdf5's: 50 188): the chunk being appended to is followed by new index blocks and moves each time it grows, and the space it leaves is too small for its next, larger version. From 8236b0e30ac965dccb1657fdaf78703b38304b04 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 18:50:23 -0500 Subject: [PATCH 15/15] edit: skip heap blocks too small for an attribute, as libhdf5 does An attribute needing a heap block larger than the next one was refused ("skipping blocks too small for an object", "a first object too large for the starting block"); once an object's move to dense storage was refused it refused every new attribute, so 24% of set_attr calls in the review's random workload failed. Following H5HF__hdr_update_iter, H5HF__man_iblock_root_create/_double and H5HF__hdr_skip_blocks, the smaller blocks are now skipped: the iterator moves past them and they become an indirect free section with a first row section (serialized, class 1, as H5HF__sect_indirect_serialize writes it) and ghost normal rows, added as returned space so it merges with a range skipped just before it (H5HF__sect_indirect_merge_row). Later objects that best-fit a row section get a block created there (H5HF__man_iblock_alloc_row / H5HF__sect_indirect_reduce_row: from the start or end of the range, or from its middle, which splits it, with libhdf5's span bookkeeping). Heaps with such sections, as libhdf5 writes them, are now read too (they were refused at open). dense_skipped_blocks_match_libhdf5 drives every path (merge, split, end, last entry, row wrap) on earliest/v110/latest files against libhdf5 doing the same edits one session each; heaps, free sections and index B-trees are equal after every phase. The refusal test now checks the skip against libhdf5 and keeps a real refusal (last object in a block); clawhdf5-written heaps get 1-4 KiB attributes too. The three tests fail on the previous fheap.rs. Random workload refusals: 24% -> 2.2%, all the documented last-object-in-a-block case. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 18 +- .../tests/edit_coverage_interop.rs | 190 +++++- crates/clawhdf5/src/edit/fheap.rs | 609 +++++++++++++++--- crates/clawhdf5/src/edit/mod.rs | 3 +- docs/known-issues.md | 20 +- 5 files changed, 730 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1949400..6b264f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,10 +63,20 @@ insertion. The heap is changed as `H5HF` changes it — best-fit free sections from its free-space manager (kept as libhdf5 keeps `FSHD`/ `FSSE`), new direct blocks through the root indirect block (created, - doubled), huge objects through the huge-object B-tree (deleted with the - last huge object), removed objects' space merged back — with libhdf5's - statistics: after the same attribute workload the heap, its free space - and both index B-trees equal libhdf5's. Attributes are encoded as libhdf5 + doubled), blocks too small for an attribute skipped as libhdf5 skips + them (`H5HF__hdr_skip_blocks`: an indirect free section with its row + sections, serialized as libhdf5 serializes them, merged with the range + skipped just before it, and later attributes given skipped blocks from + either end or the middle of a range, which splits it), huge objects + through the huge-object B-tree (deleted with the last huge object), + removed objects' space merged back — with libhdf5's statistics: after + the same attribute workload the heap, its free space and both index + B-trees equal libhdf5's (`dense_skipped_blocks_match_libhdf5` covers + every way of skipping, with libhdf5 doing one edit per session as the + editor does). In a random attribute workload (1-4 KiB attributes among + small ones) 24% of `set_attr` calls were refused before skipping was + implemented; 2.2% are now, all replacements of the last attribute in a + heap block. Attributes are encoded as libhdf5 encodes them for a file h5py opens `r+` (message version 1, 3 for non-ASCII names; simple dataspaces with their maximum dimensions). Still refused: see `docs/known-issues.md`. diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index fff256c..c2da7cf 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1230,10 +1230,11 @@ fn dense_attributes_on_clawhdf5_files() { let mut ed = FileEditor::open(&path).unwrap(); for i in 0..30u64 { for (k, o) in ["/", "d"].iter().enumerate() { - // Small attributes: a larger one needs a heap block bigger than - // the next (see dense_attribute_refusals_change_nothing). + // Strings up to 300 bytes and of 5000 (huge objects), and + // every tenth one of 1-4 KiB (a heap block larger than the + // next: blocks skipped). let v = match AV::make(i, k as u64 + 7) { - AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()), + AV::Str(s) if i % 10 == 3 => AV::Str(s.repeat(3000 / s.len().max(1) + 1)), v => v, }; ed.set_attr(o, &format!("n{i}"), &v.value()) @@ -1323,10 +1324,12 @@ fn check_root_and_attrs(path: &Path, want: &[AttrOp]) { assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); } -/// What the editor refuses in dense storage — an object larger than the -/// next heap block (libhdf5 would skip blocks and record their space as -/// free, which this editor does not do) — is `Error::Unsupported`, and the -/// file is left byte for byte as it was. +/// An attribute that needs a heap block larger than the next one: libhdf5 +/// skips the smaller blocks (`H5HF__hdr_skip_blocks`) and records them as +/// an indirect free section, and so does the editor — the heap and its +/// free space come out as libhdf5's. Replacing that attribute, the only +/// object in its block, with one of another size is still refused (libhdf5 +/// frees the block), and the refusal leaves the file byte-identical. #[test] fn dense_attribute_refusals_change_nothing() { if !tools_ok() { @@ -1334,16 +1337,29 @@ fn dense_attribute_refusals_change_nothing() { } let dir = tmpdir(); let path = dir.path().join("dense_refuse.h5"); - py(&format!( - "import h5py, numpy as np\n\ - with h5py.File({p:?}, 'w', libver='v110') as f:\n\ - \x20 g = f.create_group('g')\n\ - \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", - p = path.to_str().unwrap() - )); + let twin = dir.path().join("dense_refuse_h5py.h5"); + for p in [&path, &twin] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver='v110') as f:\n\ + \x20 g = f.create_group('g')\n\ + \x20 for i in range(12): g.attrs[f'k{{i}}'] = i\n", + p = p.to_str().unwrap() + )); + } + let big = AV::Str("x".repeat(2000)); + let mut ed = FileEditor::open(&path).unwrap(); + ed.set_attr("g", "big", &big.value()).unwrap(); + drop(ed); + py_r_plus( + &twin, + &[format!("\x20 f['g'].attrs.create('big', {})\n", big.py())], + ); + assert_eq!(dense_info(&path, "g"), dense_info(&twin, "g")); + check_tools(&path, true); let before = std::fs::read(&path).unwrap(); let mut ed = FileEditor::open(&path).unwrap(); - unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000)))); + unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("y".repeat(1500)))); drop(ed); assert!( std::fs::read(&path).unwrap() == before, @@ -1353,13 +1369,155 @@ fn dense_attribute_refusals_change_nothing() { py(&format!( "import h5py\n\ with h5py.File({p:?}, 'r+') as f:\n\ - \x20 f['g'].attrs['big'] = 'x' * 2000\n\ + \x20 f['g'].attrs['big'] = 'y' * 1500\n\ \x20 assert len(f['g'].attrs) == 13\n", p = path.to_str().unwrap() )); check_tools(&path, true); } +/// Attributes of every size up to the heap's 4 KiB managed limit, in an +/// order that makes libhdf5 skip heap blocks in every way it does: a first +/// attribute too large for the heap's starting block (at the move to dense +/// storage), a block larger than the rest of the current row, a root +/// indirect block doubled past rows, two skipped ranges merged; later small +/// attributes go into the skipped blocks (from either end of a skipped +/// range, and from its middle, which splits it). libhdf5 does the same +/// operations, one file session each (as the editor re-reads the heap for +/// each edit); after every phase the heaps, their free sections (single, +/// row and indirect) and index B-trees must be libhdf5's. +#[test] +fn dense_skipped_blocks_match_libhdf5() { + if !tools_ok() { + return; + } + for (libver, h5dump) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)] { + let dir = tmpdir(); + let a = dir.path().join("skip_h5py.h5"); + let b = dir.path().join("skip_edit.h5"); + for p in [&a, &b] { + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ + \x20 objs = [f.create_group('g'), f.create_group('t', track_order=True), \ + f.create_dataset('d', data=np.arange(4, dtype=' = Vec::new(); + for (k, o) in objs.iter().enumerate() { + want.push(( + o.to_string(), + "c0".into(), + AV::Str("s".repeat(1200 + 700 * k)), + )); + for i in 1..8i64 { + want.push((o.to_string(), format!("c{i}"), AV::Ints(vec![i]))); + } + } + for i in 0..8i64 { + want.push(("h".into(), format!("c{i}"), AV::Ints(vec![i]))); + } + // "h" starts with small attributes, so its heap has a root direct + // block and then a one-row root indirect block when the first + // large attribute comes: the rest of that row and the rows the + // doubling adds are skipped as two ranges, which merge; the next + // attributes take blocks from the start, the end and the middle + // (splitting it) of the merged range, and use up whole rows. + let h_phases: [&[usize]; 4] = [ + &[0; 20], + &[4060, 1500, 0, 3000, 0, 1500, 0, 600], + &[1000, 0, 700, 3000, 0, 0, 1500, 0, 900, 0, 0, 0, 1800], + &[0, 800, 0, 0, 0, 2000, 0, 0, 0, 0, 0, 0, 3500, 0, 0, 0, 0, 0], + ]; + // Sizes (string lengths, or 0 for a small int array) in phases. + let phases: [&[usize]; 4] = [ + &[0, 3000, 0, 700, 3900, 0, 150], + &[2500, 90, 0, 3500, 1800, 60, 3990, 0], + &[0, 0, 40, 300, 0, 900, 20, 0, 0, 500, 0, 30, 1000, 0, 0, 200], + &[0, 250, 0, 40, 600, 0, 0, 1100, 80, 0, 0, 350, 0, 0, 0, 10], + ]; + let mut n = 0; + let value = |len: usize, n: usize, k: usize| { + if len == 0 { + AV::Ints((0..1 + (n + k) as i64 % 7).collect()) + } else { + AV::Str( + (0..len + 11 * k) + .map(|j| (b'a' + ((j + n) % 26) as u8) as char) + .collect(), + ) + } + }; + for (ph, sizes) in phases.iter().enumerate() { + let mut ops: Vec = Vec::new(); + for &len in *sizes { + for (k, o) in objs.iter().enumerate() { + ops.push((o.to_string(), format!("n{n}"), value(len, n, k))); + } + n += 1; + } + for &len in h_phases[ph] { + ops.push(("h".into(), format!("n{n}"), value(len, n, 0))); + n += 1; + } + // libhdf5: one session per operation. + let lines: Vec = ops + .iter() + .map(|(o, nm, v)| { + format!( + "with h5py.File({p:?}, 'r+') as f: f[{o:?}].attrs.create({nm:?}, {})\n", + v.py(), + p = a.to_str().unwrap() + ) + }) + .collect(); + let sp = a.with_extension(format!("ops{ph}.py")); + std::fs::write(&sp, format!("import h5py, numpy as np\n{}", lines.concat())).unwrap(); + let out = Command::new(python()).arg(&sp).output().unwrap(); + assert!( + out.status.success(), + "h5py workload failed:\n{}", + text(&out) + ); + let mut ed = FileEditor::open(&b).unwrap(); + for (o, nm, v) in &ops { + ed.set_attr(o, nm, &v.value()) + .unwrap_or_else(|e| panic!("{libver} phase {ph}: set {o}/{nm}: {e}")); + set_want(&mut want, o, nm, v.clone()); + } + drop(ed); + for o in objs.iter().chain(&["h"]) { + assert_eq!( + dense_info(&b, o), + dense_info(&a, o), + "{libver}: dense storage of {o} differs from libhdf5's after phase {ph}" + ); + } + check_tools(&b, h5dump); + check_attr_values(&b, &want); + } + // libhdf5 goes on with the editor's file. + py(&format!( + "import h5py, numpy as np\n\ + with h5py.File({p:?}, 'r+') as f:\n\ + \x20 for o in ['g', 't', 'd', 'h']:\n\ + \x20 for i in range(6): f[o].attrs[f'late{{i}}'] = 'z' * (i * 700 + 5)\n\ + \x20 del f[o].attrs['c1']\n", + p = b.to_str().unwrap() + )); + want.retain(|(_, nm, _)| nm != "c1"); + check_tools(&b, h5dump); + check_attr_values(&b, &want); + } +} + /// Space one edit frees is reused by later edits of the same editor: the /// chunks a shrink removes are where the chunks of the following growth /// go, so the file does not grow; with a new editor per edit (nothing to diff --git a/crates/clawhdf5/src/edit/fheap.rs b/crates/clawhdf5/src/edit/fheap.rs index 03e6b1e..dfd3af5 100644 --- a/crates/clawhdf5/src/edit/fheap.rs +++ b/crates/clawhdf5/src/edit/fheap.rs @@ -3,23 +3,29 @@ //! best-fitting free section the heap's free-space manager records (a new //! direct block when none fits: the root direct block of an empty heap, the //! next block of the root indirect block — created from the root direct -//! block, and doubled, as needed), huge objects (larger than the heap's +//! block, and doubled, as needed — with the blocks too small for the object +//! skipped and kept as free space for later objects), huge objects (larger than the heap's //! managed maximum) into their own file space tracked by the huge-object //! version-2 B-tree; removed objects return their space to the free-space //! manager, merged with adjacent free space. //! //! The heap's free-space manager (`FSHD` header, `FSSE` section info) is -//! kept exactly as libhdf5 keeps it: sections sorted by size then offset, +//! kept exactly as libhdf5 keeps it: single sections and the indirect +//! sections of skipped blocks (with their first and normal row sections), +//! sections sorted by size then offset, //! the header's counts and sizes, section info moved when its size changes, //! the manager deleted when it tracks nothing. Header statistics (managed //! space, allocated space, free space, allocation iterator, object counts) //! follow libhdf5's arithmetic. //! +//! The heap is read afresh for each edit, so the editor does what libhdf5 +//! does with one edit per file session (libhdf5 keeps an in-memory span per +//! indirect section that it recomputes when it reloads the section info). +//! //! What libhdf5 would do differently is refused ([`Error::Unsupported`], //! before anything is written): I/O filters on the heap, child indirect -//! blocks, skipped blocks (a first object too large for the next block), -//! free sections other than "single" ones, freeing a whole direct block, -//! directly addressed huge objects, tiny objects. +//! blocks (and free space in them), freeing a whole direct block, directly +//! addressed huge objects, tiny objects. use std::collections::{BTreeMap, BTreeSet}; @@ -67,8 +73,129 @@ const FS_NCLASSES: u16 = 4; const HUGE_BT2_TYPE: u8 = 1; const HUGE_BT2_NODE: u32 = 512; -/// The heap's free-space manager: only "single" sections (free space inside -/// a direct block), keyed by heap offset. +/// The heap's doubling table as its free sections need it: the block size +/// of each direct row, the direct block overhead, the width and the size +/// of a heap offset. +#[derive(Clone, Debug)] +struct Geo { + width: usize, + /// Block size of rows 0 .. max direct rows. + rblock: Vec, + /// `H5HF_MAN_ABS_DIRECT_OVERHEAD`. + ov: u64, + heap_off_size: usize, +} + +impl Geo { + /// Free space in a block of `row` (a row section's size). + fn rfree(&self, row: usize) -> u64 { + self.rblock[row] - self.ov + } + + /// Heap offset of `row`'s first block in the root indirect block. + fn row_off(&self, row: usize) -> u64 { + (0..row).map(|r| self.rblock[r] * self.width as u64).sum() + } + + /// `H5HF__dtable_span_size`. + fn span(&self, row: usize, col: usize, n: usize) -> u64 { + let w = self.width; + let end = row * w + col + n - 1; + let (end_row, end_col) = (end / w, end % w); + if row == end_row { + return self.rblock[row] * (end_col - col + 1) as u64; + } + let mut r = row; + let mut acc = 0; + if col > 0 { + acc = self.rblock[r] * (w - col) as u64; + r += 1; + } + while r < end_row { + acc += self.rblock[r] * w as u64; + r += 1; + } + acc + self.rblock[r] * (end_col + 1) as u64 + } +} + +/// A row section (`H5HF_FSPACE_SECT_FIRST_ROW` / `NORMAL_ROW`): `n` +/// unallocated direct blocks of one row of the root indirect block, from +/// column `col`; its size is one block's free space. +#[derive(Clone, Debug)] +struct RowSect { + addr: u64, + row: usize, + col: usize, + n: usize, +} + +/// An indirect section (`H5HF_FSPACE_SECT_INDIRECT`): `n` unallocated +/// entries of the root indirect block from (`row`, `col`) — blocks skipped +/// over to allocate a larger one (`H5HF__hdr_skip_blocks`). It is never in +/// the free-space manager itself; its row sections are, the first as the +/// serialized "first row" (which stands for the indirect section on disk), +/// the others as unserialized "ghost" normal rows. `span` is kept as +/// libhdf5 keeps it (reducing a row subtracts the row section's size, not +/// the block's), since merging compares it. +#[derive(Clone, Debug)] +struct Ind { + addr: u64, + row: usize, + col: usize, + n: usize, + span: u64, + rows: Vec, +} + +impl Ind { + /// `H5HF__sect_indirect_new` + `H5HF__sect_indirect_init_rows` for the + /// root indirect block (direct rows only). + fn new(geo: &Geo, row: usize, col: usize, n: usize) -> Self { + let w = geo.width; + let end = row * w + col + n - 1; + let (end_row, end_col) = (end / w, end % w); + let addr = geo.row_off(row) + geo.rblock[row] * col as u64; + let mut rows = Vec::new(); + let mut entries = if row == end_row { + end_col - col + 1 + } else { + w - col + }; + let mut row_col = col; + let mut off = addr; + for u in row..=end_row { + rows.push(RowSect { + addr: off, + row: u, + col: row_col, + n: entries, + }); + off += entries as u64 * geo.rblock[u]; + entries = if u + 1 < end_row { w } else { end_col + 1 }; + row_col = 0; + } + Self { + addr, + row, + col, + n, + span: geo.span(row, col, n), + rows, + } + } +} + +/// A free section `find` chose. +enum Found { + Single(u64, u64), + /// Row `.1` of indirect section `.0`. + Row(usize, usize), +} + +/// The heap's free-space manager: "single" sections (free space inside a +/// direct block) keyed by heap offset, and the indirect sections (with +/// their row sections) of skipped direct blocks. struct FreeSpace { addr: u64, max_sect_addr: u16, @@ -80,6 +207,8 @@ struct FreeSpace { sect_size: u64, alloc_sect_size: u64, sects: BTreeMap, + inds: Vec, + geo: Geo, } impl FreeSpace { @@ -87,7 +216,7 @@ impl FreeSpace { 6 + 4 * ls as usize + 8 + ls as usize + os as usize + 2 * ls as usize + 4 } - fn open(img: &Image<'_>, addr: u64) -> Result { + fn open(img: &Image<'_>, addr: u64, geo: &Geo) -> Result { let (os, ls) = (img.os, img.ls); let len = Self::hdr_len(os, ls); let d = img.read(addr, len)?; @@ -120,9 +249,6 @@ impl FreeSpace { let sect_addr = next(os as usize); let sect_size = next(l); let alloc_sect_size = next(l); - if ghost != 0 || tot_count != serial { - return Err(unsupported("free space in row or indirect sections")); - } let mut fs = Self { addr, max_sect_addr, @@ -134,8 +260,13 @@ impl FreeSpace { sect_size, alloc_sect_size, sects: BTreeMap::new(), + inds: Vec::new(), + geo: geo.clone(), }; if serial == 0 { + if tot_count != 0 || ghost != 0 || tot_space != 0 { + return Err(bad("free-space counts disagree")); + } return Ok(fs); } if sect_addr == undef(os) || sect_size < 9 + os as u64 || sect_size > alloc_sect_size { @@ -154,9 +285,10 @@ impl FreeSpace { let cnt_w = enc_size(serial); let len_w = enc_size(max_sect_size); let off_w = (usize::from(max_sect_addr)).div_ceil(8); + let ind_w = geo.heap_off_size + 6; let mut q = 5 + os as usize; let mut seen = 0u64; - let mut total = 0u64; + let mut addrs = BTreeSet::new(); while seen < serial { if q + cnt_w + len_w > n - 4 { return Err(bad("section info ends early")); @@ -165,8 +297,8 @@ impl FreeSpace { q += cnt_w; let size = le(&s[q..q + len_w]); q += len_w; - if count == 0 || size == 0 { - return Err(bad("empty section size node")); + if count == 0 || size == 0 || count > serial - seen { + return Err(bad("bad section size node")); } for _ in 0..count { if q + off_w + 1 > n - 4 { @@ -175,51 +307,141 @@ impl FreeSpace { let off = le(&s[q..q + off_w]); let class = s[q + off_w]; q += off_w + 1; - if class != 0 { - return Err(unsupported("free space in row or indirect sections")); + match class { + 0 => { + if fs.sects.insert(off, size).is_some() { + return Err(bad("duplicate free section")); + } + } + 1 => { + // H5HF__sect_indirect_deserialize. + if q + ind_w > n - 4 { + return Err(bad("section info ends early")); + } + let iblock_off = le(&s[q..q + geo.heap_off_size]); + let b = &s[q + geo.heap_off_size..]; + let row = usize::from(u16::from_le_bytes([b[0], b[1]])); + let col = usize::from(u16::from_le_bytes([b[2], b[3]])); + let cnt = usize::from(u16::from_le_bytes([b[4], b[5]])); + q += ind_w; + if iblock_off != 0 { + return Err(unsupported("free space in child indirect blocks")); + } + if cnt == 0 || col >= geo.width { + return Err(bad("bad indirect free section")); + } + let end_row = (row * geo.width + col + cnt - 1) / geo.width; + if end_row >= geo.rblock.len() { + return Err(unsupported("free space in child indirect blocks")); + } + let ind = Ind::new(geo, row, col, cnt); + if ind.addr != off || geo.rfree(row) != size { + return Err(bad("bad indirect free section")); + } + fs.inds.push(ind); + } + _ => return Err(bad("free section of an unknown class")), } - if fs.sects.insert(off, size).is_some() { + if !addrs.insert(off) { return Err(bad("duplicate free section")); } seen += 1; - total += size; } } - if total != tot_space { - return Err(bad("free-space total disagrees with its sections")); + let (space, total, ser, gh) = fs.totals(); + if (space, total, ser, gh) != (tot_space, tot_count, serial, ghost) { + return Err(bad("free-space totals disagree with its sections")); } Ok(fs) } + /// Every row section: (address, size, indirect section, row). + fn rows(&self) -> impl Iterator + '_ { + self.inds.iter().enumerate().flat_map(move |(i, ind)| { + ind.rows + .iter() + .enumerate() + .map(move |(r, rs)| (rs.addr, self.geo.rfree(rs.row), i, r)) + }) + } + + /// (total space, sections, serialized sections, ghost sections). + fn totals(&self) -> (u64, u64, u64, u64) { + let nrows: u64 = self.inds.iter().map(|i| i.rows.len() as u64).sum(); + let space = self.sects.values().sum::() + self.rows().map(|r| r.1).sum::(); + let singles = self.sects.len() as u64; + let firsts = self.inds.len() as u64; + (space, singles + nrows, singles + firsts, nrows - firsts) + } + + fn is_empty(&self) -> bool { + self.sects.is_empty() && self.inds.is_empty() + } + + /// The offset of the section nearest `addr` below it (or above it) in + /// the merge list — singles and first rows, by offset — when that + /// section is a first row, the only kind a first row merges with. + fn merge_neighbour(&self, addr: u64, below: bool) -> Option { + let firsts = self.inds.iter().map(|i| (i.addr, true)); + let singles = self.sects.keys().map(|&o| (o, false)); + let all = firsts.chain(singles); + let pick = if below { + all.filter(|e| e.0 < addr).max_by_key(|e| e.0) + } else { + all.filter(|e| e.0 > addr).min_by_key(|e| e.0) + }; + pick.and_then(|(a, first)| first.then_some(a)) + } + /// Best fit (`H5FS__sect_find_node`): the smallest section of at least - /// `size` bytes, the lowest offset among equals. - fn find(&self, size: u64) -> Option<(u64, u64)> { - self.sects + /// `size` bytes, the lowest offset among equals, singles and row + /// sections alike. + fn find(&self, size: u64) -> Option { + let single = self + .sects .iter() .filter(|&(_, &s)| s >= size) - .min_by_key(|&(&o, &s)| (s, o)) - .map(|(&o, &s)| (o, s)) + .map(|(&o, &s)| ((s, o), Found::Single(o, s))); + let rows = self + .rows() + .filter(|r| r.1 >= size) + .map(|(a, s, i, r)| ((s, a), Found::Row(i, r))); + single.chain(rows).min_by_key(|(k, _)| *k).map(|(_, f)| f) + } + + /// The serialized sections, by size then offset: (size, offset, + /// indirect section if a first row). + fn serial(&self) -> Vec<(u64, u64, Option)> { + let mut v: Vec<(u64, u64, Option)> = + self.sects.iter().map(|(&o, &s)| (s, o, None)).collect(); + v.extend( + self.inds + .iter() + .enumerate() + .map(|(i, ind)| (self.geo.rfree(ind.row), ind.addr, Some(i))), + ); + v.sort_unstable(); + v } /// The serialized section info's size (`H5FS__sect_serialize_size`). fn needed(&self, os: u8) -> u64 { - let n = self.sects.len() as u64; let prefix = 4 + 1 + u64::from(os) + 4; + let serial = self.serial(); + let n = serial.len() as u64; if n == 0 { return prefix; } - let sizes: BTreeSet = self.sects.values().copied().collect(); + let sizes: BTreeSet = serial.iter().map(|s| s.0).collect(); prefix + sizes.len() as u64 * (enc_size(n) + enc_size(self.max_sect_size)) as u64 + n * (u64::from(self.max_sect_addr).div_ceil(8) + 1) + + self.inds.len() as u64 * (self.geo.heap_off_size as u64 + 6) } fn serialize(&self, os: u8) -> Vec { - let n = self.sects.len() as u64; - let mut by_size: BTreeMap> = BTreeMap::new(); - for (&o, &s) in &self.sects { - by_size.entry(s).or_default().push(o); - } + let serial = self.serial(); + let n = serial.len() as u64; let cnt_w = enc_size(n); let len_w = enc_size(self.max_sect_size); let off_w = usize::from(self.max_sect_addr).div_ceil(8); @@ -229,13 +451,28 @@ impl FreeSpace { let mut a = vec![0u8; os as usize]; put_uint(&mut a, self.addr, os); d.extend_from_slice(&a); - for (size, offs) in by_size { - d.extend_from_slice(&(offs.len() as u64).to_le_bytes()[..cnt_w]); + let mut i = 0; + while i < serial.len() { + let size = serial[i].0; + let same = serial[i..].iter().take_while(|s| s.0 == size).count(); + d.extend_from_slice(&(same as u64).to_le_bytes()[..cnt_w]); d.extend_from_slice(&size.to_le_bytes()[..len_w]); - for o in offs { + for &(_, o, ind) in &serial[i..i + same] { d.extend_from_slice(&o.to_le_bytes()[..off_w]); - d.push(0); + match ind { + None => d.push(0), + Some(k) => { + // H5HF__sect_indirect_serialize (root block: offset 0). + let ind = &self.inds[k]; + d.push(1); + d.extend_from_slice(&vec![0u8; self.geo.heap_off_size]); + for v in [ind.row, ind.col, ind.n] { + d.extend_from_slice(&(v as u16).to_le_bytes()); + } + } + } } + i += same; } d } @@ -244,13 +481,12 @@ impl FreeSpace { let (os, ls) = (img.os, img.ls); let len = Self::hdr_len(os, ls); let l = ls as usize; - let n = self.sects.len() as u64; - let tot: u64 = self.sects.values().sum(); + let (tot, n, serial, ghost) = self.totals(); let mut d = Vec::with_capacity(len); d.extend_from_slice(b"FSHD"); d.push(0); d.push(FS_CLIENT_FHEAP); - for v in [tot, n, n, 0] { + for v in [tot, n, serial, ghost] { d.extend_from_slice(&v.to_le_bytes()[..l]); } for v in [self.nclasses, self.shrink, self.expand, self.max_sect_addr] { @@ -440,7 +676,7 @@ impl Heap { } } if fs_addr != undef(os) { - let fs = FreeSpace::open(img, fs_addr)?; + let fs = FreeSpace::open(img, fs_addr, &h.geo(os))?; if fs.max_sect_addr != max_index { return Err(bad("free-space manager does not match the heap")); } @@ -678,10 +914,23 @@ impl Heap { return Err(unsupported("tiny objects")); } let (off, size) = match self.fs.as_ref().and_then(|fs| fs.find(n)) { - Some((o, s)) => { + Some(Found::Single(o, s)) => { self.fs.as_mut().expect("found above").sects.remove(&o); (o, s) } + // H5HF__man_iblock_alloc_row: a skipped block, created now. + Some(Found::Row(i, r)) => { + let entry = self.alloc_row(i, r); + self.fs_dirty = true; + let w = usize::from(self.width); + let (row, col) = (entry / w, entry % w); + let bsize = self.row_size(row); + let block_off = self.row_off(row) + bsize * col as u64; + let a = self.dblock_create(img, block_off, bsize)?; + self.ents[entry] = a; + self.iblock_dirty = true; + (block_off + self.overhead(os), bsize - self.overhead(os)) + } None => self.dblock_new(img, n)?, }; // H5HF__sect_single_reduce: the object goes at the section's start. @@ -705,7 +954,16 @@ impl Heap { /// `H5FS_ADD_RETURNED_SPACE`), creating the free-space manager if the /// heap has none. fn fs_add(&mut self, img: &mut Image<'_>, off: u64, size: u64) -> Result<(), Error> { + self.fs_mut(img)?.sects.insert(off, size); + self.fs_dirty = true; + Ok(()) + } + + /// The free-space manager, created if the heap has none + /// (`H5HF__space_start`). + fn fs_mut(&mut self, img: &mut Image<'_>) -> Result<&mut FreeSpace, Error> { if self.fs.is_none() { + let geo = self.geo(img.os); let addr = img.alloc(FreeSpace::hdr_len(img.os, img.ls) as u64)?; self.fs = Some(FreeSpace { addr, @@ -718,18 +976,160 @@ impl Heap { sect_size: 0, alloc_sect_size: 0, sects: BTreeMap::new(), + inds: Vec::new(), + geo, }); self.fs_addr = addr; } - self.fs - .as_mut() - .expect("created above") - .sects - .insert(off, size); + Ok(self.fs.as_mut().expect("created above")) + } + + /// The doubling table's direct rows, for the free sections. + fn geo(&self, os: u8) -> Geo { + Geo { + width: usize::from(self.width), + rblock: (0..self.max_direct_rows) + .map(|r| self.row_size(r)) + .collect(), + ov: self.overhead(os), + heap_off_size: self.heap_off_size, + } + } + + /// `H5HF__hdr_skip_blocks`: `n` entries of the root indirect block + /// from `start` are skipped — the iterator moves past them and they + /// become an indirect free section (`H5HF__sect_indirect_add`), whose + /// first row is added as returned space, so it merges with an indirect + /// section just below it (`H5FS__sect_merge`). + fn skip_blocks(&mut self, img: &mut Image<'_>, start: usize, n: usize) -> Result<(), Error> { + let w = usize::from(self.width); + let (row, col) = (start / w, start % w); + if (start + n - 1) / w >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + let geo = self.geo(img.os); + self.man_iter_off += geo.span(row, col, n); + let iter = self.man_iter_off; + let fs = self.fs_mut(img)?; + let mut cur = Ind::new(&geo, row, col, n); + loop { + let mut modified = false; + // The nearest sections below and above in the merge list + // (singles and first rows, by offset). + let below = fs.merge_neighbour(cur.addr, true); + let above = fs.merge_neighbour(cur.addr, false); + if let Some(a) = below + && let Some(k) = fs.inds.iter().position(|x| x.addr == a) + && fs.inds[k].addr + fs.inds[k].span == cur.addr + { + let mut lower = fs.inds.remove(k); + if cur.addr >= iter { + return Err(unsupported("free space past the heap's end")); + } + merge_ind(&mut lower, cur, w); + cur = lower; + modified = true; + } + if let Some(a) = above + && let Some(k) = fs.inds.iter().position(|x| x.addr == a) + && cur.addr + cur.span == a + { + let upper = fs.inds.remove(k); + if upper.addr >= iter { + return Err(unsupported("free space past the heap's end")); + } + merge_ind(&mut cur, upper, w); + modified = true; + } + if !modified { + break; + } + } + // H5HF__sect_row_can_shrink: a section past the iterator would + // shrink the heap; skipped blocks never are. + if cur.addr >= iter { + return Err(unsupported("free space past the heap's end")); + } + fs.inds.push(cur); self.fs_dirty = true; Ok(()) } + /// `H5HF__sect_row_reduce` with `H5HF__sect_indirect_reduce_row`: take + /// one block out of row `r` of indirect section `i`; returns the root + /// indirect block entry to create it at. + fn alloc_row(&mut self, i: usize, r: usize) -> usize { + let w = usize::from(self.width); + let fs = self.fs.as_mut().expect("section found"); + let geo = fs.geo.clone(); + let ind = &mut fs.inds[i]; + let rs = ind.rows[r].clone(); + let row_start = rs.row * w + rs.col; + let row_end = row_start + rs.n - 1; + let start = ind.row * w + ind.col; + let end = start + ind.n - 1; + let (start_row, end_row) = (ind.row, end / w); + let from_start = !(row_end == end && start_row != end_row); + let entry = if from_start { row_start } else { row_end }; + ind.span -= geo.rfree(rs.row); + let mut peer = None; + if ind.n > 1 { + if entry == start { + ind.addr += geo.rblock[ind.row]; + ind.col += 1; + if ind.col == w { + ind.row += 1; + ind.col = 0; + // The row's last block: the row goes (below). + } + ind.n -= 1; + } else if entry == end { + ind.n -= 1; + } else { + // Split: the rows before this one become a peer section. + let peer_n = entry - start; + let peer_rows = rs.row - start_row; + let rest = ind.rows.split_off(peer_rows); + let p = Ind { + addr: ind.addr, + row: ind.row, + col: ind.col, + n: peer_n, + span: rs.addr - ind.addr, + rows: std::mem::replace(&mut ind.rows, rest), + }; + ind.addr = rs.addr + geo.rblock[rs.row]; + ind.span -= p.span; + ind.row = rs.row; + ind.col = rs.col + 1; + ind.n -= peer_n + 1; + peer = Some(p); + } + } else { + ind.n -= 1; + } + // The row section itself. + let ri = if peer.is_some() { 0 } else { r }; + let row = &mut ind.rows[ri]; + if row.n == 1 { + ind.rows.remove(ri); + } else { + if from_start { + row.addr += geo.rblock[row.row]; + row.col += 1; + } + row.n -= 1; + } + let gone = ind.rows.is_empty(); + if gone { + fs.inds.remove(i); + } + if let Some(p) = peer { + fs.inds.push(p); + } + entry + } + /// `H5HF__man_dblock_new`: a direct block for an object of `request` /// bytes; returns its free section (not in the free-space manager). fn dblock_new(&mut self, img: &mut Image<'_>, request: u64) -> Result<(u64, u64), Error> { @@ -753,30 +1153,16 @@ impl Heap { self.total_man_free += self.start_block_size - self.overhead(os); return Ok((self.overhead(os), self.start_block_size - self.overhead(os))); } - if self.root == undef(os) { - return Err(unsupported( - "a first object too large for the starting block", - )); - } - // H5HF__hdr_update_iter. - if self.root_rows == 0 { - self.root_create(img, min)?; - } - let (mut row, mut col) = self.iter_pos(); - let min_row = self.size_to_row(min); - if min_row > row && row < usize::from(self.root_rows) { - return Err(unsupported("skipping blocks too small for an object")); - } - while row >= usize::from(self.root_rows) { - self.root_double(img, min)?; - (row, col) = self.iter_pos(); - } + // H5HF__hdr_update_iter, then a block at the iterator. + self.update_iter(img, min)?; + let (row, col) = self.iter_pos(); if row >= self.max_direct_rows { return Err(unsupported("child indirect blocks")); } let size = self.row_size(row); if min > size { - return Err(unsupported("skipping blocks too small for an object")); + // libhdf5: "skipping direct block sizes not supported". + return Err(unsupported("a direct block smaller than the object")); } self.man_iter_off += size; let entry = row * usize::from(self.width) + col; @@ -787,6 +1173,37 @@ impl Heap { Ok((block_off + self.overhead(os), size - self.overhead(os))) } + /// `H5HF__hdr_update_iter`: make the iterator point at a block of at + /// least `min` bytes, creating the root indirect block, skipping + /// smaller blocks and doubling the root indirect block as needed. + fn update_iter(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { + if self.root_rows == 0 { + return self.root_create(img, min); + } + let w = usize::from(self.width); + let min_row = self.size_to_row(min); + let (mut row, col) = self.iter_pos(); + let nrows = usize::from(self.root_rows); + if min_row > row && row < nrows { + let entry = row * w + col; + let skip = if min_row >= nrows { + nrows * w - entry + } else { + min_row * w - entry + }; + self.skip_blocks(img, entry, skip)?; + row = self.iter_pos().0; + } + while row >= usize::from(self.root_rows) { + self.root_double(img, min)?; + row = self.iter_pos().0; + } + if row >= self.max_direct_rows { + return Err(unsupported("child indirect blocks")); + } + Ok(()) + } + /// The allocation iterator's (row, column) in the root indirect block. fn iter_pos(&self) -> (usize, usize) { if self.man_iter_off >= self.man_size { @@ -827,8 +1244,9 @@ impl Heap { Ok(a) } - /// `H5HF__man_iblock_root_create`: the root direct block becomes entry - /// 0 of a new root indirect block. + /// `H5HF__man_iblock_root_create`: a root indirect block, with the + /// root direct block (if any) as entry 0; when the block needed is + /// larger than the starting size, the smaller blocks are skipped. fn root_create(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { let os = img.os; let mut nrows = if self.start_root_rows == 0 { @@ -846,9 +1264,6 @@ impl Heap { if nrows > self.max_direct_rows { return Err(unsupported("child indirect blocks")); } - if min > self.start_block_size { - return Err(unsupported("skipping blocks too small for an object")); - } let have_direct = self.root != undef(os); let rows = u16::try_from(nrows).map_err(|_| bad("too many rows"))?; let a = img.alloc(self.iblock_len(rows, os) as u64)?; @@ -860,43 +1275,57 @@ impl Heap { self.man_iter_off = 0; } self.iblock_dirty = true; - self.root_rows = rows; - self.root = a; let w = u64::from(self.width); let ov = self.overhead(os); let mut acc: u64 = (0..nrows).map(|u| (self.row_size(u) - ov) * w).sum(); if have_direct { acc -= self.row_size(0) - ov; } + // The header points at the new block before the skipped blocks + // are added (the free sections need the rows in place). + self.root_rows = rows; + self.root = a; + if min > self.start_block_size { + let first = usize::from(have_direct); + self.skip_blocks(img, first, (nrows - 1) * usize::from(self.width) - first)?; + } self.man_size = self.row_off(nrows); self.total_man_free += acc; Ok(()) } /// `H5HF__man_iblock_root_double`: the root indirect block gets twice - /// its rows, at a new address. + /// its rows (at least enough for a block of `min` bytes, the smaller + /// blocks before it skipped), at a new address. fn root_double(&mut self, img: &mut Image<'_>, min: u64) -> Result<(), Error> { let os = img.os; + let w = usize::from(self.width); let old = usize::from(self.root_rows); - let (row, _) = self.iter_pos(); - let next_size = self.row_size(row.min(self.max_root_rows - 1)); + let (next_row, next_col) = self.iter_pos(); + let next_entry = next_row * w + next_col; + let next_size = self.row_size(next_row.min(self.max_root_rows - 1)); + let (mut min_nrows, mut new_next_entry, mut skip) = (0, 0, false); if old < self.max_direct_rows && min > next_size { - return Err(unsupported("skipping blocks too small for an object")); + skip = true; + min_nrows = 1 + self.size_to_row(min); + new_next_entry = (min_nrows - 1) * w; } - let new = (2 * old).min(self.max_root_rows); + let new = min_nrows.max((2 * old).min(self.max_root_rows)); if new > self.max_direct_rows || new == old { return Err(unsupported("child indirect blocks")); } img.free(self.root, self.iblock_len(self.root_rows, os) as u64); let rows = u16::try_from(new).map_err(|_| bad("too many rows"))?; let a = img.alloc(self.iblock_len(rows, os) as u64)?; - let w = usize::from(self.width); self.ents.resize(new * w, undef(os)); + self.root = a; + self.iblock_dirty = true; + if skip { + self.skip_blocks(img, next_entry, new_next_entry - next_entry)?; + } let ov = self.overhead(os); let acc: u64 = (old * w..new * w).map(|u| self.row_size(u / w) - ov).sum(); self.root_rows = rows; - self.root = a; - self.iblock_dirty = true; self.man_size = 2 * self.row_off(new - 1); self.total_man_free += acc; Ok(()) @@ -1091,7 +1520,7 @@ impl Heap { self.fs_dirty = false; if let Some(mut fs) = self.fs.take() { let hdr_len = FreeSpace::hdr_len(os, img.ls) as u64; - if fs.sects.is_empty() { + if fs.is_empty() { // H5HF__space_close: a manager that tracks nothing is // deleted. img.free(fs.addr, hdr_len); @@ -1124,6 +1553,24 @@ impl Heap { } } +/// `H5HF__sect_indirect_merge_row` for two sections of the root indirect +/// block, `b` right after `a`: `b`'s rows join `a` (its first row joins +/// `a`'s last row when they share a row). +fn merge_ind(a: &mut Ind, mut b: Ind, w: usize) { + let end_row1 = (a.row * w + a.col + a.n - 1) / w; + if !b.rows.is_empty() { + if end_row1 == b.row { + let first = b.rows.remove(0); + if let Some(last) = a.rows.last_mut() { + last.n += first.n; + } + } + a.rows.append(&mut b.rows); + } + a.n += b.n; + a.span += b.span; +} + /// The ID in a huge-object B-tree record (type 1: address, length, ID). fn huge_rec_id(r: &[u8], os: u8, ls: u8) -> u64 { get_uint(&r[os as usize + ls as usize..], ls) diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index e1d301a..a5a3c1b 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -958,8 +958,7 @@ impl FileEditor { /// /// [`Error::Unsupported`] for shared attribute messages, heaps this /// editor cannot extend the way libhdf5 would (child indirect blocks, - /// skipped blocks, free space other than within direct blocks, freeing - /// a whole block), and version-1 object headers asked for an attribute + /// freeing a whole block), and version-1 object headers asked for an attribute /// larger than a header message holds. pub fn set_attr(&mut self, path: &str, name: &str, value: &AttrValue) -> Result<(), Error> { if name.is_empty() { diff --git a/docs/known-issues.md b/docs/known-issues.md index a178d47..2539f63 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -74,13 +74,19 @@ reuse were added). `clawhdf5::FileEditor` refuses, with an optional filter only when its own build lacks it, which none does for these; - attributes in dense storage when the heap cannot take them the way - libhdf5 would: an attribute that needs a heap block larger than the next - one (libhdf5 skips blocks and records them as free space — in practice an - attribute of roughly 1 to 4 KiB going into a young heap), a heap with I/O - filters or child indirect blocks (more than about 512 KiB of attributes), - free space the heap tracks outside direct blocks, replacing the last - attribute left in a heap block by one of another size (libhdf5 frees the - block), directly addressed huge objects; and shared attribute messages; + libhdf5 would: replacing the last attribute left in a heap block by one + of another size (libhdf5 frees the block), a heap with I/O filters or + child indirect blocks (more than about 512 KiB of attributes), free + space in child indirect blocks, directly addressed huge objects; and + shared attribute messages. Measured 2026-09-26 on tank with the review's + random-edit harness (120 runs of 150 random edits, `earliest`/`v110`/ + `latest`, about 5600 `set_attr` calls of 8 bytes to 6 KiB): 2.2% of + `set_attr` calls are refused, every one the last-attribute-in-a-block + replacement; before blocks could be skipped (an attribute needing a heap + block larger than the next one — any attribute of about 1 KiB or more + once a heap has started, or at the move to dense storage), 24% were, + since an object whose move to dense storage was refused kept refusing + every new attribute; - version-1 object headers asked for an attribute larger than a header message (they have no dense storage); - partial edge chunks stored unfiltered (`H5Pset_chunk_opts`), external