edit: version-2 B-tree chunk indexes, shrinking, early allocation
FileEditor can now: - add, move and resize chunks of datasets with two or more unlimited dimensions (version-2 B-tree chunk index, record types 10/11). The new edit/btree2.rs follows libhdf5's H5B2 code: H5B2_update (modify, or insert into a leaf with room, or fall back to H5B2__insert), the preemptive split/redistribute loop with its two retries, split1, split_root (depth growth, node geometry per depth), redistribute2/3, cumulative record counts and the pointer widths H5B2__hdr_init derives, a checksum per node. Removal (H5B2_remove: merge2/3, redistribution, root collapse, the internal-record swap with a leaf) is there too. A dataset without an index yet gets one from the layout message's node size and split/merge percentages. - shrink a chunked dataset along any dimension (resize to a smaller shape), as H5D__set_extent / H5D__chunk_prune_by_extent do: the same chunks visited in the same order; chunks wholly outside the new extent are removed from the index (version-1 B-tree: H5B_remove with its sibling key and link fix-ups and the empty-root case; version-2 B-tree; Fixed/Extensible Array elements reset to the fill element; an implicit index keeps its chunks, as libhdf5 does) and their space noted as free; the part of each partial edge chunk outside the extent is overwritten with the fill value, so elements that come back after a later growth read as fill. - under early allocation, allocate and fill the chunks a growth brings in (H5D__chunk_allocate), which an implicit index needs: libhdf5 refills them, and they may hold the data of chunks pruned earlier. Tests (crates/clawhdf5-tools/tests/edit_coverage_interop.rs): growth in both dimensions of v110/latest files, unfiltered and deflated, gives node-for-node the version-2 B-tree libhdf5 builds (h5py with its chunk cache off, so chunks enter the index in the editor's order), through a depth increase; random chunk order; 60 random shrink/grow/write steps on Extensible Array, version-2 B-tree, Fixed Array, 1-D and implicit datasets (earliest/v110/latest, with and without gzip+shuffle) give the values h5py gets doing the same and the same index shape (version-1 and version-2 B-tree node shapes, Extensible Array statistics); h5py r+ continues on every result; h5dump and h5rs check accept them. edit_interop's version-2 B-tree case now appends instead of expecting a refusal; shrinking is no longer an error in edit_tests. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,832 @@
|
||||
//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
|
||||
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
|
||||
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
|
||||
//! (and moved there), and space reused within an editing session. Files are
|
||||
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
|
||||
//! clawhdf5; after each edit h5py must read exactly what a model says,
|
||||
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
|
||||
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
|
||||
//! same operations can be done by libhdf5, the structures it builds (B-tree
|
||||
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
|
||||
//!
|
||||
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
|
||||
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Output};
|
||||
|
||||
use clawhdf5::{Error, File, FileEditor, Selection};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn available(cmd: &str, args: &[&str]) -> bool {
|
||||
Command::new(cmd)
|
||||
.args(args)
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Whether the tools are here; false (skip) or a panic when interop is
|
||||
/// required.
|
||||
fn tools_ok() -> bool {
|
||||
let ok =
|
||||
available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
|
||||
if !ok {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
|
||||
);
|
||||
eprintln!("SKIP: h5py/numpy or h5dump not available");
|
||||
}
|
||||
ok
|
||||
}
|
||||
|
||||
fn py(script: &str) -> String {
|
||||
let o = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("run python");
|
||||
assert!(
|
||||
o.status.success(),
|
||||
"python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&o.stdout),
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&o.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
fn text(o: &Output) -> String {
|
||||
format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&o.stdout),
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
)
|
||||
}
|
||||
|
||||
/// The file is valid for libhdf5's tools and for `h5rs check --data`.
|
||||
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
|
||||
fn check_tools(path: &Path, h5dump: bool) {
|
||||
let p = path.to_str().unwrap();
|
||||
let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
|
||||
.args(["check", "--data", "-q", p])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
|
||||
if h5dump {
|
||||
let o = Command::new("h5dump")
|
||||
.args(["-o", "/dev/null", p])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
o.status.success() && o.stderr.is_empty(),
|
||||
"h5dump {p}:\n{}",
|
||||
text(&o)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A dataset's expected contents.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Model {
|
||||
shape: Vec<u64>,
|
||||
/// Row-major values.
|
||||
data: Vec<i32>,
|
||||
}
|
||||
|
||||
impl Model {
|
||||
fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
|
||||
let n: u64 = shape.iter().product();
|
||||
Self {
|
||||
shape: shape.to_vec(),
|
||||
data: (0..n).map(f).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn index(&self, c: &[u64]) -> usize {
|
||||
c.iter()
|
||||
.zip(&self.shape)
|
||||
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
||||
}
|
||||
|
||||
/// Change the extent to `shape`: elements inside both keep their
|
||||
/// values, new ones are `fill`.
|
||||
fn resize(&mut self, shape: &[u64], fill: i32) {
|
||||
let old = self.clone();
|
||||
*self = Self::new(shape, |_| fill);
|
||||
let n: u64 = old.shape.iter().product();
|
||||
for flat in 0..n {
|
||||
let mut c = vec![0u64; old.shape.len()];
|
||||
let mut r = flat;
|
||||
for d in (0..c.len()).rev() {
|
||||
c[d] = r % old.shape[d];
|
||||
r /= old.shape[d];
|
||||
}
|
||||
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
||||
let i = self.index(&c);
|
||||
self.data[i] = old.data[flat as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a hyperslab write of `vals` (row-major over the block).
|
||||
fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
|
||||
let n: u64 = count.iter().product();
|
||||
for flat in 0..n {
|
||||
let mut c = vec![0u64; count.len()];
|
||||
let mut r = flat;
|
||||
for d in (0..c.len()).rev() {
|
||||
c[d] = start[d] + r % count[d];
|
||||
r /= count[d];
|
||||
}
|
||||
let i = self.index(&c);
|
||||
self.data[i] = vals[flat as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn block(start: &[u64], count: &[u64]) -> Selection {
|
||||
Selection::Hyperslab {
|
||||
start: start.to_vec(),
|
||||
stride: vec![1; start.len()],
|
||||
count: count.to_vec(),
|
||||
block: vec![1; start.len()],
|
||||
}
|
||||
}
|
||||
|
||||
/// h5py and our reader both read `m` from dataset `name`.
|
||||
fn verify(path: &Path, name: &str, m: &Model) {
|
||||
let f = File::open(path).unwrap();
|
||||
let ds = f.dataset(name).unwrap();
|
||||
assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
|
||||
let got = ds.read_i32().unwrap();
|
||||
if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
|
||||
panic!(
|
||||
"our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
|
||||
path.display(),
|
||||
m.shape,
|
||||
got[i],
|
||||
m.data[i]
|
||||
);
|
||||
}
|
||||
let exp = path.with_extension("expect");
|
||||
let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
std::fs::write(&exp, bytes).unwrap();
|
||||
let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
a = f[{name:?}][()]\n\
|
||||
e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
|
||||
assert a.shape == e.shape, (a.shape, e.shape)\n\
|
||||
bad = np.argwhere(a != e)\n\
|
||||
assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
|
||||
p = path.to_str().unwrap(),
|
||||
e = exp.to_str().unwrap(),
|
||||
s = shape.join(",")
|
||||
));
|
||||
}
|
||||
|
||||
/// A deterministic pseudo-random sequence (SplitMix64).
|
||||
struct Rng(u64);
|
||||
impl Rng {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.0;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
fn below(&mut self, n: u64) -> u64 {
|
||||
self.next() % n
|
||||
}
|
||||
}
|
||||
|
||||
fn tmpdir() -> tempfile::TempDir {
|
||||
let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
|
||||
tempfile::TempDir::new_in(base).unwrap()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
|
||||
match r {
|
||||
Err(Error::Unsupported(_)) => {}
|
||||
other => panic!("expected Error::Unsupported, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn le(b: &[u8]) -> u64 {
|
||||
b.iter()
|
||||
.take(8)
|
||||
.enumerate()
|
||||
.fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
|
||||
}
|
||||
|
||||
fn enc_size(v: u64) -> usize {
|
||||
((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
|
||||
}
|
||||
|
||||
/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
|
||||
/// (8-byte addresses and lengths): its depth, record count, and every
|
||||
/// node's (depth, records) breadth-first.
|
||||
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
|
||||
let h = &b[hdr..];
|
||||
assert_eq!(&h[0..4], b"BTHD");
|
||||
let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
|
||||
let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
|
||||
let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
|
||||
let root = le(&h[16..24]);
|
||||
let root_n = le(&h[24..26]);
|
||||
let total = le(&h[26..34]);
|
||||
// Node geometry, as H5B2__hdr_init computes it.
|
||||
let leaf_max = (node_size - 10) / rs;
|
||||
let nrec_w = enc_size(leaf_max);
|
||||
let mut cum = vec![(leaf_max, 0usize)];
|
||||
for d in 1..=u64::from(depth) {
|
||||
let (below, below_w) = cum[d as usize - 1];
|
||||
let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
|
||||
let max = (node_size - 10 - ptr) / (rs + ptr);
|
||||
let c = (max + 1) * below + max;
|
||||
cum.push((c, enc_size(c)));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut level = vec![(root, root_n)];
|
||||
let mut dep = depth;
|
||||
loop {
|
||||
let mut next = Vec::new();
|
||||
for &(addr, n) in &level {
|
||||
out.push((dep, n));
|
||||
if dep == 0 {
|
||||
assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
|
||||
continue;
|
||||
}
|
||||
let node = &b[addr as usize..];
|
||||
assert_eq!(&node[0..4], b"BTIN");
|
||||
let mut q = 6 + (n * rs) as usize;
|
||||
let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
|
||||
for _ in 0..=n {
|
||||
let a = le(&node[q..q + 8]);
|
||||
let c = le(&node[q + 8..q + 8 + nrec_w]);
|
||||
q += 8 + nrec_w + all_w;
|
||||
next.push((a, c));
|
||||
}
|
||||
}
|
||||
if dep == 0 {
|
||||
break;
|
||||
}
|
||||
dep -= 1;
|
||||
level = next;
|
||||
}
|
||||
(depth, total, out)
|
||||
}
|
||||
|
||||
/// The shape of the file's only chunk B-tree (record type 10 or 11).
|
||||
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
|
||||
let b = std::fs::read(path).unwrap();
|
||||
let hdrs: Vec<usize> = b
|
||||
.windows(6)
|
||||
.enumerate()
|
||||
.filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
|
||||
bt2_shape_at(&b, hdrs[0])
|
||||
}
|
||||
|
||||
/// Grow a dataset with two unlimited dimensions in both directions many
|
||||
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
|
||||
/// chunk enters the index as it is written, in the same order as the
|
||||
/// editor's) and with the editor: the same values, and the same B-tree —
|
||||
/// depth, record count, and every node's record count — through leaf and
|
||||
/// internal splits, redistributions and a depth increase.
|
||||
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
|
||||
// Extents: both dimensions grow; new columns fall between existing
|
||||
// chunks in the index's key order (row first).
|
||||
let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
|
||||
let val = |r: u64, c: u64| (r * 1000 + c) as i32;
|
||||
let create = |p: &Path, write: bool| {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
||||
\x20 d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
|
||||
dtype='<i4'{extra})\n\
|
||||
\x20 if {write}:\n\
|
||||
\x20 r0, c0 = 0, 0\n\
|
||||
\x20 for r, c in {steps:?}:\n\
|
||||
\x20 d.resize((r, c))\n\
|
||||
\x20 g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
|
||||
\x20 if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
|
||||
\x20 if r > r0: d[r0:r, :] = g[r0:r, :]\n\
|
||||
\x20 r0, c0 = r, c\n",
|
||||
p = p.to_str().unwrap(),
|
||||
write = if write { "True" } else { "False" },
|
||||
))
|
||||
};
|
||||
create(&a, true);
|
||||
create(&b, false);
|
||||
let mut m = Model::new(&[0, 0], |_| 0);
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
let (mut r0, mut c0) = (0u64, 0u64);
|
||||
for &(r, c) in &steps {
|
||||
ed.resize("x", &[r, c]).unwrap();
|
||||
m.resize(&[r, c], 0);
|
||||
if c > c0 && r0 > 0 {
|
||||
let vals: Vec<i32> = (0..r0)
|
||||
.flat_map(|i| (c0..c).map(move |j| val(i, j)))
|
||||
.collect();
|
||||
ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[0, c0], &[r0, c - c0], &vals);
|
||||
}
|
||||
if r > r0 {
|
||||
let vals: Vec<i32> = (r0..r)
|
||||
.flat_map(|i| (0..c).map(move |j| val(i, j)))
|
||||
.collect();
|
||||
ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[r0, 0], &[r - r0, c], &vals);
|
||||
}
|
||||
(r0, c0) = (r, c);
|
||||
}
|
||||
drop(ed);
|
||||
verify(&b, "x", &m);
|
||||
verify(&a, "x", &m);
|
||||
check_tools(&b, h5dump);
|
||||
let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
|
||||
assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
|
||||
assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
|
||||
// libhdf5 goes on growing what the editor built.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 d = f['x']\n\
|
||||
\x20 r, c = d.shape\n\
|
||||
\x20 d.resize((r + 3, c + 2))\n\
|
||||
\x20 d[:, c:] = -1\n\
|
||||
\x20 d[r:, :] = -2\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
let (r, c) = (m.shape[0], m.shape[1]);
|
||||
m.resize(&[r + 3, c + 2], 0);
|
||||
m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
|
||||
m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
|
||||
verify(&b, "x", &m);
|
||||
check_tools(&b, h5dump);
|
||||
}
|
||||
|
||||
/// Dataset `name`'s layout: its chunk index type (0 for a version-1
|
||||
/// B-tree), index address and chunk rank (element-size dimension
|
||||
/// included).
|
||||
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
|
||||
use clawhdf5_format::data_layout::DataLayout;
|
||||
use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
let f = File::open(path).unwrap();
|
||||
let sb = f.superblock();
|
||||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||||
let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
|
||||
let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
|
||||
let m = oh
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::DataLayout)
|
||||
.unwrap();
|
||||
match DataLayout::parse(&m.data, os, ls).unwrap() {
|
||||
DataLayout::Chunked {
|
||||
chunk_dimensions,
|
||||
btree_address,
|
||||
chunk_index_type,
|
||||
..
|
||||
} => (
|
||||
chunk_index_type.unwrap_or(0),
|
||||
btree_address,
|
||||
chunk_dimensions.len(),
|
||||
),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A version-1 chunk B-tree's shape: every node's (level, entries),
|
||||
/// breadth-first from the root.
|
||||
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
|
||||
let ks = 8 + 8 * ndims;
|
||||
let mut out = Vec::new();
|
||||
let mut level = vec![root];
|
||||
while !level.is_empty() {
|
||||
let mut next = Vec::new();
|
||||
for a in level {
|
||||
let n = &b[a as usize..];
|
||||
assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
|
||||
let lv = n[5];
|
||||
let e = le(&n[6..8]);
|
||||
out.push((lv, e));
|
||||
if lv > 0 {
|
||||
for i in 0..e as usize {
|
||||
next.push(le(
|
||||
&n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The shape of dataset `name`'s chunk index, for comparing with the one
|
||||
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
|
||||
/// nothing to compare (Fixed Array, implicit).
|
||||
fn index_shape(path: &Path, name: &str) -> String {
|
||||
let b = std::fs::read(path).unwrap();
|
||||
match chunk_index(path, name) {
|
||||
(_, None, _) => "none".into(),
|
||||
(0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
|
||||
(5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
|
||||
(4, Some(a), _) => {
|
||||
let s: Vec<u64> = (0..6)
|
||||
.map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
|
||||
.collect();
|
||||
format!("ea {s:?}")
|
||||
}
|
||||
(t, Some(_), _) => format!("type {t}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// One step of a resize workload.
|
||||
#[derive(Debug, Clone)]
|
||||
enum Op {
|
||||
Resize(Vec<u64>),
|
||||
/// A block write; values from `vals`.
|
||||
Write(Vec<u64>, Vec<u64>, u64),
|
||||
}
|
||||
|
||||
fn op_vals(seed: u64, n: u64) -> Vec<i32> {
|
||||
(0..n)
|
||||
.map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Random resizes (shrinking and growing any dimension within `max`)
|
||||
/// and block writes.
|
||||
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
|
||||
let mut shape = start.to_vec();
|
||||
let mut ops = Vec::new();
|
||||
for k in 0..n {
|
||||
if k % 3 == 0 {
|
||||
shape = (0..shape.len())
|
||||
.map(|d| {
|
||||
let cap = max[d].min(shape[d] * 2 + 6);
|
||||
rng.below(cap + 1)
|
||||
})
|
||||
.collect();
|
||||
ops.push(Op::Resize(shape.clone()));
|
||||
} else if shape.iter().all(|&s| s > 0) {
|
||||
let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
|
||||
let cnt: Vec<u64> = (0..shape.len())
|
||||
.map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
|
||||
.collect();
|
||||
ops.push(Op::Write(st, cnt, rng.next() % 1000));
|
||||
}
|
||||
}
|
||||
ops
|
||||
}
|
||||
|
||||
/// The python statements applying `ops` to dataset `d`.
|
||||
fn py_ops(ops: &[Op]) -> String {
|
||||
let mut s = String::new();
|
||||
for op in ops {
|
||||
match op {
|
||||
Op::Resize(shape) => {
|
||||
s += &format!("\x20 d.resize({shape:?})\n")
|
||||
.replace('[', "(")
|
||||
.replace(']', ",)");
|
||||
}
|
||||
Op::Write(st, cnt, seed) => {
|
||||
let n: u64 = cnt.iter().product();
|
||||
let sl: Vec<String> = st
|
||||
.iter()
|
||||
.zip(cnt)
|
||||
.map(|(a, c)| format!("{a}:{}", a + c))
|
||||
.collect();
|
||||
s += &format!(
|
||||
"\x20 d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
|
||||
.astype('<i4').reshape({cnt:?})\n",
|
||||
sl.join(", ")
|
||||
)
|
||||
.replace("reshape([", "reshape((")
|
||||
.replace("])\n", ",))\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// The same resize workload — shrinking and growing along every
|
||||
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
|
||||
/// editor on dataset `x` created by `create` (python, `f` open): the same
|
||||
/// values as a model where elements that come back after a shrink read as
|
||||
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
|
||||
/// check` accept, and h5py can go on.
|
||||
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
|
||||
let dir = tmpdir();
|
||||
let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
|
||||
let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
|
||||
let mk = |p: &Path| {
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
from h5py import h5p, h5d, h5s, h5t\n\
|
||||
with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
|
||||
{create}",
|
||||
p = p.to_str().unwrap()
|
||||
))
|
||||
};
|
||||
mk(&a);
|
||||
mk(&b);
|
||||
let (start, max) = py(&format!(
|
||||
"import h5py\n\
|
||||
d = h5py.File({p:?}, 'r')['x']\n\
|
||||
print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
|
||||
p = b.to_str().unwrap()
|
||||
))
|
||||
.split_once("] [")
|
||||
.map(|(x, y)| {
|
||||
let p = |s: &str| -> Vec<u64> {
|
||||
s.trim_matches(|c| c == '[' || c == ']')
|
||||
.split(',')
|
||||
.filter(|t| !t.trim().is_empty())
|
||||
.map(|t| t.trim().parse().unwrap())
|
||||
.collect()
|
||||
};
|
||||
(p(x), p(y))
|
||||
})
|
||||
.unwrap();
|
||||
let initial = py(&format!(
|
||||
"import h5py\n\
|
||||
d = h5py.File({p:?}, 'r')['x']\n\
|
||||
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
||||
p = b.to_str().unwrap()
|
||||
));
|
||||
let mut m = Model {
|
||||
shape: start.clone(),
|
||||
data: initial
|
||||
.split_whitespace()
|
||||
.map(|v| v.parse().unwrap())
|
||||
.collect(),
|
||||
};
|
||||
let mut rng = Rng(seed);
|
||||
let ops = random_resize_ops(&mut rng, &start, &max, 60);
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
|
||||
\x20 d = f['x']\n{o}",
|
||||
p = a.to_str().unwrap(),
|
||||
o = py_ops(&ops)
|
||||
));
|
||||
let mut ed = FileEditor::open(&b).unwrap();
|
||||
for (k, op) in ops.iter().enumerate() {
|
||||
match op {
|
||||
Op::Resize(s) => {
|
||||
ed.resize("x", s)
|
||||
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
||||
m.resize(s, fill);
|
||||
}
|
||||
Op::Write(st, cnt, seed) => {
|
||||
let vals = op_vals(*seed, cnt.iter().product());
|
||||
ed.write_values("x", &block(st, cnt), &vals)
|
||||
.unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
|
||||
m.write_block(st, cnt, &vals);
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
if tag == "implicit" {
|
||||
// An implicit index cannot drop chunks: libhdf5 leaves the data of
|
||||
// chunks wholly outside a shrunk extent in place and does not
|
||||
// refill those that come back unless they lie past the extent it
|
||||
// grows from, so they can show old values. The editor must match
|
||||
// libhdf5, not the model.
|
||||
let got = py(&format!(
|
||||
"import h5py\n\
|
||||
d = h5py.File({p:?}, 'r')['x']\n\
|
||||
print(' '.join(str(v) for v in d[()].ravel()))\n",
|
||||
p = a.to_str().unwrap()
|
||||
));
|
||||
m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
|
||||
}
|
||||
verify(&a, "x", &m);
|
||||
verify(&b, "x", &m);
|
||||
check_tools(&b, h5dump);
|
||||
assert_eq!(
|
||||
index_shape(&b, "x"),
|
||||
index_shape(&a, "x"),
|
||||
"{tag}: chunk index differs from libhdf5's"
|
||||
);
|
||||
// libhdf5 goes on: grow back to the start shape and write everything.
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 d = f['x']\n\
|
||||
\x20 d.resize({s:?})\n\
|
||||
\x20 d[...] = 3\n",
|
||||
p = b.to_str().unwrap(),
|
||||
s = start
|
||||
));
|
||||
let n: u64 = start.iter().product();
|
||||
verify(
|
||||
&b,
|
||||
"x",
|
||||
&Model {
|
||||
shape: start.clone(),
|
||||
data: vec![3; n as usize],
|
||||
},
|
||||
);
|
||||
check_tools(&b, h5dump);
|
||||
}
|
||||
|
||||
/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
|
||||
/// dimension, mixed with growth and writes, on every chunk index: the
|
||||
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
|
||||
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
|
||||
/// and implicit (early allocation), unfiltered and deflated.
|
||||
#[test]
|
||||
fn shrink_matches_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let kinds: &[(&str, &str, i32)] = &[
|
||||
(
|
||||
"ea",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
|
||||
maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
|
||||
-1,
|
||||
),
|
||||
(
|
||||
"bt2",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
|
||||
maxshape=(None, None), chunks=(3, 2){z})\n",
|
||||
0,
|
||||
),
|
||||
(
|
||||
"fa",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
|
||||
maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
|
||||
5,
|
||||
),
|
||||
(
|
||||
"1d",
|
||||
"\x20 d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
|
||||
maxshape=(None,), chunks=(4,){z})\n",
|
||||
0,
|
||||
),
|
||||
];
|
||||
let mut seed = 100;
|
||||
for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
for (kind, create, fill) in kinds {
|
||||
for (zi, z) in ["", ", compression='gzip', shuffle=True"]
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
seed += 1;
|
||||
let tag = format!("{kind}_{li}_{zi}");
|
||||
shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Implicit index: early allocation, no filters, fixed maximum shape.
|
||||
shrink_workload(
|
||||
"implicit",
|
||||
"'latest'",
|
||||
"\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
||||
\x20 dcpl.set_chunk((3, 4))\n\
|
||||
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
||||
\x20 dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
|
||||
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
|
||||
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
||||
7,
|
||||
false,
|
||||
99,
|
||||
);
|
||||
}
|
||||
|
||||
/// An implicit chunk index (early allocation, no filters, fixed maximum
|
||||
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
|
||||
/// by its position in the maximum chunk grid. The reader once used the
|
||||
/// current grid, and returned other chunks' values for every row after the
|
||||
/// first.
|
||||
#[test]
|
||||
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("implicit_max.h5");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
from h5py import h5p, h5d, h5s, h5t\n\
|
||||
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
||||
\x20 dcpl = h5p.create(h5p.DATASET_CREATE)\n\
|
||||
\x20 dcpl.set_chunk((3, 4))\n\
|
||||
\x20 dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
|
||||
\x20 h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
|
||||
\x20 f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
|
||||
verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
|
||||
// The editor places new values by the same grid.
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
|
||||
.unwrap();
|
||||
drop(ed);
|
||||
let mut m = Model::new(&[8, 9], |i| i as i32);
|
||||
m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn btree2_chunk_index_growth_matches_libhdf5() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
|
||||
bt2_growth(lv, *dump, "", &format!("plain{i}"));
|
||||
bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Random writes into a dataset with two unlimited dimensions (chunks
|
||||
/// created in any order) and growth, with a small node size so the tree
|
||||
/// gets deep, against a model; then h5py continues.
|
||||
#[test]
|
||||
fn btree2_random_chunk_order() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join("bt2_random.h5");
|
||||
// A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
|
||||
// hundred chunks).
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
from h5py import h5p, h5d, h5s, h5t\n\
|
||||
with h5py.File({p:?}, 'w', libver='latest') as f:\n\
|
||||
\x20 f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
|
||||
dtype='<i4', fillvalue=-7)\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let mut m = Model::new(&[40, 40], |_| -7);
|
||||
let mut rng = Rng(42);
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
for step in 0..400 {
|
||||
if step % 50 == 49 {
|
||||
let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
|
||||
ed.resize("x", &[r, c]).unwrap();
|
||||
m.resize(&[r, c], -7);
|
||||
}
|
||||
let r0 = rng.below(m.shape[0]);
|
||||
let c0 = rng.below(m.shape[1]);
|
||||
let cnt = [
|
||||
1 + rng.below((m.shape[0] - r0).min(3)),
|
||||
1 + rng.below((m.shape[1] - c0).min(3)),
|
||||
];
|
||||
let vals: Vec<i32> = (0..cnt[0] * cnt[1])
|
||||
.map(|_| (rng.next() % 1_000_000) as i32)
|
||||
.collect();
|
||||
ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[r0, c0], &cnt, &vals);
|
||||
if step % 100 == 99 {
|
||||
drop(ed);
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
ed = FileEditor::open(&path).unwrap();
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
verify(&path, "x", &m);
|
||||
let shape = chunk_bt2_shape(&path);
|
||||
assert!(shape.0 >= 1, "{shape:?}");
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 f['x'][...] = 5\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let n = m.data.len();
|
||||
m.data = vec![5; n];
|
||||
verify(&path, "x", &m);
|
||||
check_tools(&path, false);
|
||||
}
|
||||
@@ -752,19 +752,20 @@ fn overwrite_every_layout() {
|
||||
}
|
||||
check_tools(&path, *dump);
|
||||
}
|
||||
// A version-2 B-tree index can take new chunks only from libhdf5 for
|
||||
// now: growing works, writing the new chunks is refused and changes
|
||||
// nothing.
|
||||
// A version-2 B-tree index takes new chunks too.
|
||||
if *lv != "'earliest'" {
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
ed.resize("bt2", &[8, 6]).unwrap();
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12]));
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit changed the file"
|
||||
);
|
||||
models[12].resize(&[8, 6], 0);
|
||||
ed.resize("bt2", &[8, 7]).unwrap();
|
||||
models[12].resize(&[8, 7], 0);
|
||||
let vals: Vec<i32> = (0..23).collect();
|
||||
ed.write_values("bt2", &block(&[6, 0], &[2, 7]), &vals[..14])
|
||||
.unwrap();
|
||||
models[12].write_block(&[6, 0], &[2, 7], &vals[..14]);
|
||||
ed.write_values("bt2", &block(&[0, 6], &[6, 1]), &vals[14..20])
|
||||
.unwrap();
|
||||
models[12].write_block(&[0, 6], &[6, 1], &vals[14..20]);
|
||||
drop(ed);
|
||||
verify(&path, "bt2", &models[12]);
|
||||
}
|
||||
// libhdf5 goes on modifying what we wrote.
|
||||
py(&format!(
|
||||
@@ -1027,7 +1028,7 @@ fn refused_edits_change_nothing() {
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
unsupported(ed.write_all("s", &[0u8; 32]));
|
||||
unsupported(ed.resize("x", &[4]));
|
||||
unsupported(ed.resize("c", &[4]));
|
||||
unsupported(
|
||||
ed.resize("c", &[6, 1])
|
||||
.map_err(|_| Error::Unsupported(String::new())),
|
||||
|
||||
@@ -56,6 +56,15 @@ fn bad(why: &str) -> Error {
|
||||
))
|
||||
}
|
||||
|
||||
/// What a removal did below a node (`H5B_ins_t`), with the removed chunk's
|
||||
/// address and size.
|
||||
enum Rm {
|
||||
NotFound,
|
||||
Noop((u64, u32)),
|
||||
/// The child is gone: the parent must drop it.
|
||||
Remove((u64, u32)),
|
||||
}
|
||||
|
||||
enum Ins {
|
||||
Done,
|
||||
/// The node split; the new right sibling and its first key.
|
||||
@@ -144,7 +153,14 @@ impl BTree1 {
|
||||
put_uint(&mut d[8 + osz..], node.right, os);
|
||||
let ks = self.key_size();
|
||||
let mut p = 8 + 2 * osz;
|
||||
for (i, k) in node.keys.iter().enumerate() {
|
||||
// An empty node (a root whose last chunk was removed) stores no
|
||||
// keys, as libhdf5 writes it.
|
||||
let nkeys = if node.children.is_empty() {
|
||||
0
|
||||
} else {
|
||||
node.keys.len()
|
||||
};
|
||||
for (i, k) in node.keys.iter().take(nkeys).enumerate() {
|
||||
d[p..p + 4].copy_from_slice(&k.size.to_le_bytes());
|
||||
d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes());
|
||||
for (j, o) in k.offs.iter().enumerate() {
|
||||
@@ -210,6 +226,18 @@ impl BTree1 {
|
||||
return Err(bad("bad chunk key"));
|
||||
}
|
||||
let root = self.read(img, self.root)?;
|
||||
if root.children.is_empty() {
|
||||
// Every chunk was removed (H5B__insert_helper's first
|
||||
// insertion): the root, a leaf again, takes it.
|
||||
let right = self.right_key_after(&key);
|
||||
let node = Node {
|
||||
level: 0,
|
||||
keys: vec![key, right],
|
||||
children: vec![addr],
|
||||
..root
|
||||
};
|
||||
return self.write(img, &node);
|
||||
}
|
||||
if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? {
|
||||
// The root split: move its (left) half to a new node so the root
|
||||
// keeps its address, then make the root the parent of both.
|
||||
@@ -372,6 +400,143 @@ impl BTree1 {
|
||||
cmp(&key.offs, &right.keys[0].offs) != Ordering::Less
|
||||
}
|
||||
|
||||
/// Remove the chunk at offsets `offs` (element-size coordinate 0), as
|
||||
/// `H5B_remove` does for the chunk index (whose critical key is the
|
||||
/// left one): no rebalancing; a node left without children is deleted
|
||||
/// and its siblings relinked (the left one takes over its right key),
|
||||
/// a root left empty becomes an empty leaf. Returns the chunk's address
|
||||
/// and stored size, or `None` when the tree has no such chunk (nothing
|
||||
/// changes then). Deleted nodes are freed in `img`.
|
||||
pub(crate) fn remove(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
offs: &[u64],
|
||||
) -> Result<Option<(u64, u32)>, Error> {
|
||||
if offs.len() != self.ndims {
|
||||
return Err(bad("bad chunk key"));
|
||||
}
|
||||
let mut lt = None;
|
||||
match self.remove_at(img, self.root, 0, offs, &mut lt, 64)? {
|
||||
Rm::NotFound => Ok(None),
|
||||
Rm::Noop(c) | Rm::Remove(c) => Ok(Some(c)),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_at(
|
||||
&self,
|
||||
img: &mut Image<'_>,
|
||||
addr: u64,
|
||||
level: usize,
|
||||
offs: &[u64],
|
||||
lt_out: &mut Option<Key>,
|
||||
depth: u8,
|
||||
) -> Result<Rm, Error> {
|
||||
if depth == 0 {
|
||||
return Err(bad("tree too deep"));
|
||||
}
|
||||
let mut node = self.read(img, addr)?;
|
||||
let n = node.children.len();
|
||||
// H5D__btree_cmp3 over (keys[i], keys[i + 1]), binary search.
|
||||
let (mut lo, mut hi, mut idx) = (0usize, n, 0usize);
|
||||
let mut c = 1i32;
|
||||
while lo < hi && c != 0 {
|
||||
idx = (lo + hi) / 2;
|
||||
c = if cmp(offs, &node.keys[idx + 1].offs) != Ordering::Less {
|
||||
1
|
||||
} else if cmp(offs, &node.keys[idx].offs) == Ordering::Less {
|
||||
-1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if c < 0 {
|
||||
hi = idx;
|
||||
} else {
|
||||
lo = idx + 1;
|
||||
}
|
||||
}
|
||||
if c != 0 {
|
||||
return Ok(Rm::NotFound);
|
||||
}
|
||||
let mut lt_changed = None;
|
||||
let res = if node.level > 0 {
|
||||
let child = self.read(img, node.children[idx])?;
|
||||
if usize::from(child.level) + 1 != usize::from(node.level) {
|
||||
return Err(bad("inconsistent node levels"));
|
||||
}
|
||||
self.remove_at(
|
||||
img,
|
||||
node.children[idx],
|
||||
level + 1,
|
||||
offs,
|
||||
&mut lt_changed,
|
||||
depth - 1,
|
||||
)?
|
||||
} else {
|
||||
if node.keys[idx].offs != offs {
|
||||
return Ok(Rm::NotFound);
|
||||
}
|
||||
Rm::Remove((node.children[idx], node.keys[idx].size))
|
||||
};
|
||||
let chunk = match res {
|
||||
Rm::NotFound => return Ok(Rm::NotFound),
|
||||
Rm::Noop(c) | Rm::Remove(c) => c,
|
||||
};
|
||||
let mut dirty = false;
|
||||
if let Some(k) = lt_changed {
|
||||
node.keys[idx] = k;
|
||||
dirty = true;
|
||||
if idx == 0 {
|
||||
*lt_out = Some(node.keys[0].clone());
|
||||
}
|
||||
}
|
||||
let out = Rm::Noop(chunk);
|
||||
if let Rm::Remove(_) = res {
|
||||
let undefined = undef(img.os);
|
||||
if n == 1 {
|
||||
if level > 0 {
|
||||
if node.left != undefined {
|
||||
let mut sib = self.read(img, node.left)?;
|
||||
let last = sib.children.len();
|
||||
sib.keys[last] = node.keys[1].clone();
|
||||
sib.right = node.right;
|
||||
self.write(img, &sib)?;
|
||||
}
|
||||
if node.right != undefined {
|
||||
let mut sib = self.read(img, node.right)?;
|
||||
sib.left = node.left;
|
||||
self.write(img, &sib)?;
|
||||
}
|
||||
img.free(addr, self.node_size(img.os) as u64);
|
||||
return Ok(Rm::Remove(chunk));
|
||||
}
|
||||
node.children.clear();
|
||||
node.keys.truncate(1);
|
||||
node.level = 0;
|
||||
} else if idx == 0 {
|
||||
node.keys.remove(0);
|
||||
node.children.remove(0);
|
||||
*lt_out = Some(node.keys[0].clone());
|
||||
} else {
|
||||
// Right-most or middle child: its left key goes, the next
|
||||
// key becomes the following child's left key.
|
||||
node.keys.remove(idx);
|
||||
node.children.remove(idx);
|
||||
}
|
||||
dirty = true;
|
||||
}
|
||||
if dirty {
|
||||
self.write(img, &node)?;
|
||||
}
|
||||
// The left sibling's right key follows a changed left key.
|
||||
if lt_out.is_some() && node.left != undef(img.os) && level > 0 {
|
||||
let mut sib = self.read(img, node.left)?;
|
||||
let last = sib.children.len();
|
||||
sib.keys[last] = node.keys[0].clone();
|
||||
self.write(img, &sib)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) {
|
||||
let n = node.children.len();
|
||||
if node.level == 0 {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -383,12 +383,23 @@ impl Ea {
|
||||
}
|
||||
|
||||
/// Set element `idx` to `e`.
|
||||
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
|
||||
/// Set element `idx` to `e`, or back to the fill element (`None`: a
|
||||
/// removed chunk, `H5D__earray_idx_remove`), which creates no block.
|
||||
pub(crate) fn set(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
idx: u64,
|
||||
e: Option<Elem>,
|
||||
) -> Result<(), Error> {
|
||||
let os = img.os;
|
||||
let osz = u64::from(os);
|
||||
let es = self.slot_size(os) as u64;
|
||||
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
|
||||
let enc = encode_elem(e, self.filtered, self.elem_size, os)?;
|
||||
let clear = e.is_none();
|
||||
if self.iblock == undef(os) {
|
||||
if clear {
|
||||
return Ok(());
|
||||
}
|
||||
self.create_iblock(img)?;
|
||||
}
|
||||
let ib = self.iblock;
|
||||
@@ -420,6 +431,9 @@ impl Ea {
|
||||
let dblk_idx = l.start_dblk + local;
|
||||
let slot = dblks_at + dblk_idx * osz;
|
||||
let mut addr = get_uint(&img.read(slot, os as usize)?, os);
|
||||
if addr == undef(os) && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if addr == undef(os) {
|
||||
// libhdf5 records start_idx + (global data block index)
|
||||
// * nelmts here (H5EA__lookup_elmt), not the block's
|
||||
@@ -449,6 +463,9 @@ impl Ea {
|
||||
let sb_prefix = self.dblk_prefix_len(os);
|
||||
let sb_len = sb_prefix + bitmap_len + l.ndblks * osz;
|
||||
let mut sb = get_uint(&img.read(sslot, os as usize)?, os);
|
||||
if sb == undef(os) && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if sb == undef(os) {
|
||||
let mut d = self.block_prefix(b"EASB", l.start_idx, os);
|
||||
d.resize(d.len() + bitmap_len as usize, 0);
|
||||
@@ -471,6 +488,9 @@ impl Ea {
|
||||
let local = (rel - l.start_idx) / l.dblk_nelmts;
|
||||
let dslot = sb + sb_prefix + bitmap_len + local * osz;
|
||||
let mut addr = get_uint(&img.read(dslot, os as usize)?, os);
|
||||
if addr == undef(os) && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if addr == undef(os) {
|
||||
let off = l.start_idx + local * l.dblk_nelmts;
|
||||
addr = self.create_dblock(img, l.dblk_nelmts, off)?;
|
||||
@@ -491,6 +511,9 @@ impl Ea {
|
||||
let bpos = sb + sb_prefix + bit / 8;
|
||||
let mut byte = img.read(bpos, 1)?[0];
|
||||
let mask = 0x80u8 >> (bit % 8);
|
||||
if byte & mask == 0 && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if byte & mask == 0 {
|
||||
let fill = self.fill_elems(page, os)?;
|
||||
img.write(page_at, &fill)?;
|
||||
@@ -503,7 +526,7 @@ impl Ea {
|
||||
}
|
||||
}
|
||||
}
|
||||
if idx + 1 > self.stats[4] {
|
||||
if !clear && idx + 1 > self.stats[4] {
|
||||
self.stats[4] = idx + 1;
|
||||
self.dirty_hdr = true;
|
||||
}
|
||||
|
||||
@@ -137,13 +137,19 @@ impl Fa {
|
||||
Ok((fa, hdr))
|
||||
}
|
||||
|
||||
/// Set element `idx` to `e`.
|
||||
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
|
||||
/// Set element `idx` to `e`, or back to the fill element (`None`: a
|
||||
/// removed chunk, `H5D__farray_idx_remove`), which creates no page.
|
||||
pub(crate) fn set(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
idx: u64,
|
||||
e: Option<Elem>,
|
||||
) -> Result<(), Error> {
|
||||
let os = img.os;
|
||||
if idx >= self.nelmts {
|
||||
return Err(bad("index beyond the array"));
|
||||
}
|
||||
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
|
||||
let enc = encode_elem(e, self.filtered, self.elem_size, os)?;
|
||||
let es = self.slot(os);
|
||||
let prefix = 6 + u64::from(os);
|
||||
let page = self.page();
|
||||
@@ -162,6 +168,9 @@ impl Fa {
|
||||
let bpos = self.dblk + prefix + p / 8;
|
||||
let mut byte = img.read(bpos, 1)?[0];
|
||||
let mask = 0x80u8 >> (p % 8);
|
||||
if byte & mask == 0 && e.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if byte & mask == 0 {
|
||||
let fill = encode_elem(None, self.filtered, self.elem_size, os)?;
|
||||
img.write(page_at, &fill.repeat(count as usize))?;
|
||||
|
||||
@@ -35,6 +35,8 @@ pub(crate) struct Image<'a> {
|
||||
/// Width of addresses and lengths in the file.
|
||||
pub(crate) os: u8,
|
||||
pub(crate) ls: u8,
|
||||
/// Space the edit stopped using.
|
||||
freed: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
impl<'a> Image<'a> {
|
||||
@@ -47,6 +49,7 @@ impl<'a> Image<'a> {
|
||||
old_eoa: eoa,
|
||||
os,
|
||||
ls,
|
||||
freed: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +80,20 @@ impl<'a> Image<'a> {
|
||||
Ok(addr)
|
||||
}
|
||||
|
||||
/// Allocate `size` bytes for metadata or data, from space an earlier
|
||||
/// edit of this session freed when there is a block that fits,
|
||||
/// otherwise at the end of the file.
|
||||
pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result<u64, Error> {
|
||||
self.alloc(size)
|
||||
}
|
||||
|
||||
/// Note that the edit no longer uses `[addr, addr + len)`.
|
||||
pub(crate) fn free(&mut self, addr: u64, len: u64) {
|
||||
if len > 0 {
|
||||
self.freed.push((addr, len));
|
||||
}
|
||||
}
|
||||
|
||||
/// If `[addr, addr + old_len)` is the last allocated space, grow it to
|
||||
/// `new_len` bytes (a structure at the end of the file can grow where
|
||||
/// it is) and return true.
|
||||
|
||||
+515
-67
@@ -16,6 +16,7 @@
|
||||
//! is alive while the file changes (see `image`).
|
||||
|
||||
mod btree1;
|
||||
mod btree2;
|
||||
mod earray;
|
||||
mod farray;
|
||||
mod image;
|
||||
@@ -39,6 +40,7 @@ use crate::error::Error;
|
||||
use crate::reader::File;
|
||||
use crate::types::AttrValue;
|
||||
use btree1::{BTree1, Key};
|
||||
use btree2::Bt2;
|
||||
use earray::{Ea, EaParams, Elem};
|
||||
use farray::Fa;
|
||||
use image::{Image, get_uint, put_uint, undef};
|
||||
@@ -249,6 +251,14 @@ impl Target {
|
||||
fn dims(&self) -> &[u64] {
|
||||
&self.ds.dimensions
|
||||
}
|
||||
|
||||
/// The chunk index (or implicit chunks') address.
|
||||
fn layout_address(&self) -> Option<u64> {
|
||||
match &self.layout {
|
||||
DataLayout::Chunked { btree_address, .. } => *btree_address,
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Datatypes whose stored bytes point elsewhere in the file (variable-length
|
||||
@@ -331,7 +341,7 @@ enum IndexEdit {
|
||||
Implicit,
|
||||
Fixed(Option<Fa>),
|
||||
Extensible(Option<Ea>),
|
||||
BTree2,
|
||||
BTree2(Option<Bt2>),
|
||||
}
|
||||
|
||||
struct ChunkedEdit<'t> {
|
||||
@@ -398,7 +408,9 @@ impl<'t> ChunkedEdit<'t> {
|
||||
(_, Some(4)) => {
|
||||
IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?)
|
||||
}
|
||||
(_, Some(5)) => IndexEdit::BTree2,
|
||||
(_, Some(5)) => {
|
||||
IndexEdit::BTree2(btree_address.map(|a| Bt2::open(img, a)).transpose()?)
|
||||
}
|
||||
(v, i) => {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"chunked layout version {v}, index type {i:?}"
|
||||
@@ -520,7 +532,7 @@ impl<'t> ChunkedEdit<'t> {
|
||||
self.patch_layout_addr(img, hdr_addr)?;
|
||||
}
|
||||
if let IndexEdit::Fixed(Some(fa)) = &mut self.index {
|
||||
fa.set(img, idx, e)?;
|
||||
fa.set(img, idx, Some(e))?;
|
||||
}
|
||||
}
|
||||
IndexEdit::Extensible(ea) => {
|
||||
@@ -549,30 +561,165 @@ impl<'t> ChunkedEdit<'t> {
|
||||
self.patch_layout_addr(img, hdr_addr)?;
|
||||
}
|
||||
if let IndexEdit::Extensible(Some(ea)) = &mut self.index {
|
||||
ea.set(img, idx, e)?;
|
||||
ea.set(img, idx, Some(e))?;
|
||||
}
|
||||
}
|
||||
IndexEdit::BTree2 => {
|
||||
return Err(Error::Unsupported(
|
||||
"adding, moving or resizing a chunk in a version-2 B-tree chunk index \
|
||||
(datasets with more than one unlimited dimension)"
|
||||
.into(),
|
||||
));
|
||||
IndexEdit::BTree2(tree) => {
|
||||
if tree.is_none() {
|
||||
let at = self
|
||||
.lpos
|
||||
.params
|
||||
.ok_or_else(|| Error::Unsupported("B-tree v2 parameters".into()))?;
|
||||
let d = self.hdr.data(img, self.layout_msg)?;
|
||||
let node_size = u32::from_le_bytes([d[at], d[at + 1], d[at + 2], d[at + 3]]);
|
||||
let size_len = if filtered {
|
||||
earray::chunk_size_len(self.chunk_bytes as u64, self.lpos.version) + 4
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let rec_size = img.os as usize + size_len + 8 * self.cd.len();
|
||||
let new = Bt2::create(
|
||||
img,
|
||||
if filtered { 11 } else { 10 },
|
||||
node_size,
|
||||
rec_size,
|
||||
d[at + 4],
|
||||
d[at + 5],
|
||||
)?;
|
||||
let a = new.address();
|
||||
*tree = Some(new);
|
||||
self.patch_layout_addr(img, a)?;
|
||||
}
|
||||
let IndexEdit::BTree2(Some(tree)) = &mut self.index else {
|
||||
unreachable!("created above")
|
||||
};
|
||||
let rec = bt2_chunk_record(tree, scaled, e, filtered, img.os)?;
|
||||
let mut cmp = bt2_chunk_cmp(scaled, tree.record_size());
|
||||
tree.update(img, &mut cmp, &rec)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove chunk `scaled` (stored at `info`) from the index, as the
|
||||
/// index's `remove` operation does when libhdf5 prunes a dataset, and
|
||||
/// free its space. An implicit index cannot drop chunks: libhdf5 leaves
|
||||
/// them (and their data) where they are, and so does this.
|
||||
fn remove(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
scaled: &[u64],
|
||||
info: &ChunkInfo,
|
||||
) -> Result<(), Error> {
|
||||
let t = self.t;
|
||||
match &mut self.index {
|
||||
IndexEdit::Implicit => return Ok(()),
|
||||
IndexEdit::Single => {
|
||||
return Err(Error::Unsupported(
|
||||
"removing the chunk of a single-chunk index".into(),
|
||||
));
|
||||
}
|
||||
IndexEdit::BTree1(Some(tree)) => {
|
||||
let mut offs: Vec<u64> = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect();
|
||||
offs.push(0);
|
||||
if tree.remove(img, &offs)?.is_none() {
|
||||
return Err(Error::Unsupported("chunk missing from its B-tree".into()));
|
||||
}
|
||||
}
|
||||
IndexEdit::BTree2(Some(tree)) => {
|
||||
let mut cmp = bt2_chunk_cmp(scaled, tree.record_size());
|
||||
if tree.remove(img, &mut cmp)?.is_none() {
|
||||
return Err(Error::Unsupported("chunk missing from its B-tree".into()));
|
||||
}
|
||||
}
|
||||
IndexEdit::Fixed(Some(fa)) => {
|
||||
let max = t.ds.max_dimensions.as_deref();
|
||||
let idx = array_index(scaled, t.dims(), max, &self.cd, false)?;
|
||||
fa.set(img, idx, None)?;
|
||||
}
|
||||
IndexEdit::Extensible(Some(ea)) => {
|
||||
let max = t.ds.max_dimensions.as_deref();
|
||||
let idx = array_index(scaled, t.dims(), max, &self.cd, true)?;
|
||||
ea.set(img, idx, None)?;
|
||||
}
|
||||
_ => return Err(Error::Unsupported("chunk index missing".into())),
|
||||
}
|
||||
img.free(info.address, u64::from(info.chunk_size));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
|
||||
match &mut self.index {
|
||||
IndexEdit::Extensible(Some(ea)) => ea.finish(img)?,
|
||||
IndexEdit::Fixed(Some(fa)) => fa.finish(img)?,
|
||||
IndexEdit::BTree2(Some(t)) => t.finish(img)?,
|
||||
_ => {}
|
||||
}
|
||||
self.hdr.finish(img)
|
||||
}
|
||||
}
|
||||
|
||||
/// A version-2 B-tree chunk record (type 10, or 11 when filtered): the
|
||||
/// chunk's address, [its stored size and filter mask,] and its scaled
|
||||
/// offsets.
|
||||
fn bt2_chunk_record(
|
||||
tree: &Bt2,
|
||||
scaled: &[u64],
|
||||
e: Elem,
|
||||
filtered: bool,
|
||||
os: u8,
|
||||
) -> Result<Vec<u8>, Error> {
|
||||
let rs = tree.record_size();
|
||||
let want_type = if filtered { 11 } else { 10 };
|
||||
let osz = os as usize;
|
||||
let size_len = rs
|
||||
.checked_sub(osz + 8 * scaled.len())
|
||||
.and_then(|n| {
|
||||
if filtered {
|
||||
n.checked_sub(4).filter(|&w| (1..=8).contains(&w))
|
||||
} else {
|
||||
(n == 0).then_some(0)
|
||||
}
|
||||
})
|
||||
.filter(|_| tree.tree_type() == want_type)
|
||||
.ok_or_else(|| Error::Unsupported("version-2 B-tree chunk record layout".into()))?;
|
||||
let mut r = vec![0u8; rs];
|
||||
put_uint(&mut r, e.addr, os);
|
||||
let mut q = osz;
|
||||
if filtered {
|
||||
if size_len < 8 && e.size >> (8 * size_len) != 0 {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"filtered chunk of {} bytes does not fit the index's {size_len}-byte size field",
|
||||
e.size
|
||||
)));
|
||||
}
|
||||
r[q..q + size_len].copy_from_slice(&e.size.to_le_bytes()[..size_len]);
|
||||
q += size_len;
|
||||
r[q..q + 4].copy_from_slice(&e.mask.to_le_bytes());
|
||||
q += 4;
|
||||
}
|
||||
for &s in scaled {
|
||||
r[q..q + 8].copy_from_slice(&s.to_le_bytes());
|
||||
q += 8;
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
/// Compare chunk `scaled` with a chunk record (`H5D__bt2_compare`: the
|
||||
/// scaled offsets, the first dimension most significant).
|
||||
fn bt2_chunk_cmp(
|
||||
scaled: &[u64],
|
||||
rec_size: usize,
|
||||
) -> impl FnMut(&Image<'_>, &[u8]) -> Result<std::cmp::Ordering, Error> + '_ {
|
||||
move |_, r| {
|
||||
let at = rec_size - 8 * scaled.len();
|
||||
let theirs = (0..scaled.len()).map(|d| {
|
||||
u64::from_le_bytes(r[at + 8 * d..at + 8 * d + 8].try_into().unwrap_or([0; 8]))
|
||||
});
|
||||
Ok(scaled.iter().copied().cmp(theirs))
|
||||
}
|
||||
}
|
||||
|
||||
/// The chunk B-tree's K: the superblock's (version 1), the superblock
|
||||
/// extension's (versions 2 and 3), or libhdf5's default of 32.
|
||||
fn chunk_btree_k(f: &File) -> Result<u16, Error> {
|
||||
@@ -694,10 +841,15 @@ impl FileEditor {
|
||||
})
|
||||
}
|
||||
|
||||
/// Change a chunked dataset's current dimensions to `shape`, which may
|
||||
/// only grow each dimension, up to the dataset's maximum dimensions
|
||||
/// (h5py's `Dataset.resize`). New elements read as the fill value until
|
||||
/// written. Shrinking is [`Error::Unsupported`].
|
||||
/// Change a chunked dataset's current dimensions to `shape`: each
|
||||
/// dimension may grow up to the dataset's maximum or shrink (h5py's
|
||||
/// `Dataset.resize`), as `H5Dset_extent` does. New elements read as the
|
||||
/// fill value until written (with early allocation their chunks are
|
||||
/// allocated and filled now). Shrinking removes the chunks wholly
|
||||
/// outside the new extent from the chunk index and frees them, and
|
||||
/// overwrites the part of each partial edge chunk outside it with the
|
||||
/// fill value, so elements that come back after a later growth read as
|
||||
/// the fill value (`H5D__chunk_prune_by_extent`).
|
||||
pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> {
|
||||
self.edit(|f, img| {
|
||||
let t = Target::load(f, path)?;
|
||||
@@ -720,18 +872,16 @@ impl FileEditor {
|
||||
shape[d], max[d]
|
||||
)));
|
||||
}
|
||||
if shape[d] < dims[d] {
|
||||
return Err(Error::Unsupported("shrinking a dataset".into()));
|
||||
}
|
||||
}
|
||||
if !matches!(t.layout, DataLayout::Chunked { .. }) {
|
||||
return Err(Error::Unsupported(
|
||||
"resizing a dataset that is not chunked".into(),
|
||||
));
|
||||
}
|
||||
// Only the dataspace changes: every chunk index is keyed
|
||||
// independently of the current extent (the array indexes by the
|
||||
// maximum dimensions), and new chunks come with the writes.
|
||||
// The dataspace changes; every chunk index is keyed
|
||||
// independently of the current extent (the arrays index by the
|
||||
// maximum dimensions). Growth allocates chunks only under early
|
||||
// allocation, shrinking prunes.
|
||||
let mut hdr = Header::load(img, t.addr)?;
|
||||
let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage(
|
||||
clawhdf5_format::message_type::MessageType::Dataspace,
|
||||
@@ -754,7 +904,15 @@ impl FileEditor {
|
||||
put_uint(&mut dims_bytes[d * ls..], n, img.ls);
|
||||
}
|
||||
hdr.patch(img, i, first, &dims_bytes)?;
|
||||
hdr.finish(img)
|
||||
let fill = fill_info(img, &hdr)?;
|
||||
hdr.finish(img)?;
|
||||
let expand = shape.iter().zip(&dims).any(|(n, o)| n > o);
|
||||
let shrink = shape.iter().zip(&dims).any(|(n, o)| n < o);
|
||||
let early = expand && fill.alloc_time == ALLOC_EARLY;
|
||||
if early || shrink {
|
||||
resize_chunks(f, img, &t, &dims, shape, &fill, early, shrink)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1187,52 +1345,7 @@ fn write_selection(
|
||||
Ok(())
|
||||
})?;
|
||||
for (scaled, buf) in bufs {
|
||||
// As libhdf5 does: an optional filter that fails (LZF that
|
||||
// does not shrink the chunk) is skipped and its mask bit set.
|
||||
let (bytes, mask) = match &t.pipeline {
|
||||
Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?,
|
||||
None => (buf, 0u32),
|
||||
};
|
||||
let len = bytes.len() as u64;
|
||||
let placed = match existing.get(&scaled) {
|
||||
Some(info) if t.pipeline.is_none() => {
|
||||
if u64::from(info.chunk_size) != len {
|
||||
return Err(Error::Unsupported(
|
||||
"unfiltered chunk stored at an unexpected size".into(),
|
||||
));
|
||||
}
|
||||
img.write(info.address, &bytes)?;
|
||||
None
|
||||
}
|
||||
// Rewritten where it is when it still fits, or when it
|
||||
// is the last thing in the file (the chunk an append
|
||||
// keeps rewriting usually is) and can grow there.
|
||||
Some(info)
|
||||
if len <= u64::from(info.chunk_size)
|
||||
|| img.grow_tail(info.address, u64::from(info.chunk_size), len)? =>
|
||||
{
|
||||
img.write(info.address, &bytes)?;
|
||||
(len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(
|
||||
Elem {
|
||||
addr: info.address,
|
||||
size: len,
|
||||
mask,
|
||||
},
|
||||
)
|
||||
}
|
||||
_ => {
|
||||
let a = img.alloc(len)?;
|
||||
img.write(a, &bytes)?;
|
||||
Some(Elem {
|
||||
addr: a,
|
||||
size: len,
|
||||
mask,
|
||||
})
|
||||
}
|
||||
};
|
||||
if let Some(e) = placed {
|
||||
ce.set(img, &scaled, e)?;
|
||||
}
|
||||
store_chunk(img, &mut ce, existing.get(&scaled), &scaled, buf)?;
|
||||
}
|
||||
ce.finish(img)
|
||||
}
|
||||
@@ -1240,6 +1353,341 @@ fn write_selection(
|
||||
}
|
||||
}
|
||||
|
||||
/// `H5D_ALLOC_TIME_EARLY`.
|
||||
const ALLOC_EARLY: u8 = 1;
|
||||
|
||||
/// When a dataset's storage is allocated and filled, from its fill value
|
||||
/// message (`H5O__fill_new_decode`); a dataset without one uses the chunked
|
||||
/// defaults (incremental allocation, fill if set).
|
||||
struct FillInfo {
|
||||
alloc_time: u8,
|
||||
/// 0 on allocation, 1 never, 2 if set.
|
||||
fill_time: u8,
|
||||
/// The fill value is undefined (`H5D_FILL_VALUE_UNDEFINED`).
|
||||
undefined: bool,
|
||||
}
|
||||
|
||||
fn fill_info(img: &Image<'_>, hdr: &Header) -> Result<FillInfo, Error> {
|
||||
let mut fi = FillInfo {
|
||||
alloc_time: 3,
|
||||
fill_time: 2,
|
||||
undefined: false,
|
||||
};
|
||||
let Some(i) = hdr.find(0x05) else {
|
||||
return Ok(fi);
|
||||
};
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared fill value message".into()));
|
||||
}
|
||||
let d = hdr.data(img, i)?;
|
||||
let short = || Error::Unsupported("short fill value message".into());
|
||||
match d.first() {
|
||||
Some(1) | Some(2) => {
|
||||
fi.alloc_time = *d.get(1).ok_or_else(short)?;
|
||||
fi.fill_time = *d.get(2).ok_or_else(short)?;
|
||||
fi.undefined = d[0] == 2 && *d.get(3).ok_or_else(short)? == 0;
|
||||
}
|
||||
Some(3) => {
|
||||
let flags = *d.get(1).ok_or_else(short)?;
|
||||
fi.alloc_time = flags & 0x03;
|
||||
fi.fill_time = (flags >> 2) & 0x03;
|
||||
fi.undefined = flags & 0x10 != 0;
|
||||
}
|
||||
_ => return Err(Error::Unsupported("fill value message version".into())),
|
||||
}
|
||||
Ok(fi)
|
||||
}
|
||||
|
||||
/// Visit the chunks libhdf5 visits, in its order, when the extent changes
|
||||
/// from `old` to `new`: for allocation (`H5D__chunk_allocate`), every
|
||||
/// chunk of the new extent outside the old one; `f` gets each scaled
|
||||
/// offset.
|
||||
fn for_each_new_chunk(
|
||||
old: &[u64],
|
||||
new: &[u64],
|
||||
cd: &[u64],
|
||||
mut f: impl FnMut(&[u64]) -> Result<(), Error>,
|
||||
) -> Result<(), Error> {
|
||||
let rank = new.len();
|
||||
if rank == 0 || new.contains(&0) {
|
||||
return Ok(());
|
||||
}
|
||||
let min: Vec<u64> = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect();
|
||||
let mut max: Vec<u64> = (0..rank).map(|d| (new[d] - 1) / cd[d]).collect();
|
||||
for op in 0..rank {
|
||||
if min[op] > max[op] {
|
||||
continue;
|
||||
}
|
||||
let mut scaled = vec![0u64; rank];
|
||||
scaled[op] = min[op];
|
||||
loop {
|
||||
f(&scaled)?;
|
||||
let mut carry = true;
|
||||
for i in (0..rank).rev() {
|
||||
scaled[i] += 1;
|
||||
if scaled[i] > max[i] {
|
||||
scaled[i] = if i == op { min[i] } else { 0 };
|
||||
} else {
|
||||
carry = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if carry {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if min[op] == 0 {
|
||||
break;
|
||||
}
|
||||
max[op] = min[op] - 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What pruning does to one chunk.
|
||||
enum Prune {
|
||||
/// Overwrite the part outside the new extent with the fill value.
|
||||
Fill,
|
||||
/// Drop it from the index.
|
||||
Remove,
|
||||
}
|
||||
|
||||
/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks
|
||||
/// from `old` to `new`, in its order.
|
||||
fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec<u64>, Prune)> {
|
||||
let rank = new.len();
|
||||
let mut out = Vec::new();
|
||||
if old.contains(&0) {
|
||||
return out;
|
||||
}
|
||||
let shrunk: Vec<bool> = (0..rank).map(|d| new[d] < old[d]).collect();
|
||||
let mut max_mod: Vec<u64> = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect();
|
||||
let max_fill: Vec<i64> = (0..rank)
|
||||
.map(|d| {
|
||||
if new[d] == 0 {
|
||||
-1
|
||||
} else {
|
||||
((new[d].min(old[d]) - 1) / cd[d]) as i64
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let min_mod: Vec<u64> = (0..rank).map(|d| new[d] / cd[d]).collect();
|
||||
let fill_dim: Vec<bool> = (0..rank)
|
||||
.map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d])
|
||||
.collect();
|
||||
for op in 0..rank {
|
||||
if !shrunk[op] {
|
||||
continue;
|
||||
}
|
||||
let mut scaled = vec![0u64; rank];
|
||||
scaled[op] = min_mod[op];
|
||||
let mut outside: Vec<bool> = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect();
|
||||
let mut n_out = outside.iter().filter(|&&o| o).count();
|
||||
loop {
|
||||
if n_out == 0 {
|
||||
out.push((scaled.clone(), Prune::Fill));
|
||||
} else {
|
||||
out.push((scaled.clone(), Prune::Remove));
|
||||
}
|
||||
let mut carry = true;
|
||||
for i in (0..rank).rev() {
|
||||
scaled[i] += 1;
|
||||
if scaled[i] > max_mod[i] {
|
||||
if i == op {
|
||||
scaled[i] = min_mod[i];
|
||||
if outside[i] && fill_dim[i] {
|
||||
outside[i] = false;
|
||||
n_out -= 1;
|
||||
}
|
||||
} else {
|
||||
scaled[i] = 0;
|
||||
if outside[i] && max_fill[i] >= 0 {
|
||||
outside[i] = false;
|
||||
n_out -= 1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !outside[i] && scaled[i] as i64 > max_fill[i] {
|
||||
outside[i] = true;
|
||||
n_out += 1;
|
||||
}
|
||||
carry = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if carry {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if min_mod[op] == 0 {
|
||||
// Every chunk was visited (the dimension shrank to nothing).
|
||||
break;
|
||||
}
|
||||
max_mod[op] = min_mod[op] - 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The chunk work of a resize: under early allocation, allocate and fill
|
||||
/// the chunks the growth brings in (`H5D__chunk_allocate`); when a
|
||||
/// dimension shrank, prune (`H5D__chunk_prune_by_extent`).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn resize_chunks(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
t: &Target,
|
||||
old: &[u64],
|
||||
new: &[u64],
|
||||
fill: &FillInfo,
|
||||
early: bool,
|
||||
shrink: bool,
|
||||
) -> Result<(), Error> {
|
||||
let mut ce = ChunkedEdit::new(f, img, t)?;
|
||||
let cd = ce.cd.clone();
|
||||
let es = t.es;
|
||||
let chunk_bytes = ce.chunk_bytes;
|
||||
let existing: HashMap<Vec<u64>, ChunkInfo> = match &t.layout {
|
||||
DataLayout::Chunked {
|
||||
btree_address: Some(_),
|
||||
..
|
||||
} => {
|
||||
let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?;
|
||||
chunks
|
||||
.into_iter()
|
||||
.map(|c| {
|
||||
let s: Vec<u64> = c.offsets.iter().zip(&cd).map(|(o, c)| o / c).collect();
|
||||
(s, c)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
_ => HashMap::new(),
|
||||
};
|
||||
let fill_chunk = t.fill.repeat(chunk_bytes / es);
|
||||
if early {
|
||||
let should_fill =
|
||||
fill.fill_time == 0 || (fill.fill_time == 2 && !fill.undefined) || t.pipeline.is_some();
|
||||
let implicit = matches!(ce.index, IndexEdit::Implicit);
|
||||
let base = t.layout_address();
|
||||
for_each_new_chunk(old, new, &cd, |scaled| {
|
||||
if implicit {
|
||||
if should_fill {
|
||||
let base =
|
||||
base.ok_or_else(|| Error::Unsupported("implicit index address".into()))?;
|
||||
let max = t.ds.max_dimensions.as_deref();
|
||||
let idx = array_index(scaled, new, max, &cd, false)?;
|
||||
let at = idx
|
||||
.checked_mul(chunk_bytes as u64)
|
||||
.and_then(|o| o.checked_add(base))
|
||||
.ok_or_else(|| Error::Unsupported("chunk address overflows".into()))?;
|
||||
img.write(at, &fill_chunk)?;
|
||||
}
|
||||
Ok(())
|
||||
} else if existing.contains_key(scaled) {
|
||||
Ok(())
|
||||
} else {
|
||||
store_chunk(img, &mut ce, None, scaled, fill_chunk.clone())
|
||||
}
|
||||
})?;
|
||||
}
|
||||
if shrink && !existing.is_empty() {
|
||||
for (scaled, what) in prune_plan(old, new, &cd) {
|
||||
let Some(info) = existing.get(&scaled) else {
|
||||
continue;
|
||||
};
|
||||
match what {
|
||||
Prune::Remove => ce.remove(img, &scaled, info)?,
|
||||
Prune::Fill => {
|
||||
let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?;
|
||||
// Keep [0, count) in each dimension; fill the rest.
|
||||
let count: Vec<u64> = (0..cd.len())
|
||||
.map(|d| cd[d].min(new[d] - scaled[d] * cd[d]))
|
||||
.collect();
|
||||
let n: u64 = cd.iter().product();
|
||||
for flat in 0..n {
|
||||
let mut r = flat;
|
||||
let mut inside = true;
|
||||
for d in (0..cd.len()).rev() {
|
||||
if r % cd[d] >= count[d] {
|
||||
inside = false;
|
||||
}
|
||||
r /= cd[d];
|
||||
}
|
||||
if !inside {
|
||||
let at = flat as usize * es;
|
||||
buf[at..at + es].copy_from_slice(&t.fill);
|
||||
}
|
||||
}
|
||||
store_chunk(img, &mut ce, Some(info), &scaled, buf)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ce.finish(img)
|
||||
}
|
||||
|
||||
/// Encode chunk `scaled` (decoded bytes `buf`) and store it: an existing
|
||||
/// unfiltered chunk in place; a filtered one in place when it still fits
|
||||
/// (or can grow at the end of the file), else in new space, the old space
|
||||
/// freed; a new chunk in new space. The index is updated when the chunk's
|
||||
/// address, size or filter mask changed.
|
||||
fn store_chunk(
|
||||
img: &mut Image<'_>,
|
||||
ce: &mut ChunkedEdit<'_>,
|
||||
existing: Option<&ChunkInfo>,
|
||||
scaled: &[u64],
|
||||
buf: Vec<u8>,
|
||||
) -> Result<(), Error> {
|
||||
let t = ce.t;
|
||||
// As libhdf5 does: an optional filter that fails (LZF that does not
|
||||
// shrink the chunk) is skipped and its mask bit set.
|
||||
let (bytes, mask) = match &t.pipeline {
|
||||
Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, t.es as u32)?,
|
||||
None => (buf, 0u32),
|
||||
};
|
||||
let len = bytes.len() as u64;
|
||||
let placed = match existing {
|
||||
Some(info) if t.pipeline.is_none() => {
|
||||
if u64::from(info.chunk_size) != len {
|
||||
return Err(Error::Unsupported(
|
||||
"unfiltered chunk stored at an unexpected size".into(),
|
||||
));
|
||||
}
|
||||
img.write(info.address, &bytes)?;
|
||||
None
|
||||
}
|
||||
// Rewritten where it is when it still fits, or when it is the last
|
||||
// thing in the file (the chunk an append keeps rewriting usually is)
|
||||
// and can grow there.
|
||||
Some(info)
|
||||
if len <= u64::from(info.chunk_size)
|
||||
|| img.grow_tail(info.address, u64::from(info.chunk_size), len)? =>
|
||||
{
|
||||
img.write(info.address, &bytes)?;
|
||||
(len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(Elem {
|
||||
addr: info.address,
|
||||
size: len,
|
||||
mask,
|
||||
})
|
||||
}
|
||||
_ => {
|
||||
let a = img.alloc_reusing(len)?;
|
||||
img.write(a, &bytes)?;
|
||||
if let Some(info) = existing {
|
||||
img.free(info.address, u64::from(info.chunk_size));
|
||||
}
|
||||
Some(Elem {
|
||||
addr: a,
|
||||
size: len,
|
||||
mask,
|
||||
})
|
||||
}
|
||||
};
|
||||
if let Some(e) = placed {
|
||||
ce.set(img, scaled, e)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> {
|
||||
let start = usize::try_from(info.address)
|
||||
.map_err(|_| Error::Unsupported("chunk address out of range".into()))?;
|
||||
|
||||
@@ -98,8 +98,8 @@ fn errors_leave_the_file_untouched() {
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_))));
|
||||
assert!(ed.write_all("missing", &[0; 4]).is_err());
|
||||
// Wrong length, wrong type, outside the extent, beyond maxshape,
|
||||
// shrinking, a rank change.
|
||||
// Wrong length, wrong type, outside the extent, beyond maxshape, a
|
||||
// rank change, resizing a dataset that is not chunked.
|
||||
assert!(matches!(
|
||||
ed.write_all("flat", &[0; 7]),
|
||||
Err(Error::InvalidArgument(_))
|
||||
@@ -116,7 +116,10 @@ fn errors_leave_the_file_untouched() {
|
||||
ed.resize("raw", &[3, 5]),
|
||||
Err(Error::InvalidArgument(_))
|
||||
));
|
||||
assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_))));
|
||||
assert!(matches!(
|
||||
ed.resize("flat", &[2]),
|
||||
Err(Error::Unsupported(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
ed.resize("ext", &[4, 1]),
|
||||
Err(Error::InvalidArgument(_))
|
||||
@@ -136,3 +139,26 @@ fn errors_leave_the_file_untouched() {
|
||||
drop(ed);
|
||||
assert!(std::fs::read(&path).unwrap() == before);
|
||||
}
|
||||
|
||||
/// Shrinking and growing again on a file clawhdf5 wrote: elements that come
|
||||
/// back read as the fill value, the ones kept keep their values.
|
||||
#[test]
|
||||
fn shrink_then_grow_reads_fill() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = sample(dir.path());
|
||||
{
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
ed.resize("ext", &[2]).unwrap();
|
||||
ed.resize("ext", &[9]).unwrap();
|
||||
ed.resize("raw", &[1, 4]).unwrap();
|
||||
ed.resize("raw", &[3, 4]).unwrap();
|
||||
}
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.dataset("ext").unwrap().read_i32().unwrap(),
|
||||
[0, 1, 0, 0, 0, 0, 0, 0, 0]
|
||||
);
|
||||
let mut raw = vec![0.0f64; 12];
|
||||
raw[..4].fill(0.5);
|
||||
assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user