diff --git a/CHANGELOG.md b/CHANGELOG.md index 964390d..6ae2b02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,10 @@ partial edge chunk outside the extent is overwritten with the fill value, so it reads as fill after a later growth. Growth under early allocation now allocates and fills the new chunks (`H5D__chunk_allocate`), which an - implicit index needs. Shrinking was `Error::Unsupported`. + implicit index needs. Shrinking was `Error::Unsupported`. Only the + chunks that exist are visited (placed in libhdf5's order), so shrinking + a sparse dataset costs memory and time in its chunks, not in the + coordinates cut off (a 2 x 10^12-coordinate shrink takes 0.6 s). - **`FileEditor::set_attr` handles dense attribute storage and creation order**: objects that track (and index) attribute creation order; the move to dense storage when an object reaches its compact limit (or an diff --git a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs index 7fc309a..fff256c 100644 --- a/crates/clawhdf5-tools/tests/edit_coverage_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_coverage_interop.rs @@ -1566,3 +1566,65 @@ fn resize_clawhdf5_written_datasets() { } check_tools(&path, true); } + +/// Shrinking a huge, sparse dataset costs time and memory in the chunks +/// that exist, not in the chunk coordinates cut off. The editor once stored +/// every coordinate of the cut-off region (about 62 bytes each), so this +/// 2 x 10^12-coordinate shrink ran out of memory. +#[test] +fn shrinking_a_huge_sparse_dataset_is_bounded() { + if !tools_ok() { + return; + } + let dir = tmpdir(); + const N: u64 = 1_000_000_000_000; + for libver in ["earliest", "v110"] { + let path = dir.path().join(format!("sparse_{libver}.h5")); + py(&format!( + "import h5py\n\ + with h5py.File({p:?}, 'w', libver=({libver:?}, 'latest')) as f:\n\ + \x20 d = f.create_dataset('b', shape=(4, {N}), maxshape=(None, None), chunks=(1, 1), dtype=' = got + .as_chunks::<4>() + .0 + .iter() + .map(|c| i32::from_le_bytes(*c)) + .collect(); + assert_eq!(got, [1, 2, 3, 4, 5, 0, 0, 0, 0, 0, 0, 0], "{libver}"); + } +} diff --git a/crates/clawhdf5/src/edit/mod.rs b/crates/clawhdf5/src/edit/mod.rs index 336fbb8..e1d301a 100644 --- a/crates/clawhdf5/src/edit/mod.rs +++ b/crates/clawhdf5/src/edit/mod.rs @@ -1330,16 +1330,31 @@ enum Prune { Remove, } -/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks -/// from `old` to `new`, in its order. -fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { +/// The chunks of `existing` (keyed by scaled coordinates) that +/// `H5D__chunk_prune_by_extent` visits when the extent shrinks from `old` +/// to `new`, in its order, with what it does to each. +/// +/// libhdf5 walks every chunk coordinate of the region cut off, one pass per +/// shrunk dimension `op` in order: the coordinates with `scaled[op]` at or +/// past the new extent's chunk and every earlier shrunk dimension below its +/// own, row-major. It looks each one up in the index and stores nothing, +/// so a sparse dataset costs it time, not memory. Only chunks that exist +/// can be acted on, so this places each existing chunk in that walk (its +/// pass, then its coordinates) and sorts: the same chunks in the same +/// order, in memory and time proportional to the chunks that exist. +fn prune_plan<'c>( + old: &[u64], + new: &[u64], + cd: &[u64], + existing: &'c HashMap, ChunkInfo>, +) -> Vec<(&'c [u64], &'c ChunkInfo, Prune)> { let rank = new.len(); - let mut out = Vec::new(); - if old.contains(&0) { - return out; + if old.contains(&0) || cd.contains(&0) { + return Vec::new(); } - let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); - let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + // The last chunk index still (partly) inside the new extent; -1 when + // the dimension shrank to nothing. let max_fill: Vec = (0..rank) .map(|d| { if new[d] == 0 { @@ -1350,60 +1365,25 @@ fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { }) .collect(); let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); - let fill_dim: Vec = (0..rank) - .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) - .collect(); - for op in 0..rank { - if !shrunk[op] { - continue; - } - let mut scaled = vec![0u64; rank]; - scaled[op] = min_mod[op]; - let mut outside: Vec = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); - let mut n_out = outside.iter().filter(|&&o| o).count(); - loop { - if n_out == 0 { - out.push((scaled.clone(), Prune::Fill)); + let mut out: Vec<(usize, &[u64], &ChunkInfo, Prune)> = existing + .iter() + .filter(|(s, _)| s.len() == rank && s.iter().zip(&max_mod).all(|(c, m)| c <= m)) + .filter_map(|(s, info)| { + // The first shrunk dimension whose pass reaches this chunk; the + // passes before it covered only coordinates below their minimum. + let op = (0..rank).find(|&d| new[d] < old[d] && s[d] >= min_mod[d])?; + // Chunks with any coordinate past the last partly kept chunk go; + // the others lose the part outside the new extent. + let what = if (0..rank).all(|u| s[u] as i64 <= max_fill[u]) { + Prune::Fill } else { - out.push((scaled.clone(), Prune::Remove)); - } - let mut carry = true; - for i in (0..rank).rev() { - scaled[i] += 1; - if scaled[i] > max_mod[i] { - if i == op { - scaled[i] = min_mod[i]; - if outside[i] && fill_dim[i] { - outside[i] = false; - n_out -= 1; - } - } else { - scaled[i] = 0; - if outside[i] && max_fill[i] >= 0 { - outside[i] = false; - n_out -= 1; - } - } - } else { - if !outside[i] && scaled[i] as i64 > max_fill[i] { - outside[i] = true; - n_out += 1; - } - carry = false; - break; - } - } - if carry { - break; - } - } - if min_mod[op] == 0 { - // Every chunk was visited (the dimension shrank to nothing). - break; - } - max_mod[op] = min_mod[op] - 1; - } - out + Prune::Remove + }; + Some((op, s.as_slice(), info, what)) + }) + .collect(); + out.sort_unstable_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))); + out.into_iter().map(|(_, s, i, w)| (s, i, w)).collect() } /// The chunk work of a resize: under early allocation, allocate and fill @@ -1468,12 +1448,9 @@ fn resize_chunks( })?; } if shrink && !existing.is_empty() { - for (scaled, what) in prune_plan(old, new, &cd) { - let Some(info) = existing.get(&scaled) else { - continue; - }; + for (scaled, info, what) in prune_plan(old, new, &cd, &existing) { match what { - Prune::Remove => ce.remove(img, &scaled, info)?, + Prune::Remove => ce.remove(img, scaled, info)?, Prune::Fill => { let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?; // Keep [0, count) in each dimension; fill the rest. @@ -1495,7 +1472,7 @@ fn resize_chunks( buf[at..at + es].copy_from_slice(&t.fill); } } - store_chunk(img, &mut ce, Some(info), &scaled, buf)?; + store_chunk(img, &mut ce, Some(info), scaled, buf)?; } } } @@ -1610,6 +1587,141 @@ fn decode_chunk( #[cfg(test)] mod tests { + use super::{Prune, prune_plan}; + use clawhdf5_format::chunked_read::ChunkInfo; + use std::collections::HashMap; + + /// `H5D__chunk_prune_by_extent`'s walk over every chunk coordinate of the + /// region cut off, stored (the implementation before `prune_plan` + /// visited only existing chunks). + fn prune_walk(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec, Prune)> { + let rank = new.len(); + let mut out = Vec::new(); + if old.contains(&0) { + return out; + } + let shrunk: Vec = (0..rank).map(|d| new[d] < old[d]).collect(); + let mut max_mod: Vec = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect(); + let max_fill: Vec = (0..rank) + .map(|d| { + if new[d] == 0 { + -1 + } else { + ((new[d].min(old[d]) - 1) / cd[d]) as i64 + } + }) + .collect(); + let min_mod: Vec = (0..rank).map(|d| new[d] / cd[d]).collect(); + let fill_dim: Vec = (0..rank) + .map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d]) + .collect(); + for op in 0..rank { + if !shrunk[op] { + continue; + } + let mut scaled = vec![0u64; rank]; + scaled[op] = min_mod[op]; + let mut outside: Vec = + (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect(); + let mut n_out = outside.iter().filter(|&&o| o).count(); + loop { + if n_out == 0 { + out.push((scaled.clone(), Prune::Fill)); + } else { + out.push((scaled.clone(), Prune::Remove)); + } + let mut carry = true; + for i in (0..rank).rev() { + scaled[i] += 1; + if scaled[i] > max_mod[i] { + if i == op { + scaled[i] = min_mod[i]; + if outside[i] && fill_dim[i] { + outside[i] = false; + n_out -= 1; + } + } else { + scaled[i] = 0; + if outside[i] && max_fill[i] >= 0 { + outside[i] = false; + n_out -= 1; + } + } + } else { + if !outside[i] && scaled[i] as i64 > max_fill[i] { + outside[i] = true; + n_out += 1; + } + carry = false; + break; + } + } + if carry { + break; + } + } + if min_mod[op] == 0 { + // Every chunk was visited (the dimension shrank to nothing). + break; + } + max_mod[op] = min_mod[op] - 1; + } + out + } + + /// `prune_plan` over a random subset of chunks gives the chunks of the + /// full walk that exist, in its order and with its actions. + #[test] + fn prune_plan_follows_the_full_walk() { + let mut x: u64 = 0x1234_5678; + let mut rnd = |n: u64| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x % n + }; + for _ in 0..3000 { + let rank = 1 + rnd(3) as usize; + let cd: Vec = (0..rank).map(|_| 1 + rnd(4)).collect(); + let old: Vec = (0..rank).map(|_| 1 + rnd(14)).collect(); + let new: Vec = old + .iter() + .map(|&o| if rnd(2) == 0 { o } else { rnd(o + 1) }) + .collect(); + let grid: Vec = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect(); + let mut existing = HashMap::new(); + let total: u64 = grid.iter().product(); + for flat in 0..total { + if rnd(3) == 0 { + continue; + } + let mut r = flat; + let mut s = vec![0; rank]; + for d in (0..rank).rev() { + s[d] = r % grid[d]; + r /= grid[d]; + } + let info = ChunkInfo { + chunk_size: 1, + filter_mask: 0, + offsets: s.iter().zip(&cd).map(|(a, b)| a * b).collect(), + address: flat, + }; + existing.insert(s, info); + } + let want: Vec<(Vec, bool)> = prune_walk(&old, &new, &cd) + .into_iter() + .filter(|(s, _)| existing.contains_key(s)) + .map(|(s, w)| (s, matches!(w, Prune::Fill))) + .collect(); + let got: Vec<(Vec, bool)> = prune_plan(&old, &new, &cd, &existing) + .into_iter() + .map(|(s, _, w)| (s.to_vec(), matches!(w, Prune::Fill))) + .collect(); + assert_eq!(got, want, "old {old:?} new {new:?} chunks {cd:?}"); + } + } + use std::cell::Cell; use std::path::Path;