edit: version-2 B-tree chunk indexes, shrinking, early allocation
FileEditor can now: - add, move and resize chunks of datasets with two or more unlimited dimensions (version-2 B-tree chunk index, record types 10/11). The new edit/btree2.rs follows libhdf5's H5B2 code: H5B2_update (modify, or insert into a leaf with room, or fall back to H5B2__insert), the preemptive split/redistribute loop with its two retries, split1, split_root (depth growth, node geometry per depth), redistribute2/3, cumulative record counts and the pointer widths H5B2__hdr_init derives, a checksum per node. Removal (H5B2_remove: merge2/3, redistribution, root collapse, the internal-record swap with a leaf) is there too. A dataset without an index yet gets one from the layout message's node size and split/merge percentages. - shrink a chunked dataset along any dimension (resize to a smaller shape), as H5D__set_extent / H5D__chunk_prune_by_extent do: the same chunks visited in the same order; chunks wholly outside the new extent are removed from the index (version-1 B-tree: H5B_remove with its sibling key and link fix-ups and the empty-root case; version-2 B-tree; Fixed/Extensible Array elements reset to the fill element; an implicit index keeps its chunks, as libhdf5 does) and their space noted as free; the part of each partial edge chunk outside the extent is overwritten with the fill value, so elements that come back after a later growth read as fill. - under early allocation, allocate and fill the chunks a growth brings in (H5D__chunk_allocate), which an implicit index needs: libhdf5 refills them, and they may hold the data of chunks pruned earlier. Tests (crates/clawhdf5-tools/tests/edit_coverage_interop.rs): growth in both dimensions of v110/latest files, unfiltered and deflated, gives node-for-node the version-2 B-tree libhdf5 builds (h5py with its chunk cache off, so chunks enter the index in the editor's order), through a depth increase; random chunk order; 60 random shrink/grow/write steps on Extensible Array, version-2 B-tree, Fixed Array, 1-D and implicit datasets (earliest/v110/latest, with and without gzip+shuffle) give the values h5py gets doing the same and the same index shape (version-1 and version-2 B-tree node shapes, Extensible Array statistics); h5py r+ continues on every result; h5dump and h5rs check accept them. edit_interop's version-2 B-tree case now appends instead of expecting a refusal; shrinking is no longer an error in edit_tests. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -56,6 +56,15 @@ fn bad(why: &str) -> Error {
|
||||
))
|
||||
}
|
||||
|
||||
/// What a removal did below a node (`H5B_ins_t`), with the removed chunk's
|
||||
/// address and size.
|
||||
enum Rm {
|
||||
NotFound,
|
||||
Noop((u64, u32)),
|
||||
/// The child is gone: the parent must drop it.
|
||||
Remove((u64, u32)),
|
||||
}
|
||||
|
||||
enum Ins {
|
||||
Done,
|
||||
/// The node split; the new right sibling and its first key.
|
||||
@@ -144,7 +153,14 @@ impl BTree1 {
|
||||
put_uint(&mut d[8 + osz..], node.right, os);
|
||||
let ks = self.key_size();
|
||||
let mut p = 8 + 2 * osz;
|
||||
for (i, k) in node.keys.iter().enumerate() {
|
||||
// An empty node (a root whose last chunk was removed) stores no
|
||||
// keys, as libhdf5 writes it.
|
||||
let nkeys = if node.children.is_empty() {
|
||||
0
|
||||
} else {
|
||||
node.keys.len()
|
||||
};
|
||||
for (i, k) in node.keys.iter().take(nkeys).enumerate() {
|
||||
d[p..p + 4].copy_from_slice(&k.size.to_le_bytes());
|
||||
d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes());
|
||||
for (j, o) in k.offs.iter().enumerate() {
|
||||
@@ -210,6 +226,18 @@ impl BTree1 {
|
||||
return Err(bad("bad chunk key"));
|
||||
}
|
||||
let root = self.read(img, self.root)?;
|
||||
if root.children.is_empty() {
|
||||
// Every chunk was removed (H5B__insert_helper's first
|
||||
// insertion): the root, a leaf again, takes it.
|
||||
let right = self.right_key_after(&key);
|
||||
let node = Node {
|
||||
level: 0,
|
||||
keys: vec![key, right],
|
||||
children: vec![addr],
|
||||
..root
|
||||
};
|
||||
return self.write(img, &node);
|
||||
}
|
||||
if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? {
|
||||
// The root split: move its (left) half to a new node so the root
|
||||
// keeps its address, then make the root the parent of both.
|
||||
@@ -372,6 +400,143 @@ impl BTree1 {
|
||||
cmp(&key.offs, &right.keys[0].offs) != Ordering::Less
|
||||
}
|
||||
|
||||
/// Remove the chunk at offsets `offs` (element-size coordinate 0), as
|
||||
/// `H5B_remove` does for the chunk index (whose critical key is the
|
||||
/// left one): no rebalancing; a node left without children is deleted
|
||||
/// and its siblings relinked (the left one takes over its right key),
|
||||
/// a root left empty becomes an empty leaf. Returns the chunk's address
|
||||
/// and stored size, or `None` when the tree has no such chunk (nothing
|
||||
/// changes then). Deleted nodes are freed in `img`.
|
||||
pub(crate) fn remove(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
offs: &[u64],
|
||||
) -> Result<Option<(u64, u32)>, Error> {
|
||||
if offs.len() != self.ndims {
|
||||
return Err(bad("bad chunk key"));
|
||||
}
|
||||
let mut lt = None;
|
||||
match self.remove_at(img, self.root, 0, offs, &mut lt, 64)? {
|
||||
Rm::NotFound => Ok(None),
|
||||
Rm::Noop(c) | Rm::Remove(c) => Ok(Some(c)),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_at(
|
||||
&self,
|
||||
img: &mut Image<'_>,
|
||||
addr: u64,
|
||||
level: usize,
|
||||
offs: &[u64],
|
||||
lt_out: &mut Option<Key>,
|
||||
depth: u8,
|
||||
) -> Result<Rm, Error> {
|
||||
if depth == 0 {
|
||||
return Err(bad("tree too deep"));
|
||||
}
|
||||
let mut node = self.read(img, addr)?;
|
||||
let n = node.children.len();
|
||||
// H5D__btree_cmp3 over (keys[i], keys[i + 1]), binary search.
|
||||
let (mut lo, mut hi, mut idx) = (0usize, n, 0usize);
|
||||
let mut c = 1i32;
|
||||
while lo < hi && c != 0 {
|
||||
idx = (lo + hi) / 2;
|
||||
c = if cmp(offs, &node.keys[idx + 1].offs) != Ordering::Less {
|
||||
1
|
||||
} else if cmp(offs, &node.keys[idx].offs) == Ordering::Less {
|
||||
-1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if c < 0 {
|
||||
hi = idx;
|
||||
} else {
|
||||
lo = idx + 1;
|
||||
}
|
||||
}
|
||||
if c != 0 {
|
||||
return Ok(Rm::NotFound);
|
||||
}
|
||||
let mut lt_changed = None;
|
||||
let res = if node.level > 0 {
|
||||
let child = self.read(img, node.children[idx])?;
|
||||
if usize::from(child.level) + 1 != usize::from(node.level) {
|
||||
return Err(bad("inconsistent node levels"));
|
||||
}
|
||||
self.remove_at(
|
||||
img,
|
||||
node.children[idx],
|
||||
level + 1,
|
||||
offs,
|
||||
&mut lt_changed,
|
||||
depth - 1,
|
||||
)?
|
||||
} else {
|
||||
if node.keys[idx].offs != offs {
|
||||
return Ok(Rm::NotFound);
|
||||
}
|
||||
Rm::Remove((node.children[idx], node.keys[idx].size))
|
||||
};
|
||||
let chunk = match res {
|
||||
Rm::NotFound => return Ok(Rm::NotFound),
|
||||
Rm::Noop(c) | Rm::Remove(c) => c,
|
||||
};
|
||||
let mut dirty = false;
|
||||
if let Some(k) = lt_changed {
|
||||
node.keys[idx] = k;
|
||||
dirty = true;
|
||||
if idx == 0 {
|
||||
*lt_out = Some(node.keys[0].clone());
|
||||
}
|
||||
}
|
||||
let out = Rm::Noop(chunk);
|
||||
if let Rm::Remove(_) = res {
|
||||
let undefined = undef(img.os);
|
||||
if n == 1 {
|
||||
if level > 0 {
|
||||
if node.left != undefined {
|
||||
let mut sib = self.read(img, node.left)?;
|
||||
let last = sib.children.len();
|
||||
sib.keys[last] = node.keys[1].clone();
|
||||
sib.right = node.right;
|
||||
self.write(img, &sib)?;
|
||||
}
|
||||
if node.right != undefined {
|
||||
let mut sib = self.read(img, node.right)?;
|
||||
sib.left = node.left;
|
||||
self.write(img, &sib)?;
|
||||
}
|
||||
img.free(addr, self.node_size(img.os) as u64);
|
||||
return Ok(Rm::Remove(chunk));
|
||||
}
|
||||
node.children.clear();
|
||||
node.keys.truncate(1);
|
||||
node.level = 0;
|
||||
} else if idx == 0 {
|
||||
node.keys.remove(0);
|
||||
node.children.remove(0);
|
||||
*lt_out = Some(node.keys[0].clone());
|
||||
} else {
|
||||
// Right-most or middle child: its left key goes, the next
|
||||
// key becomes the following child's left key.
|
||||
node.keys.remove(idx);
|
||||
node.children.remove(idx);
|
||||
}
|
||||
dirty = true;
|
||||
}
|
||||
if dirty {
|
||||
self.write(img, &node)?;
|
||||
}
|
||||
// The left sibling's right key follows a changed left key.
|
||||
if lt_out.is_some() && node.left != undef(img.os) && level > 0 {
|
||||
let mut sib = self.read(img, node.left)?;
|
||||
let last = sib.children.len();
|
||||
sib.keys[last] = node.keys[0].clone();
|
||||
self.write(img, &sib)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) {
|
||||
let n = node.children.len();
|
||||
if node.level == 0 {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -383,12 +383,23 @@ impl Ea {
|
||||
}
|
||||
|
||||
/// Set element `idx` to `e`.
|
||||
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
|
||||
/// Set element `idx` to `e`, or back to the fill element (`None`: a
|
||||
/// removed chunk, `H5D__earray_idx_remove`), which creates no block.
|
||||
pub(crate) fn set(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
idx: u64,
|
||||
e: Option<Elem>,
|
||||
) -> Result<(), Error> {
|
||||
let os = img.os;
|
||||
let osz = u64::from(os);
|
||||
let es = self.slot_size(os) as u64;
|
||||
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
|
||||
let enc = encode_elem(e, self.filtered, self.elem_size, os)?;
|
||||
let clear = e.is_none();
|
||||
if self.iblock == undef(os) {
|
||||
if clear {
|
||||
return Ok(());
|
||||
}
|
||||
self.create_iblock(img)?;
|
||||
}
|
||||
let ib = self.iblock;
|
||||
@@ -420,6 +431,9 @@ impl Ea {
|
||||
let dblk_idx = l.start_dblk + local;
|
||||
let slot = dblks_at + dblk_idx * osz;
|
||||
let mut addr = get_uint(&img.read(slot, os as usize)?, os);
|
||||
if addr == undef(os) && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if addr == undef(os) {
|
||||
// libhdf5 records start_idx + (global data block index)
|
||||
// * nelmts here (H5EA__lookup_elmt), not the block's
|
||||
@@ -449,6 +463,9 @@ impl Ea {
|
||||
let sb_prefix = self.dblk_prefix_len(os);
|
||||
let sb_len = sb_prefix + bitmap_len + l.ndblks * osz;
|
||||
let mut sb = get_uint(&img.read(sslot, os as usize)?, os);
|
||||
if sb == undef(os) && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if sb == undef(os) {
|
||||
let mut d = self.block_prefix(b"EASB", l.start_idx, os);
|
||||
d.resize(d.len() + bitmap_len as usize, 0);
|
||||
@@ -471,6 +488,9 @@ impl Ea {
|
||||
let local = (rel - l.start_idx) / l.dblk_nelmts;
|
||||
let dslot = sb + sb_prefix + bitmap_len + local * osz;
|
||||
let mut addr = get_uint(&img.read(dslot, os as usize)?, os);
|
||||
if addr == undef(os) && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if addr == undef(os) {
|
||||
let off = l.start_idx + local * l.dblk_nelmts;
|
||||
addr = self.create_dblock(img, l.dblk_nelmts, off)?;
|
||||
@@ -491,6 +511,9 @@ impl Ea {
|
||||
let bpos = sb + sb_prefix + bit / 8;
|
||||
let mut byte = img.read(bpos, 1)?[0];
|
||||
let mask = 0x80u8 >> (bit % 8);
|
||||
if byte & mask == 0 && clear {
|
||||
return Ok(());
|
||||
}
|
||||
if byte & mask == 0 {
|
||||
let fill = self.fill_elems(page, os)?;
|
||||
img.write(page_at, &fill)?;
|
||||
@@ -503,7 +526,7 @@ impl Ea {
|
||||
}
|
||||
}
|
||||
}
|
||||
if idx + 1 > self.stats[4] {
|
||||
if !clear && idx + 1 > self.stats[4] {
|
||||
self.stats[4] = idx + 1;
|
||||
self.dirty_hdr = true;
|
||||
}
|
||||
|
||||
@@ -137,13 +137,19 @@ impl Fa {
|
||||
Ok((fa, hdr))
|
||||
}
|
||||
|
||||
/// Set element `idx` to `e`.
|
||||
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
|
||||
/// Set element `idx` to `e`, or back to the fill element (`None`: a
|
||||
/// removed chunk, `H5D__farray_idx_remove`), which creates no page.
|
||||
pub(crate) fn set(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
idx: u64,
|
||||
e: Option<Elem>,
|
||||
) -> Result<(), Error> {
|
||||
let os = img.os;
|
||||
if idx >= self.nelmts {
|
||||
return Err(bad("index beyond the array"));
|
||||
}
|
||||
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
|
||||
let enc = encode_elem(e, self.filtered, self.elem_size, os)?;
|
||||
let es = self.slot(os);
|
||||
let prefix = 6 + u64::from(os);
|
||||
let page = self.page();
|
||||
@@ -162,6 +168,9 @@ impl Fa {
|
||||
let bpos = self.dblk + prefix + p / 8;
|
||||
let mut byte = img.read(bpos, 1)?[0];
|
||||
let mask = 0x80u8 >> (p % 8);
|
||||
if byte & mask == 0 && e.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if byte & mask == 0 {
|
||||
let fill = encode_elem(None, self.filtered, self.elem_size, os)?;
|
||||
img.write(page_at, &fill.repeat(count as usize))?;
|
||||
|
||||
@@ -35,6 +35,8 @@ pub(crate) struct Image<'a> {
|
||||
/// Width of addresses and lengths in the file.
|
||||
pub(crate) os: u8,
|
||||
pub(crate) ls: u8,
|
||||
/// Space the edit stopped using.
|
||||
freed: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
impl<'a> Image<'a> {
|
||||
@@ -47,6 +49,7 @@ impl<'a> Image<'a> {
|
||||
old_eoa: eoa,
|
||||
os,
|
||||
ls,
|
||||
freed: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +80,20 @@ impl<'a> Image<'a> {
|
||||
Ok(addr)
|
||||
}
|
||||
|
||||
/// Allocate `size` bytes for metadata or data, from space an earlier
|
||||
/// edit of this session freed when there is a block that fits,
|
||||
/// otherwise at the end of the file.
|
||||
pub(crate) fn alloc_reusing(&mut self, size: u64) -> Result<u64, Error> {
|
||||
self.alloc(size)
|
||||
}
|
||||
|
||||
/// Note that the edit no longer uses `[addr, addr + len)`.
|
||||
pub(crate) fn free(&mut self, addr: u64, len: u64) {
|
||||
if len > 0 {
|
||||
self.freed.push((addr, len));
|
||||
}
|
||||
}
|
||||
|
||||
/// If `[addr, addr + old_len)` is the last allocated space, grow it to
|
||||
/// `new_len` bytes (a structure at the end of the file can grow where
|
||||
/// it is) and return true.
|
||||
|
||||
+515
-67
@@ -16,6 +16,7 @@
|
||||
//! is alive while the file changes (see `image`).
|
||||
|
||||
mod btree1;
|
||||
mod btree2;
|
||||
mod earray;
|
||||
mod farray;
|
||||
mod image;
|
||||
@@ -39,6 +40,7 @@ use crate::error::Error;
|
||||
use crate::reader::File;
|
||||
use crate::types::AttrValue;
|
||||
use btree1::{BTree1, Key};
|
||||
use btree2::Bt2;
|
||||
use earray::{Ea, EaParams, Elem};
|
||||
use farray::Fa;
|
||||
use image::{Image, get_uint, put_uint, undef};
|
||||
@@ -249,6 +251,14 @@ impl Target {
|
||||
fn dims(&self) -> &[u64] {
|
||||
&self.ds.dimensions
|
||||
}
|
||||
|
||||
/// The chunk index (or implicit chunks') address.
|
||||
fn layout_address(&self) -> Option<u64> {
|
||||
match &self.layout {
|
||||
DataLayout::Chunked { btree_address, .. } => *btree_address,
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Datatypes whose stored bytes point elsewhere in the file (variable-length
|
||||
@@ -331,7 +341,7 @@ enum IndexEdit {
|
||||
Implicit,
|
||||
Fixed(Option<Fa>),
|
||||
Extensible(Option<Ea>),
|
||||
BTree2,
|
||||
BTree2(Option<Bt2>),
|
||||
}
|
||||
|
||||
struct ChunkedEdit<'t> {
|
||||
@@ -398,7 +408,9 @@ impl<'t> ChunkedEdit<'t> {
|
||||
(_, Some(4)) => {
|
||||
IndexEdit::Extensible(btree_address.map(|a| Ea::open(img, a)).transpose()?)
|
||||
}
|
||||
(_, Some(5)) => IndexEdit::BTree2,
|
||||
(_, Some(5)) => {
|
||||
IndexEdit::BTree2(btree_address.map(|a| Bt2::open(img, a)).transpose()?)
|
||||
}
|
||||
(v, i) => {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"chunked layout version {v}, index type {i:?}"
|
||||
@@ -520,7 +532,7 @@ impl<'t> ChunkedEdit<'t> {
|
||||
self.patch_layout_addr(img, hdr_addr)?;
|
||||
}
|
||||
if let IndexEdit::Fixed(Some(fa)) = &mut self.index {
|
||||
fa.set(img, idx, e)?;
|
||||
fa.set(img, idx, Some(e))?;
|
||||
}
|
||||
}
|
||||
IndexEdit::Extensible(ea) => {
|
||||
@@ -549,30 +561,165 @@ impl<'t> ChunkedEdit<'t> {
|
||||
self.patch_layout_addr(img, hdr_addr)?;
|
||||
}
|
||||
if let IndexEdit::Extensible(Some(ea)) = &mut self.index {
|
||||
ea.set(img, idx, e)?;
|
||||
ea.set(img, idx, Some(e))?;
|
||||
}
|
||||
}
|
||||
IndexEdit::BTree2 => {
|
||||
return Err(Error::Unsupported(
|
||||
"adding, moving or resizing a chunk in a version-2 B-tree chunk index \
|
||||
(datasets with more than one unlimited dimension)"
|
||||
.into(),
|
||||
));
|
||||
IndexEdit::BTree2(tree) => {
|
||||
if tree.is_none() {
|
||||
let at = self
|
||||
.lpos
|
||||
.params
|
||||
.ok_or_else(|| Error::Unsupported("B-tree v2 parameters".into()))?;
|
||||
let d = self.hdr.data(img, self.layout_msg)?;
|
||||
let node_size = u32::from_le_bytes([d[at], d[at + 1], d[at + 2], d[at + 3]]);
|
||||
let size_len = if filtered {
|
||||
earray::chunk_size_len(self.chunk_bytes as u64, self.lpos.version) + 4
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let rec_size = img.os as usize + size_len + 8 * self.cd.len();
|
||||
let new = Bt2::create(
|
||||
img,
|
||||
if filtered { 11 } else { 10 },
|
||||
node_size,
|
||||
rec_size,
|
||||
d[at + 4],
|
||||
d[at + 5],
|
||||
)?;
|
||||
let a = new.address();
|
||||
*tree = Some(new);
|
||||
self.patch_layout_addr(img, a)?;
|
||||
}
|
||||
let IndexEdit::BTree2(Some(tree)) = &mut self.index else {
|
||||
unreachable!("created above")
|
||||
};
|
||||
let rec = bt2_chunk_record(tree, scaled, e, filtered, img.os)?;
|
||||
let mut cmp = bt2_chunk_cmp(scaled, tree.record_size());
|
||||
tree.update(img, &mut cmp, &rec)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove chunk `scaled` (stored at `info`) from the index, as the
|
||||
/// index's `remove` operation does when libhdf5 prunes a dataset, and
|
||||
/// free its space. An implicit index cannot drop chunks: libhdf5 leaves
|
||||
/// them (and their data) where they are, and so does this.
|
||||
fn remove(
|
||||
&mut self,
|
||||
img: &mut Image<'_>,
|
||||
scaled: &[u64],
|
||||
info: &ChunkInfo,
|
||||
) -> Result<(), Error> {
|
||||
let t = self.t;
|
||||
match &mut self.index {
|
||||
IndexEdit::Implicit => return Ok(()),
|
||||
IndexEdit::Single => {
|
||||
return Err(Error::Unsupported(
|
||||
"removing the chunk of a single-chunk index".into(),
|
||||
));
|
||||
}
|
||||
IndexEdit::BTree1(Some(tree)) => {
|
||||
let mut offs: Vec<u64> = scaled.iter().zip(&self.cd).map(|(s, c)| s * c).collect();
|
||||
offs.push(0);
|
||||
if tree.remove(img, &offs)?.is_none() {
|
||||
return Err(Error::Unsupported("chunk missing from its B-tree".into()));
|
||||
}
|
||||
}
|
||||
IndexEdit::BTree2(Some(tree)) => {
|
||||
let mut cmp = bt2_chunk_cmp(scaled, tree.record_size());
|
||||
if tree.remove(img, &mut cmp)?.is_none() {
|
||||
return Err(Error::Unsupported("chunk missing from its B-tree".into()));
|
||||
}
|
||||
}
|
||||
IndexEdit::Fixed(Some(fa)) => {
|
||||
let max = t.ds.max_dimensions.as_deref();
|
||||
let idx = array_index(scaled, t.dims(), max, &self.cd, false)?;
|
||||
fa.set(img, idx, None)?;
|
||||
}
|
||||
IndexEdit::Extensible(Some(ea)) => {
|
||||
let max = t.ds.max_dimensions.as_deref();
|
||||
let idx = array_index(scaled, t.dims(), max, &self.cd, true)?;
|
||||
ea.set(img, idx, None)?;
|
||||
}
|
||||
_ => return Err(Error::Unsupported("chunk index missing".into())),
|
||||
}
|
||||
img.free(info.address, u64::from(info.chunk_size));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
|
||||
match &mut self.index {
|
||||
IndexEdit::Extensible(Some(ea)) => ea.finish(img)?,
|
||||
IndexEdit::Fixed(Some(fa)) => fa.finish(img)?,
|
||||
IndexEdit::BTree2(Some(t)) => t.finish(img)?,
|
||||
_ => {}
|
||||
}
|
||||
self.hdr.finish(img)
|
||||
}
|
||||
}
|
||||
|
||||
/// A version-2 B-tree chunk record (type 10, or 11 when filtered): the
|
||||
/// chunk's address, [its stored size and filter mask,] and its scaled
|
||||
/// offsets.
|
||||
fn bt2_chunk_record(
|
||||
tree: &Bt2,
|
||||
scaled: &[u64],
|
||||
e: Elem,
|
||||
filtered: bool,
|
||||
os: u8,
|
||||
) -> Result<Vec<u8>, Error> {
|
||||
let rs = tree.record_size();
|
||||
let want_type = if filtered { 11 } else { 10 };
|
||||
let osz = os as usize;
|
||||
let size_len = rs
|
||||
.checked_sub(osz + 8 * scaled.len())
|
||||
.and_then(|n| {
|
||||
if filtered {
|
||||
n.checked_sub(4).filter(|&w| (1..=8).contains(&w))
|
||||
} else {
|
||||
(n == 0).then_some(0)
|
||||
}
|
||||
})
|
||||
.filter(|_| tree.tree_type() == want_type)
|
||||
.ok_or_else(|| Error::Unsupported("version-2 B-tree chunk record layout".into()))?;
|
||||
let mut r = vec![0u8; rs];
|
||||
put_uint(&mut r, e.addr, os);
|
||||
let mut q = osz;
|
||||
if filtered {
|
||||
if size_len < 8 && e.size >> (8 * size_len) != 0 {
|
||||
return Err(Error::Unsupported(format!(
|
||||
"filtered chunk of {} bytes does not fit the index's {size_len}-byte size field",
|
||||
e.size
|
||||
)));
|
||||
}
|
||||
r[q..q + size_len].copy_from_slice(&e.size.to_le_bytes()[..size_len]);
|
||||
q += size_len;
|
||||
r[q..q + 4].copy_from_slice(&e.mask.to_le_bytes());
|
||||
q += 4;
|
||||
}
|
||||
for &s in scaled {
|
||||
r[q..q + 8].copy_from_slice(&s.to_le_bytes());
|
||||
q += 8;
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
/// Compare chunk `scaled` with a chunk record (`H5D__bt2_compare`: the
|
||||
/// scaled offsets, the first dimension most significant).
|
||||
fn bt2_chunk_cmp(
|
||||
scaled: &[u64],
|
||||
rec_size: usize,
|
||||
) -> impl FnMut(&Image<'_>, &[u8]) -> Result<std::cmp::Ordering, Error> + '_ {
|
||||
move |_, r| {
|
||||
let at = rec_size - 8 * scaled.len();
|
||||
let theirs = (0..scaled.len()).map(|d| {
|
||||
u64::from_le_bytes(r[at + 8 * d..at + 8 * d + 8].try_into().unwrap_or([0; 8]))
|
||||
});
|
||||
Ok(scaled.iter().copied().cmp(theirs))
|
||||
}
|
||||
}
|
||||
|
||||
/// The chunk B-tree's K: the superblock's (version 1), the superblock
|
||||
/// extension's (versions 2 and 3), or libhdf5's default of 32.
|
||||
fn chunk_btree_k(f: &File) -> Result<u16, Error> {
|
||||
@@ -694,10 +841,15 @@ impl FileEditor {
|
||||
})
|
||||
}
|
||||
|
||||
/// Change a chunked dataset's current dimensions to `shape`, which may
|
||||
/// only grow each dimension, up to the dataset's maximum dimensions
|
||||
/// (h5py's `Dataset.resize`). New elements read as the fill value until
|
||||
/// written. Shrinking is [`Error::Unsupported`].
|
||||
/// Change a chunked dataset's current dimensions to `shape`: each
|
||||
/// dimension may grow up to the dataset's maximum or shrink (h5py's
|
||||
/// `Dataset.resize`), as `H5Dset_extent` does. New elements read as the
|
||||
/// fill value until written (with early allocation their chunks are
|
||||
/// allocated and filled now). Shrinking removes the chunks wholly
|
||||
/// outside the new extent from the chunk index and frees them, and
|
||||
/// overwrites the part of each partial edge chunk outside it with the
|
||||
/// fill value, so elements that come back after a later growth read as
|
||||
/// the fill value (`H5D__chunk_prune_by_extent`).
|
||||
pub fn resize(&mut self, path: &str, shape: &[u64]) -> Result<(), Error> {
|
||||
self.edit(|f, img| {
|
||||
let t = Target::load(f, path)?;
|
||||
@@ -720,18 +872,16 @@ impl FileEditor {
|
||||
shape[d], max[d]
|
||||
)));
|
||||
}
|
||||
if shape[d] < dims[d] {
|
||||
return Err(Error::Unsupported("shrinking a dataset".into()));
|
||||
}
|
||||
}
|
||||
if !matches!(t.layout, DataLayout::Chunked { .. }) {
|
||||
return Err(Error::Unsupported(
|
||||
"resizing a dataset that is not chunked".into(),
|
||||
));
|
||||
}
|
||||
// Only the dataspace changes: every chunk index is keyed
|
||||
// independently of the current extent (the array indexes by the
|
||||
// maximum dimensions), and new chunks come with the writes.
|
||||
// The dataspace changes; every chunk index is keyed
|
||||
// independently of the current extent (the arrays index by the
|
||||
// maximum dimensions). Growth allocates chunks only under early
|
||||
// allocation, shrinking prunes.
|
||||
let mut hdr = Header::load(img, t.addr)?;
|
||||
let i = hdr.find(MSG_DATASPACE).ok_or(Error::MissingMessage(
|
||||
clawhdf5_format::message_type::MessageType::Dataspace,
|
||||
@@ -754,7 +904,15 @@ impl FileEditor {
|
||||
put_uint(&mut dims_bytes[d * ls..], n, img.ls);
|
||||
}
|
||||
hdr.patch(img, i, first, &dims_bytes)?;
|
||||
hdr.finish(img)
|
||||
let fill = fill_info(img, &hdr)?;
|
||||
hdr.finish(img)?;
|
||||
let expand = shape.iter().zip(&dims).any(|(n, o)| n > o);
|
||||
let shrink = shape.iter().zip(&dims).any(|(n, o)| n < o);
|
||||
let early = expand && fill.alloc_time == ALLOC_EARLY;
|
||||
if early || shrink {
|
||||
resize_chunks(f, img, &t, &dims, shape, &fill, early, shrink)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1187,52 +1345,7 @@ fn write_selection(
|
||||
Ok(())
|
||||
})?;
|
||||
for (scaled, buf) in bufs {
|
||||
// As libhdf5 does: an optional filter that fails (LZF that
|
||||
// does not shrink the chunk) is skipped and its mask bit set.
|
||||
let (bytes, mask) = match &t.pipeline {
|
||||
Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, es as u32)?,
|
||||
None => (buf, 0u32),
|
||||
};
|
||||
let len = bytes.len() as u64;
|
||||
let placed = match existing.get(&scaled) {
|
||||
Some(info) if t.pipeline.is_none() => {
|
||||
if u64::from(info.chunk_size) != len {
|
||||
return Err(Error::Unsupported(
|
||||
"unfiltered chunk stored at an unexpected size".into(),
|
||||
));
|
||||
}
|
||||
img.write(info.address, &bytes)?;
|
||||
None
|
||||
}
|
||||
// Rewritten where it is when it still fits, or when it
|
||||
// is the last thing in the file (the chunk an append
|
||||
// keeps rewriting usually is) and can grow there.
|
||||
Some(info)
|
||||
if len <= u64::from(info.chunk_size)
|
||||
|| img.grow_tail(info.address, u64::from(info.chunk_size), len)? =>
|
||||
{
|
||||
img.write(info.address, &bytes)?;
|
||||
(len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(
|
||||
Elem {
|
||||
addr: info.address,
|
||||
size: len,
|
||||
mask,
|
||||
},
|
||||
)
|
||||
}
|
||||
_ => {
|
||||
let a = img.alloc(len)?;
|
||||
img.write(a, &bytes)?;
|
||||
Some(Elem {
|
||||
addr: a,
|
||||
size: len,
|
||||
mask,
|
||||
})
|
||||
}
|
||||
};
|
||||
if let Some(e) = placed {
|
||||
ce.set(img, &scaled, e)?;
|
||||
}
|
||||
store_chunk(img, &mut ce, existing.get(&scaled), &scaled, buf)?;
|
||||
}
|
||||
ce.finish(img)
|
||||
}
|
||||
@@ -1240,6 +1353,341 @@ fn write_selection(
|
||||
}
|
||||
}
|
||||
|
||||
/// `H5D_ALLOC_TIME_EARLY`.
|
||||
const ALLOC_EARLY: u8 = 1;
|
||||
|
||||
/// When a dataset's storage is allocated and filled, from its fill value
|
||||
/// message (`H5O__fill_new_decode`); a dataset without one uses the chunked
|
||||
/// defaults (incremental allocation, fill if set).
|
||||
struct FillInfo {
|
||||
alloc_time: u8,
|
||||
/// 0 on allocation, 1 never, 2 if set.
|
||||
fill_time: u8,
|
||||
/// The fill value is undefined (`H5D_FILL_VALUE_UNDEFINED`).
|
||||
undefined: bool,
|
||||
}
|
||||
|
||||
fn fill_info(img: &Image<'_>, hdr: &Header) -> Result<FillInfo, Error> {
|
||||
let mut fi = FillInfo {
|
||||
alloc_time: 3,
|
||||
fill_time: 2,
|
||||
undefined: false,
|
||||
};
|
||||
let Some(i) = hdr.find(0x05) else {
|
||||
return Ok(fi);
|
||||
};
|
||||
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
|
||||
return Err(Error::Unsupported("shared fill value message".into()));
|
||||
}
|
||||
let d = hdr.data(img, i)?;
|
||||
let short = || Error::Unsupported("short fill value message".into());
|
||||
match d.first() {
|
||||
Some(1) | Some(2) => {
|
||||
fi.alloc_time = *d.get(1).ok_or_else(short)?;
|
||||
fi.fill_time = *d.get(2).ok_or_else(short)?;
|
||||
fi.undefined = d[0] == 2 && *d.get(3).ok_or_else(short)? == 0;
|
||||
}
|
||||
Some(3) => {
|
||||
let flags = *d.get(1).ok_or_else(short)?;
|
||||
fi.alloc_time = flags & 0x03;
|
||||
fi.fill_time = (flags >> 2) & 0x03;
|
||||
fi.undefined = flags & 0x10 != 0;
|
||||
}
|
||||
_ => return Err(Error::Unsupported("fill value message version".into())),
|
||||
}
|
||||
Ok(fi)
|
||||
}
|
||||
|
||||
/// Visit the chunks libhdf5 visits, in its order, when the extent changes
|
||||
/// from `old` to `new`: for allocation (`H5D__chunk_allocate`), every
|
||||
/// chunk of the new extent outside the old one; `f` gets each scaled
|
||||
/// offset.
|
||||
fn for_each_new_chunk(
|
||||
old: &[u64],
|
||||
new: &[u64],
|
||||
cd: &[u64],
|
||||
mut f: impl FnMut(&[u64]) -> Result<(), Error>,
|
||||
) -> Result<(), Error> {
|
||||
let rank = new.len();
|
||||
if rank == 0 || new.contains(&0) {
|
||||
return Ok(());
|
||||
}
|
||||
let min: Vec<u64> = (0..rank).map(|d| old[d].div_ceil(cd[d])).collect();
|
||||
let mut max: Vec<u64> = (0..rank).map(|d| (new[d] - 1) / cd[d]).collect();
|
||||
for op in 0..rank {
|
||||
if min[op] > max[op] {
|
||||
continue;
|
||||
}
|
||||
let mut scaled = vec![0u64; rank];
|
||||
scaled[op] = min[op];
|
||||
loop {
|
||||
f(&scaled)?;
|
||||
let mut carry = true;
|
||||
for i in (0..rank).rev() {
|
||||
scaled[i] += 1;
|
||||
if scaled[i] > max[i] {
|
||||
scaled[i] = if i == op { min[i] } else { 0 };
|
||||
} else {
|
||||
carry = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if carry {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if min[op] == 0 {
|
||||
break;
|
||||
}
|
||||
max[op] = min[op] - 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// What pruning does to one chunk.
|
||||
enum Prune {
|
||||
/// Overwrite the part outside the new extent with the fill value.
|
||||
Fill,
|
||||
/// Drop it from the index.
|
||||
Remove,
|
||||
}
|
||||
|
||||
/// The chunks `H5D__chunk_prune_by_extent` visits when the extent shrinks
|
||||
/// from `old` to `new`, in its order.
|
||||
fn prune_plan(old: &[u64], new: &[u64], cd: &[u64]) -> Vec<(Vec<u64>, Prune)> {
|
||||
let rank = new.len();
|
||||
let mut out = Vec::new();
|
||||
if old.contains(&0) {
|
||||
return out;
|
||||
}
|
||||
let shrunk: Vec<bool> = (0..rank).map(|d| new[d] < old[d]).collect();
|
||||
let mut max_mod: Vec<u64> = (0..rank).map(|d| (old[d] - 1) / cd[d]).collect();
|
||||
let max_fill: Vec<i64> = (0..rank)
|
||||
.map(|d| {
|
||||
if new[d] == 0 {
|
||||
-1
|
||||
} else {
|
||||
((new[d].min(old[d]) - 1) / cd[d]) as i64
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let min_mod: Vec<u64> = (0..rank).map(|d| new[d] / cd[d]).collect();
|
||||
let fill_dim: Vec<bool> = (0..rank)
|
||||
.map(|d| shrunk[d] && min_mod[d] as i64 == max_fill[d])
|
||||
.collect();
|
||||
for op in 0..rank {
|
||||
if !shrunk[op] {
|
||||
continue;
|
||||
}
|
||||
let mut scaled = vec![0u64; rank];
|
||||
scaled[op] = min_mod[op];
|
||||
let mut outside: Vec<bool> = (0..rank).map(|u| scaled[u] as i64 > max_fill[u]).collect();
|
||||
let mut n_out = outside.iter().filter(|&&o| o).count();
|
||||
loop {
|
||||
if n_out == 0 {
|
||||
out.push((scaled.clone(), Prune::Fill));
|
||||
} else {
|
||||
out.push((scaled.clone(), Prune::Remove));
|
||||
}
|
||||
let mut carry = true;
|
||||
for i in (0..rank).rev() {
|
||||
scaled[i] += 1;
|
||||
if scaled[i] > max_mod[i] {
|
||||
if i == op {
|
||||
scaled[i] = min_mod[i];
|
||||
if outside[i] && fill_dim[i] {
|
||||
outside[i] = false;
|
||||
n_out -= 1;
|
||||
}
|
||||
} else {
|
||||
scaled[i] = 0;
|
||||
if outside[i] && max_fill[i] >= 0 {
|
||||
outside[i] = false;
|
||||
n_out -= 1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !outside[i] && scaled[i] as i64 > max_fill[i] {
|
||||
outside[i] = true;
|
||||
n_out += 1;
|
||||
}
|
||||
carry = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if carry {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if min_mod[op] == 0 {
|
||||
// Every chunk was visited (the dimension shrank to nothing).
|
||||
break;
|
||||
}
|
||||
max_mod[op] = min_mod[op] - 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The chunk work of a resize: under early allocation, allocate and fill
|
||||
/// the chunks the growth brings in (`H5D__chunk_allocate`); when a
|
||||
/// dimension shrank, prune (`H5D__chunk_prune_by_extent`).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn resize_chunks(
|
||||
f: &File,
|
||||
img: &mut Image<'_>,
|
||||
t: &Target,
|
||||
old: &[u64],
|
||||
new: &[u64],
|
||||
fill: &FillInfo,
|
||||
early: bool,
|
||||
shrink: bool,
|
||||
) -> Result<(), Error> {
|
||||
let mut ce = ChunkedEdit::new(f, img, t)?;
|
||||
let cd = ce.cd.clone();
|
||||
let es = t.es;
|
||||
let chunk_bytes = ce.chunk_bytes;
|
||||
let existing: HashMap<Vec<u64>, ChunkInfo> = match &t.layout {
|
||||
DataLayout::Chunked {
|
||||
btree_address: Some(_),
|
||||
..
|
||||
} => {
|
||||
let (chunks, _) = list_chunks(f.as_bytes(), &t.layout, &t.ds, es, img.os, img.ls)?;
|
||||
chunks
|
||||
.into_iter()
|
||||
.map(|c| {
|
||||
let s: Vec<u64> = c.offsets.iter().zip(&cd).map(|(o, c)| o / c).collect();
|
||||
(s, c)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
_ => HashMap::new(),
|
||||
};
|
||||
let fill_chunk = t.fill.repeat(chunk_bytes / es);
|
||||
if early {
|
||||
let should_fill =
|
||||
fill.fill_time == 0 || (fill.fill_time == 2 && !fill.undefined) || t.pipeline.is_some();
|
||||
let implicit = matches!(ce.index, IndexEdit::Implicit);
|
||||
let base = t.layout_address();
|
||||
for_each_new_chunk(old, new, &cd, |scaled| {
|
||||
if implicit {
|
||||
if should_fill {
|
||||
let base =
|
||||
base.ok_or_else(|| Error::Unsupported("implicit index address".into()))?;
|
||||
let max = t.ds.max_dimensions.as_deref();
|
||||
let idx = array_index(scaled, new, max, &cd, false)?;
|
||||
let at = idx
|
||||
.checked_mul(chunk_bytes as u64)
|
||||
.and_then(|o| o.checked_add(base))
|
||||
.ok_or_else(|| Error::Unsupported("chunk address overflows".into()))?;
|
||||
img.write(at, &fill_chunk)?;
|
||||
}
|
||||
Ok(())
|
||||
} else if existing.contains_key(scaled) {
|
||||
Ok(())
|
||||
} else {
|
||||
store_chunk(img, &mut ce, None, scaled, fill_chunk.clone())
|
||||
}
|
||||
})?;
|
||||
}
|
||||
if shrink && !existing.is_empty() {
|
||||
for (scaled, what) in prune_plan(old, new, &cd) {
|
||||
let Some(info) = existing.get(&scaled) else {
|
||||
continue;
|
||||
};
|
||||
match what {
|
||||
Prune::Remove => ce.remove(img, &scaled, info)?,
|
||||
Prune::Fill => {
|
||||
let mut buf = decode_chunk(img_read(f, info)?, t, info, chunk_bytes)?;
|
||||
// Keep [0, count) in each dimension; fill the rest.
|
||||
let count: Vec<u64> = (0..cd.len())
|
||||
.map(|d| cd[d].min(new[d] - scaled[d] * cd[d]))
|
||||
.collect();
|
||||
let n: u64 = cd.iter().product();
|
||||
for flat in 0..n {
|
||||
let mut r = flat;
|
||||
let mut inside = true;
|
||||
for d in (0..cd.len()).rev() {
|
||||
if r % cd[d] >= count[d] {
|
||||
inside = false;
|
||||
}
|
||||
r /= cd[d];
|
||||
}
|
||||
if !inside {
|
||||
let at = flat as usize * es;
|
||||
buf[at..at + es].copy_from_slice(&t.fill);
|
||||
}
|
||||
}
|
||||
store_chunk(img, &mut ce, Some(info), &scaled, buf)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ce.finish(img)
|
||||
}
|
||||
|
||||
/// Encode chunk `scaled` (decoded bytes `buf`) and store it: an existing
|
||||
/// unfiltered chunk in place; a filtered one in place when it still fits
|
||||
/// (or can grow at the end of the file), else in new space, the old space
|
||||
/// freed; a new chunk in new space. The index is updated when the chunk's
|
||||
/// address, size or filter mask changed.
|
||||
fn store_chunk(
|
||||
img: &mut Image<'_>,
|
||||
ce: &mut ChunkedEdit<'_>,
|
||||
existing: Option<&ChunkInfo>,
|
||||
scaled: &[u64],
|
||||
buf: Vec<u8>,
|
||||
) -> Result<(), Error> {
|
||||
let t = ce.t;
|
||||
// As libhdf5 does: an optional filter that fails (LZF that does not
|
||||
// shrink the chunk) is skipped and its mask bit set.
|
||||
let (bytes, mask) = match &t.pipeline {
|
||||
Some(p) => clawhdf5_format::filters::compress_chunk_masked(&buf, p, t.es as u32)?,
|
||||
None => (buf, 0u32),
|
||||
};
|
||||
let len = bytes.len() as u64;
|
||||
let placed = match existing {
|
||||
Some(info) if t.pipeline.is_none() => {
|
||||
if u64::from(info.chunk_size) != len {
|
||||
return Err(Error::Unsupported(
|
||||
"unfiltered chunk stored at an unexpected size".into(),
|
||||
));
|
||||
}
|
||||
img.write(info.address, &bytes)?;
|
||||
None
|
||||
}
|
||||
// Rewritten where it is when it still fits, or when it is the last
|
||||
// thing in the file (the chunk an append keeps rewriting usually is)
|
||||
// and can grow there.
|
||||
Some(info)
|
||||
if len <= u64::from(info.chunk_size)
|
||||
|| img.grow_tail(info.address, u64::from(info.chunk_size), len)? =>
|
||||
{
|
||||
img.write(info.address, &bytes)?;
|
||||
(len != u64::from(info.chunk_size) || info.filter_mask != mask).then_some(Elem {
|
||||
addr: info.address,
|
||||
size: len,
|
||||
mask,
|
||||
})
|
||||
}
|
||||
_ => {
|
||||
let a = img.alloc_reusing(len)?;
|
||||
img.write(a, &bytes)?;
|
||||
if let Some(info) = existing {
|
||||
img.free(info.address, u64::from(info.chunk_size));
|
||||
}
|
||||
Some(Elem {
|
||||
addr: a,
|
||||
size: len,
|
||||
mask,
|
||||
})
|
||||
}
|
||||
};
|
||||
if let Some(e) = placed {
|
||||
ce.set(img, scaled, e)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn img_read<'a>(f: &'a File, info: &ChunkInfo) -> Result<&'a [u8], Error> {
|
||||
let start = usize::try_from(info.address)
|
||||
.map_err(|_| Error::Unsupported("chunk address out of range".into()))?;
|
||||
|
||||
@@ -98,8 +98,8 @@ fn errors_leave_the_file_untouched() {
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_))));
|
||||
assert!(ed.write_all("missing", &[0; 4]).is_err());
|
||||
// Wrong length, wrong type, outside the extent, beyond maxshape,
|
||||
// shrinking, a rank change.
|
||||
// Wrong length, wrong type, outside the extent, beyond maxshape, a
|
||||
// rank change, resizing a dataset that is not chunked.
|
||||
assert!(matches!(
|
||||
ed.write_all("flat", &[0; 7]),
|
||||
Err(Error::InvalidArgument(_))
|
||||
@@ -116,7 +116,10 @@ fn errors_leave_the_file_untouched() {
|
||||
ed.resize("raw", &[3, 5]),
|
||||
Err(Error::InvalidArgument(_))
|
||||
));
|
||||
assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_))));
|
||||
assert!(matches!(
|
||||
ed.resize("flat", &[2]),
|
||||
Err(Error::Unsupported(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
ed.resize("ext", &[4, 1]),
|
||||
Err(Error::InvalidArgument(_))
|
||||
@@ -136,3 +139,26 @@ fn errors_leave_the_file_untouched() {
|
||||
drop(ed);
|
||||
assert!(std::fs::read(&path).unwrap() == before);
|
||||
}
|
||||
|
||||
/// Shrinking and growing again on a file clawhdf5 wrote: elements that come
|
||||
/// back read as the fill value, the ones kept keep their values.
|
||||
#[test]
|
||||
fn shrink_then_grow_reads_fill() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = sample(dir.path());
|
||||
{
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
ed.resize("ext", &[2]).unwrap();
|
||||
ed.resize("ext", &[9]).unwrap();
|
||||
ed.resize("raw", &[1, 4]).unwrap();
|
||||
ed.resize("raw", &[3, 4]).unwrap();
|
||||
}
|
||||
let f = File::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
f.dataset("ext").unwrap().read_i32().unwrap(),
|
||||
[0, 1, 0, 0, 0, 0, 0, 0, 0]
|
||||
);
|
||||
let mut raw = vec![0.0f64; 12];
|
||||
raw[..4].fill(0.5);
|
||||
assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user