quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

clawhdf5: FileEditor modifies existing files in place

Browse Source 
New clawhdf5::FileEditor opens an HDF5 file (h5py-written at any libver,
HDF5 2.0 format included, or clawhdf5-written) under an exclusive flock
and changes only what an edit touches:
- write_selection/write_all/write_values: compact, contiguous (also
  late-allocated) and chunked datasets, any selection. Chunks are decoded,
  updated and re-encoded; a filtered chunk that no longer fits moves to the
  end of the file unless it is the file's last structure, which grows in
  place. New chunks go into v1 B-tree, Extensible Array (paged data blocks
  included), Fixed Array and single-chunk indexes, created on first use.
- resize: grow chunked datasets up to maxshape.
- set_attr: add/replace compact attributes, in a NIL slot or a new
  continuation chunk.
Each edit is planned in an in-memory image and refused whole
(Error::Unsupported) when any part is unsupported (v2 B-tree / implicit
new chunks, shrinking, vlen/reference data, dense or order-tracked
attributes, cache images, paged/persistent free space). Commit writes and
syncs new space before patching existing bytes. Layout v5 (HDF5 2.0)
array indexes use 8-byte filtered chunk sizes, as libhdf5 does.

Error gains Unsupported/InvalidArgument/Locked and is #[non_exhaustive];
the Python bindings map them. build_attr_message is public.

Tests (h5py, h5dump, h5rs check --data after every round; h5py r+
afterwards): appends crossing EA super/data blocks and B-tree splits, the
same B-tree node counts and EA statistics as libhdf5 for the same writes
(in order, reversed and shuffled; paged blocks), every layout and chunk
index overwritten under random selections, attributes to continuation
chunks, random operations against a model, refused edits leave the file
byte-identical, locking.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 13:35:06 -05:00
co-authored by Claude Opus 5.5
parent f2ff2c424f
commit 3c89a31df0
14 changed files with 4657 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/type_builders.rs
+2 -1
View File
@@ -296,7 +296,8 @@ impl EnumTypeBuilder {
// ---- Attribute helper ----
pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage {
/// The attribute message the writers store for `value` under `name`.
pub fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage {
match value {
AttrValue::F64(v) => AttributeMessage {
name: name.to_string(),
crates/clawhdf5-py/src/lib.rs
+9 -2
View File
@@ -66,7 +66,9 @@ fn _panic_for_test() -> PyResult<()> {
/// - I/O errors -> `PyIOError`
/// - Format/parsing errors -> `PyValueError`
/// - Missing dataset/path errors -> `PyKeyError`
/// - Other errors -> `PyOSError`
/// - Invalid arguments -> `PyValueError`
/// - Unsupported operations -> `PyNotImplementedError`
/// - Other errors (a locked file, ...) -> `PyOSError`
pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr {
use clawhdf5_rs::Error;
match &e {
@@ -79,9 +81,14 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr {
| Error::ZeroCopyNotContiguous
| Error::ZeroCopyNonNativeEndian
| Error::ZeroCopyTypeMismatch { .. }
| Error::ZeroCopyUnaligned { .. } => {
| Error::ZeroCopyUnaligned { .. }
| Error::InvalidArgument(_) => {
PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string())
}
Error::Unsupported(_) => {
PyErr::new::<pyo3::exceptions::PyNotImplementedError, _>(e.to_string())
}
_ => PyErr::new::<pyo3::exceptions::PyOSError, _>(e.to_string()),
}
}
crates/clawhdf5-tools/tests/edit_interop.rs
+1189
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5/src/edit/btree1.rs
+403
View File
@@ -0,0 +1,403 @@
//! Inserting into (and updating) a version-1 B-tree chunk index (node type
//! 1; layout versions 1-3), as libhdf5's `H5B_insert` does:
//!
//! - keys compare lexicographically over the chunk offsets *and* the
//! element-size coordinate (0 in a chunk's own key), so a node's final
//! ("right") key after an append is the last chunk's offsets with the
//! element-size coordinate set to the element size — the smallest key
//! greater than that chunk, which is what libhdf5 writes;
//! - a full node (2K children) splits before the insertion: the right-most
//! node of a level keeps 90% of its children, the left-most 10%, any other
//! half (libhdf5's default split ratios); siblings are relinked;
//! - a full root splits by moving its left half to a new node, so the root
//! keeps its address (the layout message never changes).
//!
//! Version-1 B-tree nodes carry no checksum.
use std::cmp::Ordering;
use crate::edit::image::{Image, get_uint, put_uint, undef};
use crate::error::Error;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Key {
pub(crate) size: u32,
pub(crate) mask: u32,
/// Offsets in every dimension, the element-size one last.
pub(crate) offs: Vec<u64>,
}
fn cmp(a: &[u64], b: &[u64]) -> Ordering {
a.cmp(b)
}
#[derive(Debug, Clone)]
struct Node {
addr: u64,
level: u8,
left: u64,
right: u64,
/// `children.len() + 1` keys.
keys: Vec<Key>,
children: Vec<u64>,
}
pub(crate) struct BTree1 {
root: u64,
/// Children per node at most (2K).
two_k: usize,
ndims: usize,
elem_size: u64,
}
fn bad(why: &str) -> Error {
Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError(
format!("chunk B-tree: {why}"),
))
}
enum Ins {
Done,
/// The node split; the new right sibling and its first key.
Split(Key, u64),
}
impl BTree1 {
/// `k` is the file's chunk B-tree K (children per node are 2K);
/// `ndims` counts the element-size dimension.
pub(crate) fn new(root: u64, k: u16, ndims: usize, elem_size: u64) -> Result<Self, Error> {
if k == 0 || ndims < 2 {
return Err(bad("bad parameters"));
}
Ok(Self {
root,
two_k: 2 * k as usize,
ndims,
elem_size,
})
}
fn key_size(&self) -> usize {
8 + 8 * self.ndims
}
fn node_size(&self, os: u8) -> usize {
let os = os as usize;
8 + 2 * os + (self.two_k + 1) * self.key_size() + self.two_k * os
}
fn read(&self, img: &Image<'_>, addr: u64) -> Result<Node, Error> {
let os = img.os;
let osz = os as usize;
let d = img.read(addr, 8 + 2 * osz)?;
if &d[0..4] != b"TREE" || d[4] != 1 {
return Err(bad("not a chunk B-tree node"));
}
let level = d[5];
let n = u16::from_le_bytes([d[6], d[7]]) as usize;
if n > self.two_k {
return Err(bad("node holds more children than 2K"));
}
let left = get_uint(&d[8..], os);
let right = get_uint(&d[8 + osz..], os);
let ks = self.key_size();
let body = img.read(addr + 8 + 2 * osz as u64, (n + 1) * ks + n * osz)?;
let mut keys = Vec::with_capacity(n + 1);
let mut children = Vec::with_capacity(n);
let mut p = 0;
for i in 0..=n {
let k = &body[p..p + ks];
keys.push(Key {
size: u32::from_le_bytes([k[0], k[1], k[2], k[3]]),
mask: u32::from_le_bytes([k[4], k[5], k[6], k[7]]),
offs: (0..self.ndims)
.map(|d| {
u64::from_le_bytes(k[8 + 8 * d..16 + 8 * d].try_into().unwrap_or([0; 8]))
})
.collect(),
});
p += ks;
if i < n {
children.push(get_uint(&body[p..], os));
p += osz;
}
}
Ok(Node {
addr,
level,
left,
right,
keys,
children,
})
}
fn write(&self, img: &mut Image<'_>, node: &Node) -> Result<(), Error> {
let os = img.os;
let osz = os as usize;
let mut d = vec![0u8; self.node_size(os)];
d[0..4].copy_from_slice(b"TREE");
d[4] = 1;
d[5] = node.level;
d[6..8].copy_from_slice(&(node.children.len() as u16).to_le_bytes());
put_uint(&mut d[8..], node.left, os);
put_uint(&mut d[8 + osz..], node.right, os);
let ks = self.key_size();
let mut p = 8 + 2 * osz;
for (i, k) in node.keys.iter().enumerate() {
d[p..p + 4].copy_from_slice(&k.size.to_le_bytes());
d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes());
for (j, o) in k.offs.iter().enumerate() {
d[p + 8 + 8 * j..p + 16 + 8 * j].copy_from_slice(&o.to_le_bytes());
}
p += ks;
if i < node.children.len() {
put_uint(&mut d[p..], node.children[i], os);
p += osz;
}
}
// Unused key/child slots stay zero, as libhdf5 leaves them.
img.write(node.addr, &d)
}
/// Create a tree holding one chunk; returns it (its root is a new leaf).
pub(crate) fn create(
img: &mut Image<'_>,
k: u16,
ndims: usize,
elem_size: u64,
key: Key,
addr: u64,
) -> Result<Self, Error> {
let mut t = Self::new(0, k, ndims, elem_size)?;
let root = img.alloc(t.node_size(img.os) as u64)?;
t.root = root;
let right = t.right_key_after(&key);
let node = Node {
addr: root,
level: 0,
left: undef(img.os),
right: undef(img.os),
keys: vec![key, right],
children: vec![addr],
};
t.write(img, &node)?;
Ok(t)
}
pub(crate) fn root(&self) -> u64 {
self.root
}
/// The smallest key above chunk `key`: its offsets with the element-size
/// coordinate one element in (what libhdf5 writes as a right key).
fn right_key_after(&self, key: &Key) -> Key {
let mut offs = key.offs.clone();
if let Some(last) = offs.last_mut() {
*last = self.elem_size;
}
Key {
size: 0,
mask: 0,
offs,
}
}
/// Insert chunk `key` at address `addr`, or update it when the tree
/// already has a chunk at those offsets.
pub(crate) fn insert(&mut self, img: &mut Image<'_>, key: Key, addr: u64) -> Result<(), Error> {
if key.offs.len() != self.ndims || key.offs[self.ndims - 1] != 0 {
return Err(bad("bad chunk key"));
}
let root = self.read(img, self.root)?;
if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? {
// The root split: move its (left) half to a new node so the root
// keeps its address, then make the root the parent of both.
let old = self.read(img, self.root)?;
let right = self.read(img, right_addr)?;
let new_left = img.alloc(self.node_size(img.os) as u64)?;
let mut moved = old.clone();
moved.addr = new_left;
self.write(img, &moved)?;
let mut right = right;
right.left = new_left;
self.write(img, &right)?;
let first = old.keys[0].clone();
let last = right
.keys
.last()
.cloned()
.ok_or_else(|| bad("empty node"))?;
let new_root = Node {
addr: self.root,
level: old.level + 1,
left: undef(img.os),
right: undef(img.os),
keys: vec![first, mid, last],
children: vec![new_left, right_addr],
};
self.write(img, &new_root)?;
}
Ok(())
}
fn insert_at(
&self,
img: &mut Image<'_>,
mut node: Node,
key: &Key,
addr: u64,
depth: u8,
) -> Result<Ins, Error> {
if depth == 0 {
return Err(bad("tree too deep"));
}
let n = node.children.len();
if n == 0 {
return Err(bad("empty node"));
}
// The child whose range holds the key: the last i with
// keys[i] <= key (the first child when the key is below them all).
let mut i = node
.keys
.iter()
.take(n)
.rposition(|k| cmp(&k.offs, &key.offs) != Ordering::Greater)
.unwrap_or(0);
if node.level == 0 {
if node.keys[i].offs == key.offs {
node.keys[i].size = key.size;
node.keys[i].mask = key.mask;
node.children[i] = addr;
self.write(img, &node)?;
return Ok(Ins::Done);
}
// Insert after child i unless the key is below every child.
let pos = if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less {
0
} else {
i + 1
};
return self.add_child(img, node, pos, key.clone(), addr);
}
let child = self.read(img, node.children[i])?;
if child.level + 1 != node.level {
return Err(bad("inconsistent node levels"));
}
let ins = self.insert_at(img, child, key, addr, depth - 1)?;
let mut changed = false;
if cmp(&key.offs, &node.keys[0].offs) == Ordering::Less && i == 0 {
node.keys[0] = key.clone();
changed = true;
}
if cmp(&key.offs, &node.keys[n].offs) != Ordering::Less {
node.keys[n] = self.right_key_after(key);
changed = true;
}
match ins {
Ins::Done => {
if changed {
self.write(img, &node)?;
}
Ok(Ins::Done)
}
Ins::Split(mid, right) => {
i += 1;
self.add_child(img, node, i, mid, right)
}
}
}
/// Insert child `addr` with left key `key` at position `pos` of `node`
/// (splitting it first when full), and write what changed.
fn add_child(
&self,
img: &mut Image<'_>,
mut node: Node,
pos: usize,
key: Key,
addr: u64,
) -> Result<Ins, Error> {
let n = node.children.len();
if n < self.two_k {
Self::insert_child(self, &mut node, pos, key, addr);
self.write(img, &node)?;
return Ok(Ins::Done);
}
// Split first (H5B__split): how many children stay left.
let undefined = undef(img.os);
let mut nleft = if node.right == undefined {
(self.two_k as f64 * 0.9) as usize
} else if node.left == undefined {
(self.two_k as f64 * 0.1) as usize
} else {
self.two_k / 2
};
if pos < nleft && nleft == self.two_k {
nleft -= 1;
} else if pos >= nleft && nleft == 0 {
nleft += 1;
}
let right_addr = img.alloc(self.node_size(img.os) as u64)?;
let mut right = Node {
addr: right_addr,
level: node.level,
left: node.addr,
right: node.right,
keys: node.keys[nleft..].to_vec(),
children: node.children[nleft..].to_vec(),
};
if node.right != undefined {
let mut sib = self.read(img, node.right)?;
sib.left = right_addr;
self.write(img, &sib)?;
}
node.keys.truncate(nleft + 1);
node.children.truncate(nleft);
node.right = right_addr;
if pos <= nleft && !(pos == nleft && nleft < n && self.goes_right(&key, &right)) {
self.insert_child(&mut node, pos, key, addr);
} else {
self.insert_child(&mut right, pos - nleft, key, addr);
}
self.write(img, &node)?;
self.write(img, &right)?;
let mid = right.keys[0].clone();
Ok(Ins::Split(mid, right_addr))
}
/// For an insertion exactly at the split point: whether the key belongs
/// to the right half (it is not below the right half's first key).
fn goes_right(&self, key: &Key, right: &Node) -> bool {
cmp(&key.offs, &right.keys[0].offs) != Ordering::Less
}
fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) {
let n = node.children.len();
if node.level == 0 {
// A leaf: the new chunk's key goes at `pos`. At the end, the
// node's right key moves up to stay above the new chunk.
if pos == n {
let right = self.right_key_after(&key);
let last = node.keys.len() - 1;
if cmp(&node.keys[last].offs, &right.offs) == Ordering::Less {
node.keys[last] = right;
}
node.keys.insert(n, key);
} else {
node.keys.insert(pos, key);
}
} else {
// An internal node: `key` is the new child's left key, taking
// position `pos` (the child's range starts there).
if pos == n {
// A child split off the last child: its right key is the
// parent's right key already.
node.keys.insert(n, key);
} else {
node.keys.insert(pos, key);
}
}
node.children.insert(pos, addr);
}
}
crates/clawhdf5/src/edit/earray.rs
+512
View File
@@ -0,0 +1,512 @@
//! Setting elements of an Extensible Array chunk index (layout v4, index
//! type 4), creating the index block, super blocks, data blocks and data
//! block pages the element needs, exactly as `H5EA__lookup_elmt` creates
//! them — including the header statistics libhdf5 keeps (blocks created,
//! their bytes, elements realised, one past the highest index set) and the
//! "block offset" each data block records.
use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef};
use crate::error::Error;
/// A chunk index element.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Elem {
pub(crate) addr: u64,
pub(crate) size: u64,
pub(crate) mask: u32,
}
/// Encode an element: the address, and for a filtered array the stored
/// size (in `elem_size - os - 4` bytes) and filter mask. `None` is the
/// fill element (undefined address, zero size and mask).
pub(crate) fn encode_elem(
e: Option<Elem>,
filtered: bool,
elem_size: usize,
os: u8,
) -> Result<Vec<u8>, Error> {
let osz = os as usize;
let mut b = vec![0u8; if filtered { elem_size } else { osz }];
let addr = e.map_or(undef(os), |e| e.addr);
put_uint(&mut b, addr, os);
if filtered {
let width = elem_size - osz - 4;
if let Some(e) = e {
if width < 8 && e.size >> (8 * width) != 0 {
return Err(Error::Unsupported(format!(
"filtered chunk of {} bytes does not fit the index's {width}-byte size field",
e.size
)));
}
b[osz..osz + width].copy_from_slice(&e.size.to_le_bytes()[..width]);
b[osz + width..].copy_from_slice(&e.mask.to_le_bytes());
}
}
Ok(b)
}
/// The width libhdf5 gives the stored-size field of a filtered chunk index
/// element for chunks of `chunk_bytes` bytes (`H5D__earray_idx_create`,
/// `H5D__farray_idx_create`): one byte more than the nominal size needs —
/// except under layout message version 5 (HDF5 2.0's own format), which
/// always uses 8 bytes.
pub(crate) fn chunk_size_len(chunk_bytes: u64, layout_version: u8) -> usize {
if layout_version >= 5 {
return 8;
}
let log2 = if chunk_bytes <= 1 {
0
} else {
63 - chunk_bytes.leading_zeros()
};
(1 + ((log2 + 8) / 8) as usize).min(8)
}
/// Creation parameters, in the layout message's order.
#[derive(Debug, Clone, Copy)]
pub(crate) struct EaParams {
pub(crate) max_nelmts_bits: u8,
pub(crate) idx_blk_elmts: u8,
pub(crate) sup_blk_min_data_ptrs: u8,
pub(crate) data_blk_min_elmts: u8,
pub(crate) max_dblk_page_nelmts_bits: u8,
}
#[derive(Debug, Clone, Copy)]
struct Level {
ndblks: u64,
dblk_nelmts: u64,
/// First element of the level, counted after the index block's own.
start_idx: u64,
/// Number of data blocks in the levels before this one.
start_dblk: u64,
}
/// An open Extensible Array.
pub(crate) struct Ea {
hdr: u64,
filtered: bool,
elem_size: usize,
p: EaParams,
/// nsuper_blks, super_blk_size, ndata_blks, data_blk_size,
/// max_idx_set, nelmts.
stats: [u64; 6],
iblock: u64,
levels: Vec<Level>,
/// Levels whose data blocks the index block addresses directly.
direct_levels: usize,
ndblk_addrs: usize,
nsblk_addrs: usize,
dirty_hdr: bool,
/// Checksummed ranges changed by `set` (start -> checksum position),
/// recomputed once by `finish`.
dirty: std::collections::BTreeMap<u64, u64>,
}
fn bad(why: &str) -> Error {
Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError(
format!("Extensible Array: {why}"),
))
}
impl Ea {
fn layout(p: EaParams) -> Result<(Vec<Level>, usize, usize, usize), Error> {
let dmin = u64::from(p.data_blk_min_elmts);
if dmin == 0 || !dmin.is_power_of_two() || p.max_nelmts_bits > 64 {
return Err(bad("bad creation parameters"));
}
let nsblks =
1 + (p.max_nelmts_bits as usize).saturating_sub(dmin.trailing_zeros() as usize);
let mut levels = Vec::with_capacity(nsblks);
let (mut start_idx, mut start_dblk) = (0u64, 0u64);
for u in 0..nsblks {
let ndblks = 1u64.checked_shl((u / 2) as u32).unwrap_or(u64::MAX);
let dblk_nelmts = dmin.checked_shl(u.div_ceil(2) as u32).unwrap_or(u64::MAX);
levels.push(Level {
ndblks,
dblk_nelmts,
start_idx,
start_dblk,
});
start_idx = start_idx.saturating_add(ndblks.saturating_mul(dblk_nelmts));
start_dblk = start_dblk.saturating_add(ndblks);
}
let ndblk_addrs = 2 * (p.sup_blk_min_data_ptrs as usize).saturating_sub(1);
let mut direct_levels = 0;
let mut n = 0u64;
while n < ndblk_addrs as u64 {
if direct_levels >= levels.len() {
return Err(bad("index block holds more data blocks than the array"));
}
n += levels[direct_levels].ndblks;
direct_levels += 1;
}
if n != ndblk_addrs as u64 {
return Err(bad("index block ends mid super block"));
}
Ok((levels, direct_levels, ndblk_addrs, nsblks - direct_levels))
}
fn arr_off_size(&self) -> usize {
(self.p.max_nelmts_bits as usize).div_ceil(8)
}
fn page_nelmts(&self) -> u64 {
1u64.checked_shl(u32::from(self.p.max_dblk_page_nelmts_bits))
.unwrap_or(u64::MAX)
}
fn slot_size(&self, os: u8) -> usize {
if self.filtered {
self.elem_size
} else {
os as usize
}
}
/// Open the array whose header is at `hdr`.
pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result<Self, Error> {
let os = img.os;
let ls = img.ls as usize;
let size = 12 + 6 * ls + os as usize + 4;
let d = img.read(hdr, size)?;
if &d[0..4] != b"EAHD" || d[4] != 0 {
return Err(bad("bad header"));
}
let filtered = match d[5] {
0 => false,
1 => true,
_ => return Err(bad("unknown client")),
};
let elem_size = d[6] as usize;
if filtered && elem_size < os as usize + 5 {
return Err(bad("element too small"));
}
let p = EaParams {
max_nelmts_bits: d[7],
idx_blk_elmts: d[8],
data_blk_min_elmts: d[9],
sup_blk_min_data_ptrs: d[10],
max_dblk_page_nelmts_bits: d[11],
};
let mut stats = [0u64; 6];
for (k, s) in stats.iter_mut().enumerate() {
*s = get_uint(&d[12 + k * ls..], img.ls);
}
let iblock = get_uint(&d[12 + 6 * ls..], os);
let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4]));
if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored {
return Err(bad("header checksum mismatch"));
}
let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?;
Ok(Self {
hdr,
filtered,
elem_size,
p,
stats,
iblock,
levels,
direct_levels,
ndblk_addrs,
nsblk_addrs,
dirty_hdr: false,
dirty: Default::default(),
})
}
/// Create an empty array (header only; the index block comes with the
/// first element) and return it.
pub(crate) fn create(
img: &mut Image<'_>,
p: EaParams,
filtered: bool,
chunk_bytes: u64,
layout_version: u8,
) -> Result<Self, Error> {
let os = img.os;
let elem_size = if filtered {
os as usize + chunk_size_len(chunk_bytes, layout_version) + 4
} else {
os as usize
};
let size = 12 + 6 * img.ls as usize + os as usize + 4;
let hdr = img.alloc(size as u64)?;
let (levels, direct_levels, ndblk_addrs, nsblk_addrs) = Self::layout(p)?;
let mut ea = Self {
hdr,
filtered,
elem_size,
p,
stats: [0; 6],
iblock: undef(os),
levels,
direct_levels,
ndblk_addrs,
nsblk_addrs,
dirty_hdr: true,
dirty: Default::default(),
};
ea.write_header(img)?;
Ok(ea)
}
pub(crate) fn header_address(&self) -> u64 {
self.hdr
}
fn write_header(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
let os = img.os;
let ls = img.ls as usize;
let size = 12 + 6 * ls + os as usize + 4;
let mut d = vec![0u8; size];
d[0..4].copy_from_slice(b"EAHD");
d[4] = 0;
d[5] = u8::from(self.filtered);
d[6] = self.elem_size as u8;
d[7] = self.p.max_nelmts_bits;
d[8] = self.p.idx_blk_elmts;
d[9] = self.p.data_blk_min_elmts;
d[10] = self.p.sup_blk_min_data_ptrs;
d[11] = self.p.max_dblk_page_nelmts_bits;
for (k, s) in self.stats.iter().enumerate() {
put_uint(&mut d[12 + k * ls..], *s, img.ls);
}
put_uint(&mut d[12 + 6 * ls..], self.iblock, os);
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]);
d[size - 4..].copy_from_slice(&sum.to_le_bytes());
img.write(self.hdr, &d)?;
self.dirty_hdr = false;
Ok(())
}
/// Recompute the checksums of the blocks `set` changed; store changed
/// header statistics.
pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
for (start, end) in std::mem::take(&mut self.dirty) {
rechecksum(img, start, end)?;
}
if self.dirty_hdr {
self.write_header(img)?;
}
Ok(())
}
fn fill_elems(&self, n: u64, os: u8) -> Result<Vec<u8>, Error> {
let one = encode_elem(None, self.filtered, self.elem_size, os)?;
let n = usize::try_from(n).map_err(|_| bad("block too large"))?;
Ok(one.repeat(n))
}
fn iblock_prefix(&self, os: u8) -> u64 {
6 + u64::from(os)
}
fn iblock_len(&self, os: u8) -> u64 {
let osz = os as u64;
self.iblock_prefix(os)
+ u64::from(self.p.idx_blk_elmts) * self.slot_size(os) as u64
+ (self.ndblk_addrs + self.nsblk_addrs) as u64 * osz
}
fn create_iblock(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
let os = img.os;
let len = self.iblock_len(os);
let addr = img.alloc(len + 4)?;
let mut d = Vec::with_capacity(len as usize + 4);
d.extend_from_slice(b"EAIB");
d.push(0);
d.push(u8::from(self.filtered));
let mut a = vec![0u8; os as usize];
put_uint(&mut a, self.hdr, os);
d.extend_from_slice(&a);
d.extend_from_slice(&self.fill_elems(u64::from(self.p.idx_blk_elmts), os)?);
let u = undef(os).to_le_bytes();
for _ in 0..self.ndblk_addrs + self.nsblk_addrs {
d.extend_from_slice(&u[..os as usize]);
}
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d);
d.extend_from_slice(&sum.to_le_bytes());
img.write(addr, &d)?;
self.iblock = addr;
self.stats[5] += u64::from(self.p.idx_blk_elmts);
self.dirty_hdr = true;
Ok(())
}
fn block_prefix(&self, sig: &[u8; 4], off: u64, os: u8) -> Vec<u8> {
let mut d = Vec::new();
d.extend_from_slice(sig);
d.push(0);
d.push(u8::from(self.filtered));
let mut a = vec![0u8; os as usize];
put_uint(&mut a, self.hdr, os);
d.extend_from_slice(&a);
d.extend_from_slice(&off.to_le_bytes()[..self.arr_off_size()]);
d
}
fn dblk_prefix_len(&self, os: u8) -> u64 {
6 + u64::from(os) + self.arr_off_size() as u64
}
/// Create a data block of `nelmts` elements whose recorded block offset
/// is `off`; returns its address.
fn create_dblock(&mut self, img: &mut Image<'_>, nelmts: u64, off: u64) -> Result<u64, Error> {
let os = img.os;
let es = self.slot_size(os) as u64;
let page = self.page_nelmts();
let prefix = self.block_prefix(b"EADB", off, os);
let (size, body) = if nelmts > page {
// Paged: only the prefix (and its checksum) is written now; each
// page is written when an element in it is first set.
let npages = nelmts / page;
let size = prefix.len() as u64 + 4 + npages * (page * es + 4);
let mut d = prefix;
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d);
d.extend_from_slice(&sum.to_le_bytes());
(size, d)
} else {
let mut d = prefix;
d.extend_from_slice(&self.fill_elems(nelmts, os)?);
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d);
d.extend_from_slice(&sum.to_le_bytes());
(d.len() as u64, d)
};
let addr = img.alloc(size)?;
img.write(addr, &body)?;
self.stats[2] += 1;
self.stats[3] += size;
self.stats[5] += nelmts;
self.dirty_hdr = true;
Ok(addr)
}
/// Set element `idx` to `e`.
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
let os = img.os;
let osz = u64::from(os);
let es = self.slot_size(os) as u64;
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
if self.iblock == undef(os) {
self.create_iblock(img)?;
}
let ib = self.iblock;
let ib_len = self.iblock_len(os);
let idx_blk = u64::from(self.p.idx_blk_elmts);
if idx < idx_blk {
img.write(ib + self.iblock_prefix(os) + idx * es, &enc)?;
self.dirty.insert(ib, ib + ib_len);
} else {
let rel = idx - idx_blk;
let u = self
.levels
.iter()
.position(|l| {
rel < l
.start_idx
.saturating_add(l.ndblks.saturating_mul(l.dblk_nelmts))
})
.ok_or_else(|| bad("index beyond the array's maximum"))?;
let l = self.levels[u];
let dblks_at = ib + self.iblock_prefix(os) + idx_blk * es;
if u < self.direct_levels {
if l.dblk_nelmts > self.page_nelmts() {
return Err(Error::Unsupported(
"Extensible Array index block addressing a paged data block".into(),
));
}
let local = (rel - l.start_idx) / l.dblk_nelmts;
let dblk_idx = l.start_dblk + local;
let slot = dblks_at + dblk_idx * osz;
let mut addr = get_uint(&img.read(slot, os as usize)?, os);
if addr == undef(os) {
// libhdf5 records start_idx + (global data block index)
// * nelmts here (H5EA__lookup_elmt), not the block's
// own first element; kept for byte-for-byte parity.
let off = l.start_idx + dblk_idx * l.dblk_nelmts;
addr = self.create_dblock(img, l.dblk_nelmts, off)?;
let mut a = vec![0u8; os as usize];
put_uint(&mut a, addr, os);
img.write(slot, &a)?;
self.dirty.insert(ib, ib + ib_len);
}
let within = (rel - l.start_idx) % l.dblk_nelmts;
let at = addr + self.dblk_prefix_len(os) + within * es;
img.write(at, &enc)?;
self.dirty
.insert(addr, addr + self.dblk_prefix_len(os) + l.dblk_nelmts * es);
} else {
let s = (u - self.direct_levels) as u64;
let sslot = dblks_at + self.ndblk_addrs as u64 * osz + s * osz;
let page = self.page_nelmts();
let npages = if l.dblk_nelmts > page {
l.dblk_nelmts / page
} else {
0
};
let bitmap_len = npages.div_ceil(8) * l.ndblks;
let sb_prefix = self.dblk_prefix_len(os);
let sb_len = sb_prefix + bitmap_len + l.ndblks * osz;
let mut sb = get_uint(&img.read(sslot, os as usize)?, os);
if sb == undef(os) {
let mut d = self.block_prefix(b"EASB", l.start_idx, os);
d.resize(d.len() + bitmap_len as usize, 0);
let u8s = undef(os).to_le_bytes();
for _ in 0..l.ndblks {
d.extend_from_slice(&u8s[..os as usize]);
}
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d);
d.extend_from_slice(&sum.to_le_bytes());
sb = img.alloc(d.len() as u64)?;
img.write(sb, &d)?;
self.stats[0] += 1;
self.stats[1] += d.len() as u64;
self.dirty_hdr = true;
let mut a = vec![0u8; os as usize];
put_uint(&mut a, sb, os);
img.write(sslot, &a)?;
self.dirty.insert(ib, ib + ib_len);
}
let local = (rel - l.start_idx) / l.dblk_nelmts;
let dslot = sb + sb_prefix + bitmap_len + local * osz;
let mut addr = get_uint(&img.read(dslot, os as usize)?, os);
if addr == undef(os) {
let off = l.start_idx + local * l.dblk_nelmts;
addr = self.create_dblock(img, l.dblk_nelmts, off)?;
let mut a = vec![0u8; os as usize];
put_uint(&mut a, addr, os);
img.write(dslot, &a)?;
self.dirty.insert(sb, sb + sb_len);
}
let within = (rel - l.start_idx) % l.dblk_nelmts;
let dprefix = self.dblk_prefix_len(os);
if npages == 0 {
img.write(addr + dprefix + within * es, &enc)?;
self.dirty.insert(addr, addr + dprefix + l.dblk_nelmts * es);
} else {
let pg = within / page;
let page_at = addr + dprefix + 4 + pg * (page * es + 4);
let bit = local * npages + pg;
let bpos = sb + sb_prefix + bit / 8;
let mut byte = img.read(bpos, 1)?[0];
let mask = 0x80u8 >> (bit % 8);
if byte & mask == 0 {
let fill = self.fill_elems(page, os)?;
img.write(page_at, &fill)?;
byte |= mask;
img.write(bpos, &[byte])?;
self.dirty.insert(sb, sb + sb_len);
}
img.write(page_at + (within % page) * es, &enc)?;
self.dirty.insert(page_at, page_at + page * es);
}
}
}
if idx + 1 > self.stats[4] {
self.stats[4] = idx + 1;
self.dirty_hdr = true;
}
Ok(())
}
}
crates/clawhdf5/src/edit/farray.rs
+185
View File
@@ -0,0 +1,185 @@
//! Setting elements of a Fixed Array chunk index (layout v4, index type 3),
//! creating the array (header and data block) when the dataset has none
//! yet, and a data block page when an element in it is first set.
use crate::edit::earray::{Elem, chunk_size_len, encode_elem};
use crate::edit::image::{Image, get_uint, put_uint, rechecksum, undef};
use crate::error::Error;
pub(crate) struct Fa {
filtered: bool,
elem_size: usize,
page_bits: u8,
nelmts: u64,
dblk: u64,
/// Checksummed ranges changed by `set`, recomputed by `finish`.
dirty: std::collections::BTreeMap<u64, u64>,
}
fn bad(why: &str) -> Error {
Error::Format(clawhdf5_format::error::FormatError::ChunkedReadError(
format!("Fixed Array: {why}"),
))
}
impl Fa {
fn slot(&self, os: u8) -> u64 {
if self.filtered {
self.elem_size as u64
} else {
u64::from(os)
}
}
fn page(&self) -> u64 {
1u64.checked_shl(u32::from(self.page_bits))
.unwrap_or(u64::MAX)
}
/// Open the array whose header is at `hdr`.
pub(crate) fn open(img: &Image<'_>, hdr: u64) -> Result<Self, Error> {
let os = img.os;
let size = 8 + img.ls as usize + os as usize + 4;
let d = img.read(hdr, size)?;
if &d[0..4] != b"FAHD" || d[4] != 0 {
return Err(bad("bad header"));
}
let filtered = match d[5] {
0 => false,
1 => true,
_ => return Err(bad("unknown client")),
};
let stored = u32::from_le_bytes(d[size - 4..].try_into().unwrap_or([0; 4]));
if clawhdf5_format::checksum::jenkins_lookup3(&d[..size - 4]) != stored {
return Err(bad("header checksum mismatch"));
}
let fa = Self {
filtered,
elem_size: d[6] as usize,
page_bits: d[7],
nelmts: get_uint(&d[8..], img.ls),
dblk: get_uint(&d[8 + img.ls as usize..], os),
dirty: Default::default(),
};
if fa.filtered && fa.elem_size < os as usize + 5 {
return Err(bad("element too small"));
}
if fa.page_bits >= 64 || fa.dblk == undef(os) {
return Err(bad("bad header fields"));
}
Ok(fa)
}
/// Create an array of `nelmts` fill elements; returns it and its
/// header address.
pub(crate) fn create(
img: &mut Image<'_>,
nelmts: u64,
page_bits: u8,
filtered: bool,
chunk_bytes: u64,
layout_version: u8,
) -> Result<(Self, u64), Error> {
let os = img.os;
let osz = os as usize;
let elem_size = if filtered {
osz + chunk_size_len(chunk_bytes, layout_version) + 4
} else {
osz
};
let mut fa = Self {
filtered,
elem_size,
page_bits,
nelmts,
dblk: 0,
dirty: Default::default(),
};
let hsize = 8 + img.ls as usize + osz + 4;
let hdr = img.alloc(hsize as u64)?;
// Data block.
let fill = encode_elem(None, filtered, elem_size, os)?;
let mut d = Vec::new();
d.extend_from_slice(b"FADB");
d.push(0);
d.push(u8::from(filtered));
let mut a = vec![0u8; osz];
put_uint(&mut a, hdr, os);
d.extend_from_slice(&a);
let page = fa.page();
let n = usize::try_from(nelmts).map_err(|_| bad("too many elements"))?;
let total = if nelmts > page {
let npages = nelmts.div_ceil(page);
d.resize(d.len() + npages.div_ceil(8) as usize, 0);
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d);
d.extend_from_slice(&sum.to_le_bytes());
// Pages are written when first used; their space is reserved.
d.len() as u64 + nelmts * fa.slot(os) + npages * 4
} else {
d.extend_from_slice(&fill.repeat(n));
let sum = clawhdf5_format::checksum::jenkins_lookup3(&d);
d.extend_from_slice(&sum.to_le_bytes());
d.len() as u64
};
let dblk = img.alloc(total)?;
img.write(dblk, &d)?;
fa.dblk = dblk;
let mut h = vec![0u8; hsize];
h[0..4].copy_from_slice(b"FAHD");
h[5] = u8::from(filtered);
h[6] = elem_size as u8;
h[7] = page_bits;
put_uint(&mut h[8..], nelmts, img.ls);
put_uint(&mut h[8 + img.ls as usize..], dblk, os);
let sum = clawhdf5_format::checksum::jenkins_lookup3(&h[..hsize - 4]);
h[hsize - 4..].copy_from_slice(&sum.to_le_bytes());
img.write(hdr, &h)?;
Ok((fa, hdr))
}
/// Set element `idx` to `e`.
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
let os = img.os;
if idx >= self.nelmts {
return Err(bad("index beyond the array"));
}
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
let es = self.slot(os);
let prefix = 6 + u64::from(os);
let page = self.page();
if self.nelmts <= page {
img.write(self.dblk + prefix + idx * es, &enc)?;
self.dirty
.insert(self.dblk, self.dblk + prefix + self.nelmts * es);
return Ok(());
}
let npages = self.nelmts.div_ceil(page);
let bitmap_len = npages.div_ceil(8);
let pages_at = self.dblk + prefix + bitmap_len + 4;
let p = idx / page;
let count = page.min(self.nelmts - p * page);
let page_at = pages_at + p * (page * es + 4);
let bpos = self.dblk + prefix + p / 8;
let mut byte = img.read(bpos, 1)?[0];
let mask = 0x80u8 >> (p % 8);
if byte & mask == 0 {
let fill = encode_elem(None, self.filtered, self.elem_size, os)?;
img.write(page_at, &fill.repeat(count as usize))?;
byte |= mask;
img.write(bpos, &[byte])?;
self.dirty
.insert(self.dblk, self.dblk + prefix + bitmap_len);
}
img.write(page_at + (idx % page) * es, &enc)?;
self.dirty.insert(page_at, page_at + count * es);
Ok(())
}
/// Recompute the checksums of the blocks and pages `set` changed.
pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
for (start, end) in std::mem::take(&mut self.dirty) {
rechecksum(img, start, end)?;
}
Ok(())
}
}
crates/clawhdf5/src/edit/image.rs
+317
View File
@@ -0,0 +1,317 @@
//! The file as one edit sees it: the bytes on disk plus the edit's pending
//! writes, and an allocator that hands out space at the end of the file.
//!
//! An edit never writes to the file while it is being planned. Every change
//! is recorded here first (reads see them), so an edit that fails half-way —
//! a filter that cannot encode, a chunk index this code does not handle —
//! leaves the file exactly as it was. [`Image::commit`] then writes the
//! changes in an order that keeps the old metadata valid for as long as
//! possible (see there).
use std::collections::BTreeMap;
use std::io::{Seek, SeekFrom, Write};
use crate::error::Error;
/// Pending writes over the file's bytes, addressed as HDF5 addresses
/// (relative to the superblock).
pub(crate) struct Image<'a> {
/// The file from the superblock to its recorded end of allocation.
base: &'a [u8],
/// Pending writes: start address -> bytes. Never overlapping.
patches: BTreeMap<u64, Vec<u8>>,
/// End of allocated space (grows with [`Self::alloc`]).
eoa: u64,
/// The end of allocated space when the edit started.
old_eoa: u64,
/// Width of addresses and lengths in the file.
pub(crate) os: u8,
pub(crate) ls: u8,
}
impl<'a> Image<'a> {
pub(crate) fn new(base: &'a [u8], os: u8, ls: u8) -> Self {
let eoa = base.len() as u64;
Self {
base,
patches: BTreeMap::new(),
eoa,
old_eoa: eoa,
os,
ls,
}
}
pub(crate) fn eoa(&self) -> u64 {
self.eoa
}
pub(crate) fn old_eoa(&self) -> u64 {
self.old_eoa
}
/// Whether the edit changes anything.
pub(crate) fn is_dirty(&self) -> bool {
!self.patches.is_empty() || self.eoa != self.old_eoa
}
/// Allocate `size` bytes at the end of the file. The space reads as
/// zeros until written. Nothing is ever freed: space an edit stops
/// using (a relocated chunk, say) is leaked, as there is no free-space
/// manager.
pub(crate) fn alloc(&mut self, size: u64) -> Result<u64, Error> {
let addr = self.eoa;
let end = addr
.checked_add(size)
.filter(|&e| self.os >= 8 || e < (1u64 << (8 * u32::from(self.os))) - 1)
.ok_or_else(|| Error::Unsupported("file would exceed its address size".into()))?;
self.eoa = end;
Ok(addr)
}
/// If `[addr, addr + old_len)` is the last allocated space, grow it to
/// `new_len` bytes (a structure at the end of the file can grow where
/// it is) and return true.
pub(crate) fn grow_tail(
&mut self,
addr: u64,
old_len: u64,
new_len: u64,
) -> Result<bool, Error> {
if addr.checked_add(old_len) != Some(self.eoa) || new_len < old_len {
return Ok(false);
}
let old_end = self.eoa;
self.eoa = addr;
if let Err(e) = self.alloc(new_len) {
self.eoa = old_end;
return Err(e);
}
Ok(true)
}
/// `len` bytes at `addr`, with the pending writes applied.
pub(crate) fn read(&self, addr: u64, len: usize) -> Result<Vec<u8>, Error> {
let end = addr
.checked_add(len as u64)
.filter(|&e| e <= self.eoa)
.ok_or_else(|| {
Error::Format(clawhdf5_format::error::FormatError::UnexpectedEof {
expected: addr.saturating_add(len as u64) as usize,
available: self.eoa as usize,
})
})?;
let mut out = vec![0u8; len];
let base_len = self.base.len() as u64;
if addr < base_len {
let b_end = end.min(base_len);
out[..(b_end - addr) as usize]
.copy_from_slice(&self.base[addr as usize..b_end as usize]);
}
// Patches overlapping [addr, end): the last one starting before
// `end`, walking back while they still reach `addr`.
for (&p_start, bytes) in self.patches.range(..end).rev() {
let p_end = p_start + bytes.len() as u64;
if p_end <= addr {
break;
}
let lo = p_start.max(addr);
let hi = p_end.min(end);
out[(lo - addr) as usize..(hi - addr) as usize]
.copy_from_slice(&bytes[(lo - p_start) as usize..(hi - p_start) as usize]);
}
Ok(out)
}
/// Record a write of `bytes` at `addr` (inside allocated space).
pub(crate) fn write(&mut self, addr: u64, bytes: &[u8]) -> Result<(), Error> {
if bytes.is_empty() {
return Ok(());
}
let end = addr
.checked_add(bytes.len() as u64)
.filter(|&e| e <= self.eoa)
.ok_or_else(|| Error::Unsupported("write past the end of allocated space".into()))?;
// Fast path: inside, or extending, the one patch that starts at or
// before `addr` and reaches it (sequential writes into a block, and
// chunks allocated back to back, stay linear).
if let Some((&p_start, p)) = self.patches.range_mut(..=addr).next_back()
&& p_start + p.len() as u64 >= addr
&& self
.patches
.range(addr + 1..end.max(addr + 1))
.next()
.is_none()
{
let p = self.patches.get_mut(&p_start).expect("found above");
let off = (addr - p_start) as usize;
if off + bytes.len() > p.len() {
p.resize(off + bytes.len(), 0);
}
p[off..off + bytes.len()].copy_from_slice(bytes);
return Ok(());
}
// Patches that overlap or touch [addr, end) merge into one.
let touching: Vec<u64> = self
.patches
.range(..=end)
.rev()
.take_while(|(s, b)| **s + b.len() as u64 >= addr)
.map(|(s, _)| *s)
.collect();
if touching.is_empty() {
self.patches.insert(addr, bytes.to_vec());
return Ok(());
}
let lo = touching.iter().copied().min().map_or(addr, |s| s.min(addr));
let hi = touching
.iter()
.map(|s| s + self.patches[s].len() as u64)
.max()
.map_or(end, |e| e.max(end));
let mut merged = self.read(lo, (hi - lo) as usize)?;
merged[(addr - lo) as usize..(end - lo) as usize].copy_from_slice(bytes);
for s in touching {
self.patches.remove(&s);
}
self.patches.insert(lo, merged);
Ok(())
}
/// Write the edit to `file`, whose superblock is at `user_block`.
///
/// Order: first everything in newly allocated space (new chunks, new
/// index blocks, relocated structures), which nothing on disk refers to
/// yet, then a sync; then the changes to existing bytes — raw data
/// overwritten in place and the metadata that links the new space in
/// (superblock end of file, chunk index entries, object header
/// messages) — then a sync. A crash during the first phase leaves the
/// file as it was (plus unreferenced bytes past its end of file); a
/// crash during the second can leave it inconsistent, as with libhdf5
/// without SWMR: there is no journal.
pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> {
let old_eoa = self.old_eoa;
let mut in_place: Vec<(u64, &[u8])> = Vec::new();
for (&addr, bytes) in &self.patches {
// A patch may run from existing bytes into new space (writes
// merge); its new part goes with the new space.
let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize;
let (old, new) = bytes.split_at(split);
if !new.is_empty() {
write_at(file, user_block + addr + split as u64, new)?;
}
if !old.is_empty() {
in_place.push((addr, old));
}
}
if self.eoa > old_eoa {
let want = user_block + self.eoa;
if file.metadata()?.len() < want {
file.set_len(want)?;
}
}
file.sync_data()?;
for (addr, bytes) in in_place {
write_at(file, user_block + addr, bytes)?;
}
file.sync_all()?;
Ok(())
}
}
fn write_at(file: &mut std::fs::File, pos: u64, bytes: &[u8]) -> Result<(), Error> {
file.seek(SeekFrom::Start(pos))?;
file.write_all(bytes)?;
Ok(())
}
/// Little-endian encode of `v` in `width` bytes.
pub(crate) fn put_uint(buf: &mut [u8], v: u64, width: u8) {
let w = width as usize;
buf[..w].copy_from_slice(&v.to_le_bytes()[..w]);
}
/// Little-endian decode of `width` bytes.
pub(crate) fn get_uint(buf: &[u8], width: u8) -> u64 {
let mut b = [0u8; 8];
b[..width as usize].copy_from_slice(&buf[..width as usize]);
u64::from_le_bytes(b)
}
/// The undefined address for `os`-byte addresses.
pub(crate) fn undef(os: u8) -> u64 {
if os >= 8 {
u64::MAX
} else {
(1u64 << (8 * u32::from(os))) - 1
}
}
/// Recompute the Jenkins checksum over `[start, end)` and store it at `end`.
pub(crate) fn rechecksum(img: &mut Image<'_>, start: u64, end: u64) -> Result<(), Error> {
let bytes = img.read(start, (end - start) as usize)?;
let sum = clawhdf5_format::checksum::jenkins_lookup3(&bytes);
img.write(end, &sum.to_le_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_see_writes_and_merges() {
let base = vec![1u8; 32];
let mut img = Image::new(&base, 8, 8);
img.write(4, &[9, 9]).unwrap();
img.write(8, &[7]).unwrap();
img.write(5, &[3, 3, 3]).unwrap(); // extends the first up to the second
assert_eq!(img.read(3, 7).unwrap(), vec![1, 9, 3, 3, 3, 7, 1]);
img.write(2, &[4, 4, 4, 4, 4, 4, 4, 4]).unwrap(); // covers both: merged
assert_eq!(img.patches.len(), 1);
assert_eq!(img.read(1, 10).unwrap(), vec![1, 4, 4, 4, 4, 4, 4, 4, 4, 1]);
let a = img.alloc(10).unwrap();
assert_eq!(a, 32);
assert_eq!(img.read(30, 4).unwrap(), vec![1, 1, 0, 0]);
img.write(40, &[5]).unwrap();
assert_eq!(img.read(39, 3).unwrap(), vec![0, 5, 0]);
assert!(img.write(42, &[1]).is_err());
}
/// Random reads and writes against a flat copy of the bytes.
#[test]
fn matches_a_flat_model() {
let base: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
let mut img = Image::new(&base, 8, 8);
img.alloc(100).unwrap();
let mut flat = base.clone();
flat.resize(300, 0);
let mut x = 12345u64;
let mut next = |n: u64| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x % n
};
for step in 0..5000 {
let at = next(300);
let len = 1 + next(20).min(299 - at);
if step % 3 == 0 {
assert_eq!(
img.read(at, len as usize).unwrap(),
flat[at as usize..(at + len) as usize]
);
} else {
let bytes: Vec<u8> = (0..len).map(|_| next(256) as u8).collect();
img.write(at, &bytes).unwrap();
flat[at as usize..(at + len) as usize].copy_from_slice(&bytes);
}
}
assert_eq!(img.read(0, 300).unwrap(), flat);
// Patches never overlap.
let mut end = 0;
for (s, b) in &img.patches {
assert!(*s >= end);
end = s + b.len() as u64;
}
}
}
crates/clawhdf5/src/edit/mod.rs
+1213
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5/src/edit/ohdr.rs
+504
View File
@@ -0,0 +1,504 @@
//! An object header as an edit sees it: every chunk and every message
//! (NIL and continuation messages included) with its position in the file,
//! so single messages can be changed in place, deleted (turned into NIL
//! messages) and added (into a NIL message big enough, or into a new
//! continuation chunk at the end of the file).
//!
//! Version-2 chunks carry a checksum, recomputed by [`Header::finish`] for
//! every chunk the edit touched; a version-1 header's message count is kept
//! up to date there too.
use std::collections::BTreeSet;
use crate::edit::image::{Image, get_uint, put_uint, rechecksum};
use crate::error::Error;
use clawhdf5_format::error::FormatError;
pub(crate) const MSG_NIL: u16 = 0x00;
pub(crate) const MSG_CONTINUATION: u16 = 0x10;
pub(crate) const MSG_ATTRIBUTE: u16 = 0x0C;
/// One message: where its header and body are, and what it is.
#[derive(Debug, Clone)]
pub(crate) struct Msg {
pub(crate) chunk: usize,
pub(crate) hdr_pos: u64,
pub(crate) data_pos: u64,
pub(crate) size: usize,
pub(crate) mtype: u16,
pub(crate) flags: u8,
pub(crate) corder: Option<u16>,
}
/// One chunk of the header.
#[derive(Debug, Clone)]
struct Chunk {
/// Where the checksummed bytes start (the `OHDR`/`OCHK` signature).
start: u64,
/// Where the checksum is (version 2 only).
checksum_at: Option<u64>,
/// Where the chunk's messages end.
end: u64,
/// Bytes at the end too few for a message header (version 2 only).
/// libhdf5 refuses a chunk with both a gap and a NIL message, so a
/// NIL message made in such a chunk must absorb the gap.
gap: u64,
}
#[derive(Debug)]
pub(crate) struct Header {
pub(crate) addr: u64,
pub(crate) version: u8,
/// Version-2 header flags (0 for version 1).
pub(crate) flags: u8,
chunks: Vec<Chunk>,
pub(crate) msgs: Vec<Msg>,
dirty: BTreeSet<usize>,
/// Messages added (a split NIL message, a new chunk's messages), for a
/// version-1 header's message count.
added: usize,
}
const MAX_CHUNKS: usize = 1024;
fn corrupt(why: &'static str) -> Error {
Error::Format(FormatError::InvalidObjectHeader(why))
}
impl Header {
/// Locate every chunk and message of the header at `addr`.
pub(crate) fn load(img: &Image<'_>, addr: u64) -> Result<Self, Error> {
let sig = img.read(addr, 4)?;
let mut h = Header {
addr,
version: 0,
flags: 0,
chunks: Vec::new(),
msgs: Vec::new(),
dirty: BTreeSet::new(),
added: 0,
};
let mut pending: Vec<(u64, u64)> = Vec::new();
if sig == b"OHDR" {
let pre = img.read(addr, 6)?;
if pre[4] != 2 {
return Err(corrupt("bad object header version"));
}
h.version = 2;
h.flags = pre[5];
let mut pos = addr + 6;
if h.flags & 0x20 != 0 {
pos += 16;
}
if h.flags & 0x10 != 0 {
pos += 4;
}
let w = 1u8 << (h.flags & 0x03);
let size = get_uint(&img.read(pos, w as usize)?, w);
pos += u64::from(w);
h.chunks.push(Chunk {
start: addr,
checksum_at: Some(pos + size),
end: pos + size,
gap: 0,
});
h.scan(img, 0, pos, pos + size, &mut pending)?;
} else {
let pre = img.read(addr, 16)?;
if pre[0] != 1 {
return Err(corrupt("bad object header version"));
}
h.version = 1;
let size = u64::from(u32::from_le_bytes([pre[8], pre[9], pre[10], pre[11]]));
h.chunks.push(Chunk {
start: addr,
checksum_at: None,
end: addr + 16 + size,
gap: 0,
});
h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?;
}
while let Some((caddr, clen)) = pending.pop() {
if h.chunks.len() >= MAX_CHUNKS {
return Err(corrupt("too many object header chunks"));
}
let idx = h.chunks.len();
if h.version == 2 {
if clen < 8 || img.read(caddr, 4)? != b"OCHK" {
return Err(corrupt("bad continuation chunk"));
}
h.chunks.push(Chunk {
start: caddr,
checksum_at: Some(caddr + clen - 4),
end: caddr + clen - 4,
gap: 0,
});
h.scan(img, idx, caddr + 4, caddr + clen - 4, &mut pending)?;
} else {
h.chunks.push(Chunk {
start: caddr,
checksum_at: None,
end: caddr + clen,
gap: 0,
});
h.scan(img, idx, caddr, caddr + clen, &mut pending)?;
}
}
Ok(h)
}
/// Size of a message header in this object header.
pub(crate) fn hsize(&self) -> usize {
match (self.version, self.flags & 0x04 != 0) {
(1, _) => 8,
(_, true) => 6,
_ => 4,
}
}
fn scan(
&mut self,
img: &Image<'_>,
chunk: usize,
start: u64,
end: u64,
pending: &mut Vec<(u64, u64)>,
) -> Result<(), Error> {
let hs = self.hsize() as u64;
let bytes = img.read(start, (end - start) as usize)?;
let mut p = 0usize;
while (p as u64) + hs <= end - start {
let b = &bytes[p..];
let (mtype, size, flags, corder) = if self.version == 1 {
(
u16::from_le_bytes([b[0], b[1]]),
u16::from_le_bytes([b[2], b[3]]) as usize,
b[4],
None,
)
} else {
(
u16::from(b[0]),
u16::from_le_bytes([b[1], b[2]]) as usize,
b[3],
(hs == 6).then(|| u16::from_le_bytes([b[4], b[5]])),
)
};
let data_off = p + hs as usize;
if data_off + size > bytes.len() {
return Err(corrupt("message size exceeds buffer end"));
}
if mtype == MSG_CONTINUATION {
let d = &bytes[data_off..data_off + size];
let os = img.os as usize;
let ls = img.ls as usize;
if d.len() < os + ls {
return Err(corrupt("short continuation message"));
}
pending.push((get_uint(d, img.os), get_uint(&d[os..], img.ls)));
}
self.msgs.push(Msg {
chunk,
hdr_pos: start + p as u64,
data_pos: start + data_off as u64,
size,
mtype,
flags,
corder,
});
p = data_off + size;
}
self.chunks[chunk].gap = (end - start) - p as u64;
Ok(())
}
/// After message `i` became a NIL message: if its chunk ends in a gap,
/// grow the NIL message over it (it must be the chunk's last message).
fn absorb_gap(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> {
let m = self.msgs[i].clone();
let c = &self.chunks[m.chunk];
if c.gap == 0 {
return Ok(());
}
if m.data_pos + m.size as u64 + c.gap != c.end {
return Err(Error::Unsupported(
"object header chunk ends in a gap that a free message cannot absorb".into(),
));
}
let new_size = m.size + c.gap as usize;
if new_size > usize::from(u16::MAX) {
return Err(Error::Unsupported("object header message too large".into()));
}
self.write_msg_header(img, m.hdr_pos, MSG_NIL, new_size, 0, m.corder)?;
img.write(m.data_pos, &vec![0u8; new_size])?;
self.msgs[i].size = new_size;
self.chunks[m.chunk].gap = 0;
Ok(())
}
/// The first message of type `mtype`.
pub(crate) fn find(&self, mtype: u16) -> Option<usize> {
self.msgs.iter().position(|m| m.mtype == mtype)
}
pub(crate) fn data(&self, img: &Image<'_>, i: usize) -> Result<Vec<u8>, Error> {
let m = &self.msgs[i];
img.read(m.data_pos, m.size)
}
/// Overwrite bytes of message `i`'s body, from `offset`.
pub(crate) fn patch(
&mut self,
img: &mut Image<'_>,
i: usize,
offset: usize,
bytes: &[u8],
) -> Result<(), Error> {
let m = &self.msgs[i];
if offset + bytes.len() > m.size {
return Err(Error::Unsupported(
"change does not fit the header message".into(),
));
}
img.write(m.data_pos + offset as u64, bytes)?;
self.dirty.insert(m.chunk);
Ok(())
}
fn write_msg_header(
&mut self,
img: &mut Image<'_>,
hdr_pos: u64,
mtype: u16,
size: usize,
flags: u8,
corder: Option<u16>,
) -> Result<(), Error> {
let mut h = vec![0u8; self.hsize()];
if self.version == 1 {
h[0..2].copy_from_slice(&mtype.to_le_bytes());
h[2..4].copy_from_slice(&(size as u16).to_le_bytes());
h[4] = flags;
} else {
h[0] = mtype as u8;
h[1..3].copy_from_slice(&(size as u16).to_le_bytes());
h[3] = flags;
if h.len() == 6 {
h[4..6].copy_from_slice(&corder.unwrap_or(0).to_le_bytes());
}
}
img.write(hdr_pos, &h)
}
/// Turn message `i` into a NIL message (its space becomes free).
pub(crate) fn delete(&mut self, img: &mut Image<'_>, i: usize) -> Result<(), Error> {
let m = self.msgs[i].clone();
self.write_msg_header(img, m.hdr_pos, MSG_NIL, m.size, 0, m.corder)?;
img.write(m.data_pos, &vec![0u8; m.size])?;
self.msgs[i].mtype = MSG_NIL;
self.msgs[i].flags = 0;
self.dirty.insert(m.chunk);
self.absorb_gap(img, i)
}
/// Body size a message of `len` bytes occupies (version 1 pads to 8).
fn padded(&self, len: usize) -> usize {
if self.version == 1 {
len.next_multiple_of(8)
} else {
len
}
}
/// Whether a free slot of `slot` bytes can take a body of `need` bytes:
/// exactly, or with room left for a NIL message after it.
fn fits(&self, slot: usize, need: usize) -> bool {
slot == need || slot >= need + self.hsize()
}
/// The smallest NIL message that can take `need` body bytes.
fn best_nil(&self, need: usize) -> Option<usize> {
self.msgs
.iter()
.enumerate()
.filter(|(_, m)| m.mtype == MSG_NIL && self.fits(m.size, need))
.min_by_key(|(_, m)| m.size)
.map(|(i, _)| i)
}
/// Put a message into slot `i` (a NIL message, or a message being
/// moved away), splitting off the rest as a NIL message.
fn place(
&mut self,
img: &mut Image<'_>,
i: usize,
mtype: u16,
flags: u8,
data: &[u8],
corder: Option<u16>,
) -> Result<(), Error> {
let slot = self.msgs[i].clone();
let need = self.padded(data.len());
debug_assert!(self.fits(slot.size, need));
let mut body = data.to_vec();
body.resize(need, 0);
self.write_msg_header(img, slot.hdr_pos, mtype, need, flags, corder)?;
img.write(slot.data_pos, &body)?;
self.msgs[i] = Msg {
size: need,
mtype,
flags,
corder,
..slot.clone()
};
if slot.size > need {
let hs = self.hsize();
let nil_hdr = slot.data_pos + need as u64;
let nil_size = slot.size - need - hs;
self.write_msg_header(img, nil_hdr, MSG_NIL, nil_size, 0, Some(0))?;
img.write(nil_hdr + hs as u64, &vec![0u8; nil_size])?;
self.msgs.push(Msg {
chunk: slot.chunk,
hdr_pos: nil_hdr,
data_pos: nil_hdr + hs as u64,
size: nil_size,
mtype: MSG_NIL,
flags: 0,
corder: (hs == 6).then_some(0),
});
self.added += 1;
let nil = self.msgs.len() - 1;
self.absorb_gap(img, nil)?;
}
self.dirty.insert(slot.chunk);
Ok(())
}
/// Add a message: into free space in the header when there is some,
/// else into a new continuation chunk at the end of the file (whose
/// continuation message takes a NIL slot, or the slot of another
/// message — an attribute if possible — that moves into the new chunk
/// with it).
pub(crate) fn insert(
&mut self,
img: &mut Image<'_>,
mtype: u16,
flags: u8,
data: &[u8],
corder: Option<u16>,
) -> Result<(), Error> {
if data.len() > usize::from(u16::MAX) {
return Err(Error::Unsupported(
"message larger than 64 KiB (would need dense storage)".into(),
));
}
let need = self.padded(data.len());
if let Some(i) = self.best_nil(need) {
return self.place(img, i, mtype, flags, data, corder);
}
let os = img.os as usize;
let ls = img.ls as usize;
let cont_need = self.padded(os + ls);
// Where the continuation message goes, and the message (if any)
// that moves out of that slot into the new chunk.
let (slot, moved) = match self.best_nil(cont_need) {
Some(i) => (i, None),
None => {
// Any message but a continuation can live in any chunk;
// prefer moving an attribute, then the smallest that fits.
let i = self
.msgs
.iter()
.enumerate()
.filter(|(_, m)| {
m.mtype != MSG_NIL
&& m.mtype != MSG_CONTINUATION
&& self.fits(m.size, cont_need)
})
.min_by_key(|(_, m)| (m.mtype != MSG_ATTRIBUTE, m.size))
.map(|(i, _)| i)
.ok_or_else(|| {
Error::Unsupported(
"no room in the object header for a continuation message".into(),
)
})?;
let m = self.msgs[i].clone();
let body = img.read(m.data_pos, m.size)?;
(i, Some((m, body)))
}
};
// The new chunk: [moved message] + new message + a NIL message
// holding spare room for later additions.
let hs = self.hsize();
let spare = 64usize;
let mut payload = hs + need;
if let Some((m, _)) = &moved {
payload += hs + m.size;
}
let msgs_len = payload + hs + spare;
let (prefix, suffix) = if self.version == 2 { (4, 4) } else { (0, 0) };
let chunk_len = prefix + msgs_len + suffix;
let caddr = img.alloc(chunk_len as u64)?;
if self.version == 2 {
img.write(caddr, b"OCHK")?;
}
let cidx = self.chunks.len();
self.chunks.push(Chunk {
start: caddr,
checksum_at: (self.version == 2).then_some(caddr + (prefix + msgs_len) as u64),
end: caddr + (prefix + msgs_len) as u64,
gap: 0,
});
let first = caddr + prefix as u64;
// Lay the chunk out as one NIL message, then place into it.
self.write_msg_header(img, first, MSG_NIL, msgs_len - hs, 0, Some(0))?;
self.msgs.push(Msg {
chunk: cidx,
hdr_pos: first,
data_pos: first + hs as u64,
size: msgs_len - hs,
mtype: MSG_NIL,
flags: 0,
corder: (hs == 6).then_some(0),
});
self.added += 1;
if let Some((m, body)) = &moved {
let nil = self.msgs.len() - 1;
self.place(img, nil, m.mtype, m.flags, body, m.corder)?;
}
let nil = self.msgs.len() - 1;
self.place(img, nil, mtype, flags, data, corder)?;
// Link it in.
let mut cont = vec![0u8; os + ls];
put_uint(&mut cont, caddr, img.os);
put_uint(&mut cont[os..], chunk_len as u64, img.ls);
if moved.is_some() {
self.msgs[slot].mtype = MSG_NIL; // its content now lives in the new chunk
}
self.place(img, slot, MSG_CONTINUATION, 0, &cont, Some(0))?;
self.dirty.insert(cidx);
Ok(())
}
/// Recompute the checksum of every changed version-2 chunk; store a
/// version-1 header's new message count.
pub(crate) fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
for &c in &self.dirty {
if let Some(at) = self.chunks[c].checksum_at {
rechecksum(img, self.chunks[c].start, at)?;
}
}
if self.version == 1 && self.added > 0 {
let old = u16::from_le_bytes(img.read(self.addr + 2, 2)?.try_into().unwrap_or([0; 2]));
let new = usize::from(old) + self.added;
let new = u16::try_from(new)
.map_err(|_| Error::Unsupported("too many object header messages".into()))?;
img.write(self.addr + 2, &new.to_le_bytes())?;
}
self.dirty.clear();
self.added = 0;
Ok(())
}
}
crates/clawhdf5/src/edit/select.rs
+142
View File
@@ -0,0 +1,142 @@
//! A selection as runs of consecutive elements along the last dimension, in
//! the order the selection's elements are numbered (row-major over a
//! hyperslab, as h5py and libhdf5 number them; a point list in its order).
use clawhdf5_format::selection::Selection;
use crate::error::Error;
/// Call `f(coords, len, src)` for each run: `len` elements starting at
/// `coords` (consecutive in the last dimension), which are elements
/// `src..src + len` of the selection. Returns the number of elements.
/// The selection must already be validated against `dims`.
pub(crate) fn for_each_run(
sel: &Selection,
dims: &[u64],
mut f: impl FnMut(&[u64], u64, u64) -> Result<(), Error>,
) -> Result<u64, Error> {
let rank = dims.len();
let mut src = 0u64;
match sel {
Selection::None => {}
Selection::Points(pts) => {
for p in pts {
f(p, 1, src)?;
src += 1;
}
}
Selection::All => {
if rank == 0 {
f(&[], 1, 0)?;
return Ok(1);
}
if dims.contains(&0) {
return Ok(0);
}
let last = dims[rank - 1];
let mut coords = vec![0u64; rank];
loop {
f(&coords, last, src)?;
src += last;
if !advance(&mut coords[..rank - 1], &dims[..rank - 1]) {
break;
}
}
}
Selection::Hyperslab {
start,
stride,
count,
block,
} => {
if rank == 0 {
return Err(Error::InvalidArgument(
"hyperslab selection on a scalar dataset".into(),
));
}
if (0..rank).any(|d| count[d] == 0 || block[d] == 0) {
return Ok(0);
}
// Per-dimension extent of the selection: j in 0..count*block.
let ext: Vec<u64> = (0..rank).map(|d| count[d] * block[d]).collect();
let coord = |d: usize, j: u64| start[d] + (j / block[d]) * stride[d] + j % block[d];
let l = rank - 1;
// Along the last dimension, blocks merge when they touch.
let merged = stride[l] == block[l] || count[l] == 1;
let mut js = vec![0u64; rank - 1];
let mut coords = vec![0u64; rank];
loop {
for (d, &j) in js.iter().enumerate() {
coords[d] = coord(d, j);
}
if merged {
coords[l] = start[l];
f(&coords, ext[l], src)?;
src += ext[l];
} else {
for c in 0..count[l] {
coords[l] = start[l] + c * stride[l];
f(&coords, block[l], src)?;
src += block[l];
}
}
if !advance(&mut js, &ext[..l]) {
break;
}
}
}
}
Ok(src)
}
/// Odometer step over `0..lim[d]`; false when it wraps around.
fn advance(v: &mut [u64], lim: &[u64]) -> bool {
for d in (0..v.len()).rev() {
v[d] += 1;
if v[d] < lim[d] {
return true;
}
v[d] = 0;
}
false
}
#[cfg(test)]
mod tests {
use super::*;
fn collect(sel: &Selection, dims: &[u64]) -> Vec<(Vec<u64>, u64, u64)> {
let mut out = Vec::new();
for_each_run(sel, dims, |c, n, s| {
out.push((c.to_vec(), n, s));
Ok(())
})
.unwrap();
out
}
#[test]
fn runs() {
assert_eq!(
collect(&Selection::All, &[2, 3]),
vec![(vec![0, 0], 3, 0), (vec![1, 0], 3, 3)]
);
let h = Selection::Hyperslab {
start: vec![1, 0],
stride: vec![2, 3],
count: vec![2, 2],
block: vec![1, 2],
};
assert_eq!(
collect(&h, &[5, 6]),
vec![
(vec![1, 0], 2, 0),
(vec![1, 3], 2, 2),
(vec![3, 0], 2, 4),
(vec![3, 3], 2, 6)
]
);
assert_eq!(collect(&Selection::All, &[]), vec![(vec![], 1, 0)]);
assert_eq!(collect(&Selection::All, &[0, 4]), vec![]);
}
}
crates/clawhdf5/src/error.rs
+16
View File
@@ -6,7 +6,10 @@ use clawhdf5_format::error::FormatError;
use clawhdf5_format::message_type::MessageType;
/// Errors that can occur when using the high-level API.
///
/// Non-exhaustive: new kinds of failure may be added.
#[derive(Debug)]
#[non_exhaustive]
pub enum Error {
/// I/O error from the filesystem.
Io(std::io::Error),
@@ -36,6 +39,16 @@ pub enum Error {
/// Actual alignment of the data pointer.
actual: usize,
},
/// The requested change is valid but not supported (by
/// [`FileEditor`](crate::FileEditor): a chunk index, filter or header
/// layout it cannot modify). Nothing was written.
Unsupported(String),
/// An argument does not fit the object (a selection outside the
/// dataset, a buffer of the wrong length, a shrinking resize, ...).
InvalidArgument(String),
/// The file is locked by another writer (another [`FileEditor`](crate::FileEditor),
/// or libhdf5 with file locking on).
Locked(String),
}
impl fmt::Display for Error {
@@ -58,6 +71,9 @@ impl fmt::Display for Error {
"zero-copy type mismatch: expected {expected}, got {actual}"
)
}
Error::Unsupported(msg) => write!(f, "unsupported: {msg}"),
Error::InvalidArgument(msg) => write!(f, "invalid argument: {msg}"),
Error::Locked(msg) => write!(f, "file is locked: {msg}"),
Error::ZeroCopyUnaligned { required, actual } => {
write!(
f,
crates/clawhdf5/src/lib.rs
+18
View File
@@ -23,8 +23,25 @@
//! builder.set_attr("version", AttrValue::I64(1));
//! builder.write("output.h5").unwrap();
//! ```
//!
//! # Modifying a file in place
//!
//! ```no_run
//! use clawhdf5::{FileEditor, Selection};
//!
//! let mut ed = FileEditor::open("data.h5").unwrap();
//! ed.resize("series", &[1100]).unwrap(); // a chunked dataset, maxshape (None,)
//! let tail = Selection::Hyperslab {
//! start: vec![1000],
//! stride: vec![1],
//! count: vec![100],
//! block: vec![1],
//! };
//! ed.write_values("series", &tail, &[0.5f64; 100]).unwrap();
//! ```
mod cache_image;
mod edit;
pub mod error;
pub mod lazy;
#[cfg(feature = "mmap")]
@@ -34,6 +51,7 @@ pub mod types;
pub mod vlen;
pub mod writer;
pub use edit::FileEditor;
pub use error::Error;
pub use lazy::{LazyDataset, LazyFile, LazyGroup};
#[cfg(feature = "mmap")]
crates/clawhdf5/src/reader.rs
+9 -4
View File
@@ -1104,13 +1104,18 @@ impl<'f> Dataset<'f> {
.ok_or(Error::MissingMessage(msg_type))
}
fn datatype(&self) -> Result<Datatype, Error> {
/// The dataset's object header as parsed.
pub(crate) fn header(&self) -> &ObjectHeader {
&self.header
}
pub(crate) fn datatype(&self) -> Result<Datatype, Error> {
let data = self.required_payload(MessageType::Datatype)?;
let (dt, _) = Datatype::parse_in_header(&data, self.header.version)?;
Ok(dt)
}
fn dataspace(&self) -> Result<Dataspace, Error> {
pub(crate) fn dataspace(&self) -> Result<Dataspace, Error> {
let data = self.required_payload(MessageType::Dataspace)?;
let mut ds = Dataspace::parse(&data, self.file.length_size())?;
// libhdf5 reports a virtual dataset with unlimited or printf-style
@@ -1130,7 +1135,7 @@ impl<'f> Dataset<'f> {
Ok(ds)
}
fn data_layout(&self) -> Result<DataLayout, Error> {
pub(crate) fn data_layout(&self) -> Result<DataLayout, Error> {
let msg = find_message(&self.header, MessageType::DataLayout)?;
Ok(DataLayout::parse(
&msg.data,
@@ -1143,7 +1148,7 @@ impl<'f> Dataset<'f> {
/// that is present but unparseable is an error: treating it as "no
/// filters" would hand the caller the still-compressed bytes as if they
/// were the data.
fn filter_pipeline(&self) -> Result<Option<FilterPipeline>, Error> {
pub(crate) fn filter_pipeline(&self) -> Result<Option<FilterPipeline>, Error> {
self.message_payload(MessageType::FilterPipeline)?
.map(|data| FilterPipeline::parse(&data).map_err(Error::Format))
.transpose()
crates/clawhdf5/tests/edit_tests.rs
+138
View File
@@ -0,0 +1,138 @@
//! `FileEditor` on files clawhdf5 writes, read back with our own reader
//! (libhdf5 interop is in `clawhdf5-tools/tests/edit_interop.rs`, which can
//! also run `h5rs check`).
use clawhdf5::{AttrValue, Error, File, FileBuilder, FileEditor, Selection};
fn block(start: u64, count: u64) -> Selection {
Selection::Hyperslab {
start: vec![start],
stride: vec![1],
count: vec![count],
block: vec![1],
}
}
fn sample(dir: &std::path::Path) -> std::path::PathBuf {
let path = dir.join("f.h5");
let mut b = FileBuilder::new();
b.create_dataset("ext")
.with_i32_data(&[0, 1, 2, 3, 4])
.with_shape(&[5])
.with_maxshape(&[u64::MAX])
.with_chunks(&[4])
.with_deflate(6);
b.create_dataset("raw")
.with_f64_data(&[0.5; 8])
.with_shape(&[2, 4])
.with_maxshape(&[u64::MAX, 4])
.with_chunks(&[1, 4]);
b.create_dataset("flat").with_i64_data(&[1, 2, 3]);
b.set_attr("title", AttrValue::String("t".into()));
b.write(&path).unwrap();
path
}
#[test]
fn append_overwrite_and_attributes_round_trip() {
let dir = tempfile::tempdir().unwrap();
let path = sample(dir.path());
let len_before = std::fs::metadata(&path).unwrap().len();
let mut expect: Vec<i32> = (0..5).collect();
let mut raw = vec![0.5f64; 8];
{
let mut ed = FileEditor::open(&path).unwrap();
for k in 0..300u64 {
let n = expect.len() as u64;
let add = 1 + k % 5;
ed.resize("ext", &[n + add]).unwrap();
let vals: Vec<i32> = (0..add).map(|j| (n + j) as i32 * 2).collect();
ed.write_values("ext", &block(n, add), &vals).unwrap();
expect.extend(&vals);
}
// A filtered chunk rewritten with data that compresses worse moves.
let noisy: Vec<i32> = (0..4).map(|i| i * 7_919_993).collect();
ed.write_values("ext", &block(0, 4), &noisy).unwrap();
expect[..4].copy_from_slice(&noisy);
ed.resize("raw", &[5, 4]).unwrap();
raw.resize(20, 0.0);
let sel = Selection::Hyperslab {
start: vec![1, 1],
stride: vec![2, 2],
count: vec![2, 2],
block: vec![1, 1],
};
ed.write_values("raw", &sel, &[1.0f64, 2.0, 3.0, 4.0])
.unwrap();
for (i, (r, c)) in [(1, 1), (1, 3), (3, 1), (3, 3)].iter().enumerate() {
raw[r * 4 + c] = i as f64 + 1.0;
}
ed.write_values("flat", &Selection::Points(vec![vec![2]]), &[30i64])
.unwrap();
ed.set_attr("/", "title", &AttrValue::String("a longer title".into()))
.unwrap();
ed.set_attr("ext", "count", &AttrValue::I64(expect.len() as i64))
.unwrap();
}
let f = File::open(&path).unwrap();
assert_eq!(f.dataset("ext").unwrap().read_i32().unwrap(), expect);
assert_eq!(f.dataset("raw").unwrap().shape().unwrap(), vec![5, 4]);
assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw);
assert_eq!(
f.dataset("flat").unwrap().read_i64().unwrap(),
vec![1, 2, 30]
);
let root = f.root().attrs().unwrap();
assert!(matches!(root.get("title"), Some(AttrValue::String(s)) if s == "a longer title"));
let ext = f.dataset("ext").unwrap().attrs().unwrap();
assert!(matches!(ext.get("count"), Some(AttrValue::I64(n)) if *n == expect.len() as i64));
assert!(std::fs::metadata(&path).unwrap().len() > len_before);
}
#[test]
fn errors_leave_the_file_untouched() {
let dir = tempfile::tempdir().unwrap();
let path = sample(dir.path());
let before = std::fs::read(&path).unwrap();
let mut ed = FileEditor::open(&path).unwrap();
assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_))));
assert!(ed.write_all("missing", &[0; 4]).is_err());
// Wrong length, wrong type, outside the extent, beyond maxshape,
// shrinking, a rank change.
assert!(matches!(
ed.write_all("flat", &[0; 7]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(
ed.write_values("flat", &Selection::All, &[1i32, 2, 3]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(
ed.write_values("ext", &block(4, 2), &[1i32, 2]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(
ed.resize("raw", &[3, 5]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_))));
assert!(matches!(
ed.resize("ext", &[4, 1]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(
ed.resize("flat", &[4]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(
ed.set_attr("/", "", &AttrValue::I64(1)),
Err(Error::InvalidArgument(_))
));
// No-ops write nothing.
ed.resize("ext", &[5]).unwrap();
ed.write_values("ext", &Selection::None, &[] as &[i32])
.unwrap();
drop(ed);
assert!(std::fs::read(&path).unwrap() == before);
}