quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p3-editor-coverage' into feat/p3-remote-editor

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	CLAUDE.md
#	docs/design/range-reads.md
This commit is contained in:
osobh
2026-09-26 19:17:46 -05:00
parent 93e2d5f365 8236b0e30a
commit 7447dce121
23 changed files with 7218 additions and 533 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-accel/src/lib.rs
+4 -1
View File
@@ -237,7 +237,10 @@ pub fn f16_to_f32_batch(input: &[u16], output: &mut [f32]) {
convert::f16_to_f32_batch(input, output);
}
/// Compute Fletcher-32 checksum.
/// Compute a textbook Fletcher-32 checksum (both sums start at 0xffff).
///
/// This is not HDF5's checksum; the Fletcher-32 I/O filter uses
/// `clawhdf5_format::checksum::fletcher32`.
pub fn checksum_fletcher32(data: &[u8]) -> u32 {
checksum::checksum_fletcher32(data)
}
crates/clawhdf5-format/src/checksum.rs
+52
View File
@@ -14,6 +14,45 @@ pub fn jenkins_lookup3(data: &[u8]) -> u32 {
hashlittle(data, 0)
}
/// HDF5's Fletcher-32 checksum, as the Fletcher-32 I/O filter (filter id 3)
/// stores it after each chunk.
///
/// A line-for-line port of `H5_checksum_fletcher32` (H5checksum.c, libhdf5
/// 1.8 through 1.14): big-endian 16-bit words summed in blocks of 360, each
/// sum reduced after a block by the ones'-complement fold
/// `(s & 0xffff) + (s >> 16)` rather than `% 65535`, an odd trailing byte
/// taken as the high byte of a last word, and a final fold of both sums.
/// The fold and `% 65535` differ whenever a sum is a non-zero multiple of
/// 65535: the fold leaves 0xffff where the modulo gives 0, so the two
/// disagree on about one chunk in 32768 and libhdf5 rejects the other's
/// checksum. This must stay the only implementation.
pub fn fletcher32(data: &[u8]) -> u32 {
let mut sum1: u32 = 0;
let mut sum2: u32 = 0;
// 360 words keep both sums inside 32 bits between folds (the bound
// libhdf5 uses: after a fold sum1 < 0x10200, so sum2 stays below
// 360 * 361 / 2 * 0xffff + 360 * 0x10200 + 0x1fffe < 2^32). The adds wrap
// like the C unsigned arithmetic all the same.
let (words, odd) = data.as_chunks::<2>();
for block in words.chunks(360) {
for w in block {
sum1 = sum1.wrapping_add((u32::from(w[0]) << 8) | u32::from(w[1]));
sum2 = sum2.wrapping_add(sum1);
}
sum1 = (sum1 & 0xffff) + (sum1 >> 16);
sum2 = (sum2 & 0xffff) + (sum2 >> 16);
}
if let [last] = odd {
sum1 = sum1.wrapping_add(u32::from(*last) << 8);
sum2 = sum2.wrapping_add(sum1);
sum1 = (sum1 & 0xffff) + (sum1 >> 16);
sum2 = (sum2 & 0xffff) + (sum2 >> 16);
}
sum1 = (sum1 & 0xffff) + (sum1 >> 16);
sum2 = (sum2 & 0xffff) + (sum2 >> 16);
(sum2 << 16) | sum1
}
/// Compute CRC32 (IEEE / ISO 3309) over data.
///
/// When the `fast-checksum` feature is enabled, this uses hardware CRC32
@@ -207,6 +246,19 @@ fn hashlittle(data: &[u8], initval: u32) -> u32 {
mod tests {
use super::*;
/// Values of libhdf5's `H5_checksum_fletcher32` (h5py 3.x's bundled
/// libhdf5, called through ctypes). The first three are sums that are
/// multiples of 65535, where `% 65535` gave 0 instead of 0xffff.
#[test]
fn fletcher32_matches_libhdf5() {
assert_eq!(fletcher32(&[0x00, 0x01, 0xff, 0xfe]), 0x0001_ffff);
assert_eq!(fletcher32(&[0xff; 720]), 0xffff_ffff);
assert_eq!(fletcher32(&[0xff; 721]), 0xff00_ff00);
assert_eq!(fletcher32(&[0xff; 1441]), 0xff00_ff00);
assert_eq!(fletcher32(&[]), 0);
assert_eq!(fletcher32(&[7]), 0x0700_0700);
}
#[test]
fn empty_input() {
// Empty input should return the initial state after no mixing
crates/clawhdf5-format/src/chunked_read.rs
+47 -4
View File
@@ -1077,16 +1077,40 @@ pub fn generate_implicit_chunks(
dataset_dims: &[u64],
chunk_dimensions: &[u32],
element_size: u32,
) -> Vec<ChunkInfo> {
generate_implicit_chunks_in_grid(
base_address,
dataset_dims,
dataset_dims,
chunk_dimensions,
element_size,
)
}
/// [`generate_implicit_chunks`] for a dataset whose maximum dimensions
/// (`max_dims`) exceed its current ones: libhdf5 allocates the chunks of
/// the whole maximum extent and places chunk `scaled` at its row-major
/// position in the *maximum* chunk grid (`H5D__none_idx_get_addr`,
/// `max_down_chunks`), so the current extent's chunks are not contiguous.
/// Only the chunks of the current extent are listed.
pub fn generate_implicit_chunks_in_grid(
base_address: u64,
dataset_dims: &[u64],
max_dims: &[u64],
chunk_dimensions: &[u32],
element_size: u32,
) -> Vec<ChunkInfo> {
let rank = chunk_dimensions.len();
let chunk_byte_size: u64 =
chunk_dimensions.iter().map(|&d| d as u64).product::<u64>() * element_size as u64;
let mut num_chunks_per_dim = Vec::with_capacity(rank);
let mut grid_per_dim = Vec::with_capacity(rank);
for d in 0..rank {
let ds = dataset_dims[d];
let ch = chunk_dimensions[d] as u64;
num_chunks_per_dim.push(ds.div_ceil(ch));
let n = dataset_dims[d].div_ceil(ch);
num_chunks_per_dim.push(n);
grid_per_dim.push(max_dims.get(d).map_or(n, |m| m.div_ceil(ch)).max(n));
}
let total_chunks: u64 = num_chunks_per_dim.iter().product();
@@ -1095,18 +1119,22 @@ pub fn generate_implicit_chunks(
for linear_idx in 0..total_chunks {
let mut offsets = vec![0u64; rank];
let mut remaining = linear_idx;
let mut grid_idx = 0u64;
let mut down = 1u64;
for d in (0..rank).rev() {
let nchunks = num_chunks_per_dim[d];
let chunk_idx = remaining % nchunks;
remaining /= nchunks;
offsets[d] = chunk_idx * chunk_dimensions[d] as u64;
grid_idx = grid_idx.saturating_add(chunk_idx.saturating_mul(down));
down = down.saturating_mul(grid_per_dim[d]);
}
chunks.push(ChunkInfo {
chunk_size: chunk_byte_size as u32,
filter_mask: 0,
offsets,
address: base_address + linear_idx * chunk_byte_size,
address: base_address.saturating_add(grid_idx.saturating_mul(chunk_byte_size)),
});
}
@@ -1320,9 +1348,13 @@ pub fn list_chunks_in<S: Storage + ?Sized>(
(4, Some(2)) => {
// Implicit index — use spatial chunk dims only
let spatial_chunk_dims: &[u32] = &chunk_dimensions[..rank];
generate_implicit_chunks(
generate_implicit_chunks_in_grid(
addr,
&dataspace.dimensions,
dataspace
.max_dimensions
.as_deref()
.unwrap_or(&dataspace.dimensions),
spatial_chunk_dims,
elem_size as u32,
)
@@ -3335,6 +3367,17 @@ mod tests {
}
}
/// A dataset below its maximum extent: libhdf5 lays chunks out over the
/// maximum chunk grid, so row 1 starts after a whole maximum row (here
/// 4 chunks), not after the current row of 3.
#[test]
fn implicit_chunks_use_the_maximum_grid() {
let chunks = generate_implicit_chunks_in_grid(0x100, &[2, 3], &[4, 4], &[1, 1], 4);
let addrs: Vec<u64> = chunks.iter().map(|c| (c.address - 0x100) / 4).collect();
assert_eq!(addrs, vec![0, 1, 2, 4, 5, 6]);
assert_eq!(chunks[3].offsets, vec![1, 0]);
}
#[test]
fn implicit_chunks_partial_last() {
// 25 elements, chunk size 10 => 3 chunks (last partial)
crates/clawhdf5-format/src/filters.rs
+13 -53
View File
@@ -1716,56 +1716,6 @@ fn shuffle_compress_general(data: &[u8], n: usize, element_size: usize, result:
}
}
/// Compute HDF5 Fletcher32 checksum over data.
/// HDF5 uses a modified Fletcher32 that operates on 16-bit words.
///
/// Optimized with wider accumulators: processes blocks of 360 words before
/// taking the modulo, reducing the number of expensive modulo operations.
/// (360 is the maximum block size that avoids u32 overflow for sum2.)
fn fletcher32_compute(data: &[u8]) -> u32 {
let mut sum1: u32 = 0;
let mut sum2: u32 = 0;
// Process in blocks of 360 16-bit words (720 bytes) to delay modulo.
// Max sum1 before mod: 360 * 65535 = 23_592_600 < u32::MAX
// Max sum2 before mod: 360 * 23_592_600 ~ 8.5B > u32::MAX, but actual
// sum2 accumulates incrementally, so worst case is 360*360*65535/2 which
// fits in u64. We use u32 with block size 360 which is safe.
const BLOCK_WORDS: usize = 360;
const BLOCK_BYTES: usize = BLOCK_WORDS * 2;
let mut offset = 0;
let len = data.len();
while offset + BLOCK_BYTES <= len {
let end = offset + BLOCK_BYTES;
let mut i = offset;
while i < end {
let val = ((data[i] as u32) << 8) | (data[i + 1] as u32);
sum1 += val;
sum2 += sum1;
i += 2;
}
sum1 %= 65535;
sum2 %= 65535;
offset = end;
}
// Handle remaining bytes
while offset < len {
let val = if offset + 1 < len {
((data[offset] as u32) << 8) | (data[offset + 1] as u32)
} else {
(data[offset] as u32) << 8
};
sum1 = (sum1 + val) % 65535;
sum2 = (sum2 + sum1) % 65535;
offset += 2;
}
(sum2 << 16) | sum1
}
/// Verify Fletcher32 checksum and strip it from the data.
/// The last 4 bytes are the stored checksum.
fn fletcher32_verify(data: &[u8]) -> Result<Vec<u8>, FormatError> {
@@ -1787,8 +1737,18 @@ fn fletcher32_payload(data: &[u8]) -> Result<usize, FormatError> {
data[data.len() - 2],
data[data.len() - 1],
]);
let computed = fletcher32_compute(payload);
if stored != computed {
let computed = crate::checksum::fletcher32(payload);
// libhdf5 also accepts the checksum with the bytes of each 16-bit half
// swapped, which is how 1.6.2 and earlier stored it
// (H5Z__filter_fletcher32's `reversed_fletcher`).
let reversed = ((computed & 0x00ff_00ff) << 8) | ((computed >> 8) & 0x00ff_00ff);
// clawhdf5 v2.7.0 and earlier reduced the sums `% 65535`, which gives 0
// where libhdf5's fold gives 0xffff; accept that form too, so that files
// those releases wrote can still be read (and rewritten for libhdf5).
// It differs from `computed` only in a half that is 0xffff.
let half = |h: u32| if h == 0xffff { 0 } else { h };
let legacy = (half(computed >> 16) << 16) | half(computed & 0xffff);
if stored != computed && stored != reversed && stored != legacy {
return Err(FormatError::Fletcher32Mismatch {
expected: stored,
computed,
@@ -1799,7 +1759,7 @@ fn fletcher32_payload(data: &[u8]) -> Result<usize, FormatError> {
/// Append Fletcher32 checksum to data.
fn fletcher32_append(data: &[u8]) -> Result<Vec<u8>, FormatError> {
let checksum = fletcher32_compute(data);
let checksum = crate::checksum::fletcher32(data);
let mut result = data.to_vec();
result.extend_from_slice(&checksum.to_le_bytes());
Ok(result)
crates/clawhdf5-format/src/object_header.rs
+285 -74
View File
@@ -1,7 +1,9 @@
//! HDF5 Object Header parsing (v1 and v2).
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use alloc::{collections::BTreeSet, vec, vec::Vec};
#[cfg(feature = "std")]
use std::collections::BTreeSet;
use byteorder::{ByteOrder, LittleEndian};
@@ -196,7 +198,6 @@ impl ObjectHeader {
header_data_size,
offset_size,
length_size,
MAX_V1_CONTINUATION_DEPTH,
&mut messages,
)?;
// libhdf5 reads every message in the first chunk and refuses a header
@@ -230,79 +231,84 @@ impl ObjectHeader {
/// 8; libhdf5 refuses a message that is not aligned, that runs past the
/// end of the chunk, or leftover bytes too few for a message header (a
/// "gap", which only version 2 allows).
#[allow(clippy::too_many_arguments)]
///
/// Continuation chunks are read in the order their messages are found,
/// as `H5O_protect` loads them (so the messages keep libhdf5's order):
/// a queue of (address, length) pairs, each chunk read, parsed and
/// released before the next, so only one chunk buffer is alive at a
/// time whatever the storage. Every chunk must start at a new address
/// (else a cycle), and the chunks together may be no larger than the
/// file, so the bytes read stay within the file's size; a header of
/// more than [`MAX_V1_CHUNKS`] chunks is refused.
fn parse_v1_chunk<S: Storage + ?Sized>(
file: &S,
offset: u64,
length: usize,
offset_size: u8,
length_size: u8,
depth_remaining: u16,
messages: &mut Vec<HeaderMessage>,
) -> Result<usize, FormatError> {
if depth_remaining == 0 {
return Err(FormatError::NestingDepthExceeded);
let mut spans = ChunkSpans::new(file.len(), offset, length)?;
let mut queue: Vec<(u64, usize)> = vec![(offset, length)];
let mut chunk0_count = 0usize;
let mut next = 0usize;
while let Some(&(chunk_offset, chunk_length)) = queue.get(next) {
let chunk = read_exact_at(file, chunk_offset, chunk_length)?;
let data: &[u8] = &chunk;
let end = data.len();
let mut pos = 0usize;
let mut count = 0usize;
while pos < end {
if end - pos < V1_MSG_HEADER_SIZE {
return Err(FormatError::InvalidObjectHeader(
"gap found in early version of file format",
));
}
let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
let msg_flags = data[pos + 4];
// reserved(3) at pos+5..pos+8
pos += V1_MSG_HEADER_SIZE;
if !msg_data_size.is_multiple_of(8) {
return Err(FormatError::InvalidObjectHeader("message not aligned"));
}
if msg_data_size > end - pos {
return Err(FormatError::InvalidObjectHeader(
"message size exceeds buffer end",
));
}
let body = &data[pos..pos + msg_data_size];
check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
count += 1;
let msg_type = MessageType::from_u16(msg_type_raw);
if msg_type != MessageType::Nil {
messages.push(HeaderMessage {
msg_type,
size: msg_data_size,
flags: msg_flags,
creation_order: None,
data: body.to_vec(),
});
}
// Queue continuations (v1 continuation chunks are just raw
// messages, no signature); check_message has checked the body.
if msg_type == MessageType::ObjectHeaderContinuation {
let cont_offset = read_offset(body, 0, offset_size)?;
let cont_length =
to_usize(read_offset(body, offset_size as usize, length_size)?)?;
spans.add(cont_offset, cont_length)?;
queue.push((cont_offset, cont_length));
}
pos += msg_data_size;
}
// Only the first chunk's messages are held to the prefix count.
if next == 0 {
chunk0_count = count;
}
next += 1;
}
let chunk = read_exact_at(file, offset, length)?;
let data: &[u8] = &chunk;
let end = length;
let mut pos = 0usize;
let mut count = 0usize;
while pos < end {
if end - pos < V1_MSG_HEADER_SIZE {
return Err(FormatError::InvalidObjectHeader(
"gap found in early version of file format",
));
}
let msg_type_raw = LittleEndian::read_u16(&data[pos..pos + 2]);
let msg_data_size = LittleEndian::read_u16(&data[pos + 2..pos + 4]) as usize;
let msg_flags = data[pos + 4];
// reserved(3) at pos+5..pos+8
pos += V1_MSG_HEADER_SIZE;
if !msg_data_size.is_multiple_of(8) {
return Err(FormatError::InvalidObjectHeader("message not aligned"));
}
if msg_data_size > end - pos {
return Err(FormatError::InvalidObjectHeader(
"message size exceeds buffer end",
));
}
let body = &data[pos..pos + msg_data_size];
check_message(1, msg_type_raw, msg_flags, body, offset_size, length_size)?;
count += 1;
let msg_type = MessageType::from_u16(msg_type_raw);
if msg_type != MessageType::Nil {
messages.push(HeaderMessage {
msg_type,
size: msg_data_size,
flags: msg_flags,
creation_order: None,
data: body.to_vec(),
});
}
pos += msg_data_size;
// Follow continuations (v1 continuation chunks are just raw
// messages, no signature); check_message has checked the body.
if msg_type == MessageType::ObjectHeaderContinuation {
let cont_offset = to_usize(read_offset(body, 0, offset_size)?)?;
let cont_length = to_usize(read_offset(body, offset_size as usize, length_size)?)?;
Self::parse_v1_chunk(
file,
cont_offset as u64,
cont_length,
offset_size,
length_size,
depth_remaining - 1,
messages,
)?;
}
}
Ok(count)
Ok(chunk0_count)
}
fn parse_v2<S: Storage + ?Sized>(
@@ -425,13 +431,14 @@ impl ObjectHeader {
&mut continuations,
)?;
// Follow continuations (limit to prevent cycles in malformed data)
let mut cont_remaining = 256u16;
// Follow continuations, one chunk buffer at a time. A chunk address
// seen twice is a cycle in malformed data, and the chunks may add up
// to no more than the file; a valid header can have many chunks (libhdf5 adds one
// whenever a message no longer fits), up to the same bound as a
// version-1 header.
let mut spans = ChunkSpans::new(file.len(), base as u64, chunk0_msg_end.saturating_add(4))?;
while let Some((cont_offset, cont_length)) = continuations.pop() {
if cont_remaining == 0 {
return Err(FormatError::NestingDepthExceeded);
}
cont_remaining -= 1;
spans.add(cont_offset as u64, cont_length)?;
Self::parse_v2_continuation(
file,
cont_offset as u64,
@@ -594,8 +601,49 @@ const V2_PREFIX_MAX: usize = 34;
/// Size of a version-1 message header: type(2) + size(2) + flags(1) + reserved(3).
const V1_MSG_HEADER_SIZE: usize = 8;
/// How deep version-1 continuation chunks may chain (malformed-data guard).
const MAX_V1_CONTINUATION_DEPTH: u16 = 32;
/// The chunks of one object header read so far. A chunk starting where
/// another did is a cycle. Chunks of a valid header do not overlap, so
/// together they are no larger than the file; a header whose chunks add up
/// to more is refused, which bounds what its chunks can make a reader read
/// (a crafted chain of chunks each nested in the last would otherwise read
/// the file over and over). Overlap itself is not refused: libhdf5 reads
/// such headers (`cve-2025-7067.h5` has one).
struct ChunkSpans {
starts: BTreeSet<u64>,
/// Bytes of the chunks so far, and the most they may add up to.
total: u64,
budget: u64,
}
impl ChunkSpans {
fn new(file_len: u64, start: u64, len: usize) -> Result<Self, FormatError> {
let mut s = Self {
starts: BTreeSet::new(),
total: 0,
budget: file_len,
};
s.add(start, len)?;
Ok(s)
}
fn add(&mut self, start: u64, len: usize) -> Result<(), FormatError> {
if !self.starts.insert(start) || self.starts.len() > MAX_V1_CHUNKS {
return Err(FormatError::NestingDepthExceeded);
}
self.total = self.total.saturating_add(len as u64);
if self.total > self.budget {
return Err(FormatError::InvalidObjectHeader(
"object header chunks larger than the file",
));
}
Ok(())
}
}
/// Most chunks a version-1 object header may have (malformed-data guard;
/// libhdf5 has no limit, and a header that gains one continuation chunk per
/// attribute added can have many).
const MAX_V1_CHUNKS: usize = 1 << 16;
/// Every defined version-2 object header status flag (libhdf5
/// `H5O_HDR_ALL_FLAGS`): chunk-0 size width (bits 0-1), attribute creation
@@ -901,6 +949,169 @@ mod tests {
assert_eq!(hdr.messages[1].data[..2], [5, 6]);
}
/// A version-1 header whose continuation chunks form a chain: chunk k
/// holds a Dataspace message `[k]` and the continuation to chunk k + 1.
/// With `cycle`, the last chunk points back at the first continuation
/// chunk.
fn v1_chain(n: usize, cycle: bool) -> Vec<u8> {
// Each continuation chunk: dataspace (8 + 8) + continuation (8 + 16).
let chunk_len = 40u64;
let first = 64u64;
let cont = |addr: u64| {
let mut b = addr.to_le_bytes().to_vec();
b.extend_from_slice(&chunk_len.to_le_bytes());
b
};
let mut data = build_v1_header(&[(0x0010, &cont(first)[..], 0)], 8, 8);
data.resize(first as usize, 0);
for k in 0..n {
let mut c = Vec::new();
c.extend_from_slice(&1u16.to_le_bytes());
c.extend_from_slice(&8u16.to_le_bytes());
c.extend_from_slice(&[0; 4]);
c.extend_from_slice(&(k as u64).to_le_bytes());
let next = if k + 1 < n {
first + (k as u64 + 1) * chunk_len
} else if cycle {
first
} else {
// The last chunk ends in a NIL message instead.
c.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
c.extend_from_slice(&[0; 16]);
data.extend_from_slice(&c);
continue;
};
c.extend_from_slice(&0x10u16.to_le_bytes());
c.extend_from_slice(&16u16.to_le_bytes());
c.extend_from_slice(&[0; 4]);
c.extend_from_slice(&cont(next));
data.extend_from_slice(&c);
}
data
}
/// libhdf5 reads any chain of continuation chunks (a header grows one
/// per attribute added when full); the reader used to stop at 32.
#[test]
fn long_v1_continuation_chains_are_read() {
let data = v1_chain(200, false);
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
let spaces: Vec<u8> = hdr
.messages
.iter()
.filter(|m| m.msg_type == MessageType::Dataspace)
.map(|m| m.data[0])
.collect();
assert_eq!(spaces, (0..200).map(|k| k as u8).collect::<Vec<_>>());
}
/// A crafted version-1 header whose continuation chunks nest: each
/// chunk's continuation message points at the rest of that chunk. Read
/// depth-first with every enclosing chunk kept alive, from storage that
/// hands out owned buffers, it read n^2 bytes and held them all at once
/// (a 192 KB file read 768 MB). Chunks adding up to more than the file
/// are refused, and the bytes read stay within the file's size.
#[test]
fn nested_v1_continuation_chunks_are_bounded() {
use crate::storage::CountingStorage;
let n = 2000u64;
let a = 64u64;
let cont = |addr: u64, len: u64| {
let mut m = vec![0x10, 0, 16, 0, 0, 0, 0, 0];
m.extend_from_slice(&addr.to_le_bytes());
m.extend_from_slice(&len.to_le_bytes());
m
};
// Prefix: version 1, one message, reference count 1, 24 bytes.
let mut buf = vec![1, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0];
buf.extend_from_slice(&cont(a, 24 * n));
buf.resize(a as usize, 0);
for k in 0..n {
if k + 1 < n {
buf.extend_from_slice(&cont(a + 24 * (k + 1), 24 * (n - k - 1)));
} else {
buf.extend_from_slice(&[0, 0, 16, 0, 0, 0, 0, 0]);
buf.extend_from_slice(&[0; 16]);
}
}
let len = buf.len() as u64;
let s = CountingStorage::new(buf);
assert!(matches!(
ObjectHeader::parse_in(&s, 0, 8, 8),
Err(FormatError::InvalidObjectHeader(
"object header chunks larger than the file"
))
));
assert!(
s.bytes_read() <= 2 * len,
"read {} of {len}",
s.bytes_read()
);
}
/// libhdf5 reads a continuation chunk that overlaps the chunk holding
/// its message (`cve-2025-7067.h5` has one), and so does this reader.
#[test]
fn overlapping_v1_continuation_chunk_is_read() {
// Chunk 0 (at 16): continuation (24 bytes), then a NIL message at
// 40; the continuation chunk is that NIL message's 8-byte header.
let mut cont = 40u64.to_le_bytes().to_vec();
cont.extend_from_slice(&8u64.to_le_bytes());
let data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0000, &[][..], 0)], 8, 8);
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 1);
}
/// A valid chain over owned-buffer storage reads each chunk once.
#[test]
fn long_v1_chain_reads_each_chunk_once() {
use crate::storage::CountingStorage;
let data = v1_chain(3000, false);
let len = data.len() as u64;
let s = CountingStorage::new(data);
let hdr = ObjectHeader::parse_in(&s, 0, 8, 8).unwrap();
assert_eq!(
hdr.messages
.iter()
.filter(|m| m.msg_type == MessageType::Dataspace)
.count(),
3000
);
assert!(s.bytes_read() <= len, "read {} of {len}", s.bytes_read());
}
/// Continuation chunks are read in the order their messages are found
/// (libhdf5's `H5O_protect`), so a chunk's messages follow every
/// message of the chunk before, not the continuation message.
#[test]
fn v1_continuation_messages_keep_libhdf5_order() {
// Chunk 0: continuation to A, dataspace [1]; A: dataspace [2].
let a = 64u64;
let mut cont = a.to_le_bytes().to_vec();
cont.extend_from_slice(&16u64.to_le_bytes());
let mut data = build_v1_header(&[(0x0010, &cont[..], 0), (0x0001, &[1; 8][..], 0)], 8, 8);
data.resize(a as usize, 0);
data.extend_from_slice(&[1, 0, 8, 0, 0, 0, 0, 0]);
data.extend_from_slice(&[2; 8]);
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
let spaces: Vec<u8> = hdr
.messages
.iter()
.filter(|m| m.msg_type == MessageType::Dataspace)
.map(|m| m.data[0])
.collect();
assert_eq!(spaces, [1, 2]);
}
#[test]
fn v1_continuation_cycles_are_refused() {
let data = v1_chain(5, true);
assert!(matches!(
ObjectHeader::parse(&data, 0, 8, 8),
Err(FormatError::NestingDepthExceeded)
));
}
#[test]
fn parse_v1_unknown_message_ok() {
let messages = [(0x00FFu16, &[0xAA, 0xBB][..], 0u8)];
crates/clawhdf5-tools/tests/edit_coverage_interop.rs
+1788
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5-tools/tests/edit_interop.rs
+154 -80
View File
@@ -113,7 +113,8 @@ impl Model {
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
}
/// Grow to `shape`, new elements `fill`.
/// Change the extent to `shape`: elements inside both keep their
/// values, new ones are `fill`.
fn resize(&mut self, shape: &[u64], fill: i32) {
let old = self.clone();
*self = Self::new(shape, |_| fill);
@@ -125,8 +126,10 @@ impl Model {
c[d] = r % old.shape[d];
r /= old.shape[d];
}
let i = self.index(&c);
self.data[i] = old.data[flat as usize];
if c.iter().zip(shape).all(|(x, s)| x < s) {
let i = self.index(&c);
self.data[i] = old.data[flat as usize];
}
}
}
@@ -293,9 +296,24 @@ fn append_many_gzip() {
}
}
/// Random operations — grow, hyperslab writes, point writes, attributes —
/// on a 2-D dataset with one unlimited dimension, checked against a model
/// after every few operations.
/// A random attribute value: a scalar, an int64 array, a short string or
/// one larger than a heap's managed-object limit (a huge heap object once
/// the attributes are in dense storage).
fn random_attr(rng: &mut Rng) -> AttrValue {
match rng.below(8) {
0..=2 => AttrValue::I64(rng.next() as i64 >> 3),
3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()),
5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)),
_ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)),
}
}
/// Random operations — growth and shrinking along any dimension, hyperslab
/// and point writes, attributes (enough names to move them to dense storage
/// on version-2 object headers, replaced with values of any size) — on a
/// 2-D dataset with one unlimited dimension and one with two (a version-2
/// B-tree chunk index under `v114`/`latest`), checked against a model (and
/// through h5py, numpy) after every few operations.
fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
let dir = tmpdir();
let path = dir.path().join(format!("rand_{tag}.h5"));
@@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
\x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \
dtype='<i4', fillvalue=-9{extra})\n\
\x20 f['m'][1:3, 2:6] = 5\n",
\x20 f['m'][1:3, 2:6] = 5\n\
\x20 f.create_dataset('b', shape=(5, 6), maxshape=(None, None), chunks=(2, 4), \
dtype='<i4', fillvalue=3{extra})\n\
\x20 f['b'][0:4, 1:5] = 8\n",
p = path.to_str().unwrap()
));
let mut m = Model::new(&[4, 7], |_| -9);
m.write_block(&[1, 2], &[2, 4], &[5; 8]);
let mut attrs: Vec<(String, i64)> = Vec::new();
let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)];
models[0].write_block(&[1, 2], &[2, 4], &[5; 8]);
models[1].write_block(&[0, 1], &[4, 4], &[8; 16]);
let fills = [-9, 3];
let names = ["m", "b"];
let mut attrs: Vec<(String, AttrValue)> = Vec::new();
let mut rng = Rng(seed);
let mut ed = FileEditor::open(&path).unwrap();
for step in 0..120 {
match rng.below(10) {
0..=1 => {
let rows = m.shape[0] + 1 + rng.below(5);
ed.resize("m", &[rows, 7]).unwrap();
m.resize(&[rows, 7], -9);
for step in 0..160 {
let d = rng.below(2) as usize;
let name = names[d];
let m = &mut models[d];
match rng.below(12) {
0..=2 => {
// Grow or shrink: dimension 1 of "m" is fixed at 7.
let rows = rng.below(m.shape[0] + 6);
let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) };
ed.resize(name, &[rows, cols]).unwrap();
m.resize(&[rows, cols], fills[d]);
}
2..=6 => {
3..=7 if m.shape.iter().all(|&s| s > 0) => {
let r0 = rng.below(m.shape[0]);
let c0 = rng.below(7);
let c0 = rng.below(m.shape[1]);
let cnt = [
1 + rng.below((m.shape[0] - r0).min(6)),
1 + rng.below(7 - c0),
1 + rng.below((m.shape[1] - c0).min(6)),
];
let n = cnt[0] * cnt[1];
let vals: Vec<i32> = (0..n).map(|_| (rng.next() % 100_000) as i32).collect();
ed.write_values("m", &block(&[r0, c0], &cnt), &vals)
ed.write_values(name, &block(&[r0, c0], &cnt), &vals)
.unwrap();
m.write_block(&[r0, c0], &cnt, &vals);
}
7 => {
8 if m.shape.iter().all(|&s| s > 0) => {
let pts: Vec<Vec<u64>> = (0..1 + rng.below(4))
.map(|_| vec![rng.below(m.shape[0]), rng.below(7)])
.map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])])
.collect();
let vals: Vec<i32> = pts.iter().map(|_| rng.next() as i32).collect();
ed.write_values("m", &Selection::Points(pts.clone()), &vals)
ed.write_values(name, &Selection::Points(pts.clone()), &vals)
.unwrap();
for (p, v) in pts.iter().zip(&vals) {
let i = m.index(p);
m.data[i] = *v;
}
}
_ => {
let k = rng.below(6);
let name = format!("a{k}");
let v = rng.next() as i64;
match ed.set_attr("m", &name, &AttrValue::I64(v)) {
9..=11 => {
let k = rng.below(20);
let aname = format!("a{k}");
let v = random_attr(&mut rng);
match ed.set_attr("m", &aname, &v) {
Ok(()) => {
attrs.retain(|(n, _)| *n != name);
attrs.push((name, v));
attrs.retain(|(n, _)| *n != aname);
attrs.push((aname, v));
}
Err(e) => panic!("set_attr {name}: {e}"),
// Replacing the only attribute in a heap block with one
// of another size would have libhdf5 free the block.
Err(Error::Unsupported(msg)) if msg.contains("last object") => {}
Err(e) => panic!("set_attr {aname}: {e}"),
}
}
_ => {}
}
if step % 30 == 29 {
if step % 40 == 39 {
drop(ed);
verify(&path, "m", &m);
for (n, m) in names.iter().zip(&models) {
verify(&path, n, m);
}
check_tools(&path, h5dump);
check_attrs(&path, "m", &attrs);
ed = FileEditor::open(&path).unwrap();
}
}
drop(ed);
verify(&path, "m", &m);
for (n, m) in names.iter().zip(&models) {
verify(&path, n, m);
}
check_attrs(&path, "m", &attrs);
py(&format!(
"import h5py, numpy as np\n\
with h5py.File({p:?}, 'r+') as f:\n\
\x20 d = f['m']\n\
\x20 n = d.shape[0]\n\
\x20 d.resize((n + 3, 7))\n\
\x20 d[n:, :] = 42\n\
\x20 d.attrs['from_h5py'] = 1.5\n",
\x20 for name, cols in (('m', 7), ('b', None)):\n\
\x20 d = f[name]\n\
\x20 n, c = d.shape\n\
\x20 d.resize((n + 3, cols or c + 2))\n\
\x20 d[n:, :] = 42\n\
\x20 f['m'].attrs['from_h5py'] = 1.5\n",
p = path.to_str().unwrap()
));
let n = m.shape[0];
m.resize(&[n + 3, 7], -9);
m.write_block(&[n, 0], &[3, 7], &[42; 21]);
verify(&path, "m", &m);
for (d, m) in models.iter_mut().enumerate() {
let (n, c) = (m.shape[0], m.shape[1]);
let c2 = if d == 0 { 7 } else { c + 2 };
m.resize(&[n + 3, c2], fills[d]);
m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]);
}
for (n, m) in names.iter().zip(&models) {
verify(&path, n, m);
}
check_tools(&path, h5dump);
check_attrs(&path, "m", &attrs);
}
fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) {
/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of
/// its attributes agrees with libhdf5's object info).
fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) {
let f = File::open(path).unwrap();
let got = f.dataset(obj).unwrap().attrs().unwrap();
for (n, v) in attrs {
match got.get(n) {
Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"),
other => panic!("attribute {n}: {other:?}"),
}
let g = got
.get(n)
.unwrap_or_else(|| panic!("attribute {n} missing"));
// Our reader reports a one-element array as a scalar.
let v = match v {
AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]),
v => v,
};
assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}");
}
let want: Vec<String> = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect();
py(&format!(
let want: Vec<String> = attrs
.iter()
.map(|(n, v)| {
let pv = match v {
AttrValue::I64(x) => format!("{x}"),
AttrValue::I64Array(a) => format!("{a:?}"),
AttrValue::String(s) => format!("{s:?}"),
other => panic!("{other:?}"),
};
format!("{n:?}: {pv}")
})
.collect();
let script = format!(
"import h5py\n\
f = h5py.File({p:?}, 'r')\n\
want = {{{w}}}\n\
got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\
assert got == want, (got, want)\n",
a = f[{obj:?}].attrs\n\
def norm(v):\n\
\x20 v = v.decode() if isinstance(v, bytes) else v\n\
\x20 return v.tolist() if hasattr(v, 'tolist') else v\n\
got = {{k: norm(v) for k, v in a.items() if k in want}}\n\
assert got == want, sorted(set(want) ^ set(got))\n\
assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n",
p = path.to_str().unwrap(),
w = want.join(", ")
));
);
let sp = path.with_extension("attrs.py");
std::fs::write(&sp, script).unwrap();
let o = Command::new(python()).arg(&sp).output().unwrap();
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
}
#[test]
@@ -411,7 +482,11 @@ fn random_operations_match_a_model() {
if !tools_ok() {
return;
}
let mut seed = 1;
// CLAWHDF5_EDIT_SEED runs the same workloads with other random choices.
let mut seed = std::env::var("CLAWHDF5_EDIT_SEED")
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(1);
for (i, (lv, dump)) in LIBVERS.iter().enumerate() {
// h5dump has no LZF decoder (h5py's own filter).
for (j, (extra, lzf)) in [
@@ -752,19 +827,20 @@ fn overwrite_every_layout() {
}
check_tools(&path, *dump);
}
// A version-2 B-tree index can take new chunks only from libhdf5 for
// now: growing works, writing the new chunks is refused and changes
// nothing.
// A version-2 B-tree index takes new chunks too.
if *lv != "'earliest'" {
let mut ed = FileEditor::open(&path).unwrap();
ed.resize("bt2", &[8, 6]).unwrap();
let before = std::fs::read(&path).unwrap();
unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12]));
assert!(
std::fs::read(&path).unwrap() == before,
"a refused edit changed the file"
);
models[12].resize(&[8, 6], 0);
ed.resize("bt2", &[8, 7]).unwrap();
models[12].resize(&[8, 7], 0);
let vals: Vec<i32> = (0..23).collect();
ed.write_values("bt2", &block(&[6, 0], &[2, 7]), &vals[..14])
.unwrap();
models[12].write_block(&[6, 0], &[2, 7], &vals[..14]);
ed.write_values("bt2", &block(&[0, 6], &[6, 1]), &vals[14..20])
.unwrap();
models[12].write_block(&[0, 6], &[6, 1], &vals[14..20]);
drop(ed);
verify(&path, "bt2", &models[12]);
}
// libhdf5 goes on modifying what we wrote.
py(&format!(
@@ -835,19 +911,16 @@ fn attributes_in_place() {
.unwrap();
want.push(("d", "units".into(), AttrValue::String("km".into())));
if *lv != "'earliest'" {
// Up to the compact limit (8) and no further; attributes in
// dense storage and tracked creation order are refused, and a
// refused edit writes nothing.
// Up to the compact limit (8), then into dense storage; objects
// already in dense storage and ones tracking creation order.
ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap();
want.push(("g", "eighth".into(), AttrValue::I64(8)));
let before = std::fs::read(&path).unwrap();
unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9)));
unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1)));
unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1)));
assert!(
std::fs::read(&path).unwrap() == before,
"a refused edit changed the file"
);
ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap();
want.push(("g", "ninth".into(), AttrValue::I64(9)));
ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap();
want.push(("dense", "k0".into(), AttrValue::I64(1)));
ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap();
want.push(("tracked", "b".into(), AttrValue::I64(1)));
}
drop(ed);
check_tools(&path, *dump);
@@ -1027,7 +1100,7 @@ fn refused_edits_change_nothing() {
let before = std::fs::read(&path).unwrap();
let mut ed = FileEditor::open(&path).unwrap();
unsupported(ed.write_all("s", &[0u8; 32]));
unsupported(ed.resize("x", &[4]));
unsupported(ed.resize("c", &[4]));
unsupported(
ed.resize("c", &[6, 1])
.map_err(|_| Error::Unsupported(String::new())),
@@ -1124,9 +1197,10 @@ fn out_of_order_chunk_creation_matches_libhdf5() {
}
}
/// Not a check: prints how much space an append workload leaks (the editor
/// never reuses space), against libhdf5 doing the same appends and against
/// `h5repack` of each. Run with `--ignored --nocapture`.
/// Not a check: prints how much space an append workload leaks (one
/// editor for the whole workload, which reuses the space it frees but not
/// space it cannot fit a grown chunk into), against libhdf5 doing the same
/// appends and against `h5repack` of each. Run with `--ignored --nocapture`.
#[test]
#[ignore]
fn measure_append_waste() {
crates/clawhdf5/src/edit/attrs.rs
+513
View File
@@ -0,0 +1,513 @@
//! Setting attributes, as `H5O__attr_create` / `H5A__dense_insert` do:
//! compact attributes are object header messages (with their creation
//! index in the message header when the object tracks creation order);
//! when an object reaches its compact limit (or an attribute is too large
//! for a header message) its attributes move to dense storage — a fractal
//! heap for the encoded messages, a version-2 B-tree indexing them by name
//! hash (record type 8) and, when creation order is indexed, a second one
//! by creation index (type 9) — and the Attribute Info message points at
//! them.
use std::cmp::Ordering;
use clawhdf5_format::attribute::AttributeMessage;
use clawhdf5_format::dataspace::DataspaceType;
use crate::edit::btree2::Bt2;
use crate::edit::fheap::Heap;
use crate::edit::image::{Image, get_uint, put_uint, undef};
use crate::edit::ohdr::{Header, MSG_ATTRIBUTE};
use crate::edit::{MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, MSG_FLAG_SHARED, check_plain};
use crate::error::Error;
use crate::reader::File;
use crate::types::AttrValue;
/// `H5O_MESG_MAX_SIZE`: a larger attribute goes to dense storage.
const MESG_MAX_SIZE: usize = 65536;
/// `H5O_MAX_CRT_ORDER_IDX`: the creation index of an attribute of an object
/// that does not track creation order.
const NO_CRT_IDX: u16 = u16::MAX;
/// Name and creation-order index B-trees (`H5A_NAME_BT2_*`,
/// `H5A_CORDER_BT2_*`).
const NAME_BT2_TYPE: u8 = 8;
const CORDER_BT2_TYPE: u8 = 9;
const ATTR_BT2_NODE: u32 = 512;
/// Heap IDs in attribute records.
const ID_LEN: usize = 8;
/// An object's Attribute Info message.
#[derive(Debug, Clone)]
struct AInfo {
/// Its message index in the header.
idx: usize,
track: bool,
index: bool,
max_crt: u16,
fheap: u64,
name_bt2: u64,
corder_bt2: u64,
}
impl AInfo {
fn load(img: &Image<'_>, hdr: &Header) -> Result<Option<Self>, Error> {
let Some(idx) = hdr.find(MSG_ATTR_INFO) else {
return Ok(None);
};
if hdr.msgs[idx].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute info message".into()));
}
let d = hdr.data(img, idx)?;
let os = img.os as usize;
let short = || Error::Unsupported("short attribute info message".into());
if d.first() != Some(&0) {
return Err(Error::Unsupported("attribute info message version".into()));
}
let flags = *d.get(1).ok_or_else(short)?;
let track = flags & 0x01 != 0;
let index = flags & 0x02 != 0;
let mut p = 2;
let mut max_crt = 0;
if track {
let b = d.get(p..p + 2).ok_or_else(short)?;
max_crt = u16::from_le_bytes([b[0], b[1]]);
p += 2;
}
let n = if index { 3 } else { 2 };
if d.len() < p + n * os {
return Err(short());
}
Ok(Some(Self {
idx,
track,
index,
max_crt,
fheap: get_uint(&d[p..], img.os),
name_bt2: get_uint(&d[p + os..], img.os),
corder_bt2: if index {
get_uint(&d[p + 2 * os..], img.os)
} else {
undef(img.os)
},
}))
}
fn dense(&self, os: u8) -> bool {
self.fheap != undef(os)
}
/// Store the changeable fields back into the message.
fn store(&self, img: &mut Image<'_>, hdr: &mut Header) -> Result<(), Error> {
let os = img.os as usize;
let mut p = 2;
if self.track {
hdr.patch(img, self.idx, p, &self.max_crt.to_le_bytes())?;
p += 2;
}
let mut a = vec![0u8; os];
for (k, v) in [self.fheap, self.name_bt2, self.corder_bt2]
.into_iter()
.enumerate()
.take(if self.index { 3 } else { 2 })
{
put_uint(&mut a, v, img.os);
hdr.patch(img, self.idx, p + k * os, &a)?;
}
Ok(())
}
/// The next creation index (`H5O__attr_create`), or libhdf5's "none".
fn next_crt(&mut self) -> Result<u16, Error> {
if !self.track {
return Ok(NO_CRT_IDX);
}
if self.max_crt == NO_CRT_IDX {
return Err(Error::Unsupported(
"object's attribute creation index is exhausted".into(),
));
}
self.max_crt += 1;
Ok(self.max_crt - 1)
}
}
/// The name bytes of an attribute message body (without the NUL).
pub(super) fn attr_name(d: &[u8]) -> Result<&[u8], Error> {
let bad = || Error::Unsupported("malformed attribute message".into());
let (len, at) = match d.first() {
Some(1) | Some(2) if d.len() >= 8 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 8),
Some(3) if d.len() >= 9 => (usize::from(u16::from_le_bytes([d[2], d[3]])), 9),
_ => return Err(bad()),
};
let name = d.get(at..at + len).ok_or_else(bad)?;
Ok(name.split(|&b| b == 0).next().unwrap_or(name))
}
/// A new Attribute Info message for a version-2 header with flags
/// `hdr_flags`, as `H5O__attr_create` makes it: version 0, creation order
/// tracked / indexed as the header's flags say, the maximum creation index,
/// and no dense storage (undefined fractal heap and B-tree addresses).
fn attr_info_message(hdr_flags: u8, max_crt: u16, os: u8) -> Vec<u8> {
let track = hdr_flags & 0x04 != 0;
let index = hdr_flags & 0x08 != 0;
let mut b = vec![0u8, u8::from(track) | (u8::from(index) << 1)];
if track {
b.extend_from_slice(&max_crt.to_le_bytes());
}
let undef_addr = vec![0xffu8; os as usize];
b.extend_from_slice(&undef_addr);
b.extend_from_slice(&undef_addr);
if index {
b.extend_from_slice(&undef_addr);
}
b
}
/// A version-2 header's limit on compact attributes: stored when its flags
/// say so, else libhdf5's default of 8.
fn max_compact_attrs(img: &Image<'_>, hdr: &Header) -> Result<u16, Error> {
if hdr.flags & 0x10 == 0 {
return Ok(8);
}
let mut p = hdr.addr + 6;
if hdr.flags & 0x20 != 0 {
p += 16;
}
let b = img.read(p, 2)?;
Ok(u16::from_le_bytes([b[0], b[1]]))
}
/// A version-1 attribute message (what libhdf5 writes in a version-1 object
/// header): name, datatype and dataspace each padded to 8 bytes, the
/// dataspace as a version-1 dataspace message.
fn encode_attr_v1(a: &AttributeMessage, ls: u8) -> Vec<u8> {
let mut name = a.name.as_bytes().to_vec();
name.push(0);
let dt = a.datatype.serialize();
let mut ds = vec![1u8, a.dataspace.rank, 0, 0, 0, 0, 0, 0];
if a.dataspace.space_type == DataspaceType::Simple {
let mut b = vec![0u8; ls as usize];
for &d in &a.dataspace.dimensions {
put_uint(&mut b, d, ls);
ds.extend_from_slice(&b);
}
if let Some(max) = &a.dataspace.max_dimensions {
ds[2] = 0x01;
for &d in max {
put_uint(&mut b, d, ls);
ds.extend_from_slice(&b);
}
}
} else {
ds[1] = 0;
}
let mut out = vec![1u8, 0];
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&(dt.len() as u16).to_le_bytes());
out.extend_from_slice(&(ds.len() as u16).to_le_bytes());
for part in [&name, &dt, &ds] {
out.extend_from_slice(part);
out.resize(out.len().next_multiple_of(8), 0);
}
out.extend_from_slice(&a.raw_data);
out
}
/// Dense storage opened for changes.
struct Dense {
heap: Heap,
names: Bt2,
order: Option<Bt2>,
}
/// `H5_checksum_lookup3` of a name, as the name index keys it.
fn name_hash(name: &[u8]) -> u32 {
clawhdf5_format::checksum::jenkins_lookup3(name)
}
/// Compare attribute `name` (hash `hash`) with a name-index record
/// (`H5A__dense_btree2_name_compare`: the hash, then the stored name).
fn cmp_name(
heap: &Heap,
img: &Image<'_>,
hash: u32,
name: &[u8],
rec: &[u8],
) -> Result<Ordering, Error> {
let theirs = u32::from_le_bytes([rec[13], rec[14], rec[15], rec[16]]);
match hash.cmp(&theirs) {
Ordering::Equal => {
if rec[ID_LEN] & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported(
"shared attribute in dense storage".into(),
));
}
let obj = heap.read(img, &rec[..ID_LEN])?;
Ok(name.cmp(attr_name(&obj)?))
}
o => Ok(o),
}
}
fn corder_of(rec: &[u8]) -> u32 {
u32::from_le_bytes([rec[9], rec[10], rec[11], rec[12]])
}
impl Dense {
fn open(img: &Image<'_>, ai: &AInfo) -> Result<Self, Error> {
let heap = Heap::open(img, ai.fheap)?;
let names = Bt2::open(img, ai.name_bt2)?;
if names.tree_type() != NAME_BT2_TYPE || names.record_size() != ID_LEN + 9 {
return Err(Error::Unsupported("attribute name index layout".into()));
}
let order = if ai.index {
let t = Bt2::open(img, ai.corder_bt2)?;
if t.tree_type() != CORDER_BT2_TYPE || t.record_size() != ID_LEN + 5 {
return Err(Error::Unsupported(
"attribute creation-order index layout".into(),
));
}
Some(t)
} else {
None
};
Ok(Self { heap, names, order })
}
/// `H5A__dense_create`: heap, name index, [creation-order index].
fn create(img: &mut Image<'_>, index: bool) -> Result<Self, Error> {
let heap = Heap::create_attribute_heap(img)?;
let names = Bt2::create(img, NAME_BT2_TYPE, ATTR_BT2_NODE, ID_LEN + 9, 100, 40)?;
let order = if index {
Some(Bt2::create(
img,
CORDER_BT2_TYPE,
ATTR_BT2_NODE,
ID_LEN + 5,
100,
40,
)?)
} else {
None
};
Ok(Self { heap, names, order })
}
/// `H5A__dense_insert` of an encoded attribute message.
fn insert(&mut self, img: &mut Image<'_>, body: &[u8], crt: u16) -> Result<(), Error> {
let name = attr_name(body)?.to_vec();
let id = self.heap.insert(img, body)?;
if id.len() != ID_LEN {
return Err(Error::Unsupported("attribute heap ID length".into()));
}
let hash = name_hash(&name);
let mut rec = id.clone();
rec.push(0);
rec.extend_from_slice(&u32::from(crt).to_le_bytes());
rec.extend_from_slice(&hash.to_le_bytes());
let heap = &self.heap;
self.names
.insert(img, &mut |im, r| cmp_name(heap, im, hash, &name, r), &rec)?;
if let Some(t) = &mut self.order {
let key = u32::from(crt);
t.insert(
img,
&mut |_, r| Ok(key.cmp(&corder_of(r))),
&rec[..ID_LEN + 5],
)?;
}
Ok(())
}
fn finish(&mut self, img: &mut Image<'_>) -> Result<(), Error> {
self.heap.finish(img)?;
self.names.finish(img)?;
if let Some(t) = &mut self.order {
t.finish(img)?;
}
Ok(())
}
}
/// Set attribute `name` of the object at `path` to `value`.
pub(super) fn set_attr(
f: &File,
img: &mut Image<'_>,
path: &str,
name: &str,
value: &AttrValue,
) -> Result<(), Error> {
let addr = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), f.superblock(), path)?;
let mut hdr = Header::load(img, addr)?;
let mut msg = clawhdf5_format::type_builders::build_attr_message(name, value);
check_plain(&msg.datatype)?;
// libhdf5 encodes a simple dataspace with its maximum dimensions (the
// current ones when none were given), so an attribute takes the same
// space in a header or heap as when libhdf5 writes it.
if msg.dataspace.space_type == DataspaceType::Simple && msg.dataspace.max_dimensions.is_none() {
msg.dataspace.max_dimensions = Some(msg.dataspace.dimensions.clone());
}
// H5A__set_version: version 1 unless the name is not ASCII (then 3),
// raised to the file's low bound — which is the earliest for a file
// libhdf5 opens without a libver setting (h5py's `r+`).
let body = if hdr.version == 1 || name.is_ascii() {
encode_attr_v1(&msg, img.ls)
} else {
let mut b = msg.serialize_v3(img.ls);
if !name.is_ascii() {
b[8] = 1; // UTF-8 name
}
b
};
let mut ainfo = if hdr.version == 2 {
AInfo::load(img, &hdr)?
} else {
None
};
if let Some(ai) = ainfo.as_mut().filter(|a| a.dense(img.os)) {
let mut ai = ai.clone();
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
return hdr.finish(img);
}
let mut existing = None;
let mut count = 0usize;
for i in 0..hdr.msgs.len() {
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
continue;
}
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute message".into()));
}
count += 1;
if attr_name(&hdr.data(img, i)?)? == name.as_bytes() {
existing = Some(i);
}
}
if let Some(i) = existing {
hdr.delete(img, i)?;
count -= 1;
}
if hdr.version == 1 {
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, None)?;
return hdr.finish(img);
}
let tracked = hdr.flags & 0x04 != 0;
// H5O__attr_create: a missing Attribute Info message starts from
// nothing (and is added below, holding the new maximum creation index).
let new_ainfo = ainfo.is_none();
let mut ai = ainfo.take().unwrap_or(AInfo {
idx: usize::MAX,
track: tracked,
index: hdr.flags & 0x08 != 0,
max_crt: 0,
fheap: undef(img.os),
name_bt2: undef(img.os),
corder_bt2: undef(img.os),
});
let max_compact = usize::from(max_compact_attrs(img, &hdr)?);
if count == max_compact || body.len() >= MESG_MAX_SIZE {
if new_ainfo {
return Err(Error::Unsupported(
"dense attribute storage for an object without an Attribute Info message".into(),
));
}
to_dense(img, &mut hdr, &mut ai)?;
set_dense(img, &mut hdr, &mut ai, name.as_bytes(), &body)?;
return hdr.finish(img);
}
let crt = ai.next_crt()?;
let corder = tracked.then_some(crt);
if new_ainfo {
// libhdf5 appends the Attribute Info message before the attribute
// when free space holds both, else after it, so that a new
// continuation chunk made for the attribute has room for it too.
let a = attr_info_message(hdr.flags, ai.max_crt, img.os);
let first = hdr.has_free(a.len() + hdr.hsize() + body.len());
if first {
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
}
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
if !first {
hdr.insert(img, MSG_ATTR_INFO, MSG_FLAG_DONTSHARE, &a, Some(0))?;
}
} else {
hdr.insert(img, MSG_ATTRIBUTE, 0, &body, corder)?;
ai.store(img, &mut hdr)?;
}
hdr.finish(img)
}
/// Move every compact attribute of the object into new dense storage, in
/// header message order (`H5O__attr_to_dense_cb`), leaving free space where
/// the messages were.
fn to_dense(img: &mut Image<'_>, hdr: &mut Header, ai: &mut AInfo) -> Result<(), Error> {
let mut dense = Dense::create(img, ai.index)?;
for i in 0..hdr.msgs.len() {
if hdr.msgs[i].mtype != MSG_ATTRIBUTE {
continue;
}
if hdr.msgs[i].flags & MSG_FLAG_SHARED != 0 {
return Err(Error::Unsupported("shared attribute message".into()));
}
let body = hdr.data(img, i)?;
let crt = if ai.track {
hdr.msgs[i].corder.unwrap_or(0)
} else {
NO_CRT_IDX
};
dense.insert(img, &body, crt)?;
hdr.delete(img, i)?;
}
dense.finish(img)?;
ai.fheap = dense.heap.address();
ai.name_bt2 = dense.names.address();
if let Some(t) = &dense.order {
ai.corder_bt2 = t.address();
}
ai.store(img, hdr)
}
/// Set an attribute of an object whose attributes are in dense storage: an
/// attribute of that name whose new encoding has the old one's size is
/// rewritten in its heap object (`H5A__dense_write`); otherwise the old one
/// is removed (`H5A__dense_remove`: name index, creation-order index, heap
/// object) and the new one inserted with the next creation index.
fn set_dense(
img: &mut Image<'_>,
hdr: &mut Header,
ai: &mut AInfo,
name: &[u8],
body: &[u8],
) -> Result<(), Error> {
let mut dense = Dense::open(img, ai)?;
let hash = name_hash(name);
let found = {
let heap = &dense.heap;
dense
.names
.find(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?
};
if let Some(rec) = found {
if dense.heap.write_in_place(img, &rec[..ID_LEN], body)? {
return dense.finish(img);
}
{
let heap = &dense.heap;
dense
.names
.remove(img, &mut |im, r| cmp_name(heap, im, hash, name, r))?;
}
if let Some(t) = &mut dense.order {
let key = corder_of(&rec);
t.remove(img, &mut |_, r| Ok(key.cmp(&corder_of(r))))?
.ok_or_else(|| {
Error::Unsupported("attribute missing from its creation-order index".into())
})?;
}
dense.heap.remove(img, &rec[..ID_LEN])?;
}
let crt = ai.next_crt()?;
dense.insert(img, body, crt)?;
dense.finish(img)?;
ai.store(img, hdr)
}
crates/clawhdf5/src/edit/btree1.rs
+166 -1
View File
@@ -56,6 +56,15 @@ fn bad(why: &str) -> Error {
))
}
/// What a removal did below a node (`H5B_ins_t`), with the removed chunk's
/// address and size.
enum Rm {
NotFound,
Noop((u64, u32)),
/// The child is gone: the parent must drop it.
Remove((u64, u32)),
}
enum Ins {
Done,
/// The node split; the new right sibling and its first key.
@@ -144,7 +153,14 @@ impl BTree1 {
put_uint(&mut d[8 + osz..], node.right, os);
let ks = self.key_size();
let mut p = 8 + 2 * osz;
for (i, k) in node.keys.iter().enumerate() {
// An empty node (a root whose last chunk was removed) stores no
// keys, as libhdf5 writes it.
let nkeys = if node.children.is_empty() {
0
} else {
node.keys.len()
};
for (i, k) in node.keys.iter().take(nkeys).enumerate() {
d[p..p + 4].copy_from_slice(&k.size.to_le_bytes());
d[p + 4..p + 8].copy_from_slice(&k.mask.to_le_bytes());
for (j, o) in k.offs.iter().enumerate() {
@@ -210,6 +226,18 @@ impl BTree1 {
return Err(bad("bad chunk key"));
}
let root = self.read(img, self.root)?;
if root.children.is_empty() {
// Every chunk was removed (H5B__insert_helper's first
// insertion): the root, a leaf again, takes it.
let right = self.right_key_after(&key);
let node = Node {
level: 0,
keys: vec![key, right],
children: vec![addr],
..root
};
return self.write(img, &node);
}
if let Ins::Split(mid, right_addr) = self.insert_at(img, root, &key, addr, 64)? {
// The root split: move its (left) half to a new node so the root
// keeps its address, then make the root the parent of both.
@@ -372,6 +400,143 @@ impl BTree1 {
cmp(&key.offs, &right.keys[0].offs) != Ordering::Less
}
/// Remove the chunk at offsets `offs` (element-size coordinate 0), as
/// `H5B_remove` does for the chunk index (whose critical key is the
/// left one): no rebalancing; a node left without children is deleted
/// and its siblings relinked (the left one takes over its right key),
/// a root left empty becomes an empty leaf. Returns the chunk's address
/// and stored size, or `None` when the tree has no such chunk (nothing
/// changes then). Deleted nodes are freed in `img`.
pub(crate) fn remove(
&mut self,
img: &mut Image<'_>,
offs: &[u64],
) -> Result<Option<(u64, u32)>, Error> {
if offs.len() != self.ndims {
return Err(bad("bad chunk key"));
}
let mut lt = None;
match self.remove_at(img, self.root, 0, offs, &mut lt, 64)? {
Rm::NotFound => Ok(None),
Rm::Noop(c) | Rm::Remove(c) => Ok(Some(c)),
}
}
fn remove_at(
&self,
img: &mut Image<'_>,
addr: u64,
level: usize,
offs: &[u64],
lt_out: &mut Option<Key>,
depth: u8,
) -> Result<Rm, Error> {
if depth == 0 {
return Err(bad("tree too deep"));
}
let mut node = self.read(img, addr)?;
let n = node.children.len();
// H5D__btree_cmp3 over (keys[i], keys[i + 1]), binary search.
let (mut lo, mut hi, mut idx) = (0usize, n, 0usize);
let mut c = 1i32;
while lo < hi && c != 0 {
idx = (lo + hi) / 2;
c = if cmp(offs, &node.keys[idx + 1].offs) != Ordering::Less {
1
} else if cmp(offs, &node.keys[idx].offs) == Ordering::Less {
-1
} else {
0
};
if c < 0 {
hi = idx;
} else {
lo = idx + 1;
}
}
if c != 0 {
return Ok(Rm::NotFound);
}
let mut lt_changed = None;
let res = if node.level > 0 {
let child = self.read(img, node.children[idx])?;
if usize::from(child.level) + 1 != usize::from(node.level) {
return Err(bad("inconsistent node levels"));
}
self.remove_at(
img,
node.children[idx],
level + 1,
offs,
&mut lt_changed,
depth - 1,
)?
} else {
if node.keys[idx].offs != offs {
return Ok(Rm::NotFound);
}
Rm::Remove((node.children[idx], node.keys[idx].size))
};
let chunk = match res {
Rm::NotFound => return Ok(Rm::NotFound),
Rm::Noop(c) | Rm::Remove(c) => c,
};
let mut dirty = false;
if let Some(k) = lt_changed {
node.keys[idx] = k;
dirty = true;
if idx == 0 {
*lt_out = Some(node.keys[0].clone());
}
}
let out = Rm::Noop(chunk);
if let Rm::Remove(_) = res {
let undefined = undef(img.os);
if n == 1 {
if level > 0 {
if node.left != undefined {
let mut sib = self.read(img, node.left)?;
let last = sib.children.len();
sib.keys[last] = node.keys[1].clone();
sib.right = node.right;
self.write(img, &sib)?;
}
if node.right != undefined {
let mut sib = self.read(img, node.right)?;
sib.left = node.left;
self.write(img, &sib)?;
}
img.free(addr, self.node_size(img.os) as u64);
return Ok(Rm::Remove(chunk));
}
node.children.clear();
node.keys.truncate(1);
node.level = 0;
} else if idx == 0 {
node.keys.remove(0);
node.children.remove(0);
*lt_out = Some(node.keys[0].clone());
} else {
// Right-most or middle child: its left key goes, the next
// key becomes the following child's left key.
node.keys.remove(idx);
node.children.remove(idx);
}
dirty = true;
}
if dirty {
self.write(img, &node)?;
}
// The left sibling's right key follows a changed left key.
if lt_out.is_some() && node.left != undef(img.os) && level > 0 {
let mut sib = self.read(img, node.left)?;
let last = sib.children.len();
sib.keys[last] = node.keys[0].clone();
self.write(img, &sib)?;
}
Ok(out)
}
fn insert_child(&self, node: &mut Node, pos: usize, key: Key, addr: u64) {
let n = node.children.len();
if node.level == 0 {
crates/clawhdf5/src/edit/btree2.rs
+1121
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5/src/edit/earray.rs
+26 -3
View File
@@ -383,12 +383,23 @@ impl Ea {
}
/// Set element `idx` to `e`.
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
/// Set element `idx` to `e`, or back to the fill element (`None`: a
/// removed chunk, `H5D__earray_idx_remove`), which creates no block.
pub(crate) fn set(
&mut self,
img: &mut Image<'_>,
idx: u64,
e: Option<Elem>,
) -> Result<(), Error> {
let os = img.os;
let osz = u64::from(os);
let es = self.slot_size(os) as u64;
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
let enc = encode_elem(e, self.filtered, self.elem_size, os)?;
let clear = e.is_none();
if self.iblock == undef(os) {
if clear {
return Ok(());
}
self.create_iblock(img)?;
}
let ib = self.iblock;
@@ -420,6 +431,9 @@ impl Ea {
let dblk_idx = l.start_dblk + local;
let slot = dblks_at + dblk_idx * osz;
let mut addr = get_uint(&img.read(slot, os as usize)?, os);
if addr == undef(os) && clear {
return Ok(());
}
if addr == undef(os) {
// libhdf5 records start_idx + (global data block index)
// * nelmts here (H5EA__lookup_elmt), not the block's
@@ -449,6 +463,9 @@ impl Ea {
let sb_prefix = self.dblk_prefix_len(os);
let sb_len = sb_prefix + bitmap_len + l.ndblks * osz;
let mut sb = get_uint(&img.read(sslot, os as usize)?, os);
if sb == undef(os) && clear {
return Ok(());
}
if sb == undef(os) {
let mut d = self.block_prefix(b"EASB", l.start_idx, os);
d.resize(d.len() + bitmap_len as usize, 0);
@@ -471,6 +488,9 @@ impl Ea {
let local = (rel - l.start_idx) / l.dblk_nelmts;
let dslot = sb + sb_prefix + bitmap_len + local * osz;
let mut addr = get_uint(&img.read(dslot, os as usize)?, os);
if addr == undef(os) && clear {
return Ok(());
}
if addr == undef(os) {
let off = l.start_idx + local * l.dblk_nelmts;
addr = self.create_dblock(img, l.dblk_nelmts, off)?;
@@ -491,6 +511,9 @@ impl Ea {
let bpos = sb + sb_prefix + bit / 8;
let mut byte = img.read(bpos, 1)?[0];
let mask = 0x80u8 >> (bit % 8);
if byte & mask == 0 && clear {
return Ok(());
}
if byte & mask == 0 {
let fill = self.fill_elems(page, os)?;
img.write(page_at, &fill)?;
@@ -503,7 +526,7 @@ impl Ea {
}
}
}
if idx + 1 > self.stats[4] {
if !clear && idx + 1 > self.stats[4] {
self.stats[4] = idx + 1;
self.dirty_hdr = true;
}
crates/clawhdf5/src/edit/farray.rs
+12 -3
View File
@@ -137,13 +137,19 @@ impl Fa {
Ok((fa, hdr))
}
/// Set element `idx` to `e`.
pub(crate) fn set(&mut self, img: &mut Image<'_>, idx: u64, e: Elem) -> Result<(), Error> {
/// Set element `idx` to `e`, or back to the fill element (`None`: a
/// removed chunk, `H5D__farray_idx_remove`), which creates no page.
pub(crate) fn set(
&mut self,
img: &mut Image<'_>,
idx: u64,
e: Option<Elem>,
) -> Result<(), Error> {
let os = img.os;
if idx >= self.nelmts {
return Err(bad("index beyond the array"));
}
let enc = encode_elem(Some(e), self.filtered, self.elem_size, os)?;
let enc = encode_elem(e, self.filtered, self.elem_size, os)?;
let es = self.slot(os);
let prefix = 6 + u64::from(os);
let page = self.page();
@@ -162,6 +168,9 @@ impl Fa {
let bpos = self.dblk + prefix + p / 8;
let mut byte = img.read(bpos, 1)?[0];
let mask = 0x80u8 >> (p % 8);
if byte & mask == 0 && e.is_none() {
return Ok(());
}
if byte & mask == 0 {
let fill = encode_elem(None, self.filtered, self.elem_size, os)?;
img.write(page_at, &fill.repeat(count as usize))?;
crates/clawhdf5/src/edit/fheap.rs
+1577
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5/src/edit/image.rs
+199 -26
View File
@@ -35,6 +35,61 @@ pub(crate) struct Image<'a> {
/// Width of addresses and lengths in the file.
pub(crate) os: u8,
pub(crate) ls: u8,
/// Space the edit stopped using. Not reused by this edit: until the
/// edit is committed, the file's metadata still points at it.
freed: Vec<(u64, u64)>,
/// Space earlier edits of the session freed, available to this one.
reusable: FreeList,
/// Blocks this edit took from `reusable`: nothing on disk refers to
/// them, so they are written with the new space, before the changes
/// that link them in (see [`Plan::commit`]).
fresh: Vec<(u64, u64)>,
}
/// Free space, address -> length, adjacent blocks merged.
#[derive(Debug, Clone, Default)]
pub(crate) struct FreeList(BTreeMap<u64, u64>);
impl FreeList {
/// Add `[addr, addr + len)`, merged with neighbours it touches.
pub(crate) fn add(&mut self, addr: u64, len: u64) {
if len == 0 {
return;
}
let (mut lo, mut hi) = (addr, addr.saturating_add(len));
if let Some((&a, &l)) = self.0.range(..=lo).next_back()
&& a + l >= lo
{
lo = a;
hi = hi.max(a + l);
self.0.remove(&a);
}
while let Some((&a, &l)) = self.0.range(lo..=hi).next() {
hi = hi.max(a + l);
self.0.remove(&a);
}
self.0.insert(lo, hi - lo);
}
/// Take `size` bytes from the smallest block that holds them (the
/// lowest address among equals), from its start.
fn take(&mut self, size: u64) -> Option<u64> {
let (&a, &l) = self
.0
.iter()
.filter(|&(_, &l)| l >= size)
.min_by_key(|&(&a, &l)| (l, a))?;
self.0.remove(&a);
if l > size {
self.0.insert(a + size, l - size);
}
Some(a)
}
/// Total bytes.
pub(crate) fn total(&self) -> u64 {
self.0.values().sum()
}
}
impl<'a> Image<'a> {
@@ -47,9 +102,24 @@ impl<'a> Image<'a> {
old_eoa: eoa,
os,
ls,
freed: Vec::new(),
reusable: FreeList::default(),
fresh: Vec::new(),
}
}
/// Let the edit allocate from `free` (space earlier edits freed).
pub(crate) fn with_reusable(mut self, free: FreeList) -> Self {
// Only space inside the file as it is now.
self.reusable = FreeList(
free.0
.into_iter()
.filter(|&(a, l)| a.saturating_add(l) <= self.old_eoa)
.collect(),
);
self
}
pub(crate) fn eoa(&self) -> u64 {
self.eoa
}
@@ -63,11 +133,24 @@ impl<'a> Image<'a> {
!self.patches.is_empty() || self.eoa != self.old_eoa
}
/// Allocate `size` bytes at the end of the file. The space reads as
/// zeros until written. Nothing is ever freed: space an edit stops
/// using (a relocated chunk, say) is leaked, as there is no free-space
/// manager.
/// Allocate `size` bytes: from space an earlier edit of this session
/// freed when a block holds them (best fit), else at the end of the
/// file. The space reads as zeros until written.
pub(crate) fn alloc(&mut self, size: u64) -> Result<u64, Error> {
if size > 0
&& let Some(a) = self.reusable.take(size)
{
self.fresh.push((a, size));
let n = usize::try_from(size)
.map_err(|_| Error::Unsupported("allocation too large".into()))?;
self.write(a, &vec![0u8; n])?;
return Ok(a);
}
self.alloc_end(size)
}
/// Allocate `size` bytes at the end of the file.
fn alloc_end(&mut self, size: u64) -> Result<u64, Error> {
let addr = self.eoa;
let end = addr
.checked_add(size)
@@ -77,6 +160,14 @@ impl<'a> Image<'a> {
Ok(addr)
}
/// Note that the edit no longer uses `[addr, addr + len)`; later edits
/// of the session may reuse it.
pub(crate) fn free(&mut self, addr: u64, len: u64) {
if len > 0 {
self.freed.push((addr, len));
}
}
/// If `[addr, addr + old_len)` is the last allocated space, grow it to
/// `new_len` bytes (a structure at the end of the file can grow where
/// it is) and return true.
@@ -91,7 +182,7 @@ impl<'a> Image<'a> {
}
let old_end = self.eoa;
self.eoa = addr;
if let Err(e) = self.alloc(new_len) {
if let Err(e) = self.alloc_end(new_len) {
self.eoa = old_end;
return Err(e);
}
@@ -189,12 +280,24 @@ impl<'a> Image<'a> {
/// The edit's writes, detached from the base bytes (see the module's
/// invariant: the reader that owns them can then be dropped before
/// anything is written).
pub(crate) fn into_plan(self) -> Plan {
Plan {
patches: self.patches,
eoa: self.eoa,
old_eoa: self.old_eoa,
pub(crate) fn into_plan(self) -> (Plan, FreeList) {
// What the session may reuse once this edit is committed: what it
// did not take, and what it freed.
let mut free = self.reusable;
for (a, l) in self.freed {
free.add(a, l);
}
let mut fresh = self.fresh;
fresh.sort_unstable();
(
Plan {
patches: self.patches,
eoa: self.eoa,
old_eoa: self.old_eoa,
fresh,
},
free,
)
}
}
@@ -203,33 +306,57 @@ pub(crate) struct Plan {
patches: BTreeMap<u64, Vec<u8>>,
eoa: u64,
old_eoa: u64,
/// Reused blocks (sorted): written with the new space.
fresh: Vec<(u64, u64)>,
}
impl Plan {
/// Whether `addr` is in space nothing on disk refers to yet (past the
/// old end of file, or in a reused block), and up to where (before
/// `end`) that stays so.
fn new_space(&self, addr: u64, end: u64) -> (bool, u64) {
if addr >= self.old_eoa {
return (true, end);
}
let limit = end.min(self.old_eoa);
// The reused block holding `addr`, or the next one after it.
let i = self.fresh.partition_point(|&(a, l)| a + l <= addr);
match self.fresh.get(i) {
Some(&(a, l)) if a <= addr => (true, limit.min(a + l)),
Some(&(a, _)) => (false, limit.min(a)),
None => (false, limit),
}
}
/// Write the edit to `file`, whose superblock is at `user_block`.
///
/// Order: first everything in newly allocated space (new chunks, new
/// index blocks, relocated structures), which nothing on disk refers to
/// yet, then a sync; then the changes to existing bytes — raw data
/// overwritten in place and the metadata that links the new space in
/// (superblock end of file, chunk index entries, object header
/// index blocks, relocated structures — past the old end of file, or in
/// space an earlier edit of the session freed), which nothing on disk
/// refers to yet, then a sync; then the changes to existing bytes — raw
/// data overwritten in place and the metadata that links the new space
/// in (superblock end of file, chunk index entries, object header
/// messages) — then a sync. A crash during the first phase leaves the
/// file as it was (plus unreferenced bytes past its end of file); a
/// crash during the second can leave it inconsistent, as with libhdf5
/// without SWMR: there is no journal.
/// file as it was (plus unreferenced bytes); a crash during the second
/// can leave it inconsistent, as with libhdf5 without SWMR: there is no
/// journal.
pub(crate) fn commit(self, file: &mut std::fs::File, user_block: u64) -> Result<(), Error> {
let old_eoa = self.old_eoa;
let mut in_place: Vec<(u64, &[u8])> = Vec::new();
for (&addr, bytes) in &self.patches {
// A patch may run from existing bytes into new space (writes
// merge); its new part goes with the new space.
let split = old_eoa.saturating_sub(addr).min(bytes.len() as u64) as usize;
let (old, new) = bytes.split_at(split);
if !new.is_empty() {
write_at(file, user_block + addr + split as u64, new)?;
}
if !old.is_empty() {
in_place.push((addr, old));
// A patch may run across new and existing space (writes
// merge): split it where that changes.
let end = addr + bytes.len() as u64;
let mut at = addr;
while at < end {
let (new, upto) = self.new_space(at, end);
let part = &bytes[(at - addr) as usize..(upto - addr) as usize];
if new {
write_at(file, user_block + at, part)?;
} else {
in_place.push((at, part));
}
at = upto;
}
}
if self.eoa > old_eoa {
@@ -305,6 +432,52 @@ mod tests {
assert!(img.write(42, &[1]).is_err());
}
#[test]
fn free_list_merges_and_takes_best_fit() {
let mut f = FreeList::default();
f.add(100, 10);
f.add(120, 5);
f.add(110, 10); // joins both neighbours
assert_eq!(
f.0.iter().map(|(&a, &l)| (a, l)).collect::<Vec<_>>(),
[(100, 25)]
);
f.add(300, 8);
f.add(200, 40);
// Best fit: the 8-byte block for 6 bytes, from its start.
assert_eq!(f.take(6), Some(300));
assert_eq!(f.take(30), Some(200));
assert_eq!(f.take(26), None);
assert_eq!(f.total(), 25 + 2 + 10);
}
/// An edit allocates from space earlier edits freed (zeroed), never
/// from what it frees itself; the plan writes reused blocks with the
/// new space.
#[test]
fn reuse_across_edits_only() {
let base = vec![7u8; 64];
let mut free = FreeList::default();
free.add(8, 16);
let mut img = Image::new(&base, 8, 8).with_reusable(free);
img.free(32, 16); // freed by this edit: not reusable yet
let a = img.alloc(16).unwrap();
assert_eq!(a, 8);
assert_eq!(img.read(8, 16).unwrap(), vec![0u8; 16]);
let b = img.alloc(8).unwrap();
assert_eq!(b, 64, "the edit's own freed space is not reused");
img.write(4, &[1; 8]).unwrap(); // existing bytes 4..8, reused 8..12
let (plan, next) = img.into_plan();
assert_eq!(plan.new_space(4, 12), (false, 8));
assert_eq!(plan.new_space(8, 12), (true, 12));
assert_eq!(plan.new_space(30, 40), (false, 40));
assert_eq!(plan.new_space(64, 72), (true, 72));
assert_eq!(
next.0.iter().map(|(&a, &l)| (a, l)).collect::<Vec<_>>(),
[(32, 16)]
);
}
/// Random reads and writes against a flat copy of the bytes.
#[test]
fn matches_a_flat_model() {
crates/clawhdf5/src/edit/mod.rs
+687 -250
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5/src/edit/ohdr.rs
+6 -2
View File
@@ -59,7 +59,7 @@ pub(crate) struct Header {
added: usize,
}
const MAX_CHUNKS: usize = 1024;
const MAX_CHUNKS: usize = 1 << 16;
fn corrupt(why: &'static str) -> Error {
Error::Format(FormatError::InvalidObjectHeader(why))
@@ -118,7 +118,11 @@ impl Header {
});
h.scan(img, 0, addr + 16, addr + 16 + size, &mut pending)?;
}
while let Some((caddr, clen)) = pending.pop() {
// Continuation chunks in the order their messages are found, as
// H5O_protect loads them (so messages keep libhdf5's order).
let mut next = 0;
while let Some(&(caddr, clen)) = pending.get(next) {
next += 1;
if h.chunks.len() >= MAX_CHUNKS {
return Err(corrupt("too many object header chunks"));
}
crates/clawhdf5/tests/edit_tests.rs
+29 -3
View File
@@ -98,8 +98,8 @@ fn errors_leave_the_file_untouched() {
let mut ed = FileEditor::open(&path).unwrap();
assert!(matches!(FileEditor::open(&path), Err(Error::Locked(_))));
assert!(ed.write_all("missing", &[0; 4]).is_err());
// Wrong length, wrong type, outside the extent, beyond maxshape,
// shrinking, a rank change.
// Wrong length, wrong type, outside the extent, beyond maxshape, a
// rank change, resizing a dataset that is not chunked.
assert!(matches!(
ed.write_all("flat", &[0; 7]),
Err(Error::InvalidArgument(_))
@@ -116,7 +116,10 @@ fn errors_leave_the_file_untouched() {
ed.resize("raw", &[3, 5]),
Err(Error::InvalidArgument(_))
));
assert!(matches!(ed.resize("ext", &[4]), Err(Error::Unsupported(_))));
assert!(matches!(
ed.resize("flat", &[2]),
Err(Error::Unsupported(_))
));
assert!(matches!(
ed.resize("ext", &[4, 1]),
Err(Error::InvalidArgument(_))
@@ -136,3 +139,26 @@ fn errors_leave_the_file_untouched() {
drop(ed);
assert!(std::fs::read(&path).unwrap() == before);
}
/// Shrinking and growing again on a file clawhdf5 wrote: elements that come
/// back read as the fill value, the ones kept keep their values.
#[test]
fn shrink_then_grow_reads_fill() {
let dir = tempfile::tempdir().unwrap();
let path = sample(dir.path());
{
let mut ed = FileEditor::open(&path).unwrap();
ed.resize("ext", &[2]).unwrap();
ed.resize("ext", &[9]).unwrap();
ed.resize("raw", &[1, 4]).unwrap();
ed.resize("raw", &[3, 4]).unwrap();
}
let f = File::open(&path).unwrap();
assert_eq!(
f.dataset("ext").unwrap().read_i32().unwrap(),
[0, 1, 0, 0, 0, 0, 0, 0, 0]
);
let mut raw = vec![0.0f64; 12];
raw[..4].fill(0.5);
assert_eq!(f.dataset("raw").unwrap().read_f64().unwrap(), raw);
}
crates/clawhdf5/tests/fletcher32_interop.rs
+321
View File
@@ -0,0 +1,321 @@
//! Fletcher-32 against libhdf5.
//!
//! libhdf5's `H5_checksum_fletcher32` reduces its sums with the
//! ones'-complement fold `(s & 0xffff) + (s >> 16)`, which leaves 0xffff
//! where `% 65535` leaves 0. Our checksum once used `% 65535`, so on about
//! one chunk in 32768 (a sum that is a non-zero multiple of 65535) libhdf5
//! rejected the chunks we wrote and we rejected the chunks it wrote.
//!
//! - The checksum is compared with libhdf5's own `H5_checksum_fletcher32`,
//! called through ctypes from the library h5py loads, over every one-byte
//! and two-byte input and a large corpus of random and fold-heavy inputs.
//! - Chunks engineered to hit the fold are written by `FileBuilder` and by
//! `FileEditor` and read by h5py, and written by h5py and read by us.
//!
//! Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
use std::process::Command;
use clawhdf5::{File, FileBuilder, FileEditor};
use clawhdf5_format::checksum::fletcher32;
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn have_h5py() -> bool {
let ok = Command::new(python())
.args(["-c", "import h5py, numpy"])
.output()
.is_ok_and(|o| o.status.success());
if !ok {
assert!(
std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
}
ok
}
fn run_python(script: &str, args: &[&str]) -> String {
let out = Command::new(python())
.arg("-c")
.arg(script)
.args(args)
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\nSTDOUT: {}\nSTDERR: {}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).trim().to_string()
}
fn tmp(name: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!("clawhdf5_fletcher32_{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
dir.join(name)
}
/// The checksum our code computed before it was fixed: each sum reduced
/// `% 65535`. Only used to show that the test data hits the disagreement.
fn fletcher32_mod(data: &[u8]) -> u32 {
let (mut s1, mut s2) = (0u64, 0u64);
for w in data.chunks(2) {
let v = (u64::from(w[0]) << 8) | w.get(1).map_or(0, |&b| u64::from(b));
s1 = (s1 + v) % 65535;
s2 = (s2 + s1) % 65535;
}
((s2 as u32) << 16) | s1 as u32
}
/// Splitmix64, so the data is the same on every run.
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15);
let mut z = self.0;
z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
z ^ (z >> 31)
}
}
/// Writes every case of the file `argv[1]` (u32 LE length + bytes) back to
/// `argv[2]` as libhdf5's checksum of each, u32 LE.
const LIBHDF5_CHECKSUMS: &str = r#"
import ctypes, glob, os, struct, sys
import h5py
cands = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, 'h5py.libs', 'libhdf5-*.so*'))
cands += glob.glob(os.path.join(os.path.dirname(h5py.__file__), '.dylibs', 'libhdf5*.dylib'))
if cands:
lib = ctypes.CDLL(cands[0])
else:
# A system h5py links the system libhdf5, already loaded.
import h5py.h5
lib = ctypes.CDLL(h5py.h5.__file__)
f = lib.H5_checksum_fletcher32
f.restype = ctypes.c_uint32
f.argtypes = [ctypes.c_char_p, ctypes.c_size_t]
data = open(sys.argv[1], 'rb').read()
out = bytearray()
i = 0
while i < len(data):
(n,) = struct.unpack_from('<I', data, i)
i += 4
b = data[i:i + n]
i += n
out += struct.pack('<I', f(b, n))
open(sys.argv[2], 'wb').write(out)
"#;
#[test]
fn checksum_matches_libhdf5() {
if !have_h5py() {
return;
}
let mut cases: Vec<Vec<u8>> = Vec::new();
// Every one-byte input (the odd-length path alone) and every one-word
// input (65535 = 0xffff is the smallest fold).
cases.extend((0..=255u8).map(|b| vec![b]));
cases.extend((0..=u16::MAX).map(|w| w.to_be_bytes().to_vec()));
let mut rng = Rng(0x5eed_f1e7);
// Words drawn from values that make multiples of 65535 frequent, at
// lengths around the 360-word block boundaries, odd and even.
const FOLDY: [u16; 6] = [0, 1, 0xfffe, 0xffff, 0x8000, 0x7fff];
for _ in 0..40_000 {
let len = match rng.next() % 4 {
0 => (rng.next() % 16) as usize,
1 => 718 + (rng.next() % 6) as usize,
2 => 1438 + (rng.next() % 6) as usize,
_ => (rng.next() % 3000) as usize,
};
let foldy = rng.next().is_multiple_of(2);
let mut v = Vec::with_capacity(len + 1);
while v.len() < len {
let w = if foldy {
FOLDY[(rng.next() % 6) as usize]
} else {
rng.next() as u16
};
v.extend_from_slice(&w.to_be_bytes());
}
v.truncate(len);
cases.push(v);
}
// Long runs of 0xff: sums are multiples of 65535 at every block.
for len in [720, 721, 1440, 1441, 7200, 65536, 65537] {
cases.push(vec![0xff; len]);
}
let mut blob = Vec::new();
for c in &cases {
blob.extend_from_slice(&(c.len() as u32).to_le_bytes());
blob.extend_from_slice(c);
}
let input = tmp("cases.bin");
let output = tmp("sums.bin");
std::fs::write(&input, &blob).unwrap();
run_python(
LIBHDF5_CHECKSUMS,
&[input.to_str().unwrap(), output.to_str().unwrap()],
);
let sums = std::fs::read(&output).unwrap();
assert_eq!(sums.len(), cases.len() * 4);
let mut folds = 0;
for (c, s) in cases.iter().zip(sums.as_chunks::<4>().0) {
let want = u32::from_le_bytes(*s);
assert_eq!(
fletcher32(c),
want,
"checksum of {} bytes {:02x?}...",
c.len(),
&c[..c.len().min(16)]
);
if fletcher32_mod(c) != want {
folds += 1;
}
}
// The corpus must exercise the case `% 65535` got wrong.
assert!(folds > 500, "only {folds} fold cases");
}
const CHUNK: usize = 8;
/// `n` chunks of `CHUNK` bytes, each one a chunk on which the old
/// `% 65535` checksum and libhdf5's differ (sum1, sum2 or both a non-zero
/// multiple of 65535), with an ordinary chunk between them.
fn fold_chunks(n: usize) -> Vec<u8> {
let mut rng = Rng(42);
let mut out = Vec::new();
let mut found = 0;
while found < n {
// Build a chunk whose sum1 is a multiple of 65535 half the time,
// otherwise search at random for a sum2 fold.
let mut c: Vec<u8> = (0..CHUNK).map(|_| rng.next() as u8).collect();
if found % 2 == 0 {
let words: u64 = c[..CHUNK - 2]
.chunks(2)
.map(|w| (u64::from(w[0]) << 8) | u64::from(w[1]))
.sum();
let last = ((65535 - words % 65535) % 65535) as u16;
c[CHUNK - 2..].copy_from_slice(&last.to_be_bytes());
}
if fletcher32(&c) != fletcher32_mod(&c) {
out.extend_from_slice(&c);
out.extend((0..CHUNK).map(|i| i as u8 + 1));
found += 1;
}
}
out
}
#[test]
fn h5py_reads_fold_case_chunks_we_write() {
if !have_h5py() {
return;
}
let data = fold_chunks(32);
// FileBuilder.
let built = tmp("built.h5");
let mut b = FileBuilder::new();
b.create_dataset("d")
.with_u8_data(&data)
.with_chunks(&[CHUNK as u64])
.with_fletcher32();
b.write(&built).unwrap();
// FileEditor, into a dataset h5py created.
let edited = tmp("edited.h5");
run_python(
"import sys, h5py, numpy as np\n\
with h5py.File(sys.argv[1], 'w') as f:\n\
\x20 f.create_dataset('d', data=np.zeros(int(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)",
&[edited.to_str().unwrap(), &data.len().to_string()],
);
FileEditor::open(&edited)
.unwrap()
.write_all("d", &data)
.unwrap();
for path in [&built, &edited] {
let got = run_python(
"import sys, h5py\n\
with h5py.File(sys.argv[1], 'r') as f:\n\
\x20 assert f['d'].fletcher32\n\
\x20 print(f['d'][:].tobytes().hex())",
&[path.to_str().unwrap()],
);
assert_eq!(got, hex(&data), "{}", path.display());
}
}
#[test]
fn we_read_fold_case_chunks_h5py_writes() {
if !have_h5py() {
return;
}
let data = fold_chunks(32);
let path = tmp("h5py.h5");
run_python(
"import sys, h5py, numpy as np\n\
with h5py.File(sys.argv[1], 'w') as f:\n\
\x20 f.create_dataset('d', data=np.frombuffer(bytes.fromhex(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)",
&[path.to_str().unwrap(), &hex(&data)],
);
let file = File::open(&path).unwrap();
let ds = file.dataset("d").unwrap();
assert_eq!(
ds.read_selection(&clawhdf5_format::selection::Selection::All)
.unwrap(),
data
);
}
/// A checksum stored with the bytes of each 16-bit half swapped, as
/// libhdf5 1.6.2 and earlier wrote it, is accepted as libhdf5 accepts it;
/// so is the `% 65535` form clawhdf5 v2.7.0 and earlier wrote, so that
/// their files stay readable.
#[test]
fn legacy_checksums_are_accepted() {
use clawhdf5_format::filter_pipeline::{FILTER_FLETCHER32, FilterDescription, FilterPipeline};
let payload = [1u8, 2, 3, 4, 5];
let sum = fletcher32(&payload);
let swapped = ((sum & 0x00ff_00ff) << 8) | ((sum >> 8) & 0x00ff_00ff);
assert_ne!(sum, swapped);
let pipeline = FilterPipeline {
version: 2,
filters: vec![FilterDescription {
filter_id: FILTER_FLETCHER32,
name: None,
client_data: vec![],
flags: 0,
}],
};
for stored in [sum, swapped] {
let mut chunk = payload.to_vec();
chunk.extend_from_slice(&stored.to_le_bytes());
let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1)
.unwrap();
assert_eq!(out, payload);
}
// Our old checksum of a fold-case chunk.
let fold = fold_chunks(1);
let fold = &fold[..CHUNK];
let old = fletcher32_mod(fold);
assert_ne!(old, fletcher32(fold));
let mut chunk = fold.to_vec();
chunk.extend_from_slice(&old.to_le_bytes());
let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, CHUNK, 1).unwrap();
assert_eq!(out, fold);
let mut chunk = payload.to_vec();
chunk.extend_from_slice(&(sum ^ 1).to_le_bytes());
assert!(
clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1).is_err()
);
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}