Performance, security and provenance hardening (ann/io/migrate/agent) + two audit fixes #2
@@ -34,6 +34,14 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
|
||||
the cache and self-heals on drift). Build the agent with
|
||||
`--no-default-features --features float16` to force the exact linear cosine scan.
|
||||
- WAL (write-ahead log) for crash-safe persistence, with a CRC32 trailer per entry so a corrupted entry stops replay cleanly instead of loading bad data
|
||||
- `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by
|
||||
default) recomputes a dataset's SHA-256 and compares it against the
|
||||
`_provenance_sha256` attribute written automatically on save when
|
||||
`DatasetBuilder::with_provenance` is used. It's opt-in per call, not run
|
||||
automatically on open — it decodes and hashes the whole dataset. The hash
|
||||
is unkeyed (tamper-*evident*, not tamper-*proof*): it detects accidental
|
||||
corruption, not a deliberate actor able to modify both the data and the
|
||||
stored hash.
|
||||
- `clawhdf5-agent`'s `HDF5Memory::save`/`save_batch`/`save_or_update` run every
|
||||
write through an in-memory (session-scoped, not persisted to disk)
|
||||
provenance ledger and write-anomaly detector: a content hash per record
|
||||
|
||||
@@ -30,7 +30,7 @@ name = "parallel_bench"
|
||||
harness = false
|
||||
|
||||
[features]
|
||||
default = ["mmap", "fast-deflate"]
|
||||
default = ["mmap", "fast-deflate", "provenance"]
|
||||
mmap = ["clawhdf5-io/mmap"]
|
||||
parallel = ["clawhdf5-format/parallel", "rayon"]
|
||||
fast-deflate = ["clawhdf5-format/fast-deflate"]
|
||||
@@ -39,6 +39,10 @@ zstd = ["clawhdf5-format/zstd"]
|
||||
blake3_hash = ["clawhdf5-format/blake3_hash"]
|
||||
lz4 = ["clawhdf5-format/lz4"]
|
||||
pcodec = ["clawhdf5-format/pcodec"]
|
||||
# Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
|
||||
# against its stored _provenance_sha256 attribute. On by default, matching
|
||||
# clawhdf5-format's own default-on `provenance` feature.
|
||||
provenance = ["clawhdf5-format/provenance"]
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
features = ["mmap"]
|
||||
|
||||
@@ -51,6 +51,8 @@ pub use clawhdf5_format::property_list::{
|
||||
pub use clawhdf5_format::selection::Selection;
|
||||
pub use clawhdf5_format::superblock::swmr_flags;
|
||||
pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime};
|
||||
#[cfg(feature = "provenance")]
|
||||
pub use clawhdf5_format::provenance;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
@@ -698,6 +698,31 @@ impl<'f> Dataset<'f> {
|
||||
))
|
||||
}
|
||||
|
||||
/// Verify this dataset's content against its stored provenance hash
|
||||
/// (`_provenance_sha256`, written automatically on save when a
|
||||
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
|
||||
/// that module's docs). Returns `VerifyResult::NoHash` if the dataset
|
||||
/// was never written with one.
|
||||
///
|
||||
/// This decodes and hashes the *entire* dataset, so unlike the other
|
||||
/// read methods it is not run automatically on `open()`/`dataset()` —
|
||||
/// call it explicitly where the cost of a full read is acceptable (e.g.
|
||||
/// a periodic integrity sweep, not the hot read path).
|
||||
///
|
||||
/// The hash is unkeyed and stored alongside the data it protects, so
|
||||
/// this only detects *accidental* corruption — anyone able to modify the
|
||||
/// dataset can also recompute and overwrite the stored hash. A `VerifyResult::Ok`
|
||||
/// result is not a tamper-evidence or authenticity guarantee.
|
||||
#[cfg(feature = "provenance")]
|
||||
pub fn verify_provenance(&self) -> Result<clawhdf5_format::provenance::VerifyResult, Error> {
|
||||
Ok(clawhdf5_format::provenance::verify_dataset(
|
||||
self.file.as_bytes(),
|
||||
&self.header,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)?)
|
||||
}
|
||||
|
||||
fn datatype(&self) -> Result<Datatype, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&msg.data)?;
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! Tests for `Dataset::verify_provenance` — the facade-crate wiring of
|
||||
//! `clawhdf5_format::provenance::verify_dataset` into the read path (INT-08:
|
||||
//! the write-side hash existed and was tested, but nothing in `clawhdf5-io`
|
||||
//! or the `clawhdf5` facade ever called `verify_dataset`).
|
||||
|
||||
#![cfg(feature = "provenance")]
|
||||
|
||||
use clawhdf5::provenance::VerifyResult;
|
||||
use clawhdf5::{File, FileBuilder};
|
||||
|
||||
#[test]
|
||||
fn verify_provenance_ok_on_intact_dataset() {
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("sensor")
|
||||
.with_f64_data(&[1.0, 2.0, 3.0, 4.0])
|
||||
.with_provenance("test-suite", "2026-08-17T00:00:00Z", None);
|
||||
let bytes = b.finish().unwrap();
|
||||
|
||||
let file = File::from_bytes(bytes).unwrap();
|
||||
let ds = file.dataset("sensor").unwrap();
|
||||
assert_eq!(ds.verify_provenance().unwrap(), VerifyResult::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_provenance_no_hash_when_not_written_with_provenance() {
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("plain").with_f64_data(&[1.0, 2.0]);
|
||||
let bytes = b.finish().unwrap();
|
||||
|
||||
let file = File::from_bytes(bytes).unwrap();
|
||||
let ds = file.dataset("plain").unwrap();
|
||||
assert_eq!(ds.verify_provenance().unwrap(), VerifyResult::NoHash);
|
||||
}
|
||||
|
||||
/// A corrupted dataset (raw bytes flipped after write, stored hash left
|
||||
/// stale) must surface as a typed `Mismatch`, not be silently readable.
|
||||
#[test]
|
||||
fn verify_provenance_detects_corruption() {
|
||||
let mut b = FileBuilder::new();
|
||||
b.create_dataset("sensor")
|
||||
.with_f64_data(&[1.0, 2.0, 3.0, 4.0])
|
||||
.with_provenance("test-suite", "2026-08-17T00:00:00Z", None);
|
||||
let mut bytes = b.finish().unwrap();
|
||||
|
||||
// Flip a byte inside the dataset's raw f64 payload (well past the
|
||||
// superblock/header region) without touching the stored hash attribute,
|
||||
// simulating corruption that occurred after the hash was written.
|
||||
let needle = 2.0f64.to_le_bytes();
|
||||
let pos = bytes
|
||||
.windows(needle.len())
|
||||
.position(|w| w == needle)
|
||||
.expect("expected to find the f64 payload for 2.0 in the file bytes");
|
||||
bytes[pos] ^= 0xFF;
|
||||
|
||||
let file = File::from_bytes(bytes).unwrap();
|
||||
let ds = file.dataset("sensor").unwrap();
|
||||
match ds.verify_provenance().unwrap() {
|
||||
VerifyResult::Mismatch { .. } => {}
|
||||
other => panic!("expected Mismatch for corrupted data, got {other:?}"),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user