security(format): wire provenance verify_dataset into the clawhdf5 read path

verify_dataset existed and was tested, but was only ever called from
clawhdf5-format's own test files — no reader path in clawhdf5-io or the
clawhdf5 facade called it, so a corrupted dataset was silently readable
even though the write-side SHA-256 hash machinery (gated on the
provenance feature) had already written what it needed to detect that.

Add Dataset::verify_provenance() to the clawhdf5 facade, gated behind a
new `provenance` feature (on by default, forwarding to
clawhdf5-format/provenance which is already default-on). It surfaces a
typed VerifyResult (Ok/Mismatch/NoHash) via the existing Error type
rather than panicking. Deliberately NOT called automatically on
open()/dataset() — it decodes and hashes the entire dataset, which would
regress every read path (including the zero-copy/mmap ones) if run
unconditionally; callers opt in per dataset where the cost is
acceptable (e.g. a periodic integrity sweep).

Also re-export clawhdf5_format::provenance from the facade crate so
VerifyResult is reachable without depending on clawhdf5-format directly.

INT-08
This commit is contained in:
ClawHDF5 Coding Agent
2026-08-17 00:54:44 +00:00
parent ab283d2759
commit 5db1008eb7
5 changed files with 101 additions and 1 deletions
+8
View File
@@ -34,6 +34,14 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
the cache and self-heals on drift). Build the agent with the cache and self-heals on drift). Build the agent with
`--no-default-features --features float16` to force the exact linear cosine scan. `--no-default-features --features float16` to force the exact linear cosine scan.
- WAL (write-ahead log) for crash-safe persistence, with a CRC32 trailer per entry so a corrupted entry stops replay cleanly instead of loading bad data - WAL (write-ahead log) for crash-safe persistence, with a CRC32 trailer per entry so a corrupted entry stops replay cleanly instead of loading bad data
- `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by
default) recomputes a dataset's SHA-256 and compares it against the
`_provenance_sha256` attribute written automatically on save when
`DatasetBuilder::with_provenance` is used. It's opt-in per call, not run
automatically on open — it decodes and hashes the whole dataset. The hash
is unkeyed (tamper-*evident*, not tamper-*proof*): it detects accidental
corruption, not a deliberate actor able to modify both the data and the
stored hash.
- `clawhdf5-agent`'s `HDF5Memory::save`/`save_batch`/`save_or_update` run every - `clawhdf5-agent`'s `HDF5Memory::save`/`save_batch`/`save_or_update` run every
write through an in-memory (session-scoped, not persisted to disk) write through an in-memory (session-scoped, not persisted to disk)
provenance ledger and write-anomaly detector: a content hash per record provenance ledger and write-anomaly detector: a content hash per record
+5 -1
View File
@@ -30,7 +30,7 @@ name = "parallel_bench"
harness = false harness = false
[features] [features]
default = ["mmap", "fast-deflate"] default = ["mmap", "fast-deflate", "provenance"]
mmap = ["clawhdf5-io/mmap"] mmap = ["clawhdf5-io/mmap"]
parallel = ["clawhdf5-format/parallel", "rayon"] parallel = ["clawhdf5-format/parallel", "rayon"]
fast-deflate = ["clawhdf5-format/fast-deflate"] fast-deflate = ["clawhdf5-format/fast-deflate"]
@@ -39,6 +39,10 @@ zstd = ["clawhdf5-format/zstd"]
blake3_hash = ["clawhdf5-format/blake3_hash"] blake3_hash = ["clawhdf5-format/blake3_hash"]
lz4 = ["clawhdf5-format/lz4"] lz4 = ["clawhdf5-format/lz4"]
pcodec = ["clawhdf5-format/pcodec"] pcodec = ["clawhdf5-format/pcodec"]
# Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
# against its stored _provenance_sha256 attribute. On by default, matching
# clawhdf5-format's own default-on `provenance` feature.
provenance = ["clawhdf5-format/provenance"]
[package.metadata.docs.rs] [package.metadata.docs.rs]
features = ["mmap"] features = ["mmap"]
+2
View File
@@ -51,6 +51,8 @@ pub use clawhdf5_format::property_list::{
pub use clawhdf5_format::selection::Selection; pub use clawhdf5_format::selection::Selection;
pub use clawhdf5_format::superblock::swmr_flags; pub use clawhdf5_format::superblock::swmr_flags;
pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime}; pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime};
#[cfg(feature = "provenance")]
pub use clawhdf5_format::provenance;
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
+25
View File
@@ -698,6 +698,31 @@ impl<'f> Dataset<'f> {
)) ))
} }
/// Verify this dataset's content against its stored provenance hash
/// (`_provenance_sha256`, written automatically on save when a
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
/// that module's docs). Returns `VerifyResult::NoHash` if the dataset
/// was never written with one.
///
/// This decodes and hashes the *entire* dataset, so unlike the other
/// read methods it is not run automatically on `open()`/`dataset()` —
/// call it explicitly where the cost of a full read is acceptable (e.g.
/// a periodic integrity sweep, not the hot read path).
///
/// The hash is unkeyed and stored alongside the data it protects, so
/// this only detects *accidental* corruption — anyone able to modify the
/// dataset can also recompute and overwrite the stored hash. A `VerifyResult::Ok`
/// result is not a tamper-evidence or authenticity guarantee.
#[cfg(feature = "provenance")]
pub fn verify_provenance(&self) -> Result<clawhdf5_format::provenance::VerifyResult, Error> {
Ok(clawhdf5_format::provenance::verify_dataset(
self.file.as_bytes(),
&self.header,
self.file.offset_size(),
self.file.length_size(),
)?)
}
fn datatype(&self) -> Result<Datatype, Error> { fn datatype(&self) -> Result<Datatype, Error> {
let msg = find_message(&self.header, MessageType::Datatype)?; let msg = find_message(&self.header, MessageType::Datatype)?;
let (dt, _) = Datatype::parse(&msg.data)?; let (dt, _) = Datatype::parse(&msg.data)?;
+61
View File
@@ -0,0 +1,61 @@
//! Tests for `Dataset::verify_provenance` — the facade-crate wiring of
//! `clawhdf5_format::provenance::verify_dataset` into the read path (INT-08:
//! the write-side hash existed and was tested, but nothing in `clawhdf5-io`
//! or the `clawhdf5` facade ever called `verify_dataset`).
#![cfg(feature = "provenance")]
use clawhdf5::provenance::VerifyResult;
use clawhdf5::{File, FileBuilder};
#[test]
fn verify_provenance_ok_on_intact_dataset() {
let mut b = FileBuilder::new();
b.create_dataset("sensor")
.with_f64_data(&[1.0, 2.0, 3.0, 4.0])
.with_provenance("test-suite", "2026-08-17T00:00:00Z", None);
let bytes = b.finish().unwrap();
let file = File::from_bytes(bytes).unwrap();
let ds = file.dataset("sensor").unwrap();
assert_eq!(ds.verify_provenance().unwrap(), VerifyResult::Ok);
}
#[test]
fn verify_provenance_no_hash_when_not_written_with_provenance() {
let mut b = FileBuilder::new();
b.create_dataset("plain").with_f64_data(&[1.0, 2.0]);
let bytes = b.finish().unwrap();
let file = File::from_bytes(bytes).unwrap();
let ds = file.dataset("plain").unwrap();
assert_eq!(ds.verify_provenance().unwrap(), VerifyResult::NoHash);
}
/// A corrupted dataset (raw bytes flipped after write, stored hash left
/// stale) must surface as a typed `Mismatch`, not be silently readable.
#[test]
fn verify_provenance_detects_corruption() {
let mut b = FileBuilder::new();
b.create_dataset("sensor")
.with_f64_data(&[1.0, 2.0, 3.0, 4.0])
.with_provenance("test-suite", "2026-08-17T00:00:00Z", None);
let mut bytes = b.finish().unwrap();
// Flip a byte inside the dataset's raw f64 payload (well past the
// superblock/header region) without touching the stored hash attribute,
// simulating corruption that occurred after the hash was written.
let needle = 2.0f64.to_le_bytes();
let pos = bytes
.windows(needle.len())
.position(|w| w == needle)
.expect("expected to find the f64 payload for 2.0 in the file bytes");
bytes[pos] ^= 0xFF;
let file = File::from_bytes(bytes).unwrap();
let ds = file.dataset("sensor").unwrap();
match ds.verify_provenance().unwrap() {
VerifyResult::Mismatch { .. } => {}
other => panic!("expected Mismatch for corrupted data, got {other:?}"),
}
}