security(format): wire provenance verify_dataset into the clawhdf5 read path
verify_dataset existed and was tested, but was only ever called from clawhdf5-format's own test files — no reader path in clawhdf5-io or the clawhdf5 facade called it, so a corrupted dataset was silently readable even though the write-side SHA-256 hash machinery (gated on the provenance feature) had already written what it needed to detect that. Add Dataset::verify_provenance() to the clawhdf5 facade, gated behind a new `provenance` feature (on by default, forwarding to clawhdf5-format/provenance which is already default-on). It surfaces a typed VerifyResult (Ok/Mismatch/NoHash) via the existing Error type rather than panicking. Deliberately NOT called automatically on open()/dataset() — it decodes and hashes the entire dataset, which would regress every read path (including the zero-copy/mmap ones) if run unconditionally; callers opt in per dataset where the cost is acceptable (e.g. a periodic integrity sweep). Also re-export clawhdf5_format::provenance from the facade crate so VerifyResult is reachable without depending on clawhdf5-format directly. INT-08
This commit is contained in:
@@ -34,6 +34,14 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
|
|||||||
the cache and self-heals on drift). Build the agent with
|
the cache and self-heals on drift). Build the agent with
|
||||||
`--no-default-features --features float16` to force the exact linear cosine scan.
|
`--no-default-features --features float16` to force the exact linear cosine scan.
|
||||||
- WAL (write-ahead log) for crash-safe persistence, with a CRC32 trailer per entry so a corrupted entry stops replay cleanly instead of loading bad data
|
- WAL (write-ahead log) for crash-safe persistence, with a CRC32 trailer per entry so a corrupted entry stops replay cleanly instead of loading bad data
|
||||||
|
- `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by
|
||||||
|
default) recomputes a dataset's SHA-256 and compares it against the
|
||||||
|
`_provenance_sha256` attribute written automatically on save when
|
||||||
|
`DatasetBuilder::with_provenance` is used. It's opt-in per call, not run
|
||||||
|
automatically on open — it decodes and hashes the whole dataset. The hash
|
||||||
|
is unkeyed (tamper-*evident*, not tamper-*proof*): it detects accidental
|
||||||
|
corruption, not a deliberate actor able to modify both the data and the
|
||||||
|
stored hash.
|
||||||
- `clawhdf5-agent`'s `HDF5Memory::save`/`save_batch`/`save_or_update` run every
|
- `clawhdf5-agent`'s `HDF5Memory::save`/`save_batch`/`save_or_update` run every
|
||||||
write through an in-memory (session-scoped, not persisted to disk)
|
write through an in-memory (session-scoped, not persisted to disk)
|
||||||
provenance ledger and write-anomaly detector: a content hash per record
|
provenance ledger and write-anomaly detector: a content hash per record
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ name = "parallel_bench"
|
|||||||
harness = false
|
harness = false
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["mmap", "fast-deflate"]
|
default = ["mmap", "fast-deflate", "provenance"]
|
||||||
mmap = ["clawhdf5-io/mmap"]
|
mmap = ["clawhdf5-io/mmap"]
|
||||||
parallel = ["clawhdf5-format/parallel", "rayon"]
|
parallel = ["clawhdf5-format/parallel", "rayon"]
|
||||||
fast-deflate = ["clawhdf5-format/fast-deflate"]
|
fast-deflate = ["clawhdf5-format/fast-deflate"]
|
||||||
@@ -39,6 +39,10 @@ zstd = ["clawhdf5-format/zstd"]
|
|||||||
blake3_hash = ["clawhdf5-format/blake3_hash"]
|
blake3_hash = ["clawhdf5-format/blake3_hash"]
|
||||||
lz4 = ["clawhdf5-format/lz4"]
|
lz4 = ["clawhdf5-format/lz4"]
|
||||||
pcodec = ["clawhdf5-format/pcodec"]
|
pcodec = ["clawhdf5-format/pcodec"]
|
||||||
|
# Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
|
||||||
|
# against its stored _provenance_sha256 attribute. On by default, matching
|
||||||
|
# clawhdf5-format's own default-on `provenance` feature.
|
||||||
|
provenance = ["clawhdf5-format/provenance"]
|
||||||
|
|
||||||
[package.metadata.docs.rs]
|
[package.metadata.docs.rs]
|
||||||
features = ["mmap"]
|
features = ["mmap"]
|
||||||
|
|||||||
@@ -51,6 +51,8 @@ pub use clawhdf5_format::property_list::{
|
|||||||
pub use clawhdf5_format::selection::Selection;
|
pub use clawhdf5_format::selection::Selection;
|
||||||
pub use clawhdf5_format::superblock::swmr_flags;
|
pub use clawhdf5_format::superblock::swmr_flags;
|
||||||
pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime};
|
pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime};
|
||||||
|
#[cfg(feature = "provenance")]
|
||||||
|
pub use clawhdf5_format::provenance;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|||||||
@@ -698,6 +698,31 @@ impl<'f> Dataset<'f> {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Verify this dataset's content against its stored provenance hash
|
||||||
|
/// (`_provenance_sha256`, written automatically on save when a
|
||||||
|
/// [`Provenance`](clawhdf5_format::provenance::Provenance) is set — see
|
||||||
|
/// that module's docs). Returns `VerifyResult::NoHash` if the dataset
|
||||||
|
/// was never written with one.
|
||||||
|
///
|
||||||
|
/// This decodes and hashes the *entire* dataset, so unlike the other
|
||||||
|
/// read methods it is not run automatically on `open()`/`dataset()` —
|
||||||
|
/// call it explicitly where the cost of a full read is acceptable (e.g.
|
||||||
|
/// a periodic integrity sweep, not the hot read path).
|
||||||
|
///
|
||||||
|
/// The hash is unkeyed and stored alongside the data it protects, so
|
||||||
|
/// this only detects *accidental* corruption — anyone able to modify the
|
||||||
|
/// dataset can also recompute and overwrite the stored hash. A `VerifyResult::Ok`
|
||||||
|
/// result is not a tamper-evidence or authenticity guarantee.
|
||||||
|
#[cfg(feature = "provenance")]
|
||||||
|
pub fn verify_provenance(&self) -> Result<clawhdf5_format::provenance::VerifyResult, Error> {
|
||||||
|
Ok(clawhdf5_format::provenance::verify_dataset(
|
||||||
|
self.file.as_bytes(),
|
||||||
|
&self.header,
|
||||||
|
self.file.offset_size(),
|
||||||
|
self.file.length_size(),
|
||||||
|
)?)
|
||||||
|
}
|
||||||
|
|
||||||
fn datatype(&self) -> Result<Datatype, Error> {
|
fn datatype(&self) -> Result<Datatype, Error> {
|
||||||
let msg = find_message(&self.header, MessageType::Datatype)?;
|
let msg = find_message(&self.header, MessageType::Datatype)?;
|
||||||
let (dt, _) = Datatype::parse(&msg.data)?;
|
let (dt, _) = Datatype::parse(&msg.data)?;
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
//! Tests for `Dataset::verify_provenance` — the facade-crate wiring of
|
||||||
|
//! `clawhdf5_format::provenance::verify_dataset` into the read path (INT-08:
|
||||||
|
//! the write-side hash existed and was tested, but nothing in `clawhdf5-io`
|
||||||
|
//! or the `clawhdf5` facade ever called `verify_dataset`).
|
||||||
|
|
||||||
|
#![cfg(feature = "provenance")]
|
||||||
|
|
||||||
|
use clawhdf5::provenance::VerifyResult;
|
||||||
|
use clawhdf5::{File, FileBuilder};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verify_provenance_ok_on_intact_dataset() {
|
||||||
|
let mut b = FileBuilder::new();
|
||||||
|
b.create_dataset("sensor")
|
||||||
|
.with_f64_data(&[1.0, 2.0, 3.0, 4.0])
|
||||||
|
.with_provenance("test-suite", "2026-08-17T00:00:00Z", None);
|
||||||
|
let bytes = b.finish().unwrap();
|
||||||
|
|
||||||
|
let file = File::from_bytes(bytes).unwrap();
|
||||||
|
let ds = file.dataset("sensor").unwrap();
|
||||||
|
assert_eq!(ds.verify_provenance().unwrap(), VerifyResult::Ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn verify_provenance_no_hash_when_not_written_with_provenance() {
|
||||||
|
let mut b = FileBuilder::new();
|
||||||
|
b.create_dataset("plain").with_f64_data(&[1.0, 2.0]);
|
||||||
|
let bytes = b.finish().unwrap();
|
||||||
|
|
||||||
|
let file = File::from_bytes(bytes).unwrap();
|
||||||
|
let ds = file.dataset("plain").unwrap();
|
||||||
|
assert_eq!(ds.verify_provenance().unwrap(), VerifyResult::NoHash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A corrupted dataset (raw bytes flipped after write, stored hash left
|
||||||
|
/// stale) must surface as a typed `Mismatch`, not be silently readable.
|
||||||
|
#[test]
|
||||||
|
fn verify_provenance_detects_corruption() {
|
||||||
|
let mut b = FileBuilder::new();
|
||||||
|
b.create_dataset("sensor")
|
||||||
|
.with_f64_data(&[1.0, 2.0, 3.0, 4.0])
|
||||||
|
.with_provenance("test-suite", "2026-08-17T00:00:00Z", None);
|
||||||
|
let mut bytes = b.finish().unwrap();
|
||||||
|
|
||||||
|
// Flip a byte inside the dataset's raw f64 payload (well past the
|
||||||
|
// superblock/header region) without touching the stored hash attribute,
|
||||||
|
// simulating corruption that occurred after the hash was written.
|
||||||
|
let needle = 2.0f64.to_le_bytes();
|
||||||
|
let pos = bytes
|
||||||
|
.windows(needle.len())
|
||||||
|
.position(|w| w == needle)
|
||||||
|
.expect("expected to find the f64 payload for 2.0 in the file bytes");
|
||||||
|
bytes[pos] ^= 0xFF;
|
||||||
|
|
||||||
|
let file = File::from_bytes(bytes).unwrap();
|
||||||
|
let ds = file.dataset("sensor").unwrap();
|
||||||
|
match ds.verify_provenance().unwrap() {
|
||||||
|
VerifyResult::Mismatch { .. } => {}
|
||||||
|
other => panic!("expected Mismatch for corrupted data, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user