use clawhdf5::reader::File as Hdf5File; use clawhdf5_format::provenance::VerifyResult; use crate::hdf5_reader::read_hdf5; use crate::sqlite_reader::SqliteData; type BoxErr = Box; /// Summary of a migration validation. #[derive(Debug)] pub struct ValidationSummary { pub chunks: u64, pub sessions: u64, pub entities: u64, pub relations: u64, pub embedding_dim: u64, /// Number of rows whose full content was compared against the source. pub rows_checked: u64, /// Whether the `chunks/text` and `chunks/embeddings` SHINES provenance /// hashes (written via [`crate::hdf5_writer`]) were both present and /// matched their recomputed SHA-256 on read-back. `false` when either /// dataset has no provenance metadata (e.g. an older output file) or /// there are zero chunks to check. pub provenance_verified: bool, } /// Validate a migrated HDF5 file against the source data. /// /// Reads the written file back and compares actual content — chunk text, /// embeddings, and every session/entity/relation field — to the source, not /// just the row counts. When `full` is false a representative sample of chunk /// rows is content-checked (counts and all other groups are always checked in /// full); when `full` is true every chunk row is compared too. `float16` widens /// the embedding tolerance to allow for half-precision quantization. pub fn validate_hdf5( path: &str, source: &SqliteData, full: bool, float16: bool, ) -> Result { let got = read_hdf5(path)?; let provenance_verified = verify_chunk_provenance(path)?; // ---- Counts ---- check_count("chunk", got.chunks.len(), source.chunks.len())?; check_count("session", got.sessions.len(), source.sessions.len())?; check_count("entity", got.entities.len(), source.entities.len())?; check_count("relation", got.relations.len(), source.relations.len())?; if got.embedding_dim != source.embedding_dim { return Err(format!( "embedding_dim mismatch: HDF5 has {}, source has {}", got.embedding_dim, source.embedding_dim ) .into()); } // ---- Chunk content (sampled or full) ---- let (emb_abs, emb_rel) = if float16 { (1e-2, 1e-2) } else { (1e-4, 0.0) }; let mut rows_checked = 0u64; for i in sample_indices(source.chunks.len(), full) { let (s, g) = (&source.chunks[i], &got.chunks[i]); if s.id != g.id { return Err(field_err("chunk", i, "id", s.id, g.id)); } if s.chunk != g.chunk { return Err(format!( "chunk[{i}].text mismatch: source {:?}, HDF5 {:?}", truncate(&s.chunk), truncate(&g.chunk) ) .into()); } if s.session_id != g.session_id || s.source_channel != g.source_channel || s.tags != g.tags { return Err(format!("chunk[{i}] string field mismatch").into()); } if s.deleted != g.deleted { return Err(field_err("chunk", i, "deleted", s.deleted, g.deleted)); } if s.embedding.len() != g.embedding.len() { return Err(format!( "chunk[{i}] embedding length mismatch: {} vs {}", s.embedding.len(), g.embedding.len() ) .into()); } for (k, (&a, &b)) in s.embedding.iter().zip(g.embedding.iter()).enumerate() { if (a - b).abs() > emb_abs + emb_rel * a.abs() { return Err( format!("chunk[{i}].embedding[{k}] mismatch: source {a}, HDF5 {b}").into(), ); } } rows_checked += 1; } // ---- Other groups (always full — they are small) ---- for (i, (s, g)) in source.sessions.iter().zip(got.sessions.iter()).enumerate() { if s.id != g.id || s.start_idx != g.start_idx || s.end_idx != g.end_idx || s.channel != g.channel || s.summary != g.summary { return Err(format!("session[{i}] mismatch").into()); } rows_checked += 1; } for (i, (s, g)) in source.entities.iter().zip(got.entities.iter()).enumerate() { if s.id != g.id || s.name != g.name || s.entity_type != g.entity_type || s.embedding_idx != g.embedding_idx { return Err(format!("entity[{i}] mismatch").into()); } rows_checked += 1; } for (i, (s, g)) in source .relations .iter() .zip(got.relations.iter()) .enumerate() { if s.src != g.src || s.tgt != g.tgt || s.relation != g.relation { return Err(format!("relation[{i}] mismatch").into()); } rows_checked += 1; } Ok(ValidationSummary { chunks: got.chunks.len() as u64, sessions: got.sessions.len() as u64, entities: got.entities.len() as u64, relations: got.relations.len() as u64, embedding_dim: got.embedding_dim as u64, rows_checked, provenance_verified, }) } fn check_count(kind: &str, got: usize, expected: usize) -> Result<(), BoxErr> { if got != expected { return Err(format!("{kind} count mismatch: HDF5 has {got}, source has {expected}").into()); } Ok(()) } /// Re-verify the SHA-256 provenance hash of `chunks/text` and /// `chunks/embeddings` against their actual stored bytes, catching /// post-write corruption that a plain content comparison against the /// in-memory source wouldn't (the source is compared against what /// `read_hdf5` decoded, not against the raw bytes on disk). /// /// Returns `Ok(true)` only if both datasets exist and both hashes match. /// Returns `Ok(false)` (not an error) if a dataset has no provenance /// attributes at all (e.g. a file written before this check existed) or /// there are zero chunks. Returns an error only on an actual hash mismatch — /// that indicates real corruption. fn verify_chunk_provenance(path: &str) -> Result { let file = Hdf5File::open(path)?; let Ok(chunks) = file.group("chunks") else { return Ok(false); }; let mut all_present = true; for name in ["text", "embeddings"] { let Ok(ds) = chunks.dataset(name) else { all_present = false; continue; }; match ds.verify_provenance()? { VerifyResult::Ok => {} VerifyResult::NoHash => all_present = false, VerifyResult::Mismatch { stored, computed } => { return Err(format!( "provenance hash mismatch on chunks/{name}: stored {stored}, recomputed {computed} — data may be corrupted" ) .into()); } } } Ok(all_present) } fn field_err(kind: &str, i: usize, field: &str, s: T, g: T) -> BoxErr { format!("{kind}[{i}].{field} mismatch: source {s}, HDF5 {g}").into() } fn truncate(s: &str) -> String { if s.len() <= 40 { s.to_string() } else { let cut = s.char_indices().nth(40).map(|(i, _)| i).unwrap_or(s.len()); format!("{}…", &s[..cut]) } } /// Indices of chunk rows to content-check. Full = all; otherwise a spread of /// representative rows (first/last and evenly-spaced interior samples). fn sample_indices(n: usize, full: bool) -> Vec { if n == 0 { return Vec::new(); } if full || n <= 16 { return (0..n).collect(); } let mut idx: Vec = (0..16).map(|k| k * (n - 1) / 15).collect(); idx.dedup(); idx } #[cfg(test)] mod tests { use super::*; #[test] fn truncate_short_string_unchanged() { assert_eq!(truncate("hello"), "hello"); } /// A multi-byte character straddling byte offset 40 must not panic a /// byte-index slice — this is arbitrary UTF-8 chunk text from an /// untrusted source database, not test-only input. #[test] fn truncate_multibyte_char_at_boundary_does_not_panic() { // 39 ASCII bytes then a 4-byte emoji straddling the byte-40 cut point. let s = format!("{}{}", "a".repeat(39), "😀".repeat(5)); let result = truncate(&s); assert!(result.ends_with('…')); assert!(result.chars().count() < s.chars().count()); } #[test] fn truncate_exactly_at_limit_unchanged() { let s = "a".repeat(40); assert_eq!(truncate(&s), s); } }