CLAUDE.md

@@ -33,7 +33,13 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F
   the approximate `clawhdf5-ann` index for the vector stage (the index mirrors
   the cache and self-heals on drift). Build the agent with
   `--no-default-features --features float16` to force the exact linear cosine scan.
-- WAL (write-ahead log) for crash-safe persistence, with a CRC32 trailer per entry so a corrupted entry stops replay cleanly instead of loading bad data
+- WAL (write-ahead log) for crash-safe persistence, with a chained CRC32
+  trailer per entry (each entry's CRC folds in the previous entry's CRC) so a
+  corrupted, reordered, duplicated, or spliced entry stops replay cleanly
+  instead of loading bad or tampered data. The pre-chaining per-entry-CRC
+  format (v2) is still fully readable; the oldest no-CRC format (v1) is only
+  reachable through the one-time migration path in `HDF5Memory::open`, not
+  through the public `WalFile::read_entries`.
 - `Dataset::verify_provenance()` (clawhdf5 facade, `provenance` feature, on by
   default) recomputes a dataset's SHA-256 and compares it against the
   `_provenance_sha256` attribute written automatically on save when

crates/clawhdf5-agent/src/lib.rs

@@ -292,7 +292,10 @@ impl HDF5Memory {
         // Replay WAL if present
         let wal_path = path.with_extension("h5.wal");
         let wal = if wal_path.exists() {
-            let entries = wal::WalFile::read_entries(&wal_path)?;
+            // Uses the migration-only reader since this is the one legitimate
+            // path that may need to read a legacy (pre-CRC) WAL file — see
+            // WalFile::read_entries_for_migration.
+            let entries = wal::WalFile::read_entries_for_migration(&wal_path)?;
             wal::replay_into_cache(&entries, &mut cache);
             Some(wal::WalFile::open(&wal_path)?)
         } else if config.wal_enabled {

crates/clawhdf5-agent/src/wal.rs

@@ -13,16 +13,40 @@ use crate::MemoryError;
 
 const WAL_MAGIC: [u8; 4] = [0x45, 0x48, 0x57, 0x4C]; // "EHWL"
 
-/// Current WAL format version: every entry ends with a 4-byte CRC32 trailer
-/// (see [`TeeReader`]) so a bit-flip is detected and replay stops there
-/// instead of silently accepting corrupted data.
-const WAL_VERSION: u8 = 2;
+/// Current WAL format version: every entry's CRC32 trailer is computed over
+/// its own bytes *chained with the previous entry's stored CRC*
+/// (`crc32(entry_bytes ++ prev_crc.to_le_bytes())`, seeded with 0 for the
+/// first entry after a truncation). A per-entry CRC alone only detects a
+/// bit-flip within that entry; chaining additionally detects entries being
+/// reordered, duplicated, or spliced (e.g. a Tombstone moved before/after
+/// its target Save) — the moved/inserted entry's stored CRC was computed
+/// against a different predecessor than the one now in front of it on disk,
+/// so the chain breaks at that point and replay stops there.
+const WAL_VERSION: u8 = 3;
 
-/// The only other WAL version this crate still knows how to *read*: no
-/// per-entry CRC trailer. Written by versions of this crate before the CRC32
-/// hardening. `WalFile::open` migrates a legacy file to [`WAL_VERSION`] by
-/// recreating it fresh — safe because every real call site reads existing
-/// entries via [`WalFile::read_entries`] before calling `open` (see
+/// The previous WAL format version: still a CRC32 per entry (so a bit-flip
+/// within one entry is caught), but not chained to the previous entry's CRC
+/// (so reordering/splicing whole entries is not detected). Written by
+/// versions of this crate before the chaining hardening. Fully supported for
+/// reading via [`WalFile::read_entries`] — not restricted like
+/// [`WAL_VERSION_LEGACY_NO_CRC`], since it still verifies each entry
+/// individually. `WalFile::open` migrates it to [`WAL_VERSION`] by
+/// recreating the file fresh, the same as the legacy-no-CRC migration below.
+const WAL_VERSION_CRC_UNCHAINED: u8 = 2;
+
+/// The oldest WAL version this crate still knows how to *read*: no
+/// per-entry CRC trailer at all, so a bit-flip anywhere is silently
+/// accepted. Written by versions of this crate before the CRC32 hardening.
+/// Because of that — unlike [`WAL_VERSION_CRC_UNCHAINED`] — this version is
+/// deliberately *not* reachable through the public [`WalFile::read_entries`]
+/// API; only [`WalFile::read_entries_for_migration`] (used exclusively by
+/// `HDF5Memory::open`'s one-time migration path) will parse it. Flipping a
+/// version byte from 2/3 down to 1 no longer silently downgrades a file to
+/// the fully-unverified parser for an arbitrary caller.
+///
+/// `WalFile::open` migrates a legacy file to [`WAL_VERSION`] by recreating
+/// it fresh — safe because every real call site reads existing entries via
+/// [`WalFile::read_entries_for_migration`] before calling `open` (see
 /// `HDF5Memory::open`), so no data is lost.
 const WAL_VERSION_LEGACY_NO_CRC: u8 = 1;
 
@@ -77,15 +101,21 @@ pub struct WalFile {
     entry_count: u32,
     /// Entries written since the last header count update.
     pending_header_sync: u32,
+    /// CRC32 chain state: the previous entry's stored CRC (0 if this file
+    /// has no entries yet), folded into the next entry's CRC computation.
+    /// Reset to 0 by `truncate()`/`create_fresh_wal_file`, and re-derived by
+    /// scanning existing entries when `open()` attaches to a non-empty file.
+    running_crc: u32,
 }
 
 impl WalFile {
     /// Open or create a WAL file. If it exists, read the header and entry count.
     ///
-    /// A legacy (pre-CRC) WAL file is migrated to the current format by
-    /// recreating it fresh — see [`WAL_VERSION_LEGACY_NO_CRC`]. Callers that
-    /// need the legacy file's entries must call [`WalFile::read_entries`]
-    /// first, before calling `open`.
+    /// A pre-chaining WAL file ([`WAL_VERSION_CRC_UNCHAINED`] or
+    /// [`WAL_VERSION_LEGACY_NO_CRC`]) is migrated to the current format by
+    /// recreating it fresh. Callers that need an existing file's entries must
+    /// call [`WalFile::read_entries`] (or, for a legacy-no-CRC file,
+    /// [`WalFile::read_entries_for_migration`]) first, before calling `open`.
     pub fn open(path: &Path) -> Result<Self, MemoryError> {
         if path.exists() {
             // Read existing header
@@ -105,17 +135,28 @@ impl WalFile {
                 WAL_VERSION => {
                     let mut count_buf = [0u8; 4];
                     f.read_exact(&mut count_buf)?;
-                    let entry_count = u32::from_le_bytes(count_buf);
-                    // Seek to end for appending
+                    let header_count = u32::from_le_bytes(count_buf);
+                    // Scan any existing entries to resume the CRC chain
+                    // correctly for further appends (the header's count may
+                    // be stale from deferred group-commit sync, same
+                    // tolerance `read_entries` already has, so the scanned
+                    // count is also the more accurate of the two).
+                    let (entries, running_crc) = read_chained_entries(&mut f, 0);
+                    let entry_count = if entries.is_empty() {
+                        header_count
+                    } else {
+                        entries.len() as u32
+                    };
                     f.seek(SeekFrom::End(0))?;
                     Ok(Self {
                         path: path.to_path_buf(),
                         file: Some(f),
                         entry_count,
                         pending_header_sync: 0,
+                        running_crc,
                     })
                 }
-                WAL_VERSION_LEGACY_NO_CRC => {
+                WAL_VERSION_CRC_UNCHAINED | WAL_VERSION_LEGACY_NO_CRC => {
                     drop(f);
                     let f = create_fresh_wal_file(path)?;
                     Ok(Self {
@@ -123,6 +164,7 @@ impl WalFile {
                         file: Some(f),
                         entry_count: 0,
                         pending_header_sync: 0,
+                        running_crc: 0,
                     })
                 }
                 v => Err(MemoryError::Schema(format!("unsupported WAL version {v}"))),
@@ -134,6 +176,7 @@ impl WalFile {
                 file: Some(f),
                 entry_count: 0,
                 pending_header_sync: 0,
+                running_crc: 0,
             })
         }
     }
@@ -168,7 +211,10 @@ impl WalFile {
         serialize_str(&mut buf, &entry.session_id);
         serialize_str(&mut buf, &entry.tags);
 
-        let crc = crc32(&buf);
+        // Chain this entry's CRC to the previous one's so reordering/
+        // splicing entries (not just flipping a bit within one) is detected
+        // on replay — see WAL_VERSION's doc comment.
+        let crc = chained_crc(&buf, self.running_crc);
         buf.extend_from_slice(&crc.to_le_bytes());
 
         let f = self
@@ -177,6 +223,7 @@ impl WalFile {
             .ok_or_else(|| MemoryError::Io(std::io::Error::other("WAL file not open")))?;
         f.write_all(&buf)?;
 
+        self.running_crc = crc;
         self.entry_count += 1;
         self.pending_header_sync += 1;
         if self.pending_header_sync >= GROUP_COMMIT_SIZE {
@@ -191,7 +238,7 @@ impl WalFile {
         buf[0] = WalEntryType::Tombstone as u8;
         buf[1..9].copy_from_slice(&timestamp.to_le_bytes());
         buf[9..13].copy_from_slice(&(index as u32).to_le_bytes());
-        let crc = crc32(&buf[..13]);
+        let crc = chained_crc(&buf[..13], self.running_crc);
         buf[13..17].copy_from_slice(&crc.to_le_bytes());
 
         let f = self
@@ -200,6 +247,7 @@ impl WalFile {
             .ok_or_else(|| MemoryError::Io(std::io::Error::other("WAL file not open")))?;
         f.write_all(&buf)?;
 
+        self.running_crc = crc;
         self.entry_count += 1;
         self.pending_header_sync += 1;
         if self.pending_header_sync >= GROUP_COMMIT_SIZE {
@@ -214,9 +262,36 @@ impl WalFile {
     /// (and may be stale if written with deferred group-commit updates). This
     /// tolerates both truncated files (crash mid-write) and stale header counts
     /// (crash before the next group-commit header sync). On a `WAL_VERSION`
-    /// file, a CRC32 mismatch on an entry is treated the same way — replay
-    /// stops there rather than accepting corrupted data.
+    /// file, a broken CRC chain (bit-flip, or an entry reordered/duplicated/
+    /// spliced in) is treated the same way — replay stops there rather than
+    /// accepting corrupted or tampered data. `WAL_VERSION_CRC_UNCHAINED`
+    /// files are read the same way minus the chain check (each entry's own
+    /// CRC is still verified).
+    ///
+    /// Does **not** read [`WAL_VERSION_LEGACY_NO_CRC`] files — that format has
+    /// no integrity verification at all, so it's only reachable through
+    /// [`WalFile::read_entries_for_migration`], used exclusively by
+    /// `HDF5Memory::open`'s one-time migration path. Calling this on a
+    /// legacy-no-CRC file returns a typed error instead of silently
+    /// downgrading to the unverified parser.
     pub fn read_entries(path: &Path) -> Result<Vec<WalEntry>, MemoryError> {
+        Self::read_entries_impl(path, false)
+    }
+
+    /// Like [`WalFile::read_entries`], but also accepts
+    /// [`WAL_VERSION_LEGACY_NO_CRC`] files (no per-entry integrity check at
+    /// all). Restricted to `pub(crate)` and named accordingly: the only
+    /// legitimate caller is `HDF5Memory::open`'s one-time migration of a
+    /// pre-CRC WAL file, which immediately recreates it in the current
+    /// format afterward. Do not use this for anything else.
+    pub(crate) fn read_entries_for_migration(path: &Path) -> Result<Vec<WalEntry>, MemoryError> {
+        Self::read_entries_impl(path, true)
+    }
+
+    fn read_entries_impl(
+        path: &Path,
+        allow_legacy_no_crc: bool,
+    ) -> Result<Vec<WalEntry>, MemoryError> {
         if !path.exists() {
             return Ok(Vec::new());
         }
@@ -229,10 +304,15 @@ impl WalFile {
         }
         // entry_count is a pre-allocation hint only — we read until EOF.
         let entry_count_hint = u32::from_le_bytes([header[5], header[6], header[7], header[8]]);
-        let mut entries = Vec::with_capacity(entry_count_hint as usize);
 
         match header[4] {
-            WAL_VERSION => loop {
+            WAL_VERSION => {
+                let (entries, _final_crc) = read_chained_entries(&mut f, 0);
+                Ok(entries)
+            }
+            WAL_VERSION_CRC_UNCHAINED => {
+                let mut entries = Vec::with_capacity(entry_count_hint as usize);
+                loop {
                     let raw_and_result = {
                         let mut tee = TeeReader::new(&mut f);
                         let result = read_one_entry(&mut tee);
@@ -249,27 +329,37 @@ impl WalFile {
                     }
                     let stored_crc = u32::from_le_bytes(crc_buf);
                     if crc32(&raw) != stored_crc {
-                    // Corruption detected — stop replay here, same as a clean
-                    // truncation/EOF, rather than accepting the bad entry.
+                        // Corruption detected — stop replay here, same as a
+                        // clean truncation/EOF, rather than accepting the bad
+                        // entry.
                         break;
                     }
                     if let Some(entry) = entry_opt {
                         entries.push(entry);
                     }
-            },
-            WAL_VERSION_LEGACY_NO_CRC => loop {
+                }
+                Ok(entries)
+            }
+            WAL_VERSION_LEGACY_NO_CRC if allow_legacy_no_crc => {
+                let mut entries = Vec::with_capacity(entry_count_hint as usize);
+                loop {
                     match read_one_entry(&mut f) {
                         Err(()) => break,
                         Ok(Some(entry)) => entries.push(entry),
                         Ok(None) => {}
                     }
-            },
-            v => {
-                return Err(MemoryError::Schema(format!("unsupported WAL version {v}")));
-            }
                 }
                 Ok(entries)
             }
+            WAL_VERSION_LEGACY_NO_CRC => Err(MemoryError::Schema(
+                "WAL file is in the legacy no-CRC format (version 1), which read_entries() no \
+                 longer accepts — it has no per-entry integrity verification. Only the one-time \
+                 migration path (WalFile::open) can read and upgrade it."
+                    .into(),
+            )),
+            v => Err(MemoryError::Schema(format!("unsupported WAL version {v}"))),
+        }
+    }
 
     /// Truncate the WAL (after merge into .h5).
     pub fn truncate(&mut self) -> Result<(), MemoryError> {
@@ -279,6 +369,7 @@ impl WalFile {
         self.file = Some(f);
         self.entry_count = 0;
         self.pending_header_sync = 0;
+        self.running_crc = 0;
         Ok(())
     }
 
@@ -373,6 +464,55 @@ fn read_embedding<R: Read>(f: &mut R) -> Result<Vec<f32>, MemoryError> {
     Ok(vals)
 }
 
+/// Compute the CRC32 trailer for a `WAL_VERSION` entry, chaining in the
+/// previous entry's stored CRC (0 for the first entry after a truncation).
+fn chained_crc(entry_bytes: &[u8], prev_crc: u32) -> u32 {
+    let mut chained = Vec::with_capacity(entry_bytes.len() + 4);
+    chained.extend_from_slice(entry_bytes);
+    chained.extend_from_slice(&prev_crc.to_le_bytes());
+    crc32(&chained)
+}
+
+/// Read and verify all entries from a `WAL_VERSION` (chained-CRC) stream
+/// starting at the reader's current position, given the chain state to
+/// resume from (0 for a stream starting at the beginning of a fresh WAL).
+///
+/// Returns the parsed entries and the final running CRC — the chain state to
+/// continue from for further appends. Stops (without erroring) at the first
+/// entry that fails to parse or whose stored CRC doesn't match the expected
+/// chain value — a bit-flip, truncation/EOF, or an entry having been
+/// reordered/duplicated/spliced all produce a chain mismatch at that point,
+/// and are all handled the same way: replay stops there.
+fn read_chained_entries<R: Read>(f: &mut R, start_crc: u32) -> (Vec<WalEntry>, u32) {
+    let mut entries = Vec::new();
+    let mut running_crc = start_crc;
+    loop {
+        let raw_and_result = {
+            let mut tee = TeeReader::new(f);
+            let result = read_one_entry(&mut tee);
+            (tee.into_buf(), result)
+        };
+        let (raw, result) = raw_and_result;
+        let entry_opt = match result {
+            Err(()) => break,
+            Ok(v) => v,
+        };
+        let mut crc_buf = [0u8; 4];
+        if f.read_exact(&mut crc_buf).is_err() {
+            break;
+        }
+        let stored_crc = u32::from_le_bytes(crc_buf);
+        if chained_crc(&raw, running_crc) != stored_crc {
+            break;
+        }
+        running_crc = stored_crc;
+        if let Some(entry) = entry_opt {
+            entries.push(entry);
+        }
+    }
+    (entries, running_crc)
+}
+
 /// Create a fresh WAL file at `path` with the current-version header,
 /// truncating/overwriting anything already there.
 fn create_fresh_wal_file(path: &Path) -> Result<File, MemoryError> {
@@ -912,16 +1052,113 @@ mod tests {
         assert_eq!(entries[0].chunk, "first");
     }
 
+    /// Reordering two entries on disk must break the CRC chain — the
+    /// second entry's stored CRC was computed against the first entry's
+    /// real CRC, not against the chain state a reader sees after swapping
+    /// them, so replay stops immediately instead of accepting the tampered
+    /// order (INT-09).
     #[test]
-    fn test_wal_reads_legacy_v1_format_without_crc() {
+    fn test_wal_detects_reordered_entries() {
         let dir = TempDir::new().unwrap();
-        let wal_path = dir.path().join("legacy.h5.wal");
+        let wal_path = dir.path().join("test.h5.wal");
+        let mut wal = WalFile::open(&wal_path).unwrap();
+        wal.append_save(&make_wal_entry("first", &[1.0, 2.0]))
+            .unwrap();
+        let len_after_first = std::fs::metadata(&wal_path).unwrap().len() as usize;
+        wal.append_save(&make_wal_entry("second", &[3.0, 4.0]))
+            .unwrap();
+        let len_after_second = std::fs::metadata(&wal_path).unwrap().len() as usize;
+        drop(wal);
+
+        let bytes = std::fs::read(&wal_path).unwrap();
+        let header_len = 9usize;
+        let entry1_bytes = bytes[header_len..len_after_first].to_vec();
+        let entry2_bytes = bytes[len_after_first..len_after_second].to_vec();
+
+        let mut spliced = bytes[..header_len].to_vec();
+        spliced.extend_from_slice(&entry2_bytes);
+        spliced.extend_from_slice(&entry1_bytes);
+        std::fs::write(&wal_path, &spliced).unwrap();
+
+        let entries = WalFile::read_entries(&wal_path).unwrap();
+        assert!(
+            entries.is_empty(),
+            "reordered entries must break the CRC chain and stop replay, got {} entries",
+            entries.len()
+        );
+    }
+
+    /// Splicing a third-party entry in between two legitimate entries (e.g.
+    /// moving a Tombstone in front of the Save it's meant to follow) must
+    /// also break the chain for everything after the splice point.
+    #[test]
+    fn test_wal_detects_spliced_entry() {
+        let dir = TempDir::new().unwrap();
+        let wal_path = dir.path().join("test.h5.wal");
+        let mut wal = WalFile::open(&wal_path).unwrap();
+        wal.append_save(&make_wal_entry("first", &[1.0])).unwrap();
+        let len_after_first = std::fs::metadata(&wal_path).unwrap().len() as usize;
+        wal.append_save(&make_wal_entry("second", &[2.0])).unwrap();
+        let len_after_second = std::fs::metadata(&wal_path).unwrap().len() as usize;
+        wal.append_save(&make_wal_entry("third", &[3.0])).unwrap();
+        drop(wal);
+
+        let bytes = std::fs::read(&wal_path).unwrap();
+        let entry2_bytes = bytes[len_after_first..len_after_second].to_vec();
+
+        // Duplicate "second" right after itself: [first][second][second][third]
+        let mut spliced = bytes[..len_after_second].to_vec();
+        spliced.extend_from_slice(&entry2_bytes);
+        spliced.extend_from_slice(&bytes[len_after_second..]);
+        std::fs::write(&wal_path, &spliced).unwrap();
+
+        let entries = WalFile::read_entries(&wal_path).unwrap();
+        assert_eq!(
+            entries.len(),
+            2,
+            "replay must stop at the spliced duplicate, keeping only the entries before it"
+        );
+        assert_eq!(entries[0].chunk, "first");
+        assert_eq!(entries[1].chunk, "second");
+    }
+
+    /// A WAL closed (without truncating) and reopened must continue the CRC
+    /// chain correctly for newly appended entries — this is the normal
+    /// crash-restart-without-flush scenario (`HDF5Memory::open` replays
+    /// existing entries, then reopens the same file for further appends
+    /// without clearing it), and must not produce a false "reordering"
+    /// detection for its own legitimately-appended entries.
+    #[test]
+    fn test_wal_chain_continues_across_reopen() {
+        let dir = TempDir::new().unwrap();
+        let wal_path = dir.path().join("test.h5.wal");
+
+        let mut wal = WalFile::open(&wal_path).unwrap();
+        wal.append_save(&make_wal_entry("first", &[1.0])).unwrap();
+        drop(wal); // simulate a restart without ever truncating the WAL
+
+        let mut wal2 = WalFile::open(&wal_path).unwrap();
+        wal2.append_save(&make_wal_entry("second", &[2.0]))
+            .unwrap();
+        drop(wal2);
+
+        let entries = WalFile::read_entries(&wal_path).unwrap();
+        assert_eq!(
+            entries.len(),
+            2,
+            "both pre- and post-reopen entries must replay cleanly"
+        );
+        assert_eq!(entries[0].chunk, "first");
+        assert_eq!(entries[1].chunk, "second");
+    }
+
+    /// Build a legacy (WAL_VERSION_LEGACY_NO_CRC) WAL file containing one
+    /// Save entry, with no trailing CRC32.
+    fn build_legacy_v1_wal_bytes() -> Vec<u8> {
         let mut buf = Vec::new();
         buf.extend_from_slice(&WAL_MAGIC);
         buf.push(WAL_VERSION_LEGACY_NO_CRC);
         buf.extend_from_slice(&1u32.to_le_bytes());
-        // One Save entry in the old format: type + timestamp + fields, with
-        // no trailing CRC32.
         buf.push(WalEntryType::Save as u8);
         buf.extend_from_slice(&42.0f64.to_le_bytes());
         serialize_str(&mut buf, "legacy-chunk");
@@ -933,14 +1170,39 @@ mod tests {
         serialize_str(&mut buf, "chan");
         serialize_str(&mut buf, "sess");
         serialize_str(&mut buf, "tags");
-        std::fs::write(&wal_path, &buf).unwrap();
+        buf
+    }
 
-        let entries = WalFile::read_entries(&wal_path).unwrap();
+    #[test]
+    fn test_wal_reads_legacy_v1_format_without_crc() {
+        let dir = TempDir::new().unwrap();
+        let wal_path = dir.path().join("legacy.h5.wal");
+        std::fs::write(&wal_path, build_legacy_v1_wal_bytes()).unwrap();
+
+        // Only the migration-only reader may read a legacy no-CRC file.
+        let entries = WalFile::read_entries_for_migration(&wal_path).unwrap();
         assert_eq!(entries.len(), 1);
         assert_eq!(entries[0].chunk, "legacy-chunk");
         assert_eq!(entries[0].embedding, vec![1.0, 2.0]);
     }
 
+    /// The public `read_entries` must reject a legacy no-CRC file instead of
+    /// silently downgrading to the fully-unverified parser (INT-09) — flipping
+    /// a version byte from 2/3 down to 1 must not be a way to bypass every
+    /// integrity check for an arbitrary caller of the public API.
+    #[test]
+    fn test_wal_read_entries_rejects_legacy_v1_format() {
+        let dir = TempDir::new().unwrap();
+        let wal_path = dir.path().join("legacy.h5.wal");
+        std::fs::write(&wal_path, build_legacy_v1_wal_bytes()).unwrap();
+
+        let result = WalFile::read_entries(&wal_path);
+        assert!(
+            result.is_err(),
+            "read_entries() must reject a legacy no-CRC WAL file, not silently parse it"
+        );
+    }
+
     #[test]
     fn test_wal_open_migrates_legacy_v1_to_current_version() {
         let dir = TempDir::new().unwrap();