fix(migrate): write a real clawhdf5-agent store

clawhdf5-migrate wrote a layout of its own (/chunks, /sessions,
/entities, /relations, root attributes, no /meta or schema_version) that
HDF5Memory::open rejects, so a "migrated" SQLite database could not be
used as agent memory — contrary to the README.

It now writes through the agent's own API (HDF5Memory::create/open,
save_batch, the session cache and the knowledge graph), so there is no
second copy of the schema:

- sessions and entities/relations carry over; deleted rows become
  deleted records (or are left out with --skip-deleted);
- embeddings follow the library default (float16), --f32 opts out and
  --float16 is a hidden no-op, as in clawhdf5-cli; the `half`-based
  conversion is gone;
- every source row is checked before the output is created: a wrong
  embedding length, an empty embedding, a dimension that differs from
  an existing store's, or a float16 value beyond +-65504 is an error
  naming the chunk id, and an existing store is left untouched;
- --incremental opens the existing store, adds only rows it does not
  hold (matched by content) and follows the source's deleted flags;
- a source with no memory rows needs --embedding-dim;
- validation reads the result back with HDF5Memory::open_read_only,
  compares every field (embeddings bit for bit, round_to_f16 of the
  source for float16) and checks a migrated record is found by search.

clawhdf5-agent gains HDF5Memory::sessions()/sessions_mut(),
HDF5Memory::delete_batch (one save, all-or-nothing, no auto-compact),
SessionCache::add_at, and re-exports SessionCache/SessionEntry.

The old layout's per-dataset SHA-256 provenance attributes have no place
in the agent schema and are gone. An adversarial review found two
blockers (silent truncation of long embeddings; an --incremental
dimension check that could never fire) and four majors (a failed run
wiping the existing store, dim-0 stores, deleted-flag drift); all are
fixed with regression tests. 42 migrate tests, incl. h5py opening a
migrated store.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-agent/src/lib.rs

@@ -73,7 +73,7 @@ pub use ephemeral::{EphemeralEntry, EphemeralStats};
 use knowledge::KnowledgeCache;
 use memory_strategy::{Exchange, MemoryStrategy, StrategyOutput};
 pub use search::SearchOptions;
-use session::SessionCache;
+pub use session::{SessionCache, SessionEntry};
 
 // --- Error type ---
 
@@ -1028,6 +1028,18 @@ impl HDF5Memory {
         &self.config
     }
 
+    /// The sessions recorded in this store.
+    pub fn sessions(&self) -> &SessionCache {
+        &self.sessions
+    }
+
+    /// Mutable access to the sessions, e.g. to add many at once. Changes
+    /// reach the disk at the next checkpoint (any flushing call, such as
+    /// [`HDF5Memory::flush_wal`] or `save_batch`), not immediately.
+    pub fn sessions_mut(&mut self) -> &mut SessionCache {
+        &mut self.sessions
+    }
+
     /// Get a reference to the knowledge cache.
     pub fn knowledge(&self) -> &KnowledgeCache {
         &self.knowledge
@@ -1401,6 +1413,35 @@ impl HDF5Memory {
 }
 
 impl HDF5Memory {
+    /// Delete many records with a single checkpoint, where
+    /// [`AgentMemory::delete`] checkpoints once per record.
+    ///
+    /// All or nothing: if any id is out of range or already deleted (or
+    /// repeated), nothing is deleted and `MemoryError::NotFound` is returned.
+    /// Unlike `delete`, this never auto-compacts, so the records stay in the
+    /// store as tombstones (their indices unchanged) until [`AgentMemory::compact`]
+    /// is called — importers use it to carry over records that were already
+    /// deleted in the source.
+    pub fn delete_batch(&mut self, ids: &[usize]) -> Result<()> {
+        let mut seen = std::collections::HashSet::with_capacity(ids.len());
+        for &id in ids {
+            if self.cache.tombstones.get(id).copied() != Some(0) || !seen.insert(id) {
+                return Err(MemoryError::NotFound(format!(
+                    "entry {id} not found or already deleted"
+                )));
+            }
+        }
+        if ids.is_empty() {
+            return Ok(());
+        }
+        for &id in ids {
+            self.cache.mark_deleted(id);
+            self.hnsw_on_delete(id);
+            self.bm25_on_delete(id);
+        }
+        self.flush()
+    }
+
     pub fn tick_session(&mut self) -> Result<()> {
         let d = self.config.decay_factor;
         for w in self.cache.activation_weights.iter_mut() {
@@ -1599,6 +1640,79 @@ mod tests {
         }
     }
 
+    #[test]
+    fn delete_batch_tombstones_without_compacting() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.h5");
+        let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
+        mem.save_batch(
+            (0..4)
+                .map(|i| make_entry(&format!("record {i}"), &[i as f32, 1.0, 0.0, 0.0]))
+                .collect(),
+        )
+        .unwrap();
+        // 3 of 4 is far past compact_threshold (0.3): delete() would compact.
+        mem.delete_batch(&[0, 1, 3]).unwrap();
+        assert_eq!(mem.count(), 4);
+        assert_eq!(mem.count_active(), 1);
+        drop(mem);
+
+        let mut mem = HDF5Memory::open(&path).unwrap();
+        assert_eq!(mem.cache.tombstones, vec![1, 1, 0, 1]);
+        let hits = mem.hybrid_search(&[0.0, 1.0, 0.0, 0.0], "record", 0.5, 0.5, 10);
+        assert!(
+            hits.iter().all(|r| r.index == 2),
+            "tombstoned record returned"
+        );
+    }
+
+    #[test]
+    fn delete_batch_is_all_or_nothing() {
+        let dir = TempDir::new().unwrap();
+        let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
+        mem.save_batch(vec![
+            make_entry("a", &[1.0, 0.0, 0.0, 0.0]),
+            make_entry("b", &[0.0, 1.0, 0.0, 0.0]),
+        ])
+        .unwrap();
+        for bad in [&[0, 5][..], &[1, 1][..]] {
+            assert!(matches!(
+                mem.delete_batch(bad),
+                Err(MemoryError::NotFound(_))
+            ));
+            assert_eq!(mem.count_active(), 2, "{bad:?} deleted something");
+        }
+        mem.delete_batch(&[]).unwrap();
+        assert_eq!(mem.count_active(), 2);
+    }
+
+    #[test]
+    fn sessions_mut_add_at_keeps_timestamp_across_reopen() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.h5");
+        let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
+        mem.sessions_mut()
+            .add_at("s-old", 2, 7, "discord", "old summary", 1.7e15);
+        mem.flush_wal().unwrap();
+        drop(mem);
+
+        let mem = HDF5Memory::open_read_only(&path).unwrap();
+        let s = mem.sessions();
+        assert_eq!(s.len(), 1);
+        let e = &s.entries[0];
+        assert_eq!(
+            (
+                e.id.as_str(),
+                e.start_idx,
+                e.end_idx,
+                e.channel.as_str(),
+                e.ts
+            ),
+            ("s-old", 2, 7, "discord", 1.7e15)
+        );
+        assert_eq!(s.summaries[0], "old summary");
+    }
+
     #[test]
     fn create_new_file() {
         let dir = TempDir::new().unwrap();

crates/clawhdf5-agent/src/session.rs

@@ -33,7 +33,7 @@ impl SessionCache {
         self.entries.is_empty()
     }
 
-    /// Add a new session with its summary.
+    /// Add a new session with its summary, timestamped now.
     pub fn add(
         &mut self,
         id: &str,
@@ -47,6 +47,21 @@ impl SessionCache {
             .unwrap_or_default()
             .as_secs_f64()
             * 1_000_000.0; // microseconds
+        self.add_at(id, start_idx, end_idx, channel, summary, ts);
+    }
+
+    /// Add a session with an explicit timestamp (Unix **microseconds**, the
+    /// unit [`SessionEntry::ts`] uses) — for importers carrying sessions over
+    /// from another store, whose original time should be kept.
+    pub fn add_at(
+        &mut self,
+        id: &str,
+        start_idx: usize,
+        end_idx: usize,
+        channel: &str,
+        summary: &str,
+        ts: f64,
+    ) {
         self.entries.push(SessionEntry {
             id: id.to_string(),
             start_idx: start_idx as u64,

crates/clawhdf5-migrate/Cargo.toml

@@ -3,7 +3,7 @@ name = "clawhdf5-migrate"
 version = "2.7.0"
 edition = "2024"
 rust-version.workspace = true
-description = "CLI to migrate SQLite agent memory databases to HDF5 format"
+description = "CLI to migrate SQLite agent memory databases to clawhdf5-agent stores"
 license = "MIT"
 repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
 readme = "README.md"
@@ -17,10 +17,8 @@ path = "src/main.rs"
 [dependencies]
 clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.7.0" }
 clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
-clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
 rusqlite = { version = "0.31", features = ["bundled"] }
 clap = { version = "4", features = ["derive"] }
-half = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }

crates/clawhdf5-migrate/README.md

@@ -3,9 +3,12 @@
 [![crates.io](https://img.shields.io/crates/v/clawhdf5-migrate.svg)](https://crates.io/crates/clawhdf5-migrate)
 [![docs.rs](https://img.shields.io/docsrs/clawhdf5-migrate)](https://docs.rs/clawhdf5-migrate)
 
-CLI tool to migrate SQLite agent memory databases to HDF5 format.
+CLI tool to migrate SQLite agent memory databases (the ZeroClaw layout) to a
+[clawhdf5-agent](https://crates.io/crates/clawhdf5-agent) store.
 
-Converts existing SQLite-based agent memory stores (embeddings, text chunks, metadata) into the HDF5 format used by [clawhdf5-agent](https://crates.io/crates/clawhdf5-agent).
+The output is written through `clawhdf5-agent`'s own API, so it opens with
+`HDF5Memory::open` and is searchable immediately: memory records, sessions and
+the knowledge graph (entities and relations) are carried over.
 
 ## Installation
 
@@ -16,9 +19,19 @@ cargo install clawhdf5-migrate
 ## Usage
 
 ```bash
-clawhdf5-migrate --input agent.db --output agent.h5
+clawhdf5-migrate --sqlite agent.db --hdf5 agent.h5 --agent-id my-agent
 ```
 
+Embeddings are stored as float16 (the library default for new stores); pass
+`--f32` for full precision. Every embedding must have the same dimension
+(the first row's, or `--embedding-dim`, which a source with no memory records
+requires); rows are never truncated, and the whole source is checked before an
+existing output store is replaced. `--incremental` adds only new rows to an
+existing store of the same dimension and carries over changes to rows'
+deleted flags, `--skip-deleted` leaves out tombstoned rows, and `--dry-run`
+only counts.
+See `clawhdf5-migrate --help` for every option.
+
 ## License
 
 MIT

crates/clawhdf5-migrate/src/hdf5_reader.rs

@@ -1,163 +0,0 @@
-//! Read a migration HDF5 file back into the in-memory data model.
-//!
-//! Used to verify migrated content (real validation) and to merge new rows into
-//! an existing output (incremental migration). Mirrors the layout produced by
-//! [`crate::hdf5_writer`].
-
-use clawhdf5::reader::{File, Group};
-use clawhdf5_format::type_builders::AttrValue;
-
-use crate::sqlite_reader::{Entity, MemoryChunk, Relation, Session, SqliteData};
-
-type BoxErr = Box<dyn std::error::Error>;
-
-fn read_strings(group: &Group<'_>, name: &str) -> Result<Vec<String>, BoxErr> {
-    Ok(group.dataset(name)?.read_string()?)
-}
-
-fn read_i64s(group: &Group<'_>, name: &str) -> Result<Vec<i64>, BoxErr> {
-    Ok(group.dataset(name)?.read_i64()?)
-}
-
-fn read_f64s(group: &Group<'_>, name: &str) -> Result<Vec<f64>, BoxErr> {
-    Ok(group.dataset(name)?.read_f64()?)
-}
-
-/// Read the embeddings dataset as a flat `Vec<f32>` of `n * dim` values,
-/// handling both f32 and (lossy) f16 storage.
-fn read_embeddings_flat(group: &Group<'_>) -> Result<Vec<f32>, BoxErr> {
-    Ok(group.dataset("embeddings")?.read_f32()?)
-}
-
-/// Read a migration HDF5 file into a [`SqliteData`].
-pub fn read_hdf5(path: &str) -> Result<SqliteData, BoxErr> {
-    let file = File::open(path)?;
-
-    let embedding_dim = match file.root().attrs()?.get("embedding_dim") {
-        Some(AttrValue::I64(d)) => *d as usize,
-        _ => 0,
-    };
-
-    let chunks = read_chunks(&file, embedding_dim)?;
-    let sessions = read_sessions(&file)?;
-    let entities = read_entities(&file)?;
-    let relations = read_relations(&file)?;
-
-    Ok(SqliteData {
-        chunks,
-        sessions,
-        entities,
-        relations,
-        embedding_dim,
-        // Not a SQLite read — the caller (incremental migration) carries
-        // forward the current run's actual `source_path` from the fresh
-        // SQLite read instead of using this placeholder.
-        source_path: String::new(),
-    })
-}
-
-fn read_chunks(file: &File, dim: usize) -> Result<Vec<MemoryChunk>, BoxErr> {
-    let g = file.group("chunks")?;
-    let count = group_count(&g)?;
-    if count == 0 {
-        return Ok(Vec::new());
-    }
-    let ids = read_i64s(&g, "id")?;
-    let texts = read_strings(&g, "text")?;
-    let channels = read_strings(&g, "source_channel")?;
-    let timestamps = read_f64s(&g, "timestamp")?;
-    let session_ids = read_strings(&g, "session_id")?;
-    let tags = read_strings(&g, "tags")?;
-    let deleted = g.dataset("deleted")?.read_i32()?;
-    let emb_flat = read_embeddings_flat(&g)?;
-    let dim = dim.max(1);
-
-    let mut chunks = Vec::with_capacity(ids.len());
-    for (i, &id) in ids.iter().enumerate() {
-        let embedding = emb_flat
-            .get(i * dim..(i + 1) * dim)
-            .map(|s| s.to_vec())
-            .unwrap_or_default();
-        chunks.push(MemoryChunk {
-            id,
-            chunk: texts.get(i).cloned().unwrap_or_default(),
-            embedding,
-            source_channel: channels.get(i).cloned().unwrap_or_default(),
-            timestamp: timestamps.get(i).copied().unwrap_or(0.0),
-            session_id: session_ids.get(i).cloned().unwrap_or_default(),
-            tags: tags.get(i).cloned().unwrap_or_default(),
-            deleted: deleted.get(i).copied().unwrap_or(0),
-        });
-    }
-    Ok(chunks)
-}
-
-fn read_sessions(file: &File) -> Result<Vec<Session>, BoxErr> {
-    let g = file.group("sessions")?;
-    if group_count(&g)? == 0 {
-        return Ok(Vec::new());
-    }
-    let ids = read_strings(&g, "id")?;
-    let starts = read_i64s(&g, "start_idx")?;
-    let ends = read_i64s(&g, "end_idx")?;
-    let channels = read_strings(&g, "channel")?;
-    let timestamps = read_f64s(&g, "timestamp")?;
-    let summaries = read_strings(&g, "summary")?;
-    Ok((0..ids.len())
-        .map(|i| Session {
-            id: ids[i].clone(),
-            start_idx: starts.get(i).copied().unwrap_or(0),
-            end_idx: ends.get(i).copied().unwrap_or(0),
-            channel: channels.get(i).cloned().unwrap_or_default(),
-            timestamp: timestamps.get(i).copied().unwrap_or(0.0),
-            summary: summaries.get(i).cloned().unwrap_or_default(),
-        })
-        .collect())
-}
-
-fn read_entities(file: &File) -> Result<Vec<Entity>, BoxErr> {
-    let g = file.group("entities")?;
-    if group_count(&g)? == 0 {
-        return Ok(Vec::new());
-    }
-    let ids = read_i64s(&g, "id")?;
-    let names = read_strings(&g, "name")?;
-    let types = read_strings(&g, "type")?;
-    let emb_idxs = read_i64s(&g, "embedding_idx")?;
-    Ok((0..ids.len())
-        .map(|i| Entity {
-            id: ids[i],
-            name: names.get(i).cloned().unwrap_or_default(),
-            entity_type: types.get(i).cloned().unwrap_or_default(),
-            embedding_idx: emb_idxs.get(i).copied().unwrap_or(-1),
-        })
-        .collect())
-}
-
-fn read_relations(file: &File) -> Result<Vec<Relation>, BoxErr> {
-    let g = file.group("relations")?;
-    if group_count(&g)? == 0 {
-        return Ok(Vec::new());
-    }
-    let srcs = read_i64s(&g, "src")?;
-    let tgts = read_i64s(&g, "tgt")?;
-    let rels = read_strings(&g, "relation")?;
-    let weights = read_f64s(&g, "weight")?;
-    let timestamps = read_f64s(&g, "timestamp")?;
-    Ok((0..srcs.len())
-        .map(|i| Relation {
-            src: srcs[i],
-            tgt: tgts.get(i).copied().unwrap_or(0),
-            relation: rels.get(i).cloned().unwrap_or_default(),
-            weight: weights.get(i).copied().unwrap_or(1.0),
-            timestamp: timestamps.get(i).copied().unwrap_or(0.0),
-        })
-        .collect())
-}
-
-fn group_count(group: &Group<'_>) -> Result<u64, BoxErr> {
-    match group.attrs()?.get("count") {
-        Some(AttrValue::I64(n)) => Ok(*n as u64),
-        _ => Ok(0),
-    }
-}

crates/clawhdf5-migrate/src/hdf5_writer.rs

@@ -1,366 +0,0 @@
-use clawhdf5::writer::FileBuilder;
-use clawhdf5_format::datatype::{CharacterSet, Datatype, StringPadding};
-use clawhdf5_format::type_builders::AttrValue;
-
-use crate::sqlite_reader::SqliteData;
-
-/// Options controlling HDF5 output.
-pub struct WriteOptions {
-    pub agent_id: String,
-    pub embedder: String,
-    pub compression: bool,
-    pub compression_level: u32,
-    pub float16: bool,
-}
-
-/// Write SQLite data to an HDF5 file.
-pub fn write_hdf5(
-    path: &str,
-    data: &SqliteData,
-    opts: &WriteOptions,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let mut builder = FileBuilder::new();
-    let timestamp = iso8601_now();
-
-    // Root-level metadata attributes
-    builder.set_attr("agent_id", AttrValue::String(opts.agent_id.clone()));
-    builder.set_attr("embedder", AttrValue::String(opts.embedder.clone()));
-    builder.set_attr("embedding_dim", AttrValue::I64(data.embedding_dim as i64));
-    builder.set_attr("source", AttrValue::String("sqlite-migration".into()));
-    builder.set_attr("version", AttrValue::I64(1));
-    // Lineage: which SQLite database this output was migrated from and when,
-    // plus the migrator tool version — so a chain of `--incremental` runs
-    // still has an audit trail instead of every run overwriting the same
-    // static attributes (see research/03_provenance.md, INT-03).
-    builder.set_attr("source_path", AttrValue::String(data.source_path.clone()));
-    builder.set_attr("migrated_at", AttrValue::String(timestamp.clone()));
-    builder.set_attr(
-        "migrator_version",
-        AttrValue::String(env!("CARGO_PKG_VERSION").to_owned()),
-    );
-
-    write_chunks_group(&mut builder, data, opts, &timestamp);
-    write_sessions_group(&mut builder, data);
-    write_entities_group(&mut builder, data);
-    write_relations_group(&mut builder, data);
-
-    builder.write(path)?;
-    Ok(())
-}
-
-/// Current UTC time formatted as an ISO-8601 / RFC-3339 timestamp
-/// (`YYYY-MM-DDTHH:MM:SSZ`), with no external date/time dependency.
-fn iso8601_now() -> String {
-    let secs = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap_or_default()
-        .as_secs();
-    let days = (secs / 86_400) as i64;
-    let time_of_day = secs % 86_400;
-    let (h, m, s) = (
-        time_of_day / 3600,
-        (time_of_day % 3600) / 60,
-        time_of_day % 60,
-    );
-    let (y, mo, d) = civil_from_days(days);
-    format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z")
-}
-
-/// Days-since-epoch to (year, month, day), Howard Hinnant's `civil_from_days`
-/// algorithm (proleptic Gregorian calendar, valid for the full `i64` range).
-fn civil_from_days(z: i64) -> (i64, u32, u32) {
-    let z = z + 719_468;
-    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
-    let doe = (z - era * 146_097) as u64; // [0, 146096]
-    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399]
-    let y = yoe as i64 + era * 400;
-    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365]
-    let mp = (5 * doy + 2) / 153; // [0, 11]
-    let d = (doy - (153 * mp + 2) / 5 + 1) as u32; // [1, 31]
-    let m = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32; // [1, 12]
-    let y = if m <= 2 { y + 1 } else { y };
-    (y, m, d)
-}
-
-/// Build a fixed-length string Datatype from the max byte length of the items.
-fn string_dtype(max_len: usize) -> Datatype {
-    Datatype::String {
-        size: max_len.max(1) as u32,
-        padding: StringPadding::NullPad,
-        charset: CharacterSet::Utf8,
-    }
-}
-
-/// Pack a slice of strings into null-padded raw bytes of uniform width.
-fn pack_strings(strings: &[String]) -> (Vec<u8>, usize) {
-    let max_len = strings.iter().map(|s| s.len()).max().unwrap_or(0).max(1);
-    let mut buf = vec![0u8; strings.len() * max_len];
-    for (i, s) in strings.iter().enumerate() {
-        let start = i * max_len;
-        let bytes = s.as_bytes();
-        let copy_len = bytes.len().min(max_len);
-        buf[start..start + copy_len].copy_from_slice(&bytes[..copy_len]);
-    }
-    (buf, max_len)
-}
-
-fn apply_compression(ds: &mut clawhdf5_format::type_builders::DatasetBuilder, opts: &WriteOptions) {
-    if opts.compression {
-        ds.with_deflate(opts.compression_level);
-        ds.with_shuffle();
-    }
-}
-
-fn write_chunks_group(
-    builder: &mut FileBuilder,
-    data: &SqliteData,
-    opts: &WriteOptions,
-    timestamp: &str,
-) {
-    let mut group = builder.create_group("chunks");
-    let n = data.chunks.len() as u64;
-
-    if n == 0 {
-        group.set_attr("count", AttrValue::I64(0));
-        builder.add_group(group.finish());
-        return;
-    }
-
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    // Source attribution attached directly to the content-bearing datasets
-    // (SHA-256 of the raw bytes + creator/timestamp/source), so the chunk
-    // text and embeddings each carry their own verifiable provenance
-    // (see clawhdf5_format::provenance / `Dataset::verify_provenance`).
-    let source_opt = if data.source_path.is_empty() {
-        None
-    } else {
-        Some(data.source_path.as_str())
-    };
-
-    // ids
-    let ids: Vec<i64> = data.chunks.iter().map(|c| c.id).collect();
-    group.create_dataset("id").with_i64_data(&ids);
-
-    // text
-    let texts: Vec<String> = data.chunks.iter().map(|c| c.chunk.clone()).collect();
-    let (text_raw, text_len) = pack_strings(&texts);
-    group
-        .create_dataset("text")
-        .with_compound_data(string_dtype(text_len), text_raw, n)
-        .with_provenance("clawhdf5-migrate", timestamp, source_opt);
-
-    // embeddings - flatten to [N, dim]
-    let dim = data.embedding_dim;
-    if opts.float16 {
-        let f16_data: Vec<u16> = data
-            .chunks
-            .iter()
-            .flat_map(|c| {
-                c.embedding
-                    .iter()
-                    .map(|&v| half::f16::from_f32(v).to_bits())
-            })
-            .collect();
-        let raw: Vec<u8> = f16_data.iter().flat_map(|v| v.to_le_bytes()).collect();
-        let f16_dtype = Datatype::FloatingPoint {
-            size: 2,
-            byte_order: clawhdf5_format::datatype::DatatypeByteOrder::LittleEndian,
-            bit_offset: 0,
-            bit_precision: 16,
-            exponent_location: 10,
-            exponent_size: 5,
-            mantissa_location: 0,
-            mantissa_size: 10,
-            exponent_bias: 15,
-        };
-        let ds = group
-            .create_dataset("embeddings")
-            .with_compound_data(f16_dtype, raw, n)
-            .with_shape(&[n, dim as u64])
-            .with_provenance("clawhdf5-migrate", timestamp, source_opt);
-        apply_compression(ds, opts);
-    } else {
-        let flat: Vec<f32> = data
-            .chunks
-            .iter()
-            .flat_map(|c| c.embedding.iter().copied())
-            .collect();
-        let ds = group
-            .create_dataset("embeddings")
-            .with_f32_data(&flat)
-            .with_shape(&[n, dim as u64])
-            .with_provenance("clawhdf5-migrate", timestamp, source_opt);
-        apply_compression(ds, opts);
-    }
-
-    // source_channel
-    let channels: Vec<String> = data
-        .chunks
-        .iter()
-        .map(|c| c.source_channel.clone())
-        .collect();
-    let (ch_raw, ch_len) = pack_strings(&channels);
-    group
-        .create_dataset("source_channel")
-        .with_compound_data(string_dtype(ch_len), ch_raw, n);
-
-    // timestamp
-    let timestamps: Vec<f64> = data.chunks.iter().map(|c| c.timestamp).collect();
-    group.create_dataset("timestamp").with_f64_data(&timestamps);
-
-    // session_id
-    let sess_ids: Vec<String> = data.chunks.iter().map(|c| c.session_id.clone()).collect();
-    let (sid_raw, sid_len) = pack_strings(&sess_ids);
-    group
-        .create_dataset("session_id")
-        .with_compound_data(string_dtype(sid_len), sid_raw, n);
-
-    // tags
-    let tags: Vec<String> = data.chunks.iter().map(|c| c.tags.clone()).collect();
-    let (tag_raw, tag_len) = pack_strings(&tags);
-    group
-        .create_dataset("tags")
-        .with_compound_data(string_dtype(tag_len), tag_raw, n);
-
-    // deleted
-    let deleted: Vec<i32> = data.chunks.iter().map(|c| c.deleted).collect();
-    group.create_dataset("deleted").with_i32_data(&deleted);
-
-    builder.add_group(group.finish());
-}
-
-fn write_sessions_group(builder: &mut FileBuilder, data: &SqliteData) {
-    let mut group = builder.create_group("sessions");
-    let n = data.sessions.len() as u64;
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    if n == 0 {
-        builder.add_group(group.finish());
-        return;
-    }
-
-    let ids: Vec<String> = data.sessions.iter().map(|s| s.id.clone()).collect();
-    let (id_raw, id_len) = pack_strings(&ids);
-    group
-        .create_dataset("id")
-        .with_compound_data(string_dtype(id_len), id_raw, n);
-
-    let start_idxs: Vec<i64> = data.sessions.iter().map(|s| s.start_idx).collect();
-    group.create_dataset("start_idx").with_i64_data(&start_idxs);
-
-    let end_idxs: Vec<i64> = data.sessions.iter().map(|s| s.end_idx).collect();
-    group.create_dataset("end_idx").with_i64_data(&end_idxs);
-
-    let channels: Vec<String> = data.sessions.iter().map(|s| s.channel.clone()).collect();
-    let (ch_raw, ch_len) = pack_strings(&channels);
-    group
-        .create_dataset("channel")
-        .with_compound_data(string_dtype(ch_len), ch_raw, n);
-
-    let timestamps: Vec<f64> = data.sessions.iter().map(|s| s.timestamp).collect();
-    group.create_dataset("timestamp").with_f64_data(&timestamps);
-
-    let summaries: Vec<String> = data.sessions.iter().map(|s| s.summary.clone()).collect();
-    let (sum_raw, sum_len) = pack_strings(&summaries);
-    group
-        .create_dataset("summary")
-        .with_compound_data(string_dtype(sum_len), sum_raw, n);
-
-    builder.add_group(group.finish());
-}
-
-fn write_entities_group(builder: &mut FileBuilder, data: &SqliteData) {
-    let mut group = builder.create_group("entities");
-    let n = data.entities.len() as u64;
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    if n == 0 {
-        builder.add_group(group.finish());
-        return;
-    }
-
-    let ids: Vec<i64> = data.entities.iter().map(|e| e.id).collect();
-    group.create_dataset("id").with_i64_data(&ids);
-
-    let names: Vec<String> = data.entities.iter().map(|e| e.name.clone()).collect();
-    let (name_raw, name_len) = pack_strings(&names);
-    group
-        .create_dataset("name")
-        .with_compound_data(string_dtype(name_len), name_raw, n);
-
-    let types: Vec<String> = data
-        .entities
-        .iter()
-        .map(|e| e.entity_type.clone())
-        .collect();
-    let (type_raw, type_len) = pack_strings(&types);
-    group
-        .create_dataset("type")
-        .with_compound_data(string_dtype(type_len), type_raw, n);
-
-    let emb_idxs: Vec<i64> = data.entities.iter().map(|e| e.embedding_idx).collect();
-    group
-        .create_dataset("embedding_idx")
-        .with_i64_data(&emb_idxs);
-
-    builder.add_group(group.finish());
-}
-
-fn write_relations_group(builder: &mut FileBuilder, data: &SqliteData) {
-    let mut group = builder.create_group("relations");
-    let n = data.relations.len() as u64;
-    group.set_attr("count", AttrValue::I64(n as i64));
-
-    if n == 0 {
-        builder.add_group(group.finish());
-        return;
-    }
-
-    let srcs: Vec<i64> = data.relations.iter().map(|r| r.src).collect();
-    group.create_dataset("src").with_i64_data(&srcs);
-
-    let tgts: Vec<i64> = data.relations.iter().map(|r| r.tgt).collect();
-    group.create_dataset("tgt").with_i64_data(&tgts);
-
-    let rels: Vec<String> = data.relations.iter().map(|r| r.relation.clone()).collect();
-    let (rel_raw, rel_len) = pack_strings(&rels);
-    group
-        .create_dataset("relation")
-        .with_compound_data(string_dtype(rel_len), rel_raw, n);
-
-    let weights: Vec<f64> = data.relations.iter().map(|r| r.weight).collect();
-    group.create_dataset("weight").with_f64_data(&weights);
-
-    let timestamps: Vec<f64> = data.relations.iter().map(|r| r.timestamp).collect();
-    group.create_dataset("timestamp").with_f64_data(&timestamps);
-
-    builder.add_group(group.finish());
-}
-
-#[cfg(test)]
-mod time_tests {
-    use super::civil_from_days;
-
-    #[test]
-    fn epoch_day_zero_is_1970_01_01() {
-        assert_eq!(civil_from_days(0), (1970, 1, 1));
-    }
-
-    #[test]
-    fn known_dates_roundtrip() {
-        // 2026-08-16 is 20,681 days after 1970-01-01.
-        assert_eq!(civil_from_days(20_681), (2026, 8, 16));
-        // 2000-02-29 (leap day itself) and 2000-03-01 (the day after).
-        assert_eq!(civil_from_days(11_016), (2000, 2, 29));
-        assert_eq!(civil_from_days(11_017), (2000, 3, 1));
-    }
-
-    #[test]
-    fn iso8601_now_has_expected_shape() {
-        let ts = super::iso8601_now();
-        assert_eq!(ts.len(), "2026-08-16T00:00:00Z".len());
-        assert!(ts.starts_with("20")); // sanity: 21st-century year
-        assert!(ts.ends_with('Z'));
-    }
-}

crates/clawhdf5-migrate/src/sqlite_reader.rs

@@ -50,12 +50,8 @@ pub struct SqliteData {
     pub sessions: Vec<Session>,
     pub entities: Vec<Entity>,
     pub relations: Vec<Relation>,
+    /// `--embedding-dim`, or the first row's; 0 when neither exists.
     pub embedding_dim: usize,
-    /// Filesystem path of the SQLite database this data was read from, for
-    /// provenance attribution on the HDF5 output. Empty when the data did
-    /// not come directly from a SQLite read (e.g. re-read of a prior HDF5
-    /// migration output for an incremental merge).
-    pub source_path: String,
 }
 
 /// A table name plus the ordered column names the reader maps by position.
@@ -167,11 +163,13 @@ pub fn read_counts(
     })
 }
 
-/// Auto-detect embedding dimension from the first chunk's BLOB size.
+/// Auto-detect embedding dimension from the BLOB size of the first chunk (in
+/// id order, deleted or not).
 fn detect_embedding_dim(conn: &Connection, config: &SchemaConfig) -> SqlResult<Option<usize>> {
     let emb_col = config.chunks.columns.get(2).copied().unwrap_or("embedding");
+    let id_col = config.chunks.columns.first().copied().unwrap_or("id");
     let mut stmt = conn.prepare(&format!(
-        "SELECT {emb_col} FROM {} LIMIT 1",
+        "SELECT {emb_col} FROM {} ORDER BY {id_col} LIMIT 1",
         config.chunks.table
     ))?;
     let mut rows = stmt.query([])?;
@@ -195,24 +193,16 @@ fn blob_to_f32(blob: &[u8]) -> Vec<f32> {
 /// Read all data from a ZeroClaw SQLite database.
 ///
 /// If `skip_deleted` is true, rows with `deleted=1` are excluded from chunks.
-/// If `embedding_dim` is `None`, auto-detect from the first row.
+/// If `embedding_dim` is `None`, auto-detect from the first row (0 when there
+/// are no rows). Embeddings are returned at their full stored length whatever
+/// the dimension: checking that every row matches it is the writer's job
+/// (`store_writer::write_store`), so a mismatch is an error, not silent
+/// truncation.
 pub fn read_sqlite(
     path: &str,
     skip_deleted: bool,
     embedding_dim: Option<usize>,
     config: &SchemaConfig,
-) -> Result<SqliteData, Box<dyn std::error::Error>> {
-    read_sqlite_filtered(path, skip_deleted, embedding_dim, config, 0)
-}
-
-/// Like [`read_sqlite`] but only reads chunks whose id is greater than
-/// `min_chunk_id` (0 = all). Used for incremental migration.
-pub fn read_sqlite_filtered(
-    path: &str,
-    skip_deleted: bool,
-    embedding_dim: Option<usize>,
-    config: &SchemaConfig,
-    min_chunk_id: i64,
 ) -> Result<SqliteData, Box<dyn std::error::Error>> {
     let conn = Connection::open(path)?;
 
@@ -221,7 +211,7 @@ pub fn read_sqlite_filtered(
         None => detect_embedding_dim(&conn, config)?.unwrap_or(0),
     };
 
-    let chunks = read_chunks(&conn, skip_deleted, dim, config, min_chunk_id)?;
+    let chunks = read_chunks(&conn, skip_deleted, config)?;
     let sessions = read_sessions(&conn, config)?;
     let entities = read_entities(&conn, config)?;
     let relations = read_relations(&conn, config)?;
@@ -232,42 +222,43 @@ pub fn read_sqlite_filtered(
         entities,
         relations,
         embedding_dim: dim,
-        source_path: path.to_owned(),
     })
 }
 
 fn read_chunks(
     conn: &Connection,
     skip_deleted: bool,
-    expected_dim: usize,
     config: &SchemaConfig,
-    min_chunk_id: i64,
 ) -> SqlResult<Vec<MemoryChunk>> {
     let id_col = config.chunks.columns.first().copied().unwrap_or("id");
     let deleted_col = config.chunks.columns.get(7).copied().unwrap_or("deleted");
-    let mut conds = Vec::new();
+    let mut where_clause = String::new();
     if skip_deleted {
-        conds.push(format!("{deleted_col} = 0"));
+        where_clause = format!(" WHERE {deleted_col} = 0");
     }
-    if min_chunk_id > 0 {
+    // In id order, so the store's records follow the source's order.
-        conds.push(format!("{id_col} > {min_chunk_id}"));
+    where_clause.push_str(&format!(" ORDER BY {id_col}"));
-    }
-    let where_clause = if conds.is_empty() {
-        String::new()
-    } else {
-        format!(" WHERE {}", conds.join(" AND "))
-    };
     let sql = config.chunks.select(&where_clause);
 
     let mut stmt = conn.prepare(&sql)?;
     let rows = stmt.query_map([], |row| {
         let blob: Vec<u8> = row.get(2)?;
-        let mut embedding = blob_to_f32(&blob);
+        if !blob.len().is_multiple_of(4) {
-
+            let id: i64 = row.get(0)?;
-        // Validate/truncate to expected dimension
+            return Err(rusqlite::Error::FromSqlConversionFailure(
-        if expected_dim > 0 {
+                2,
-            embedding.truncate(expected_dim);
+                rusqlite::types::Type::Blob,
+                format!(
+                    "chunk id {id}: embedding BLOB is {} bytes, not a whole number of \
+                     little-endian f32 values",
+                    blob.len()
+                )
+                .into(),
+            ));
         }
+        // Read at full length: rows of the wrong dimension are rejected by
+        // the writer, never truncated to fit.
+        let embedding = blob_to_f32(&blob);
 
         Ok(MemoryChunk {
             id: row.get(0)?,

crates/clawhdf5-migrate/src/store_writer.rs

@@ -0,0 +1,407 @@
+//! Write migrated SQLite data into a clawhdf5-agent store.
+//!
+//! Everything goes through `clawhdf5-agent`'s own API — `HDF5Memory::create`
+//! (or `open` for `--incremental`), `save_batch`, `delete_batch`, the session
+//! cache and the knowledge graph — so the result is an ordinary agent store
+//! that `HDF5Memory::open` accepts, not a second hand-built copy of its schema.
+
+use std::collections::{HashMap, HashSet};
+use std::path::Path;
+
+use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
+use clawhdf5_format::float16::round_to_f16;
+
+use crate::sqlite_reader::{MemoryChunk, SqliteData};
+
+type BoxErr = Box<dyn std::error::Error>;
+
+/// SQLite timestamps are Unix seconds; the agent's session and relation
+/// timestamps are Unix microseconds (memory records stay in seconds).
+pub const US_PER_SEC: f64 = 1_000_000.0;
+
+/// Options controlling the output store.
+#[derive(Debug, Clone)]
+pub struct WriteOptions {
+    pub agent_id: String,
+    pub embedder: String,
+    pub compression: bool,
+    pub compression_level: u32,
+    /// Store full-precision `f32` embeddings instead of the library default
+    /// (half precision). Only applies to a newly created store: an existing
+    /// store keeps the precision it was created with.
+    pub f32: bool,
+    /// Add to the store at the output path if there is one, instead of
+    /// replacing it.
+    pub incremental: bool,
+    /// Leave out deleted source rows that are not in the store. (A deleted
+    /// row that matches an active store record still tombstones it, so pass
+    /// deleted rows in `data` for an incremental run.)
+    pub skip_deleted: bool,
+}
+
+/// What the migration wrote, and where each source row went, so validation
+/// can compare the store with the source row by row.
+#[derive(Debug, Default)]
+pub struct Migration {
+    /// Whether the output store existed and was added to (`--incremental`).
+    pub appended_to_existing: bool,
+    /// The store's embedding precision.
+    pub float16: bool,
+    pub embedding_dim: usize,
+    /// Records in the store after the migration (including tombstones).
+    pub store_count: usize,
+    /// `(store index, source chunk index)` of every record written.
+    pub records: Vec<(usize, usize)>,
+    /// Source chunks already in the store (incremental), not written again.
+    pub chunks_present: usize,
+    /// `(store index, source chunk index)` of records that were active in
+    /// the store but whose source row is now deleted (incremental): they were
+    /// tombstoned by this run.
+    pub deleted_in_store: Vec<(usize, usize)>,
+    /// Source rows that were deleted in the store but are active in the
+    /// source (incremental): the agent has no un-delete, so each was written
+    /// again as a new record (counted in `records` too).
+    pub restored: usize,
+    /// Deleted source rows left out because of `skip_deleted`.
+    pub deleted_skipped: usize,
+    /// `(store session index, source session index)` of each session written.
+    pub sessions: Vec<(usize, usize)>,
+    pub sessions_present: usize,
+    /// `(store entity id, source entity index)` of each entity written.
+    pub entities: Vec<(u64, usize)>,
+    pub entities_present: usize,
+    /// SQLite entity id -> store entity id, for every source entity.
+    pub entity_ids: HashMap<i64, u64>,
+    /// `(store relation index, source relation index)` of each relation written.
+    pub relations: Vec<(usize, usize)>,
+    pub relations_present: usize,
+    /// Source relations naming an entity id that is not in the entities
+    /// table; the knowledge graph cannot hold them, so they are skipped.
+    pub dangling_relations: Vec<usize>,
+    /// Messages of the write-anomaly alerts the agent raised while importing
+    /// (informational; they never block a save — a bulk import typically
+    /// trips the write-rate check).
+    pub anomaly_alerts: Vec<String>,
+}
+
+/// Identity of a memory record for incremental de-duplication: every field
+/// the agent stores except the embedding (whose stored form depends on the
+/// store's precision).
+type RecordKey = (String, String, String, String, u64);
+
+fn record_key(
+    chunk: &str,
+    source_channel: &str,
+    session_id: &str,
+    tags: &str,
+    ts: f64,
+) -> RecordKey {
+    (
+        chunk.to_owned(),
+        source_channel.to_owned(),
+        session_id.to_owned(),
+        tags.to_owned(),
+        ts.to_bits(),
+    )
+}
+
+/// Reject rows the agent would otherwise store differently from the source,
+/// or not at all: an embedding of a different length from the store's
+/// dimension (the agent pads/truncates silently), an empty embedding, or, in
+/// a float16 store, a value beyond the half-precision range.
+///
+/// Every source row is checked, including ones that end up not being written
+/// (already in the store, or deleted and skipped): the source must be
+/// consistent as a whole, and the check runs before the store is touched.
+fn check_chunks(chunks: &[MemoryChunk], dim: usize, float16: bool) -> Result<(), BoxErr> {
+    for c in chunks {
+        if c.embedding.is_empty() {
+            return Err(format!(
+                "chunk id {}: the embedding is empty; an agent store needs an embedding \
+                 for every record",
+                c.id
+            )
+            .into());
+        }
+        if c.embedding.len() != dim {
+            return Err(format!(
+                "chunk id {}: embedding has {} values, expected {dim}; every row must have \
+                 the store's dimension (detected from the first row unless --embedding-dim \
+                 is given), and rows are never truncated or padded to fit",
+                c.id,
+                c.embedding.len()
+            )
+            .into());
+        }
+        if float16
+            && let Some((k, v)) = c
+                .embedding
+                .iter()
+                .enumerate()
+                .find(|&(_, &v)| v.is_finite() && round_to_f16(v).is_infinite())
+        {
+            return Err(format!(
+                "chunk id {}: embedding[{k}] = {v} is outside the half-precision range \
+                 (±65504) of a float16 store; migrate with --f32",
+                c.id
+            )
+            .into());
+        }
+    }
+    Ok(())
+}
+
+/// Migrate `data` into the agent store at `path`.
+///
+/// Without `opts.incremental` (or when nothing exists at `path`) a new store
+/// is created, replacing any file there — but only once every source row has
+/// passed [`check_chunks`], so a source that cannot be migrated leaves an
+/// existing store untouched. With it, the existing store is opened and only
+/// source rows it does not already hold are added: memory records are
+/// matched on their content, sessions on their id, entities on name and
+/// type, relations on (source, target, relation). A matched record then
+/// takes the source row's deleted flag: see [`Migration::deleted_in_store`]
+/// and [`Migration::restored`].
+pub fn write_store(
+    path: &Path,
+    data: &SqliteData,
+    opts: &WriteOptions,
+) -> Result<Migration, BoxErr> {
+    let existing = opts.incremental && path.exists();
+    let mut mem = if existing {
+        // `open` does not modify the store beyond what the agent itself does
+        // on open; the checks below run before anything is written.
+        let mem = HDF5Memory::open(path)?;
+        let dim = mem.config().embedding_dim;
+        // `data.embedding_dim` is 0 only for a source with no records and no
+        // --embedding-dim, which has no dimension to disagree with.
+        if data.embedding_dim != 0 && dim != data.embedding_dim {
+            let hint = if dim == 0 {
+                " (a store created from a source with no memory records; re-create it \
+                 with --embedding-dim)"
+            } else {
+                ""
+            };
+            return Err(format!(
+                "the store at {} has embedding_dim {dim}{hint}, the source {}; \
+                 embeddings of a different dimension cannot be added to it",
+                path.display(),
+                data.embedding_dim
+            )
+            .into());
+        }
+        check_chunks(&data.chunks, dim, mem.config().float16)?;
+        mem
+    } else {
+        // (With records, a dimension of 0 means an empty first embedding,
+        // which `check_chunks` reports more precisely.)
+        if data.embedding_dim == 0 && data.chunks.is_empty() {
+            return Err(
+                "the source has no memory records to detect the embedding dimension \
+                        from; pass --embedding-dim (the dimension of the agent's embedder), \
+                        or the store could never hold a record"
+                    .into(),
+            );
+        }
+        let mut config = MemoryConfig::new(path.to_path_buf(), &opts.agent_id, data.embedding_dim);
+        config.embedder = opts.embedder.clone();
+        config.compression = opts.compression;
+        config.compression_level = opts.compression_level;
+        // Only ever switch the library default off (as `clawhdf5-cli create`).
+        if opts.f32 {
+            config.float16 = false;
+        }
+        // Before `create`, which replaces whatever is at `path`.
+        check_chunks(&data.chunks, config.embedding_dim, config.float16)?;
+        HDF5Memory::create(config)?
+    };
+    let float16 = mem.config().float16;
+    let dim = mem.config().embedding_dim;
+
+    let mut m = Migration {
+        appended_to_existing: existing,
+        float16,
+        embedding_dim: dim,
+        ..Migration::default()
+    };
+
+    // ---- Memory records --------------------------------------------------
+    // Store indices of every record the store already holds, by content, so
+    // a source row that appears twice is only treated as present as often
+    // as the store has it.
+    let mut present: HashMap<RecordKey, Vec<usize>> = HashMap::new();
+    if existing {
+        let c = &mem.cache;
+        for i in 0..c.len() {
+            let key = record_key(
+                &c.chunks[i],
+                &c.source_channels[i],
+                &c.session_ids[i],
+                &c.tags[i],
+                c.timestamps[i],
+            );
+            present.entry(key).or_default().push(i);
+        }
+    }
+    let key_of = |c: &MemoryChunk| {
+        record_key(
+            &c.chunk,
+            &c.source_channel,
+            &c.session_id,
+            &c.tags,
+            c.timestamp,
+        )
+    };
+    let tombstoned = |idx: usize| mem.cache.tombstones[idx] != 0;
+    // Pass 1: a store record in the same deleted state as the source row.
+    let mut unmatched: Vec<usize> = Vec::new();
+    for (i, c) in data.chunks.iter().enumerate() {
+        let src_deleted = c.deleted != 0;
+        let hit = present.get_mut(&key_of(c)).and_then(|idxs| {
+            let at = idxs.iter().position(|&x| tombstoned(x) == src_deleted)?;
+            Some(idxs.remove(at))
+        });
+        match hit {
+            Some(_) => m.chunks_present += 1,
+            None => unmatched.push(i),
+        }
+    }
+    // Pass 2: a store record whose deleted state differs — the source row
+    // was deleted or restored since the last migration. The source wins.
+    let mut new_chunks: Vec<usize> = Vec::with_capacity(unmatched.len());
+    let mut delete_in_store: Vec<usize> = Vec::new();
+    for i in unmatched {
+        let c = &data.chunks[i];
+        let hit = present
+            .get_mut(&key_of(c))
+            .and_then(|idxs| (!idxs.is_empty()).then(|| idxs.remove(0)));
+        match hit {
+            // Active in the store, deleted in the source: tombstone it.
+            Some(idx) if c.deleted != 0 => {
+                m.deleted_in_store.push((idx, i));
+                delete_in_store.push(idx);
+            }
+            // Deleted in the store, active in the source. The agent has no
+            // un-delete, so the row is written again as a new active record
+            // (the tombstone stays until the store is compacted).
+            Some(_) => {
+                m.restored += 1;
+                new_chunks.push(i);
+            }
+            None if c.deleted != 0 && opts.skip_deleted => m.deleted_skipped += 1,
+            None => new_chunks.push(i),
+        }
+    }
+    new_chunks.sort_unstable();
+    let to_write: Vec<&MemoryChunk> = new_chunks.iter().map(|&i| &data.chunks[i]).collect();
+
+    // ---- Sessions (in the cache; persisted by the save_batch checkpoint) ---
+    let known_sessions: HashSet<String> = mem
+        .sessions()
+        .entries
+        .iter()
+        .map(|e| e.id.clone())
+        .collect();
+    for (i, s) in data.sessions.iter().enumerate() {
+        if known_sessions.contains(&s.id) {
+            m.sessions_present += 1;
+            continue;
+        }
+        let sessions = mem.sessions_mut();
+        let at = sessions.len();
+        sessions.add_at(
+            &s.id,
+            s.start_idx.max(0) as usize,
+            s.end_idx.max(0) as usize,
+            &s.channel,
+            &s.summary,
+            s.timestamp * US_PER_SEC,
+        );
+        m.sessions.push((at, i));
+    }
+
+    // ---- Knowledge graph -------------------------------------------------
+    let kg = mem.knowledge_mut();
+    // Matched only against what the store held before this run: the source
+    // itself is copied as it is, duplicates included.
+    let by_name_type: HashMap<(String, String), u64> = kg
+        .entities
+        .iter()
+        .map(|e| ((e.name.clone(), e.entity_type.clone()), e.id))
+        .collect();
+    for (i, e) in data.entities.iter().enumerate() {
+        let key = (e.name.clone(), e.entity_type.clone());
+        let id = match by_name_type.get(&key) {
+            Some(&id) => {
+                m.entities_present += 1;
+                id
+            }
+            None => {
+                let id = kg.add_entity(&e.name, &e.entity_type, e.embedding_idx);
+                m.entities.push((id, i));
+                id
+            }
+        };
+        m.entity_ids.insert(e.id, id);
+    }
+    let known_relations: HashSet<(u64, u64, String)> = kg
+        .relations
+        .iter()
+        .map(|r| (r.src, r.tgt, r.relation.clone()))
+        .collect();
+    for (i, r) in data.relations.iter().enumerate() {
+        let (Some(&src), Some(&tgt)) = (m.entity_ids.get(&r.src), m.entity_ids.get(&r.tgt)) else {
+            m.dangling_relations.push(i);
+            continue;
+        };
+        if known_relations.contains(&(src, tgt, r.relation.clone())) {
+            m.relations_present += 1;
+            continue;
+        }
+        let at = kg.relations.len();
+        kg.add_relation(src, tgt, &r.relation, r.weight as f32);
+        kg.relations[at].ts = r.timestamp * US_PER_SEC;
+        m.relations.push((at, i));
+    }
+
+    // ---- Write: one checkpoint for records, sessions and graph -----------
+    let entries: Vec<MemoryEntry> = to_write
+        .iter()
+        .map(|c| MemoryEntry {
+            chunk: c.chunk.clone(),
+            embedding: c.embedding.clone(),
+            source_channel: c.source_channel.clone(),
+            timestamp: c.timestamp,
+            session_id: c.session_id.clone(),
+            tags: c.tags.clone(),
+        })
+        .collect();
+    let indices = mem.save_batch(entries)?;
+    m.records = indices
+        .iter()
+        .copied()
+        .zip(new_chunks.iter().copied())
+        .collect();
+
+    // Rows deleted in the source stay deleted: tombstones, as the agent's own
+    // `delete` leaves them (not compacted away).
+    // Records matched in the store whose source row has since been deleted
+    // are tombstoned too.
+    let tombstones: Vec<usize> = m
+        .records
+        .iter()
+        .filter(|&&(_, src)| data.chunks[src].deleted != 0)
+        .map(|&(idx, _)| idx)
+        .chain(delete_in_store)
+        .collect();
+    mem.delete_batch(&tombstones)?;
+
+    m.anomaly_alerts = mem
+        .take_anomaly_alerts()
+        .into_iter()
+        .map(|a| a.message)
+        .collect();
+    m.store_count = mem.count();
+    drop(mem); // release the single-writer lock before anyone re-opens it
+    Ok(m)
+}

crates/clawhdf5-migrate/src/validate.rs

@@ -1,192 +1,266 @@
-use clawhdf5::reader::File as Hdf5File;
+//! Validate a migration by reading the store back the way an agent would:
-use clawhdf5_format::provenance::VerifyResult;
+//! through `HDF5Memory::open_read_only`, comparing what it loads with the
+//! SQLite source, and running a search for a migrated record.
+
+use std::path::Path;
+
+use clawhdf5_agent::{AgentMemory, HDF5Memory, SearchOptions};
+use clawhdf5_format::float16::round_to_f16;
 
-use crate::hdf5_reader::read_hdf5;
 use crate::sqlite_reader::SqliteData;
+use crate::store_writer::{Migration, US_PER_SEC};
 
 type BoxErr = Box<dyn std::error::Error>;
 
 /// Summary of a migration validation.
 #[derive(Debug)]
 pub struct ValidationSummary {
-    pub chunks: u64,
+    /// Records in the store (including tombstones).
-    pub sessions: u64,
+    pub count: usize,
-    pub entities: u64,
+    /// Records in the store that are not deleted.
-    pub relations: u64,
+    pub active: usize,
-    pub embedding_dim: u64,
+    pub sessions: usize,
-    /// Number of rows whose full content was compared against the source.
+    pub entities: usize,
+    pub relations: usize,
+    pub embedding_dim: usize,
+    pub float16: bool,
+    /// Rows whose full content was compared against the source.
     pub rows_checked: u64,
-    /// Whether the `chunks/text` and `chunks/embeddings` SHINES provenance
+    /// Whether a search for a migrated record found it (`false` when there
-    /// hashes (written via [`crate::hdf5_writer`]) were both present and
+    /// was no active migrated record with an embedding to search for).
-    /// matched their recomputed SHA-256 on read-back. `false` when either
+    pub search_checked: bool,
-    /// dataset has no provenance metadata (e.g. an older output file) or
-    /// there are zero chunks to check.
-    pub provenance_verified: bool,
 }
 
-/// Validate a migrated HDF5 file against the source data.
+/// Validate the store at `path` against the source rows `migration` wrote.
 ///
-/// Reads the written file back and compares actual content — chunk text,
+/// Counts and the session / entity / relation rows are always checked in
-/// embeddings, and every session/entity/relation field — to the source, not
+/// full. Memory records are content-checked on a representative sample, or
-/// just the row counts. When `full` is false a representative sample of chunk
+/// all of them with `full`. Embeddings must match exactly: the source values
-/// rows is content-checked (counts and all other groups are always checked in
+/// themselves in an `f32` store, their [`round_to_f16`] in a `float16` one.
-/// full); when `full` is true every chunk row is compared too. `float16` widens
+pub fn validate_store(
-/// the embedding tolerance to allow for half-precision quantization.
+    path: &Path,
-pub fn validate_hdf5(
-    path: &str,
     source: &SqliteData,
+    migration: &Migration,
     full: bool,
-    float16: bool,
 ) -> Result<ValidationSummary, BoxErr> {
-    let got = read_hdf5(path)?;
+    let mut mem = HDF5Memory::open_read_only(path)?;
-    let provenance_verified = verify_chunk_provenance(path)?;
+    let float16 = mem.config().float16;
+    let dim = mem.config().embedding_dim;
 
     // ---- Counts ----
-    check_count("chunk", got.chunks.len(), source.chunks.len())?;
+    check_count("record", mem.count(), migration.store_count)?;
-    check_count("session", got.sessions.len(), source.sessions.len())?;
+    if float16 != migration.float16 {
-    check_count("entity", got.entities.len(), source.entities.len())?;
-    check_count("relation", got.relations.len(), source.relations.len())?;
-    if got.embedding_dim != source.embedding_dim {
         return Err(format!(
-            "embedding_dim mismatch: HDF5 has {}, source has {}",
+            "float16 mismatch: store {float16}, expected {}",
-            got.embedding_dim, source.embedding_dim
+            migration.float16
         )
         .into());
     }
+    if dim != migration.embedding_dim {
+        return Err(format!(
+            "embedding_dim mismatch: store has {dim}, expected {}",
+            migration.embedding_dim
+        )
+        .into());
+    }
+    if !migration.appended_to_existing {
+        check_count("record", mem.count(), migration.records.len())?;
+        check_count("session", mem.sessions().len(), migration.sessions.len())?;
+        check_count(
+            "entity",
+            mem.knowledge().entities.len(),
+            migration.entities.len(),
+        )?;
+        check_count(
+            "relation",
+            mem.knowledge().relations.len(),
+            migration.relations.len(),
+        )?;
+    }
 
-    // ---- Chunk content (sampled or full) ----
+    // ---- Memory records (sampled or full) ----
-    let (emb_abs, emb_rel) = if float16 { (1e-2, 1e-2) } else { (1e-4, 0.0) };
     let mut rows_checked = 0u64;
-    for i in sample_indices(source.chunks.len(), full) {
+    let expected_value = |v: f32| if float16 { round_to_f16(v) } else { v };
-        let (s, g) = (&source.chunks[i], &got.chunks[i]);
+    for k in sample_indices(migration.records.len(), full) {
-        if s.id != g.id {
+        let (idx, src) = migration.records[k];
-            return Err(field_err("chunk", i, "id", s.id, g.id));
+        let s = &source.chunks[src];
+        let c = &mem.cache;
+        if idx >= c.len() {
+            return Err(
+                format!("record {idx} (chunk id {}) is missing from the store", s.id).into(),
+            );
         }
-        if s.chunk != g.chunk {
+        let id = s.id;
+        if c.chunks[idx] != s.chunk {
             return Err(format!(
-                "chunk[{i}].text mismatch: source {:?}, HDF5 {:?}",
+                "record {idx} (chunk id {id}) text mismatch: source {:?}, store {:?}",
                 truncate(&s.chunk),
-                truncate(&g.chunk)
+                truncate(&c.chunks[idx])
             )
             .into());
         }
-        if s.session_id != g.session_id || s.source_channel != g.source_channel || s.tags != g.tags
+        if c.source_channels[idx] != s.source_channel
+            || c.session_ids[idx] != s.session_id
+            || c.tags[idx] != s.tags
         {
-            return Err(format!("chunk[{i}] string field mismatch").into());
+            return Err(format!("record {idx} (chunk id {id}) string field mismatch").into());
         }
-        if s.deleted != g.deleted {
+        if c.timestamps[idx].to_bits() != s.timestamp.to_bits() {
-            return Err(field_err("chunk", i, "deleted", s.deleted, g.deleted));
-        }
-        if s.embedding.len() != g.embedding.len() {
             return Err(format!(
-                "chunk[{i}] embedding length mismatch: {} vs {}",
+                "record {idx} (chunk id {id}) timestamp mismatch: source {}, store {}",
-                s.embedding.len(),
+                s.timestamp, c.timestamps[idx]
-                g.embedding.len()
             )
             .into());
         }
-        for (k, (&a, &b)) in s.embedding.iter().zip(g.embedding.iter()).enumerate() {
+        let deleted = c.tombstones[idx] != 0;
-            if (a - b).abs() > emb_abs + emb_rel * a.abs() {
+        if deleted != (s.deleted != 0) {
-                return Err(
+            return Err(format!(
-                    format!("chunk[{i}].embedding[{k}] mismatch: source {a}, HDF5 {b}").into(),
+                "record {idx} (chunk id {id}) deleted mismatch: source {}, store {deleted}",
-                );
+                s.deleted != 0
+            )
+            .into());
+        }
+        let got = c.embeddings.get(idx).unwrap_or(&[]);
+        if got.len() != s.embedding.len() {
+            return Err(format!(
+                "record {idx} (chunk id {id}) embedding length mismatch: source {}, store {}",
+                s.embedding.len(),
+                got.len()
+            )
+            .into());
+        }
+        for (j, (&a, &b)) in s.embedding.iter().zip(got).enumerate() {
+            let want = expected_value(a);
+            if want.to_bits() != b.to_bits() && !(want.is_nan() && b.is_nan()) {
+                return Err(format!(
+                    "record {idx} (chunk id {id}) embedding[{j}] mismatch: source {a}, \
+                     expected {want}, store {b}"
+                )
+                .into());
             }
         }
         rows_checked += 1;
     }
 
-    // ---- Other groups (always full — they are small) ----
+    // ---- Records tombstoned because their source row was deleted ----
-    for (i, (s, g)) in source.sessions.iter().zip(got.sessions.iter()).enumerate() {
+    for &(idx, src) in &migration.deleted_in_store {
-        if s.id != g.id
+        let s = &source.chunks[src];
-            || s.start_idx != g.start_idx
+        let c = &mem.cache;
-            || s.end_idx != g.end_idx
+        if idx >= c.len() || c.chunks[idx] != s.chunk || c.timestamps[idx] != s.timestamp {
-            || s.channel != g.channel
+            return Err(format!("record {idx} (chunk id {}) mismatch or missing", s.id).into());
-            || s.summary != g.summary
-        {
-            return Err(format!("session[{i}] mismatch").into());
         }
-        rows_checked += 1;
+        if c.tombstones[idx] == 0 {
-    }
+            return Err(format!(
-    for (i, (s, g)) in source.entities.iter().zip(got.entities.iter()).enumerate() {
+                "record {idx} (chunk id {}) is deleted in the source but active in the store",
-        if s.id != g.id
+                s.id
-            || s.name != g.name
+            )
-            || s.entity_type != g.entity_type
+            .into());
-            || s.embedding_idx != g.embedding_idx
-        {
-            return Err(format!("entity[{i}] mismatch").into());
-        }
-        rows_checked += 1;
-    }
-    for (i, (s, g)) in source
-        .relations
-        .iter()
-        .zip(got.relations.iter())
-        .enumerate()
-    {
-        if s.src != g.src || s.tgt != g.tgt || s.relation != g.relation {
-            return Err(format!("relation[{i}] mismatch").into());
         }
         rows_checked += 1;
     }
 
+    // ---- Sessions ----
+    let sessions = mem.sessions();
+    for &(at, src) in &migration.sessions {
+        let s = &source.sessions[src];
+        let (Some(e), Some(summary)) = (sessions.entries.get(at), sessions.summaries.get(at))
+        else {
+            return Err(format!("session {:?} is missing from the store", s.id).into());
+        };
+        if e.id != s.id
+            || e.start_idx != s.start_idx.max(0) as u64
+            || e.end_idx != s.end_idx.max(0) as u64
+            || e.channel != s.channel
+            || *summary != s.summary
+            || e.ts != s.timestamp * US_PER_SEC
+        {
+            return Err(format!("session {:?} mismatch", s.id).into());
+        }
+        rows_checked += 1;
+    }
+
+    // ---- Knowledge graph ----
+    let kg = mem.knowledge();
+    for &(id, src) in &migration.entities {
+        let s = &source.entities[src];
+        let Some(e) = kg.get_entity(id) else {
+            return Err(format!(
+                "entity {:?} (id {}) is missing from the store",
+                s.name, s.id
+            )
+            .into());
+        };
+        if e.name != s.name || e.entity_type != s.entity_type || e.embedding_idx != s.embedding_idx
+        {
+            return Err(format!("entity {:?} (id {}) mismatch", s.name, s.id).into());
+        }
+        rows_checked += 1;
+    }
+    for &(at, src) in &migration.relations {
+        let s = &source.relations[src];
+        let r = kg.relations.get(at);
+        let ok = r.is_some_and(|r| {
+            Some(&r.src) == migration.entity_ids.get(&s.src)
+                && Some(&r.tgt) == migration.entity_ids.get(&s.tgt)
+                && r.relation == s.relation
+                && r.weight == s.weight as f32
+                && r.ts == s.timestamp * US_PER_SEC
+        });
+        if !ok {
+            return Err(format!(
+                "relation {} -[{}]-> {} mismatch or missing",
+                s.src, s.relation, s.tgt
+            )
+            .into());
+        }
+        rows_checked += 1;
+    }
+
+    // ---- A migrated record must be findable by search ----
+    let probe = migration
+        .records
+        .iter()
+        .copied()
+        .find(|&(idx, _)| dim > 0 && mem.cache.tombstones[idx] == 0);
+    let search_checked = match probe {
+        None => false,
+        Some((idx, _)) => {
+            let query = mem.cache.embeddings[idx].to_vec();
+            let text = mem.cache.chunks[idx].clone();
+            let hits = mem.search(&query, &text, &SearchOptions::new(10));
+            // A record with the same text is as good a hit: the source may
+            // hold duplicates, and they tie.
+            if !hits.iter().any(|h| h.index == idx || h.chunk == text) {
+                return Err(format!(
+                    "search for migrated record {idx} ({:?}) did not return it",
+                    truncate(&text)
+                )
+                .into());
+            }
+            true
+        }
+    };
+
     Ok(ValidationSummary {
-        chunks: got.chunks.len() as u64,
+        count: mem.count(),
-        sessions: got.sessions.len() as u64,
+        active: mem.count_active(),
-        entities: got.entities.len() as u64,
+        sessions: mem.sessions().len(),
-        relations: got.relations.len() as u64,
+        entities: mem.knowledge().entities.len(),
-        embedding_dim: got.embedding_dim as u64,
+        relations: mem.knowledge().relations.len(),
+        embedding_dim: dim,
+        float16,
         rows_checked,
-        provenance_verified,
+        search_checked,
     })
 }
 
 fn check_count(kind: &str, got: usize, expected: usize) -> Result<(), BoxErr> {
     if got != expected {
-        return Err(format!("{kind} count mismatch: HDF5 has {got}, source has {expected}").into());
+        return Err(format!("{kind} count mismatch: store has {got}, expected {expected}").into());
     }
     Ok(())
 }
 
-/// Re-verify the SHA-256 provenance hash of `chunks/text` and
-/// `chunks/embeddings` against their actual stored bytes, catching
-/// post-write corruption that a plain content comparison against the
-/// in-memory source wouldn't (the source is compared against what
-/// `read_hdf5` decoded, not against the raw bytes on disk).
-///
-/// Returns `Ok(true)` only if both datasets exist and both hashes match.
-/// Returns `Ok(false)` (not an error) if a dataset has no provenance
-/// attributes at all (e.g. a file written before this check existed) or
-/// there are zero chunks. Returns an error only on an actual hash mismatch —
-/// that indicates real corruption.
-fn verify_chunk_provenance(path: &str) -> Result<bool, BoxErr> {
-    let file = Hdf5File::open(path)?;
-    let Ok(chunks) = file.group("chunks") else {
-        return Ok(false);
-    };
-    let mut all_present = true;
-    for name in ["text", "embeddings"] {
-        let Ok(ds) = chunks.dataset(name) else {
-            all_present = false;
-            continue;
-        };
-        match ds.verify_provenance()? {
-            VerifyResult::Ok => {}
-            VerifyResult::NoHash => all_present = false,
-            VerifyResult::Mismatch { stored, computed } => {
-                return Err(format!(
-                    "provenance hash mismatch on chunks/{name}: stored {stored}, recomputed {computed} — data may be corrupted"
-                )
-                .into());
-            }
-        }
-    }
-    Ok(all_present)
-}
-
-fn field_err<T: std::fmt::Display>(kind: &str, i: usize, field: &str, s: T, g: T) -> BoxErr {
-    format!("{kind}[{i}].{field} mismatch: source {s}, HDF5 {g}").into()
-}
-
 fn truncate(s: &str) -> String {
     if s.len() <= 40 {
         s.to_string()
@@ -196,7 +270,7 @@ fn truncate(s: &str) -> String {
     }
 }
 
-/// Indices of chunk rows to content-check. Full = all; otherwise a spread of
+/// Indices of records to content-check. Full = all; otherwise a spread of
 /// representative rows (first/last and evenly-spaced interior samples).
 fn sample_indices(n: usize, full: bool) -> Vec<usize> {
     if n == 0 {