fix(migrate): write a real clawhdf5-agent store

clawhdf5-migrate wrote a layout of its own (/chunks, /sessions,
/entities, /relations, root attributes, no /meta or schema_version) that
HDF5Memory::open rejects, so a "migrated" SQLite database could not be
used as agent memory — contrary to the README.

It now writes through the agent's own API (HDF5Memory::create/open,
save_batch, the session cache and the knowledge graph), so there is no
second copy of the schema:

- sessions and entities/relations carry over; deleted rows become
  deleted records (or are left out with --skip-deleted);
- embeddings follow the library default (float16), --f32 opts out and
  --float16 is a hidden no-op, as in clawhdf5-cli; the `half`-based
  conversion is gone;
- every source row is checked before the output is created: a wrong
  embedding length, an empty embedding, a dimension that differs from
  an existing store's, or a float16 value beyond +-65504 is an error
  naming the chunk id, and an existing store is left untouched;
- --incremental opens the existing store, adds only rows it does not
  hold (matched by content) and follows the source's deleted flags;
- a source with no memory rows needs --embedding-dim;
- validation reads the result back with HDF5Memory::open_read_only,
  compares every field (embeddings bit for bit, round_to_f16 of the
  source for float16) and checks a migrated record is found by search.

clawhdf5-agent gains HDF5Memory::sessions()/sessions_mut(),
HDF5Memory::delete_batch (one save, all-or-nothing, no auto-compact),
SessionCache::add_at, and re-exports SessionCache/SessionEntry.

The old layout's per-dataset SHA-256 provenance attributes have no place
in the agent schema and are gone. An adversarial review found two
blockers (silent truncation of long embeddings; an --incremental
dimension check that could never fire) and four majors (a failed run
wiping the existing store, dim-0 stores, deleted-flag drift); all are
fixed with regression tests. 42 migrate tests, incl. h5py opening a
migrated store.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-migrate/src/sqlite_reader.rs

@@ -50,12 +50,8 @@ pub struct SqliteData {
     pub sessions: Vec<Session>,
     pub entities: Vec<Entity>,
     pub relations: Vec<Relation>,
+    /// `--embedding-dim`, or the first row's; 0 when neither exists.
     pub embedding_dim: usize,
-    /// Filesystem path of the SQLite database this data was read from, for
-    /// provenance attribution on the HDF5 output. Empty when the data did
-    /// not come directly from a SQLite read (e.g. re-read of a prior HDF5
-    /// migration output for an incremental merge).
-    pub source_path: String,
 }
 
 /// A table name plus the ordered column names the reader maps by position.
@@ -167,11 +163,13 @@ pub fn read_counts(
     })
 }
 
-/// Auto-detect embedding dimension from the first chunk's BLOB size.
+/// Auto-detect embedding dimension from the BLOB size of the first chunk (in
+/// id order, deleted or not).
 fn detect_embedding_dim(conn: &Connection, config: &SchemaConfig) -> SqlResult<Option<usize>> {
     let emb_col = config.chunks.columns.get(2).copied().unwrap_or("embedding");
+    let id_col = config.chunks.columns.first().copied().unwrap_or("id");
     let mut stmt = conn.prepare(&format!(
-        "SELECT {emb_col} FROM {} LIMIT 1",
+        "SELECT {emb_col} FROM {} ORDER BY {id_col} LIMIT 1",
         config.chunks.table
     ))?;
     let mut rows = stmt.query([])?;
@@ -195,24 +193,16 @@ fn blob_to_f32(blob: &[u8]) -> Vec<f32> {
 /// Read all data from a ZeroClaw SQLite database.
 ///
 /// If `skip_deleted` is true, rows with `deleted=1` are excluded from chunks.
-/// If `embedding_dim` is `None`, auto-detect from the first row.
+/// If `embedding_dim` is `None`, auto-detect from the first row (0 when there
+/// are no rows). Embeddings are returned at their full stored length whatever
+/// the dimension: checking that every row matches it is the writer's job
+/// (`store_writer::write_store`), so a mismatch is an error, not silent
+/// truncation.
 pub fn read_sqlite(
     path: &str,
     skip_deleted: bool,
     embedding_dim: Option<usize>,
     config: &SchemaConfig,
-) -> Result<SqliteData, Box<dyn std::error::Error>> {
-    read_sqlite_filtered(path, skip_deleted, embedding_dim, config, 0)
-}
-
-/// Like [`read_sqlite`] but only reads chunks whose id is greater than
-/// `min_chunk_id` (0 = all). Used for incremental migration.
-pub fn read_sqlite_filtered(
-    path: &str,
-    skip_deleted: bool,
-    embedding_dim: Option<usize>,
-    config: &SchemaConfig,
-    min_chunk_id: i64,
 ) -> Result<SqliteData, Box<dyn std::error::Error>> {
     let conn = Connection::open(path)?;
 
@@ -221,7 +211,7 @@ pub fn read_sqlite_filtered(
         None => detect_embedding_dim(&conn, config)?.unwrap_or(0),
     };
 
-    let chunks = read_chunks(&conn, skip_deleted, dim, config, min_chunk_id)?;
+    let chunks = read_chunks(&conn, skip_deleted, config)?;
     let sessions = read_sessions(&conn, config)?;
     let entities = read_entities(&conn, config)?;
     let relations = read_relations(&conn, config)?;
@@ -232,42 +222,43 @@ pub fn read_sqlite_filtered(
         entities,
         relations,
         embedding_dim: dim,
-        source_path: path.to_owned(),
     })
 }
 
 fn read_chunks(
     conn: &Connection,
     skip_deleted: bool,
-    expected_dim: usize,
     config: &SchemaConfig,
-    min_chunk_id: i64,
 ) -> SqlResult<Vec<MemoryChunk>> {
     let id_col = config.chunks.columns.first().copied().unwrap_or("id");
     let deleted_col = config.chunks.columns.get(7).copied().unwrap_or("deleted");
-    let mut conds = Vec::new();
+    let mut where_clause = String::new();
     if skip_deleted {
-        conds.push(format!("{deleted_col} = 0"));
+        where_clause = format!(" WHERE {deleted_col} = 0");
     }
-    if min_chunk_id > 0 {
-        conds.push(format!("{id_col} > {min_chunk_id}"));
-    }
-    let where_clause = if conds.is_empty() {
-        String::new()
-    } else {
-        format!(" WHERE {}", conds.join(" AND "))
-    };
+    // In id order, so the store's records follow the source's order.
+    where_clause.push_str(&format!(" ORDER BY {id_col}"));
     let sql = config.chunks.select(&where_clause);
 
     let mut stmt = conn.prepare(&sql)?;
     let rows = stmt.query_map([], |row| {
         let blob: Vec<u8> = row.get(2)?;
-        let mut embedding = blob_to_f32(&blob);
-
-        // Validate/truncate to expected dimension
-        if expected_dim > 0 {
-            embedding.truncate(expected_dim);
+        if !blob.len().is_multiple_of(4) {
+            let id: i64 = row.get(0)?;
+            return Err(rusqlite::Error::FromSqlConversionFailure(
+                2,
+                rusqlite::types::Type::Blob,
+                format!(
+                    "chunk id {id}: embedding BLOB is {} bytes, not a whole number of \
+                     little-endian f32 values",
+                    blob.len()
+                )
+                .into(),
+            ));
         }
+        // Read at full length: rows of the wrong dimension are rejected by
+        // the writer, never truncated to fit.
+        let embedding = blob_to_f32(&blob);
 
         Ok(MemoryChunk {
             id: row.get(0)?,