quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

fix(migrate): write a real clawhdf5-agent store

Browse Source 
clawhdf5-migrate wrote a layout of its own (/chunks, /sessions,
/entities, /relations, root attributes, no /meta or schema_version) that
HDF5Memory::open rejects, so a "migrated" SQLite database could not be
used as agent memory — contrary to the README.

It now writes through the agent's own API (HDF5Memory::create/open,
save_batch, the session cache and the knowledge graph), so there is no
second copy of the schema:

- sessions and entities/relations carry over; deleted rows become
  deleted records (or are left out with --skip-deleted);
- embeddings follow the library default (float16), --f32 opts out and
  --float16 is a hidden no-op, as in clawhdf5-cli; the `half`-based
  conversion is gone;
- every source row is checked before the output is created: a wrong
  embedding length, an empty embedding, a dimension that differs from
  an existing store's, or a float16 value beyond +-65504 is an error
  naming the chunk id, and an existing store is left untouched;
- --incremental opens the existing store, adds only rows it does not
  hold (matched by content) and follows the source's deleted flags;
- a source with no memory rows needs --embedding-dim;
- validation reads the result back with HDF5Memory::open_read_only,
  compares every field (embeddings bit for bit, round_to_f16 of the
  source for float16) and checks a migrated record is found by search.

clawhdf5-agent gains HDF5Memory::sessions()/sessions_mut(),
HDF5Memory::delete_batch (one save, all-or-nothing, no auto-compact),
SessionCache::add_at, and re-exports SessionCache/SessionEntry.

The old layout's per-dataset SHA-256 provenance attributes have no place
in the agent schema and are gone. An adversarial review found two
blockers (silent truncation of long embeddings; an --incremental
dimension check that could never fire) and four majors (a failed run
wiping the existing store, dim-0 stores, deleted-flag drift); all are
fixed with regression tests. 42 migrate tests, incl. h5py opening a
migrated store.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-24 23:45:15 -05:00
co-authored by Claude Opus 5.5
parent 5c8323cb1e
commit a8fb758489
10 changed files with 1834 additions and 1131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-agent/src/lib.rs
+115 -1
View File
@@ -73,7 +73,7 @@ pub use ephemeral::{EphemeralEntry, EphemeralStats};
use knowledge::KnowledgeCache;
use memory_strategy::{Exchange, MemoryStrategy, StrategyOutput};
pub use search::SearchOptions;
use session::SessionCache;
pub use session::{SessionCache, SessionEntry};
// --- Error type ---
@@ -1028,6 +1028,18 @@ impl HDF5Memory {
&self.config
}
/// The sessions recorded in this store.
pub fn sessions(&self) -> &SessionCache {
&self.sessions
}
/// Mutable access to the sessions, e.g. to add many at once. Changes
/// reach the disk at the next checkpoint (any flushing call, such as
/// [`HDF5Memory::flush_wal`] or `save_batch`), not immediately.
pub fn sessions_mut(&mut self) -> &mut SessionCache {
&mut self.sessions
}
/// Get a reference to the knowledge cache.
pub fn knowledge(&self) -> &KnowledgeCache {
&self.knowledge
@@ -1401,6 +1413,35 @@ impl HDF5Memory {
}
impl HDF5Memory {
/// Delete many records with a single checkpoint, where
/// [`AgentMemory::delete`] checkpoints once per record.
///
/// All or nothing: if any id is out of range or already deleted (or
/// repeated), nothing is deleted and `MemoryError::NotFound` is returned.
/// Unlike `delete`, this never auto-compacts, so the records stay in the
/// store as tombstones (their indices unchanged) until [`AgentMemory::compact`]
/// is called — importers use it to carry over records that were already
/// deleted in the source.
pub fn delete_batch(&mut self, ids: &[usize]) -> Result<()> {
let mut seen = std::collections::HashSet::with_capacity(ids.len());
for &id in ids {
if self.cache.tombstones.get(id).copied() != Some(0) || !seen.insert(id) {
return Err(MemoryError::NotFound(format!(
"entry {id} not found or already deleted"
)));
}
}
if ids.is_empty() {
return Ok(());
}
for &id in ids {
self.cache.mark_deleted(id);
self.hnsw_on_delete(id);
self.bm25_on_delete(id);
}
self.flush()
}
pub fn tick_session(&mut self) -> Result<()> {
let d = self.config.decay_factor;
for w in self.cache.activation_weights.iter_mut() {
@@ -1599,6 +1640,79 @@ mod tests {
}
}
#[test]
fn delete_batch_tombstones_without_compacting() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("test.h5");
let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
mem.save_batch(
(0..4)
.map(|i| make_entry(&format!("record {i}"), &[i as f32, 1.0, 0.0, 0.0]))
.collect(),
)
.unwrap();
// 3 of 4 is far past compact_threshold (0.3): delete() would compact.
mem.delete_batch(&[0, 1, 3]).unwrap();
assert_eq!(mem.count(), 4);
assert_eq!(mem.count_active(), 1);
drop(mem);
let mut mem = HDF5Memory::open(&path).unwrap();
assert_eq!(mem.cache.tombstones, vec![1, 1, 0, 1]);
let hits = mem.hybrid_search(&[0.0, 1.0, 0.0, 0.0], "record", 0.5, 0.5, 10);
assert!(
hits.iter().all(|r| r.index == 2),
"tombstoned record returned"
);
}
#[test]
fn delete_batch_is_all_or_nothing() {
let dir = TempDir::new().unwrap();
let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
mem.save_batch(vec![
make_entry("a", &[1.0, 0.0, 0.0, 0.0]),
make_entry("b", &[0.0, 1.0, 0.0, 0.0]),
])
.unwrap();
for bad in [&[0, 5][..], &[1, 1][..]] {
assert!(matches!(
mem.delete_batch(bad),
Err(MemoryError::NotFound(_))
));
assert_eq!(mem.count_active(), 2, "{bad:?} deleted something");
}
mem.delete_batch(&[]).unwrap();
assert_eq!(mem.count_active(), 2);
}
#[test]
fn sessions_mut_add_at_keeps_timestamp_across_reopen() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("test.h5");
let mut mem = HDF5Memory::create(make_config(&dir)).unwrap();
mem.sessions_mut()
.add_at("s-old", 2, 7, "discord", "old summary", 1.7e15);
mem.flush_wal().unwrap();
drop(mem);
let mem = HDF5Memory::open_read_only(&path).unwrap();
let s = mem.sessions();
assert_eq!(s.len(), 1);
let e = &s.entries[0];
assert_eq!(
(
e.id.as_str(),
e.start_idx,
e.end_idx,
e.channel.as_str(),
e.ts
),
("s-old", 2, 7, "discord", 1.7e15)
);
assert_eq!(s.summaries[0], "old summary");
}
#[test]
fn create_new_file() {
let dir = TempDir::new().unwrap();
crates/clawhdf5-agent/src/session.rs
+16 -1
View File
@@ -33,7 +33,7 @@ impl SessionCache {
self.entries.is_empty()
}
/// Add a new session with its summary.
/// Add a new session with its summary, timestamped now.
pub fn add(
&mut self,
id: &str,
@@ -47,6 +47,21 @@ impl SessionCache {
.unwrap_or_default()
.as_secs_f64()
* 1_000_000.0; // microseconds
self.add_at(id, start_idx, end_idx, channel, summary, ts);
}
/// Add a session with an explicit timestamp (Unix **microseconds**, the
/// unit [`SessionEntry::ts`] uses) — for importers carrying sessions over
/// from another store, whose original time should be kept.
pub fn add_at(
&mut self,
id: &str,
start_idx: usize,
end_idx: usize,
channel: &str,
summary: &str,
ts: f64,
) {
self.entries.push(SessionEntry {
id: id.to_string(),
start_idx: start_idx as u64,
crates/clawhdf5-migrate/Cargo.toml
+1 -3
View File
@@ -3,7 +3,7 @@ name = "clawhdf5-migrate"
version = "2.7.0"
edition = "2024"
rust-version.workspace = true
description = "CLI to migrate SQLite agent memory databases to HDF5 format"
description = "CLI to migrate SQLite agent memory databases to clawhdf5-agent stores"
license = "MIT"
repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
readme = "README.md"
@@ -17,10 +17,8 @@ path = "src/main.rs"
[dependencies]
clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.7.0" }
clawhdf5-format = { path = "../clawhdf5-format", version = "2.7.0" }
clawhdf5 = { path = "../clawhdf5", version = "2.7.0" }
rusqlite = { version = "0.31", features = ["bundled"] }
clap = { version = "4", features = ["derive"] }
half = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
crates/clawhdf5-migrate/README.md
+16 -3
View File
@@ -3,9 +3,12 @@
[![crates.io](https://img.shields.io/crates/v/clawhdf5-migrate.svg)](https://crates.io/crates/clawhdf5-migrate)
[![docs.rs](https://img.shields.io/docsrs/clawhdf5-migrate)](https://docs.rs/clawhdf5-migrate)
CLI tool to migrate SQLite agent memory databases to HDF5 format.
CLI tool to migrate SQLite agent memory databases (the ZeroClaw layout) to a
[clawhdf5-agent](https://crates.io/crates/clawhdf5-agent) store.
Converts existing SQLite-based agent memory stores (embeddings, text chunks, metadata) into the HDF5 format used by [clawhdf5-agent](https://crates.io/crates/clawhdf5-agent).
The output is written through `clawhdf5-agent`'s own API, so it opens with
`HDF5Memory::open` and is searchable immediately: memory records, sessions and
the knowledge graph (entities and relations) are carried over.
## Installation
@@ -16,9 +19,19 @@ cargo install clawhdf5-migrate
## Usage
```bash
clawhdf5-migrate --input agent.db --output agent.h5
clawhdf5-migrate --sqlite agent.db --hdf5 agent.h5 --agent-id my-agent
```
Embeddings are stored as float16 (the library default for new stores); pass
`--f32` for full precision. Every embedding must have the same dimension
(the first row's, or `--embedding-dim`, which a source with no memory records
requires); rows are never truncated, and the whole source is checked before an
existing output store is replaced. `--incremental` adds only new rows to an
existing store of the same dimension and carries over changes to rows'
deleted flags, `--skip-deleted` leaves out tombstoned rows, and `--dry-run`
only counts.
See `clawhdf5-migrate --help` for every option.
## License
MIT
crates/clawhdf5-migrate/src/hdf5_reader.rs
-163
View File
@@ -1,163 +0,0 @@
//! Read a migration HDF5 file back into the in-memory data model.
//!
//! Used to verify migrated content (real validation) and to merge new rows into
//! an existing output (incremental migration). Mirrors the layout produced by
//! [`crate::hdf5_writer`].
use clawhdf5::reader::{File, Group};
use clawhdf5_format::type_builders::AttrValue;
use crate::sqlite_reader::{Entity, MemoryChunk, Relation, Session, SqliteData};
type BoxErr = Box<dyn std::error::Error>;
fn read_strings(group: &Group<'_>, name: &str) -> Result<Vec<String>, BoxErr> {
Ok(group.dataset(name)?.read_string()?)
}
fn read_i64s(group: &Group<'_>, name: &str) -> Result<Vec<i64>, BoxErr> {
Ok(group.dataset(name)?.read_i64()?)
}
fn read_f64s(group: &Group<'_>, name: &str) -> Result<Vec<f64>, BoxErr> {
Ok(group.dataset(name)?.read_f64()?)
}
/// Read the embeddings dataset as a flat `Vec<f32>` of `n * dim` values,
/// handling both f32 and (lossy) f16 storage.
fn read_embeddings_flat(group: &Group<'_>) -> Result<Vec<f32>, BoxErr> {
Ok(group.dataset("embeddings")?.read_f32()?)
}
/// Read a migration HDF5 file into a [`SqliteData`].
pub fn read_hdf5(path: &str) -> Result<SqliteData, BoxErr> {
let file = File::open(path)?;
let embedding_dim = match file.root().attrs()?.get("embedding_dim") {
Some(AttrValue::I64(d)) => *d as usize,
_ => 0,
};
let chunks = read_chunks(&file, embedding_dim)?;
let sessions = read_sessions(&file)?;
let entities = read_entities(&file)?;
let relations = read_relations(&file)?;
Ok(SqliteData {
chunks,
sessions,
entities,
relations,
embedding_dim,
// Not a SQLite read — the caller (incremental migration) carries
// forward the current run's actual `source_path` from the fresh
// SQLite read instead of using this placeholder.
source_path: String::new(),
})
}
fn read_chunks(file: &File, dim: usize) -> Result<Vec<MemoryChunk>, BoxErr> {
let g = file.group("chunks")?;
let count = group_count(&g)?;
if count == 0 {
return Ok(Vec::new());
}
let ids = read_i64s(&g, "id")?;
let texts = read_strings(&g, "text")?;
let channels = read_strings(&g, "source_channel")?;
let timestamps = read_f64s(&g, "timestamp")?;
let session_ids = read_strings(&g, "session_id")?;
let tags = read_strings(&g, "tags")?;
let deleted = g.dataset("deleted")?.read_i32()?;
let emb_flat = read_embeddings_flat(&g)?;
let dim = dim.max(1);
let mut chunks = Vec::with_capacity(ids.len());
for (i, &id) in ids.iter().enumerate() {
let embedding = emb_flat
.get(i * dim..(i + 1) * dim)
.map(|s| s.to_vec())
.unwrap_or_default();
chunks.push(MemoryChunk {
id,
chunk: texts.get(i).cloned().unwrap_or_default(),
embedding,
source_channel: channels.get(i).cloned().unwrap_or_default(),
timestamp: timestamps.get(i).copied().unwrap_or(0.0),
session_id: session_ids.get(i).cloned().unwrap_or_default(),
tags: tags.get(i).cloned().unwrap_or_default(),
deleted: deleted.get(i).copied().unwrap_or(0),
});
}
Ok(chunks)
}
fn read_sessions(file: &File) -> Result<Vec<Session>, BoxErr> {
let g = file.group("sessions")?;
if group_count(&g)? == 0 {
return Ok(Vec::new());
}
let ids = read_strings(&g, "id")?;
let starts = read_i64s(&g, "start_idx")?;
let ends = read_i64s(&g, "end_idx")?;
let channels = read_strings(&g, "channel")?;
let timestamps = read_f64s(&g, "timestamp")?;
let summaries = read_strings(&g, "summary")?;
Ok((0..ids.len())
.map(|i| Session {
id: ids[i].clone(),
start_idx: starts.get(i).copied().unwrap_or(0),
end_idx: ends.get(i).copied().unwrap_or(0),
channel: channels.get(i).cloned().unwrap_or_default(),
timestamp: timestamps.get(i).copied().unwrap_or(0.0),
summary: summaries.get(i).cloned().unwrap_or_default(),
})
.collect())
}
fn read_entities(file: &File) -> Result<Vec<Entity>, BoxErr> {
let g = file.group("entities")?;
if group_count(&g)? == 0 {
return Ok(Vec::new());
}
let ids = read_i64s(&g, "id")?;
let names = read_strings(&g, "name")?;
let types = read_strings(&g, "type")?;
let emb_idxs = read_i64s(&g, "embedding_idx")?;
Ok((0..ids.len())
.map(|i| Entity {
id: ids[i],
name: names.get(i).cloned().unwrap_or_default(),
entity_type: types.get(i).cloned().unwrap_or_default(),
embedding_idx: emb_idxs.get(i).copied().unwrap_or(-1),
})
.collect())
}
fn read_relations(file: &File) -> Result<Vec<Relation>, BoxErr> {
let g = file.group("relations")?;
if group_count(&g)? == 0 {
return Ok(Vec::new());
}
let srcs = read_i64s(&g, "src")?;
let tgts = read_i64s(&g, "tgt")?;
let rels = read_strings(&g, "relation")?;
let weights = read_f64s(&g, "weight")?;
let timestamps = read_f64s(&g, "timestamp")?;
Ok((0..srcs.len())
.map(|i| Relation {
src: srcs[i],
tgt: tgts.get(i).copied().unwrap_or(0),
relation: rels.get(i).cloned().unwrap_or_default(),
weight: weights.get(i).copied().unwrap_or(1.0),
timestamp: timestamps.get(i).copied().unwrap_or(0.0),
})
.collect())
}
fn group_count(group: &Group<'_>) -> Result<u64, BoxErr> {
match group.attrs()?.get("count") {
Some(AttrValue::I64(n)) => Ok(*n as u64),
_ => Ok(0),
}
}
crates/clawhdf5-migrate/src/hdf5_writer.rs
-366
View File
@@ -1,366 +0,0 @@
use clawhdf5::writer::FileBuilder;
use clawhdf5_format::datatype::{CharacterSet, Datatype, StringPadding};
use clawhdf5_format::type_builders::AttrValue;
use crate::sqlite_reader::SqliteData;
/// Options controlling HDF5 output.
pub struct WriteOptions {
pub agent_id: String,
pub embedder: String,
pub compression: bool,
pub compression_level: u32,
pub float16: bool,
}
/// Write SQLite data to an HDF5 file.
pub fn write_hdf5(
path: &str,
data: &SqliteData,
opts: &WriteOptions,
) -> Result<(), Box<dyn std::error::Error>> {
let mut builder = FileBuilder::new();
let timestamp = iso8601_now();
// Root-level metadata attributes
builder.set_attr("agent_id", AttrValue::String(opts.agent_id.clone()));
builder.set_attr("embedder", AttrValue::String(opts.embedder.clone()));
builder.set_attr("embedding_dim", AttrValue::I64(data.embedding_dim as i64));
builder.set_attr("source", AttrValue::String("sqlite-migration".into()));
builder.set_attr("version", AttrValue::I64(1));
// Lineage: which SQLite database this output was migrated from and when,
// plus the migrator tool version — so a chain of `--incremental` runs
// still has an audit trail instead of every run overwriting the same
// static attributes (see research/03_provenance.md, INT-03).
builder.set_attr("source_path", AttrValue::String(data.source_path.clone()));
builder.set_attr("migrated_at", AttrValue::String(timestamp.clone()));
builder.set_attr(
"migrator_version",
AttrValue::String(env!("CARGO_PKG_VERSION").to_owned()),
);
write_chunks_group(&mut builder, data, opts, &timestamp);
write_sessions_group(&mut builder, data);
write_entities_group(&mut builder, data);
write_relations_group(&mut builder, data);
builder.write(path)?;
Ok(())
}
/// Current UTC time formatted as an ISO-8601 / RFC-3339 timestamp
/// (`YYYY-MM-DDTHH:MM:SSZ`), with no external date/time dependency.
fn iso8601_now() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let days = (secs / 86_400) as i64;
let time_of_day = secs % 86_400;
let (h, m, s) = (
time_of_day / 3600,
(time_of_day % 3600) / 60,
time_of_day % 60,
);
let (y, mo, d) = civil_from_days(days);
format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z")
}
/// Days-since-epoch to (year, month, day), Howard Hinnant's `civil_from_days`
/// algorithm (proleptic Gregorian calendar, valid for the full `i64` range).
fn civil_from_days(z: i64) -> (i64, u32, u32) {
let z = z + 719_468;
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
let doe = (z - era * 146_097) as u64; // [0, 146096]
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399]
let y = yoe as i64 + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365]
let mp = (5 * doy + 2) / 153; // [0, 11]
let d = (doy - (153 * mp + 2) / 5 + 1) as u32; // [1, 31]
let m = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32; // [1, 12]
let y = if m <= 2 { y + 1 } else { y };
(y, m, d)
}
/// Build a fixed-length string Datatype from the max byte length of the items.
fn string_dtype(max_len: usize) -> Datatype {
Datatype::String {
size: max_len.max(1) as u32,
padding: StringPadding::NullPad,
charset: CharacterSet::Utf8,
}
}
/// Pack a slice of strings into null-padded raw bytes of uniform width.
fn pack_strings(strings: &[String]) -> (Vec<u8>, usize) {
let max_len = strings.iter().map(|s| s.len()).max().unwrap_or(0).max(1);
let mut buf = vec![0u8; strings.len() * max_len];
for (i, s) in strings.iter().enumerate() {
let start = i * max_len;
let bytes = s.as_bytes();
let copy_len = bytes.len().min(max_len);
buf[start..start + copy_len].copy_from_slice(&bytes[..copy_len]);
}
(buf, max_len)
}
fn apply_compression(ds: &mut clawhdf5_format::type_builders::DatasetBuilder, opts: &WriteOptions) {
if opts.compression {
ds.with_deflate(opts.compression_level);
ds.with_shuffle();
}
}
fn write_chunks_group(
builder: &mut FileBuilder,
data: &SqliteData,
opts: &WriteOptions,
timestamp: &str,
) {
let mut group = builder.create_group("chunks");
let n = data.chunks.len() as u64;
if n == 0 {
group.set_attr("count", AttrValue::I64(0));
builder.add_group(group.finish());
return;
}
group.set_attr("count", AttrValue::I64(n as i64));
// Source attribution attached directly to the content-bearing datasets
// (SHA-256 of the raw bytes + creator/timestamp/source), so the chunk
// text and embeddings each carry their own verifiable provenance
// (see clawhdf5_format::provenance / `Dataset::verify_provenance`).
let source_opt = if data.source_path.is_empty() {
None
} else {
Some(data.source_path.as_str())
};
// ids
let ids: Vec<i64> = data.chunks.iter().map(|c| c.id).collect();
group.create_dataset("id").with_i64_data(&ids);
// text
let texts: Vec<String> = data.chunks.iter().map(|c| c.chunk.clone()).collect();
let (text_raw, text_len) = pack_strings(&texts);
group
.create_dataset("text")
.with_compound_data(string_dtype(text_len), text_raw, n)
.with_provenance("clawhdf5-migrate", timestamp, source_opt);
// embeddings - flatten to [N, dim]
let dim = data.embedding_dim;
if opts.float16 {
let f16_data: Vec<u16> = data
.chunks
.iter()
.flat_map(|c| {
c.embedding
.iter()
.map(|&v| half::f16::from_f32(v).to_bits())
})
.collect();
let raw: Vec<u8> = f16_data.iter().flat_map(|v| v.to_le_bytes()).collect();
let f16_dtype = Datatype::FloatingPoint {
size: 2,
byte_order: clawhdf5_format::datatype::DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 16,
exponent_location: 10,
exponent_size: 5,
mantissa_location: 0,
mantissa_size: 10,
exponent_bias: 15,
};
let ds = group
.create_dataset("embeddings")
.with_compound_data(f16_dtype, raw, n)
.with_shape(&[n, dim as u64])
.with_provenance("clawhdf5-migrate", timestamp, source_opt);
apply_compression(ds, opts);
} else {
let flat: Vec<f32> = data
.chunks
.iter()
.flat_map(|c| c.embedding.iter().copied())
.collect();
let ds = group
.create_dataset("embeddings")
.with_f32_data(&flat)
.with_shape(&[n, dim as u64])
.with_provenance("clawhdf5-migrate", timestamp, source_opt);
apply_compression(ds, opts);
}
// source_channel
let channels: Vec<String> = data
.chunks
.iter()
.map(|c| c.source_channel.clone())
.collect();
let (ch_raw, ch_len) = pack_strings(&channels);
group
.create_dataset("source_channel")
.with_compound_data(string_dtype(ch_len), ch_raw, n);
// timestamp
let timestamps: Vec<f64> = data.chunks.iter().map(|c| c.timestamp).collect();
group.create_dataset("timestamp").with_f64_data(&timestamps);
// session_id
let sess_ids: Vec<String> = data.chunks.iter().map(|c| c.session_id.clone()).collect();
let (sid_raw, sid_len) = pack_strings(&sess_ids);
group
.create_dataset("session_id")
.with_compound_data(string_dtype(sid_len), sid_raw, n);
// tags
let tags: Vec<String> = data.chunks.iter().map(|c| c.tags.clone()).collect();
let (tag_raw, tag_len) = pack_strings(&tags);
group
.create_dataset("tags")
.with_compound_data(string_dtype(tag_len), tag_raw, n);
// deleted
let deleted: Vec<i32> = data.chunks.iter().map(|c| c.deleted).collect();
group.create_dataset("deleted").with_i32_data(&deleted);
builder.add_group(group.finish());
}
fn write_sessions_group(builder: &mut FileBuilder, data: &SqliteData) {
let mut group = builder.create_group("sessions");
let n = data.sessions.len() as u64;
group.set_attr("count", AttrValue::I64(n as i64));
if n == 0 {
builder.add_group(group.finish());
return;
}
let ids: Vec<String> = data.sessions.iter().map(|s| s.id.clone()).collect();
let (id_raw, id_len) = pack_strings(&ids);
group
.create_dataset("id")
.with_compound_data(string_dtype(id_len), id_raw, n);
let start_idxs: Vec<i64> = data.sessions.iter().map(|s| s.start_idx).collect();
group.create_dataset("start_idx").with_i64_data(&start_idxs);
let end_idxs: Vec<i64> = data.sessions.iter().map(|s| s.end_idx).collect();
group.create_dataset("end_idx").with_i64_data(&end_idxs);
let channels: Vec<String> = data.sessions.iter().map(|s| s.channel.clone()).collect();
let (ch_raw, ch_len) = pack_strings(&channels);
group
.create_dataset("channel")
.with_compound_data(string_dtype(ch_len), ch_raw, n);
let timestamps: Vec<f64> = data.sessions.iter().map(|s| s.timestamp).collect();
group.create_dataset("timestamp").with_f64_data(&timestamps);
let summaries: Vec<String> = data.sessions.iter().map(|s| s.summary.clone()).collect();
let (sum_raw, sum_len) = pack_strings(&summaries);
group
.create_dataset("summary")
.with_compound_data(string_dtype(sum_len), sum_raw, n);
builder.add_group(group.finish());
}
fn write_entities_group(builder: &mut FileBuilder, data: &SqliteData) {
let mut group = builder.create_group("entities");
let n = data.entities.len() as u64;
group.set_attr("count", AttrValue::I64(n as i64));
if n == 0 {
builder.add_group(group.finish());
return;
}
let ids: Vec<i64> = data.entities.iter().map(|e| e.id).collect();
group.create_dataset("id").with_i64_data(&ids);
let names: Vec<String> = data.entities.iter().map(|e| e.name.clone()).collect();
let (name_raw, name_len) = pack_strings(&names);
group
.create_dataset("name")
.with_compound_data(string_dtype(name_len), name_raw, n);
let types: Vec<String> = data
.entities
.iter()
.map(|e| e.entity_type.clone())
.collect();
let (type_raw, type_len) = pack_strings(&types);
group
.create_dataset("type")
.with_compound_data(string_dtype(type_len), type_raw, n);
let emb_idxs: Vec<i64> = data.entities.iter().map(|e| e.embedding_idx).collect();
group
.create_dataset("embedding_idx")
.with_i64_data(&emb_idxs);
builder.add_group(group.finish());
}
fn write_relations_group(builder: &mut FileBuilder, data: &SqliteData) {
let mut group = builder.create_group("relations");
let n = data.relations.len() as u64;
group.set_attr("count", AttrValue::I64(n as i64));
if n == 0 {
builder.add_group(group.finish());
return;
}
let srcs: Vec<i64> = data.relations.iter().map(|r| r.src).collect();
group.create_dataset("src").with_i64_data(&srcs);
let tgts: Vec<i64> = data.relations.iter().map(|r| r.tgt).collect();
group.create_dataset("tgt").with_i64_data(&tgts);
let rels: Vec<String> = data.relations.iter().map(|r| r.relation.clone()).collect();
let (rel_raw, rel_len) = pack_strings(&rels);
group
.create_dataset("relation")
.with_compound_data(string_dtype(rel_len), rel_raw, n);
let weights: Vec<f64> = data.relations.iter().map(|r| r.weight).collect();
group.create_dataset("weight").with_f64_data(&weights);
let timestamps: Vec<f64> = data.relations.iter().map(|r| r.timestamp).collect();
group.create_dataset("timestamp").with_f64_data(&timestamps);
builder.add_group(group.finish());
}
#[cfg(test)]
mod time_tests {
use super::civil_from_days;
#[test]
fn epoch_day_zero_is_1970_01_01() {
assert_eq!(civil_from_days(0), (1970, 1, 1));
}
#[test]
fn known_dates_roundtrip() {
// 2026-08-16 is 20,681 days after 1970-01-01.
assert_eq!(civil_from_days(20_681), (2026, 8, 16));
// 2000-02-29 (leap day itself) and 2000-03-01 (the day after).
assert_eq!(civil_from_days(11_016), (2000, 2, 29));
assert_eq!(civil_from_days(11_017), (2000, 3, 1));
}
#[test]
fn iso8601_now_has_expected_shape() {
let ts = super::iso8601_now();
assert_eq!(ts.len(), "2026-08-16T00:00:00Z".len());
assert!(ts.starts_with("20")); // sanity: 21st-century year
assert!(ts.ends_with('Z'));
}
}
crates/clawhdf5-migrate/src/main.rs
+1041 -421
View File
File diff suppressed because it is too large Load Diff
crates/clawhdf5-migrate/src/sqlite_reader.rs
+30 -39
View File
@@ -50,12 +50,8 @@ pub struct SqliteData {
pub sessions: Vec<Session>,
pub entities: Vec<Entity>,
pub relations: Vec<Relation>,
/// `--embedding-dim`, or the first row's; 0 when neither exists.
pub embedding_dim: usize,
/// Filesystem path of the SQLite database this data was read from, for
/// provenance attribution on the HDF5 output. Empty when the data did
/// not come directly from a SQLite read (e.g. re-read of a prior HDF5
/// migration output for an incremental merge).
pub source_path: String,
}
/// A table name plus the ordered column names the reader maps by position.
@@ -167,11 +163,13 @@ pub fn read_counts(
})
}
/// Auto-detect embedding dimension from the first chunk's BLOB size.
/// Auto-detect embedding dimension from the BLOB size of the first chunk (in
/// id order, deleted or not).
fn detect_embedding_dim(conn: &Connection, config: &SchemaConfig) -> SqlResult<Option<usize>> {
let emb_col = config.chunks.columns.get(2).copied().unwrap_or("embedding");
let id_col = config.chunks.columns.first().copied().unwrap_or("id");
let mut stmt = conn.prepare(&format!(
"SELECT {emb_col} FROM {} LIMIT 1",
"SELECT {emb_col} FROM {} ORDER BY {id_col} LIMIT 1",
config.chunks.table
))?;
let mut rows = stmt.query([])?;
@@ -195,24 +193,16 @@ fn blob_to_f32(blob: &[u8]) -> Vec<f32> {
/// Read all data from a ZeroClaw SQLite database.
///
/// If `skip_deleted` is true, rows with `deleted=1` are excluded from chunks.
/// If `embedding_dim` is `None`, auto-detect from the first row.
/// If `embedding_dim` is `None`, auto-detect from the first row (0 when there
/// are no rows). Embeddings are returned at their full stored length whatever
/// the dimension: checking that every row matches it is the writer's job
/// (`store_writer::write_store`), so a mismatch is an error, not silent
/// truncation.
pub fn read_sqlite(
path: &str,
skip_deleted: bool,
embedding_dim: Option<usize>,
config: &SchemaConfig,
) -> Result<SqliteData, Box<dyn std::error::Error>> {
read_sqlite_filtered(path, skip_deleted, embedding_dim, config, 0)
}
/// Like [`read_sqlite`] but only reads chunks whose id is greater than
/// `min_chunk_id` (0 = all). Used for incremental migration.
pub fn read_sqlite_filtered(
path: &str,
skip_deleted: bool,
embedding_dim: Option<usize>,
config: &SchemaConfig,
min_chunk_id: i64,
) -> Result<SqliteData, Box<dyn std::error::Error>> {
let conn = Connection::open(path)?;
@@ -221,7 +211,7 @@ pub fn read_sqlite_filtered(
None => detect_embedding_dim(&conn, config)?.unwrap_or(0),
};
let chunks = read_chunks(&conn, skip_deleted, dim, config, min_chunk_id)?;
let chunks = read_chunks(&conn, skip_deleted, config)?;
let sessions = read_sessions(&conn, config)?;
let entities = read_entities(&conn, config)?;
let relations = read_relations(&conn, config)?;
@@ -232,42 +222,43 @@ pub fn read_sqlite_filtered(
entities,
relations,
embedding_dim: dim,
source_path: path.to_owned(),
})
}
fn read_chunks(
conn: &Connection,
skip_deleted: bool,
expected_dim: usize,
config: &SchemaConfig,
min_chunk_id: i64,
) -> SqlResult<Vec<MemoryChunk>> {
let id_col = config.chunks.columns.first().copied().unwrap_or("id");
let deleted_col = config.chunks.columns.get(7).copied().unwrap_or("deleted");
let mut conds = Vec::new();
let mut where_clause = String::new();
if skip_deleted {
conds.push(format!("{deleted_col} = 0"));
where_clause = format!(" WHERE {deleted_col} = 0");
}
if min_chunk_id > 0 {
conds.push(format!("{id_col} > {min_chunk_id}"));
}
let where_clause = if conds.is_empty() {
String::new()
} else {
format!(" WHERE {}", conds.join(" AND "))
};
// In id order, so the store's records follow the source's order.
where_clause.push_str(&format!(" ORDER BY {id_col}"));
let sql = config.chunks.select(&where_clause);
let mut stmt = conn.prepare(&sql)?;
let rows = stmt.query_map([], |row| {
let blob: Vec<u8> = row.get(2)?;
let mut embedding = blob_to_f32(&blob);
// Validate/truncate to expected dimension
if expected_dim > 0 {
embedding.truncate(expected_dim);
if !blob.len().is_multiple_of(4) {
let id: i64 = row.get(0)?;
return Err(rusqlite::Error::FromSqlConversionFailure(
2,
rusqlite::types::Type::Blob,
format!(
"chunk id {id}: embedding BLOB is {} bytes, not a whole number of \
little-endian f32 values",
blob.len()
)
.into(),
));
}
// Read at full length: rows of the wrong dimension are rejected by
// the writer, never truncated to fit.
let embedding = blob_to_f32(&blob);
Ok(MemoryChunk {
id: row.get(0)?,
crates/clawhdf5-migrate/src/store_writer.rs
+407
View File
@@ -0,0 +1,407 @@
//! Write migrated SQLite data into a clawhdf5-agent store.
//!
//! Everything goes through `clawhdf5-agent`'s own API — `HDF5Memory::create`
//! (or `open` for `--incremental`), `save_batch`, `delete_batch`, the session
//! cache and the knowledge graph — so the result is an ordinary agent store
//! that `HDF5Memory::open` accepts, not a second hand-built copy of its schema.
use std::collections::{HashMap, HashSet};
use std::path::Path;
use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
use clawhdf5_format::float16::round_to_f16;
use crate::sqlite_reader::{MemoryChunk, SqliteData};
type BoxErr = Box<dyn std::error::Error>;
/// SQLite timestamps are Unix seconds; the agent's session and relation
/// timestamps are Unix microseconds (memory records stay in seconds).
pub const US_PER_SEC: f64 = 1_000_000.0;
/// Options controlling the output store.
#[derive(Debug, Clone)]
pub struct WriteOptions {
pub agent_id: String,
pub embedder: String,
pub compression: bool,
pub compression_level: u32,
/// Store full-precision `f32` embeddings instead of the library default
/// (half precision). Only applies to a newly created store: an existing
/// store keeps the precision it was created with.
pub f32: bool,
/// Add to the store at the output path if there is one, instead of
/// replacing it.
pub incremental: bool,
/// Leave out deleted source rows that are not in the store. (A deleted
/// row that matches an active store record still tombstones it, so pass
/// deleted rows in `data` for an incremental run.)
pub skip_deleted: bool,
}
/// What the migration wrote, and where each source row went, so validation
/// can compare the store with the source row by row.
#[derive(Debug, Default)]
pub struct Migration {
/// Whether the output store existed and was added to (`--incremental`).
pub appended_to_existing: bool,
/// The store's embedding precision.
pub float16: bool,
pub embedding_dim: usize,
/// Records in the store after the migration (including tombstones).
pub store_count: usize,
/// `(store index, source chunk index)` of every record written.
pub records: Vec<(usize, usize)>,
/// Source chunks already in the store (incremental), not written again.
pub chunks_present: usize,
/// `(store index, source chunk index)` of records that were active in
/// the store but whose source row is now deleted (incremental): they were
/// tombstoned by this run.
pub deleted_in_store: Vec<(usize, usize)>,
/// Source rows that were deleted in the store but are active in the
/// source (incremental): the agent has no un-delete, so each was written
/// again as a new record (counted in `records` too).
pub restored: usize,
/// Deleted source rows left out because of `skip_deleted`.
pub deleted_skipped: usize,
/// `(store session index, source session index)` of each session written.
pub sessions: Vec<(usize, usize)>,
pub sessions_present: usize,
/// `(store entity id, source entity index)` of each entity written.
pub entities: Vec<(u64, usize)>,
pub entities_present: usize,
/// SQLite entity id -> store entity id, for every source entity.
pub entity_ids: HashMap<i64, u64>,
/// `(store relation index, source relation index)` of each relation written.
pub relations: Vec<(usize, usize)>,
pub relations_present: usize,
/// Source relations naming an entity id that is not in the entities
/// table; the knowledge graph cannot hold them, so they are skipped.
pub dangling_relations: Vec<usize>,
/// Messages of the write-anomaly alerts the agent raised while importing
/// (informational; they never block a save — a bulk import typically
/// trips the write-rate check).
pub anomaly_alerts: Vec<String>,
}
/// Identity of a memory record for incremental de-duplication: every field
/// the agent stores except the embedding (whose stored form depends on the
/// store's precision).
type RecordKey = (String, String, String, String, u64);
fn record_key(
chunk: &str,
source_channel: &str,
session_id: &str,
tags: &str,
ts: f64,
) -> RecordKey {
(
chunk.to_owned(),
source_channel.to_owned(),
session_id.to_owned(),
tags.to_owned(),
ts.to_bits(),
)
}
/// Reject rows the agent would otherwise store differently from the source,
/// or not at all: an embedding of a different length from the store's
/// dimension (the agent pads/truncates silently), an empty embedding, or, in
/// a float16 store, a value beyond the half-precision range.
///
/// Every source row is checked, including ones that end up not being written
/// (already in the store, or deleted and skipped): the source must be
/// consistent as a whole, and the check runs before the store is touched.
fn check_chunks(chunks: &[MemoryChunk], dim: usize, float16: bool) -> Result<(), BoxErr> {
for c in chunks {
if c.embedding.is_empty() {
return Err(format!(
"chunk id {}: the embedding is empty; an agent store needs an embedding \
for every record",
c.id
)
.into());
}
if c.embedding.len() != dim {
return Err(format!(
"chunk id {}: embedding has {} values, expected {dim}; every row must have \
the store's dimension (detected from the first row unless --embedding-dim \
is given), and rows are never truncated or padded to fit",
c.id,
c.embedding.len()
)
.into());
}
if float16
&& let Some((k, v)) = c
.embedding
.iter()
.enumerate()
.find(|&(_, &v)| v.is_finite() && round_to_f16(v).is_infinite())
{
return Err(format!(
"chunk id {}: embedding[{k}] = {v} is outside the half-precision range \
(±65504) of a float16 store; migrate with --f32",
c.id
)
.into());
}
}
Ok(())
}
/// Migrate `data` into the agent store at `path`.
///
/// Without `opts.incremental` (or when nothing exists at `path`) a new store
/// is created, replacing any file there — but only once every source row has
/// passed [`check_chunks`], so a source that cannot be migrated leaves an
/// existing store untouched. With it, the existing store is opened and only
/// source rows it does not already hold are added: memory records are
/// matched on their content, sessions on their id, entities on name and
/// type, relations on (source, target, relation). A matched record then
/// takes the source row's deleted flag: see [`Migration::deleted_in_store`]
/// and [`Migration::restored`].
pub fn write_store(
path: &Path,
data: &SqliteData,
opts: &WriteOptions,
) -> Result<Migration, BoxErr> {
let existing = opts.incremental && path.exists();
let mut mem = if existing {
// `open` does not modify the store beyond what the agent itself does
// on open; the checks below run before anything is written.
let mem = HDF5Memory::open(path)?;
let dim = mem.config().embedding_dim;
// `data.embedding_dim` is 0 only for a source with no records and no
// --embedding-dim, which has no dimension to disagree with.
if data.embedding_dim != 0 && dim != data.embedding_dim {
let hint = if dim == 0 {
" (a store created from a source with no memory records; re-create it \
with --embedding-dim)"
} else {
""
};
return Err(format!(
"the store at {} has embedding_dim {dim}{hint}, the source {}; \
embeddings of a different dimension cannot be added to it",
path.display(),
data.embedding_dim
)
.into());
}
check_chunks(&data.chunks, dim, mem.config().float16)?;
mem
} else {
// (With records, a dimension of 0 means an empty first embedding,
// which `check_chunks` reports more precisely.)
if data.embedding_dim == 0 && data.chunks.is_empty() {
return Err(
"the source has no memory records to detect the embedding dimension \
from; pass --embedding-dim (the dimension of the agent's embedder), \
or the store could never hold a record"
.into(),
);
}
let mut config = MemoryConfig::new(path.to_path_buf(), &opts.agent_id, data.embedding_dim);
config.embedder = opts.embedder.clone();
config.compression = opts.compression;
config.compression_level = opts.compression_level;
// Only ever switch the library default off (as `clawhdf5-cli create`).
if opts.f32 {
config.float16 = false;
}
// Before `create`, which replaces whatever is at `path`.
check_chunks(&data.chunks, config.embedding_dim, config.float16)?;
HDF5Memory::create(config)?
};
let float16 = mem.config().float16;
let dim = mem.config().embedding_dim;
let mut m = Migration {
appended_to_existing: existing,
float16,
embedding_dim: dim,
..Migration::default()
};
// ---- Memory records --------------------------------------------------
// Store indices of every record the store already holds, by content, so
// a source row that appears twice is only treated as present as often
// as the store has it.
let mut present: HashMap<RecordKey, Vec<usize>> = HashMap::new();
if existing {
let c = &mem.cache;
for i in 0..c.len() {
let key = record_key(
&c.chunks[i],
&c.source_channels[i],
&c.session_ids[i],
&c.tags[i],
c.timestamps[i],
);
present.entry(key).or_default().push(i);
}
}
let key_of = |c: &MemoryChunk| {
record_key(
&c.chunk,
&c.source_channel,
&c.session_id,
&c.tags,
c.timestamp,
)
};
let tombstoned = |idx: usize| mem.cache.tombstones[idx] != 0;
// Pass 1: a store record in the same deleted state as the source row.
let mut unmatched: Vec<usize> = Vec::new();
for (i, c) in data.chunks.iter().enumerate() {
let src_deleted = c.deleted != 0;
let hit = present.get_mut(&key_of(c)).and_then(|idxs| {
let at = idxs.iter().position(|&x| tombstoned(x) == src_deleted)?;
Some(idxs.remove(at))
});
match hit {
Some(_) => m.chunks_present += 1,
None => unmatched.push(i),
}
}
// Pass 2: a store record whose deleted state differs — the source row
// was deleted or restored since the last migration. The source wins.
let mut new_chunks: Vec<usize> = Vec::with_capacity(unmatched.len());
let mut delete_in_store: Vec<usize> = Vec::new();
for i in unmatched {
let c = &data.chunks[i];
let hit = present
.get_mut(&key_of(c))
.and_then(|idxs| (!idxs.is_empty()).then(|| idxs.remove(0)));
match hit {
// Active in the store, deleted in the source: tombstone it.
Some(idx) if c.deleted != 0 => {
m.deleted_in_store.push((idx, i));
delete_in_store.push(idx);
}
// Deleted in the store, active in the source. The agent has no
// un-delete, so the row is written again as a new active record
// (the tombstone stays until the store is compacted).
Some(_) => {
m.restored += 1;
new_chunks.push(i);
}
None if c.deleted != 0 && opts.skip_deleted => m.deleted_skipped += 1,
None => new_chunks.push(i),
}
}
new_chunks.sort_unstable();
let to_write: Vec<&MemoryChunk> = new_chunks.iter().map(|&i| &data.chunks[i]).collect();
// ---- Sessions (in the cache; persisted by the save_batch checkpoint) ---
let known_sessions: HashSet<String> = mem
.sessions()
.entries
.iter()
.map(|e| e.id.clone())
.collect();
for (i, s) in data.sessions.iter().enumerate() {
if known_sessions.contains(&s.id) {
m.sessions_present += 1;
continue;
}
let sessions = mem.sessions_mut();
let at = sessions.len();
sessions.add_at(
&s.id,
s.start_idx.max(0) as usize,
s.end_idx.max(0) as usize,
&s.channel,
&s.summary,
s.timestamp * US_PER_SEC,
);
m.sessions.push((at, i));
}
// ---- Knowledge graph -------------------------------------------------
let kg = mem.knowledge_mut();
// Matched only against what the store held before this run: the source
// itself is copied as it is, duplicates included.
let by_name_type: HashMap<(String, String), u64> = kg
.entities
.iter()
.map(|e| ((e.name.clone(), e.entity_type.clone()), e.id))
.collect();
for (i, e) in data.entities.iter().enumerate() {
let key = (e.name.clone(), e.entity_type.clone());
let id = match by_name_type.get(&key) {
Some(&id) => {
m.entities_present += 1;
id
}
None => {
let id = kg.add_entity(&e.name, &e.entity_type, e.embedding_idx);
m.entities.push((id, i));
id
}
};
m.entity_ids.insert(e.id, id);
}
let known_relations: HashSet<(u64, u64, String)> = kg
.relations
.iter()
.map(|r| (r.src, r.tgt, r.relation.clone()))
.collect();
for (i, r) in data.relations.iter().enumerate() {
let (Some(&src), Some(&tgt)) = (m.entity_ids.get(&r.src), m.entity_ids.get(&r.tgt)) else {
m.dangling_relations.push(i);
continue;
};
if known_relations.contains(&(src, tgt, r.relation.clone())) {
m.relations_present += 1;
continue;
}
let at = kg.relations.len();
kg.add_relation(src, tgt, &r.relation, r.weight as f32);
kg.relations[at].ts = r.timestamp * US_PER_SEC;
m.relations.push((at, i));
}
// ---- Write: one checkpoint for records, sessions and graph -----------
let entries: Vec<MemoryEntry> = to_write
.iter()
.map(|c| MemoryEntry {
chunk: c.chunk.clone(),
embedding: c.embedding.clone(),
source_channel: c.source_channel.clone(),
timestamp: c.timestamp,
session_id: c.session_id.clone(),
tags: c.tags.clone(),
})
.collect();
let indices = mem.save_batch(entries)?;
m.records = indices
.iter()
.copied()
.zip(new_chunks.iter().copied())
.collect();
// Rows deleted in the source stay deleted: tombstones, as the agent's own
// `delete` leaves them (not compacted away).
// Records matched in the store whose source row has since been deleted
// are tombstoned too.
let tombstones: Vec<usize> = m
.records
.iter()
.filter(|&&(_, src)| data.chunks[src].deleted != 0)
.map(|&(idx, _)| idx)
.chain(delete_in_store)
.collect();
mem.delete_batch(&tombstones)?;
m.anomaly_alerts = mem
.take_anomaly_alerts()
.into_iter()
.map(|a| a.message)
.collect();
m.store_count = mem.count();
drop(mem); // release the single-writer lock before anyone re-opens it
Ok(m)
}
crates/clawhdf5-migrate/src/validate.rs
+208 -134
View File
@@ -1,192 +1,266 @@
use clawhdf5::reader::File as Hdf5File;
use clawhdf5_format::provenance::VerifyResult;
//! Validate a migration by reading the store back the way an agent would:
//! through `HDF5Memory::open_read_only`, comparing what it loads with the
//! SQLite source, and running a search for a migrated record.
use std::path::Path;
use clawhdf5_agent::{AgentMemory, HDF5Memory, SearchOptions};
use clawhdf5_format::float16::round_to_f16;
use crate::hdf5_reader::read_hdf5;
use crate::sqlite_reader::SqliteData;
use crate::store_writer::{Migration, US_PER_SEC};
type BoxErr = Box<dyn std::error::Error>;
/// Summary of a migration validation.
#[derive(Debug)]
pub struct ValidationSummary {
pub chunks: u64,
pub sessions: u64,
pub entities: u64,
pub relations: u64,
pub embedding_dim: u64,
/// Number of rows whose full content was compared against the source.
/// Records in the store (including tombstones).
pub count: usize,
/// Records in the store that are not deleted.
pub active: usize,
pub sessions: usize,
pub entities: usize,
pub relations: usize,
pub embedding_dim: usize,
pub float16: bool,
/// Rows whose full content was compared against the source.
pub rows_checked: u64,
/// Whether the `chunks/text` and `chunks/embeddings` SHINES provenance
/// hashes (written via [`crate::hdf5_writer`]) were both present and
/// matched their recomputed SHA-256 on read-back. `false` when either
/// dataset has no provenance metadata (e.g. an older output file) or
/// there are zero chunks to check.
pub provenance_verified: bool,
/// Whether a search for a migrated record found it (`false` when there
/// was no active migrated record with an embedding to search for).
pub search_checked: bool,
}
/// Validate a migrated HDF5 file against the source data.
/// Validate the store at `path` against the source rows `migration` wrote.
///
/// Reads the written file back and compares actual content — chunk text,
/// embeddings, and every session/entity/relation field — to the source, not
/// just the row counts. When `full` is false a representative sample of chunk
/// rows is content-checked (counts and all other groups are always checked in
/// full); when `full` is true every chunk row is compared too. `float16` widens
/// the embedding tolerance to allow for half-precision quantization.
pub fn validate_hdf5(
path: &str,
/// Counts and the session / entity / relation rows are always checked in
/// full. Memory records are content-checked on a representative sample, or
/// all of them with `full`. Embeddings must match exactly: the source values
/// themselves in an `f32` store, their [`round_to_f16`] in a `float16` one.
pub fn validate_store(
path: &Path,
source: &SqliteData,
migration: &Migration,
full: bool,
float16: bool,
) -> Result<ValidationSummary, BoxErr> {
let got = read_hdf5(path)?;
let provenance_verified = verify_chunk_provenance(path)?;
let mut mem = HDF5Memory::open_read_only(path)?;
let float16 = mem.config().float16;
let dim = mem.config().embedding_dim;
// ---- Counts ----
check_count("chunk", got.chunks.len(), source.chunks.len())?;
check_count("session", got.sessions.len(), source.sessions.len())?;
check_count("entity", got.entities.len(), source.entities.len())?;
check_count("relation", got.relations.len(), source.relations.len())?;
if got.embedding_dim != source.embedding_dim {
check_count("record", mem.count(), migration.store_count)?;
if float16 != migration.float16 {
return Err(format!(
"embedding_dim mismatch: HDF5 has {}, source has {}",
got.embedding_dim, source.embedding_dim
"float16 mismatch: store {float16}, expected {}",
migration.float16
)
.into());
}
if dim != migration.embedding_dim {
return Err(format!(
"embedding_dim mismatch: store has {dim}, expected {}",
migration.embedding_dim
)
.into());
}
if !migration.appended_to_existing {
check_count("record", mem.count(), migration.records.len())?;
check_count("session", mem.sessions().len(), migration.sessions.len())?;
check_count(
"entity",
mem.knowledge().entities.len(),
migration.entities.len(),
)?;
check_count(
"relation",
mem.knowledge().relations.len(),
migration.relations.len(),
)?;
}
// ---- Chunk content (sampled or full) ----
let (emb_abs, emb_rel) = if float16 { (1e-2, 1e-2) } else { (1e-4, 0.0) };
// ---- Memory records (sampled or full) ----
let mut rows_checked = 0u64;
for i in sample_indices(source.chunks.len(), full) {
let (s, g) = (&source.chunks[i], &got.chunks[i]);
if s.id != g.id {
return Err(field_err("chunk", i, "id", s.id, g.id));
let expected_value = |v: f32| if float16 { round_to_f16(v) } else { v };
for k in sample_indices(migration.records.len(), full) {
let (idx, src) = migration.records[k];
let s = &source.chunks[src];
let c = &mem.cache;
if idx >= c.len() {
return Err(
format!("record {idx} (chunk id {}) is missing from the store", s.id).into(),
);
}
if s.chunk != g.chunk {
let id = s.id;
if c.chunks[idx] != s.chunk {
return Err(format!(
"chunk[{i}].text mismatch: source {:?}, HDF5 {:?}",
"record {idx} (chunk id {id}) text mismatch: source {:?}, store {:?}",
truncate(&s.chunk),
truncate(&g.chunk)
truncate(&c.chunks[idx])
)
.into());
}
if s.session_id != g.session_id || s.source_channel != g.source_channel || s.tags != g.tags
if c.source_channels[idx] != s.source_channel
|| c.session_ids[idx] != s.session_id
|| c.tags[idx] != s.tags
{
return Err(format!("chunk[{i}] string field mismatch").into());
return Err(format!("record {idx} (chunk id {id}) string field mismatch").into());
}
if s.deleted != g.deleted {
return Err(field_err("chunk", i, "deleted", s.deleted, g.deleted));
}
if s.embedding.len() != g.embedding.len() {
if c.timestamps[idx].to_bits() != s.timestamp.to_bits() {
return Err(format!(
"chunk[{i}] embedding length mismatch: {} vs {}",
s.embedding.len(),
g.embedding.len()
"record {idx} (chunk id {id}) timestamp mismatch: source {}, store {}",
s.timestamp, c.timestamps[idx]
)
.into());
}
for (k, (&a, &b)) in s.embedding.iter().zip(g.embedding.iter()).enumerate() {
if (a - b).abs() > emb_abs + emb_rel * a.abs() {
return Err(
format!("chunk[{i}].embedding[{k}] mismatch: source {a}, HDF5 {b}").into(),
);
let deleted = c.tombstones[idx] != 0;
if deleted != (s.deleted != 0) {
return Err(format!(
"record {idx} (chunk id {id}) deleted mismatch: source {}, store {deleted}",
s.deleted != 0
)
.into());
}
let got = c.embeddings.get(idx).unwrap_or(&[]);
if got.len() != s.embedding.len() {
return Err(format!(
"record {idx} (chunk id {id}) embedding length mismatch: source {}, store {}",
s.embedding.len(),
got.len()
)
.into());
}
for (j, (&a, &b)) in s.embedding.iter().zip(got).enumerate() {
let want = expected_value(a);
if want.to_bits() != b.to_bits() && !(want.is_nan() && b.is_nan()) {
return Err(format!(
"record {idx} (chunk id {id}) embedding[{j}] mismatch: source {a}, \
expected {want}, store {b}"
)
.into());
}
}
rows_checked += 1;
}
// ---- Other groups (always full — they are small) ----
for (i, (s, g)) in source.sessions.iter().zip(got.sessions.iter()).enumerate() {
if s.id != g.id
|| s.start_idx != g.start_idx
|| s.end_idx != g.end_idx
|| s.channel != g.channel
|| s.summary != g.summary
{
return Err(format!("session[{i}] mismatch").into());
// ---- Records tombstoned because their source row was deleted ----
for &(idx, src) in &migration.deleted_in_store {
let s = &source.chunks[src];
let c = &mem.cache;
if idx >= c.len() || c.chunks[idx] != s.chunk || c.timestamps[idx] != s.timestamp {
return Err(format!("record {idx} (chunk id {}) mismatch or missing", s.id).into());
}
rows_checked += 1;
}
for (i, (s, g)) in source.entities.iter().zip(got.entities.iter()).enumerate() {
if s.id != g.id
|| s.name != g.name
|| s.entity_type != g.entity_type
|| s.embedding_idx != g.embedding_idx
{
return Err(format!("entity[{i}] mismatch").into());
}
rows_checked += 1;
}
for (i, (s, g)) in source
.relations
.iter()
.zip(got.relations.iter())
.enumerate()
{
if s.src != g.src || s.tgt != g.tgt || s.relation != g.relation {
return Err(format!("relation[{i}] mismatch").into());
if c.tombstones[idx] == 0 {
return Err(format!(
"record {idx} (chunk id {}) is deleted in the source but active in the store",
s.id
)
.into());
}
rows_checked += 1;
}
// ---- Sessions ----
let sessions = mem.sessions();
for &(at, src) in &migration.sessions {
let s = &source.sessions[src];
let (Some(e), Some(summary)) = (sessions.entries.get(at), sessions.summaries.get(at))
else {
return Err(format!("session {:?} is missing from the store", s.id).into());
};
if e.id != s.id
|| e.start_idx != s.start_idx.max(0) as u64
|| e.end_idx != s.end_idx.max(0) as u64
|| e.channel != s.channel
|| *summary != s.summary
|| e.ts != s.timestamp * US_PER_SEC
{
return Err(format!("session {:?} mismatch", s.id).into());
}
rows_checked += 1;
}
// ---- Knowledge graph ----
let kg = mem.knowledge();
for &(id, src) in &migration.entities {
let s = &source.entities[src];
let Some(e) = kg.get_entity(id) else {
return Err(format!(
"entity {:?} (id {}) is missing from the store",
s.name, s.id
)
.into());
};
if e.name != s.name || e.entity_type != s.entity_type || e.embedding_idx != s.embedding_idx
{
return Err(format!("entity {:?} (id {}) mismatch", s.name, s.id).into());
}
rows_checked += 1;
}
for &(at, src) in &migration.relations {
let s = &source.relations[src];
let r = kg.relations.get(at);
let ok = r.is_some_and(|r| {
Some(&r.src) == migration.entity_ids.get(&s.src)
&& Some(&r.tgt) == migration.entity_ids.get(&s.tgt)
&& r.relation == s.relation
&& r.weight == s.weight as f32
&& r.ts == s.timestamp * US_PER_SEC
});
if !ok {
return Err(format!(
"relation {} -[{}]-> {} mismatch or missing",
s.src, s.relation, s.tgt
)
.into());
}
rows_checked += 1;
}
// ---- A migrated record must be findable by search ----
let probe = migration
.records
.iter()
.copied()
.find(|&(idx, _)| dim > 0 && mem.cache.tombstones[idx] == 0);
let search_checked = match probe {
None => false,
Some((idx, _)) => {
let query = mem.cache.embeddings[idx].to_vec();
let text = mem.cache.chunks[idx].clone();
let hits = mem.search(&query, &text, &SearchOptions::new(10));
// A record with the same text is as good a hit: the source may
// hold duplicates, and they tie.
if !hits.iter().any(|h| h.index == idx || h.chunk == text) {
return Err(format!(
"search for migrated record {idx} ({:?}) did not return it",
truncate(&text)
)
.into());
}
true
}
};
Ok(ValidationSummary {
chunks: got.chunks.len() as u64,
sessions: got.sessions.len() as u64,
entities: got.entities.len() as u64,
relations: got.relations.len() as u64,
embedding_dim: got.embedding_dim as u64,
count: mem.count(),
active: mem.count_active(),
sessions: mem.sessions().len(),
entities: mem.knowledge().entities.len(),
relations: mem.knowledge().relations.len(),
embedding_dim: dim,
float16,
rows_checked,
provenance_verified,
search_checked,
})
}
fn check_count(kind: &str, got: usize, expected: usize) -> Result<(), BoxErr> {
if got != expected {
return Err(format!("{kind} count mismatch: HDF5 has {got}, source has {expected}").into());
return Err(format!("{kind} count mismatch: store has {got}, expected {expected}").into());
}
Ok(())
}
/// Re-verify the SHA-256 provenance hash of `chunks/text` and
/// `chunks/embeddings` against their actual stored bytes, catching
/// post-write corruption that a plain content comparison against the
/// in-memory source wouldn't (the source is compared against what
/// `read_hdf5` decoded, not against the raw bytes on disk).
///
/// Returns `Ok(true)` only if both datasets exist and both hashes match.
/// Returns `Ok(false)` (not an error) if a dataset has no provenance
/// attributes at all (e.g. a file written before this check existed) or
/// there are zero chunks. Returns an error only on an actual hash mismatch —
/// that indicates real corruption.
fn verify_chunk_provenance(path: &str) -> Result<bool, BoxErr> {
let file = Hdf5File::open(path)?;
let Ok(chunks) = file.group("chunks") else {
return Ok(false);
};
let mut all_present = true;
for name in ["text", "embeddings"] {
let Ok(ds) = chunks.dataset(name) else {
all_present = false;
continue;
};
match ds.verify_provenance()? {
VerifyResult::Ok => {}
VerifyResult::NoHash => all_present = false,
VerifyResult::Mismatch { stored, computed } => {
return Err(format!(
"provenance hash mismatch on chunks/{name}: stored {stored}, recomputed {computed} — data may be corrupted"
)
.into());
}
}
}
Ok(all_present)
}
fn field_err<T: std::fmt::Display>(kind: &str, i: usize, field: &str, s: T, g: T) -> BoxErr {
format!("{kind}[{i}].{field} mismatch: source {s}, HDF5 {g}").into()
}
fn truncate(s: &str) -> String {
if s.len() <= 40 {
s.to_string()
@@ -196,7 +270,7 @@ fn truncate(s: &str) -> String {
}
}
/// Indices of chunk rows to content-check. Full = all; otherwise a spread of
/// Indices of records to content-check. Full = all; otherwise a spread of
/// representative rows (first/last and evenly-spaced interior samples).
fn sample_indices(n: usize, full: bool) -> Vec<usize> {
if n == 0 {