From 4f2975d7e3da6533a89097f8394de4f5a91f203e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 06:00:32 -0700 Subject: [PATCH] fix(agent): persist behavioural config; make compression actually work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight MemoryConfig fields (float16, compression, compression_level, compact_threshold, hebbian_boost, decay_factor, wal_enabled, wal_max_entries) were never written to /meta, so reopening a store silently reset them to defaults — a compressed store was rewritten uncompressed by the first checkpoint after a reopen, and wal_enabled=false flipped back to true. They are now stored as /meta attributes; each is optional on load so older files keep opening with the previous defaults, and non-finite floats are ignored. Writing the round-trip test exposed that `compression = true` never worked in a default build: the embeddings dataset called with_zstd() unconditionally but the agent crate never enabled the zstd feature, so every checkpoint failed with "unsupported filter: 32015". The default build now compresses with deflate (always available, pure Rust path); Zstd is opt-in via a new `zstd` agent feature. Co-Authored-By: Claude Fable 5.1 --- crates/clawhdf5-agent/Cargo.toml | 3 + crates/clawhdf5-agent/src/schema.rs | 126 ++++++++++++++++++++++++---- 2 files changed, 113 insertions(+), 16 deletions(-) diff --git a/crates/clawhdf5-agent/Cargo.toml b/crates/clawhdf5-agent/Cargo.toml index d7ba0ff..4c4caad 100644 --- a/crates/clawhdf5-agent/Cargo.toml +++ b/crates/clawhdf5-agent/Cargo.toml @@ -48,6 +48,9 @@ harness = false default = ["float16", "hnsw"] float16 = ["half"] parallel = ["rayon"] +# Compress embeddings with Zstd instead of deflate when +# `MemoryConfig::compression` is on. Off by default: it links libzstd (C). +zstd = ["clawhdf5/zstd"] # HNSW approximate-nearest-neighbour acceleration for the vector stage of # hybrid_search. On by default; the index is rebuilt from the cache on demand # and stays self-consistent with the persisted memory store. Disable with diff --git a/crates/clawhdf5-agent/src/schema.rs b/crates/clawhdf5-agent/src/schema.rs index 8c234eb..5376d13 100644 --- a/crates/clawhdf5-agent/src/schema.rs +++ b/crates/clawhdf5-agent/src/schema.rs @@ -54,6 +54,27 @@ pub fn build_hdf5_file_with_mark( meta.set_attr("embedding_dim", AttrValue::I64(config.embedding_dim as i64)); meta.set_attr("chunk_size", AttrValue::I64(config.chunk_size as i64)); meta.set_attr("overlap", AttrValue::I64(config.overlap as i64)); + // Behavioural settings. These used to live only in memory, so reopening a + // store silently reset them to defaults — e.g. a compressed store was + // rewritten uncompressed by the first checkpoint after a reopen. Loaders + // treat each one as optional so older files keep opening. + meta.set_attr("float16", AttrValue::I64(config.float16.into())); + meta.set_attr("compression", AttrValue::I64(config.compression.into())); + meta.set_attr( + "compression_level", + AttrValue::I64(config.compression_level.into()), + ); + meta.set_attr( + "compact_threshold", + AttrValue::F64(config.compact_threshold.into()), + ); + meta.set_attr("hebbian_boost", AttrValue::F64(config.hebbian_boost.into())); + meta.set_attr("decay_factor", AttrValue::F64(config.decay_factor.into())); + meta.set_attr("wal_enabled", AttrValue::I64(config.wal_enabled.into())); + meta.set_attr( + "wal_max_entries", + AttrValue::I64(config.wal_max_entries as i64), + ); meta.set_attr( "edgehdf5_version", AttrValue::String(ZEROCLAW_VERSION.into()), @@ -107,15 +128,33 @@ fn build_memory_group( let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n); ds.with_chunks(&[rows_per_chunk, d]); - // Compression: Zstd for embeddings — faster than deflate at same ratio. - // Shuffle is applied automatically (auto-shuffle pre-filter). + // Compression. Shuffle is applied automatically (auto-shuffle + // pre-filter). Zstd is faster than deflate at the same ratio but + // pulls in libzstd, so it is opt-in via the `zstd` feature; the + // default build uses deflate, which is always available. (This + // used to call `with_zstd` unconditionally, so without the + // feature every checkpoint of a compressed store failed with + // "unsupported filter: 32015".) Both are standard HDF5 filters; + // reading a zstd-compressed store needs a zstd-enabled build. if config.compression { - let level = if config.compression_level > 0 { - config.compression_level.min(22) - } else { - 3 // Zstd level 3: fast + good ratio for f32 embeddings - }; - ds.with_zstd(level); + #[cfg(feature = "zstd")] + { + let level = if config.compression_level > 0 { + config.compression_level.min(22) + } else { + 3 // fast + good ratio for f32 embeddings + }; + ds.with_zstd(level); + } + #[cfg(not(feature = "zstd"))] + { + let level = if config.compression_level > 0 { + config.compression_level.min(9) + } else { + 4 + }; + ds.with_deflate(level); + } } } @@ -382,15 +421,19 @@ pub fn validate_and_load( embedding_dim, chunk_size, overlap, - float16: false, - compression: false, - compression_level: 0, - compact_threshold: 0.3, - hebbian_boost: 0.15, - decay_factor: 0.98, + float16: optional_bool_attr(&attrs, "float16", false), + compression: optional_bool_attr(&attrs, "compression", false), + compression_level: optional_i64_attr(&attrs, "compression_level") + .and_then(|v| u32::try_from(v).ok()) + .unwrap_or(0), + compact_threshold: optional_f32_attr(&attrs, "compact_threshold", 0.3), + hebbian_boost: optional_f32_attr(&attrs, "hebbian_boost", 0.15), + decay_factor: optional_f32_attr(&attrs, "decay_factor", 0.98), created_at, - wal_enabled: true, - wal_max_entries: 500, + wal_enabled: optional_bool_attr(&attrs, "wal_enabled", true), + wal_max_entries: optional_i64_attr(&attrs, "wal_max_entries") + .and_then(|v| usize::try_from(v).ok()) + .unwrap_or(500), }; // Load /memory group @@ -595,6 +638,27 @@ fn extract_string_attr( } } +type MetaAttrs = std::collections::HashMap; + +fn optional_i64_attr(attrs: &MetaAttrs, name: &str) -> Option { + match attrs.get(name) { + Some(AttrValue::I64(v)) => Some(*v), + _ => None, + } +} + +fn optional_bool_attr(attrs: &MetaAttrs, name: &str, default: bool) -> bool { + optional_i64_attr(attrs, name).map_or(default, |v| v != 0) +} + +/// Finite values only: a NaN threshold/decay would poison every comparison. +fn optional_f32_attr(attrs: &MetaAttrs, name: &str, default: f32) -> f32 { + match attrs.get(name) { + Some(AttrValue::F64(v)) if v.is_finite() => *v as f32, + _ => default, + } +} + fn extract_i64_attr( attrs: &std::collections::HashMap, name: &str, @@ -716,6 +780,36 @@ mod tests { validate_and_load(&file).map(|(_, cache, _, _)| cache) } + #[test] + fn behavioural_config_survives_a_reopen() { + let mut cfg = config(); + cfg.compression = true; + cfg.compression_level = 7; + cfg.compact_threshold = 0.5; + cfg.hebbian_boost = 0.25; + cfg.decay_factor = 0.9; + cfg.wal_enabled = false; + cfg.wal_max_entries = 42; + let bytes = build_hdf5_file( + &cfg, + &cache_with(2), + &SessionCache::new(), + &KnowledgeCache::new(), + ) + .unwrap(); + let file = clawhdf5::File::from_bytes(bytes).unwrap(); + let (loaded, loaded_cache, ..) = validate_and_load(&file).unwrap(); + // The compressed embeddings must also read back intact. + assert_eq!(loaded_cache.embeddings, cache_with(2).embeddings); + assert!(loaded.compression); + assert_eq!(loaded.compression_level, 7); + assert_eq!(loaded.compact_threshold, 0.5); + assert_eq!(loaded.hebbian_boost, 0.25); + assert_eq!(loaded.decay_factor, 0.9); + assert!(!loaded.wal_enabled); + assert_eq!(loaded.wal_max_entries, 42); + } + #[test] fn consistent_store_loads() { let loaded = roundtrip(&cache_with(3)).unwrap();