ci: lint all targets, run interop suites for real, compile benches
- clippy --all-targets plus a clawhdf5-format feature matrix (parallel, lz4, zstd, pcodec, fast-checksum); fix the accumulated lint backlog in test, bench and feature-gated code (no behaviour changes). - Install python3 + h5py/numpy/netCDF4/xarray in the CI container and set CLAWHDF5_REQUIRE_INTEROP=1, which makes a missing interop dependency a test failure. Every h5py/netCDF4 interop test used to skip silently in CI. Run the #[ignore]d writer_h5py_tests suite explicitly. - cargo bench --no-run so benches can't rot; fix bench.rs and memory_bench.rs, which no longer compiled against the current strategy/consolidation APIs. - Optional fuzz smoke run via CLAWHDF5_FUZZ_SECONDS. - CHANGELOG and docs/known-issues.md updated. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
706189c3ef
commit
bbe1baa208
+193
-193
@@ -839,6 +839,199 @@ fn is_leap(y: i64) -> bool {
|
||||
(y % 4 == 0 && y % 100 != 0) || y % 400 == 0
|
||||
}
|
||||
|
||||
impl HDF5Memory {
|
||||
pub fn set_strategy(&mut self, s: Box<dyn MemoryStrategy>) {
|
||||
self.strategy = Some(s);
|
||||
}
|
||||
pub fn record(&mut self, exchange: Exchange) -> Result<StrategyOutput> {
|
||||
let strat = self.strategy.as_ref().ok_or_else(|| {
|
||||
MemoryError::Schema(
|
||||
"strategy not initialized: call set_strategy() before record()".to_owned(),
|
||||
)
|
||||
})?;
|
||||
let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge);
|
||||
let output = strat.evaluate(&exchange, &view);
|
||||
for e in &output.entries {
|
||||
self.cache.push(
|
||||
e.chunk.clone(),
|
||||
e.embedding.clone(),
|
||||
e.source_channel.clone(),
|
||||
e.timestamp,
|
||||
e.session_id.clone(),
|
||||
e.tags.clone(),
|
||||
);
|
||||
}
|
||||
for eu in &output.entity_updates {
|
||||
let id = self.knowledge.add_entity(&eu.name, &eu.entity_type, -1);
|
||||
for a in &eu.aliases {
|
||||
self.knowledge.add_alias(a, id as i64);
|
||||
}
|
||||
}
|
||||
if !output.entries.is_empty() || !output.entity_updates.is_empty() {
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
impl HDF5Memory {
|
||||
pub fn tick_session(&mut self) -> Result<()> {
|
||||
let d = self.config.decay_factor;
|
||||
for w in self.cache.activation_weights.iter_mut() {
|
||||
*w *= d;
|
||||
}
|
||||
self.flush()?;
|
||||
if let Some(ref mut w) = self.wal {
|
||||
w.truncate()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Number of pending WAL entries (0 if WAL disabled).
|
||||
pub fn wal_pending_count(&self) -> usize {
|
||||
self.wal.as_ref().map_or(0, |w| w.pending_count() as usize)
|
||||
}
|
||||
|
||||
/// Explicit WAL merge: flush .h5, truncate WAL.
|
||||
pub fn flush_wal(&mut self) -> Result<()> {
|
||||
self.flush()?;
|
||||
if let Some(ref mut w) = self.wal {
|
||||
w.truncate()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Ephemeral tier integration
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
impl HDF5Memory {
|
||||
/// Enable the ephemeral working memory tier with the given configuration.
|
||||
pub fn enable_ephemeral(&mut self, config: EphemeralConfig) {
|
||||
self.ephemeral = Some(EphemeralStore::new(config));
|
||||
}
|
||||
|
||||
/// Return a shared reference to the ephemeral store, if enabled.
|
||||
pub fn ephemeral(&self) -> Option<&EphemeralStore> {
|
||||
self.ephemeral.as_ref()
|
||||
}
|
||||
|
||||
/// Return a mutable reference to the ephemeral store, if enabled.
|
||||
pub fn ephemeral_mut(&mut self) -> Option<&mut EphemeralStore> {
|
||||
self.ephemeral.as_mut()
|
||||
}
|
||||
|
||||
/// Promote frequently-accessed ephemeral entries into the persistent cache.
|
||||
///
|
||||
/// Every entry whose `access_count >= min_access_count` is removed from the
|
||||
/// ephemeral store and written to the HDF5 cache, then the file is flushed.
|
||||
/// Returns the number of entries promoted.
|
||||
pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result<usize> {
|
||||
let candidates = match &self.ephemeral {
|
||||
None => return Ok(0),
|
||||
Some(s) => s.promotion_candidates(min_access_count),
|
||||
};
|
||||
|
||||
if candidates.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let dim = self.config.embedding_dim;
|
||||
let mut promoted = 0;
|
||||
|
||||
for key in candidates {
|
||||
let entry = match self
|
||||
.ephemeral
|
||||
.as_mut()
|
||||
.and_then(|s| s.take_for_promotion(&key))
|
||||
{
|
||||
Some(e) => e,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let chunk = entry
|
||||
.text
|
||||
.clone()
|
||||
.unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned());
|
||||
let embedding = entry.embedding.clone().unwrap_or_else(|| vec![0.0f32; dim]);
|
||||
|
||||
self.cache.push(
|
||||
chunk,
|
||||
embedding,
|
||||
format!("ephemeral::{key}"),
|
||||
entry.created_at,
|
||||
String::new(),
|
||||
entry.tags.join(","),
|
||||
);
|
||||
promoted += 1;
|
||||
}
|
||||
|
||||
if promoted > 0 {
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(promoted)
|
||||
}
|
||||
|
||||
/// Search both the persistent HDF5 tier and the ephemeral tier, returning
|
||||
/// the top `k` results sorted by score descending.
|
||||
///
|
||||
/// Ephemeral results are boosted by a factor of 1.2 to surface recent
|
||||
/// in-context information above older persisted data.
|
||||
pub fn unified_search(
|
||||
&mut self,
|
||||
query_embedding: &[f32],
|
||||
query_text: &str,
|
||||
k: usize,
|
||||
) -> Vec<SearchResult> {
|
||||
// Persistent tier.
|
||||
let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k);
|
||||
const EPHEMERAL_BOOST: f32 = 1.2;
|
||||
let mut results = persistent;
|
||||
|
||||
if self.ephemeral.is_none() {
|
||||
return results;
|
||||
}
|
||||
|
||||
let eph = self.ephemeral.as_mut().unwrap();
|
||||
|
||||
// Collect (key, score) pairs from ephemeral — borrow ends before we
|
||||
// access entries again below.
|
||||
let eph_hits: Vec<(String, f32)> = if !query_embedding.is_empty() {
|
||||
eph.search_embedding(query_embedding, k)
|
||||
} else if !query_text.is_empty() {
|
||||
eph.search_text(query_text, k)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
for (key, score) in &eph_hits {
|
||||
if let Some(entry) = eph.get_entry(key) {
|
||||
let chunk = entry
|
||||
.text
|
||||
.clone()
|
||||
.unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned());
|
||||
results.push(SearchResult {
|
||||
score: score * EPHEMERAL_BOOST,
|
||||
chunk,
|
||||
index: usize::MAX,
|
||||
timestamp: entry.created_at,
|
||||
source_channel: format!("ephemeral::{key}"),
|
||||
activation: 1.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| {
|
||||
b.score
|
||||
.partial_cmp(&a.score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
results.truncate(k);
|
||||
results
|
||||
}
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1676,196 +1869,3 @@ mod tests {
|
||||
assert!((mem.cache.tombstone_fraction() - 0.50).abs() < 0.01);
|
||||
}
|
||||
}
|
||||
|
||||
impl HDF5Memory {
|
||||
pub fn set_strategy(&mut self, s: Box<dyn MemoryStrategy>) {
|
||||
self.strategy = Some(s);
|
||||
}
|
||||
pub fn record(&mut self, exchange: Exchange) -> Result<StrategyOutput> {
|
||||
let strat = self.strategy.as_ref().ok_or_else(|| {
|
||||
MemoryError::Schema(
|
||||
"strategy not initialized: call set_strategy() before record()".to_owned(),
|
||||
)
|
||||
})?;
|
||||
let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge);
|
||||
let output = strat.evaluate(&exchange, &view);
|
||||
for e in &output.entries {
|
||||
self.cache.push(
|
||||
e.chunk.clone(),
|
||||
e.embedding.clone(),
|
||||
e.source_channel.clone(),
|
||||
e.timestamp,
|
||||
e.session_id.clone(),
|
||||
e.tags.clone(),
|
||||
);
|
||||
}
|
||||
for eu in &output.entity_updates {
|
||||
let id = self.knowledge.add_entity(&eu.name, &eu.entity_type, -1);
|
||||
for a in &eu.aliases {
|
||||
self.knowledge.add_alias(a, id as i64);
|
||||
}
|
||||
}
|
||||
if !output.entries.is_empty() || !output.entity_updates.is_empty() {
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
impl HDF5Memory {
|
||||
pub fn tick_session(&mut self) -> Result<()> {
|
||||
let d = self.config.decay_factor;
|
||||
for w in self.cache.activation_weights.iter_mut() {
|
||||
*w *= d;
|
||||
}
|
||||
self.flush()?;
|
||||
if let Some(ref mut w) = self.wal {
|
||||
w.truncate()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Number of pending WAL entries (0 if WAL disabled).
|
||||
pub fn wal_pending_count(&self) -> usize {
|
||||
self.wal.as_ref().map_or(0, |w| w.pending_count() as usize)
|
||||
}
|
||||
|
||||
/// Explicit WAL merge: flush .h5, truncate WAL.
|
||||
pub fn flush_wal(&mut self) -> Result<()> {
|
||||
self.flush()?;
|
||||
if let Some(ref mut w) = self.wal {
|
||||
w.truncate()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Ephemeral tier integration
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
impl HDF5Memory {
|
||||
/// Enable the ephemeral working memory tier with the given configuration.
|
||||
pub fn enable_ephemeral(&mut self, config: EphemeralConfig) {
|
||||
self.ephemeral = Some(EphemeralStore::new(config));
|
||||
}
|
||||
|
||||
/// Return a shared reference to the ephemeral store, if enabled.
|
||||
pub fn ephemeral(&self) -> Option<&EphemeralStore> {
|
||||
self.ephemeral.as_ref()
|
||||
}
|
||||
|
||||
/// Return a mutable reference to the ephemeral store, if enabled.
|
||||
pub fn ephemeral_mut(&mut self) -> Option<&mut EphemeralStore> {
|
||||
self.ephemeral.as_mut()
|
||||
}
|
||||
|
||||
/// Promote frequently-accessed ephemeral entries into the persistent cache.
|
||||
///
|
||||
/// Every entry whose `access_count >= min_access_count` is removed from the
|
||||
/// ephemeral store and written to the HDF5 cache, then the file is flushed.
|
||||
/// Returns the number of entries promoted.
|
||||
pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result<usize> {
|
||||
let candidates = match &self.ephemeral {
|
||||
None => return Ok(0),
|
||||
Some(s) => s.promotion_candidates(min_access_count),
|
||||
};
|
||||
|
||||
if candidates.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let dim = self.config.embedding_dim;
|
||||
let mut promoted = 0;
|
||||
|
||||
for key in candidates {
|
||||
let entry = match self
|
||||
.ephemeral
|
||||
.as_mut()
|
||||
.and_then(|s| s.take_for_promotion(&key))
|
||||
{
|
||||
Some(e) => e,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let chunk = entry
|
||||
.text
|
||||
.clone()
|
||||
.unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned());
|
||||
let embedding = entry.embedding.clone().unwrap_or_else(|| vec![0.0f32; dim]);
|
||||
|
||||
self.cache.push(
|
||||
chunk,
|
||||
embedding,
|
||||
format!("ephemeral::{key}"),
|
||||
entry.created_at,
|
||||
String::new(),
|
||||
entry.tags.join(","),
|
||||
);
|
||||
promoted += 1;
|
||||
}
|
||||
|
||||
if promoted > 0 {
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(promoted)
|
||||
}
|
||||
|
||||
/// Search both the persistent HDF5 tier and the ephemeral tier, returning
|
||||
/// the top `k` results sorted by score descending.
|
||||
///
|
||||
/// Ephemeral results are boosted by a factor of 1.2 to surface recent
|
||||
/// in-context information above older persisted data.
|
||||
pub fn unified_search(
|
||||
&mut self,
|
||||
query_embedding: &[f32],
|
||||
query_text: &str,
|
||||
k: usize,
|
||||
) -> Vec<SearchResult> {
|
||||
// Persistent tier.
|
||||
let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k);
|
||||
const EPHEMERAL_BOOST: f32 = 1.2;
|
||||
let mut results = persistent;
|
||||
|
||||
if self.ephemeral.is_none() {
|
||||
return results;
|
||||
}
|
||||
|
||||
let eph = self.ephemeral.as_mut().unwrap();
|
||||
|
||||
// Collect (key, score) pairs from ephemeral — borrow ends before we
|
||||
// access entries again below.
|
||||
let eph_hits: Vec<(String, f32)> = if !query_embedding.is_empty() {
|
||||
eph.search_embedding(query_embedding, k)
|
||||
} else if !query_text.is_empty() {
|
||||
eph.search_text(query_text, k)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
for (key, score) in &eph_hits {
|
||||
if let Some(entry) = eph.get_entry(key) {
|
||||
let chunk = entry
|
||||
.text
|
||||
.clone()
|
||||
.unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned());
|
||||
results.push(SearchResult {
|
||||
score: score * EPHEMERAL_BOOST,
|
||||
chunk,
|
||||
index: usize::MAX,
|
||||
timestamp: entry.created_at,
|
||||
source_channel: format!("ephemeral::{key}"),
|
||||
activation: 1.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| {
|
||||
b.score
|
||||
.partial_cmp(&a.score)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
results.truncate(k);
|
||||
results
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user