diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 39751d6..289d7cc 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -22,5 +22,19 @@ jobs: run: rustup component add rustfmt clippy - name: Install thumbv7em-none-eabihf target run: rustup target add thumbv7em-none-eabihf + - name: Install Python interop dependencies + # The interop suites used to skip silently when python3/h5py were + # missing, so they never ran in CI. Install them and make a missing + # dependency a failure (CLAWHDF5_REQUIRE_INTEROP below). + run: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-venv + python3 -m venv /opt/interop + /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray + echo "/opt/interop/bin" >> "$GITHUB_PATH" + - name: Show interop library versions + run: python3 -c "import h5py, netCDF4; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__)" - name: Run CI script + env: + CLAWHDF5_REQUIRE_INTEROP: "1" run: bash scripts/ci-test.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b04e0..7cc517b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## Unreleased + +### Bug Fixes +- `clawhdf5-format`: compound datatypes written with **default libver bounds** + (datatype message version 1 — what plain `h5py.File(path, 'w')` produces) + were mis-parsed. The v1 member layout carries 28 bytes of legacy array + fields after the byte offset (the parser skipped 24), and v2 pads member + names to 8 bytes and has no array fields at all (the parser did neither), so + every member after the first byte offset was read from the wrong position — + typically surfacing as `Overflow("compound member ...")` on read. Found by + adding a default-libver axis to the h5py interop tests; byte-level regression + tests for v1 and v2 added. +- `clawhdf5-gpu`: `gpu_tests` could hang forever under the default parallel + test runner — every test created its own wgpu instance and device at once. + Tests now serialise GPU access, and GPU→CPU readback waits are bounded + (30 s) so a wedged driver returns `GpuError::BufferMap` instead of blocking. +- `clawhdf5-agent`: `benches/bench.rs` and `benches/memory_bench.rs` no longer + compiled against the current `strategy`/`consolidation` APIs. + +### CI / Testing +- CI now lints every target (`cargo clippy --all-targets`) plus + `clawhdf5-format`'s optional features, compiles all benches, and tests the + format feature matrix. Previously test/bench code and feature-gated modules + were never linted; the accumulated clippy backlog is fixed. +- CI installs python3 + h5py/numpy/netCDF4/xarray and sets + `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a missing interop dependency into a + test **failure**. Until now every h5py/netCDF4 interop test silently skipped + in CI, which is how the HDF5 2.0 compound bug fixed in v2.2.0 reached a user. + The `#[ignore]`d `writer_h5py_tests` suite is run explicitly. +- h5py-generated-file tests now cover default libver bounds as well as + `libver='latest'` (HDF5 2.0 raised the default low bound to 1.8). +- Optional fuzz smoke run (`CLAWHDF5_FUZZ_SECONDS=N scripts/ci-test.sh`); new + datatype corpus seeds for v1 compound and native complex messages. + ## v2.2.0 (2026-09-18) ### Security diff --git a/crates/clawhdf5-agent/benches/bench.rs b/crates/clawhdf5-agent/benches/bench.rs index 60b4758..ef5606b 100644 --- a/crates/clawhdf5-agent/benches/bench.rs +++ b/crates/clawhdf5-agent/benches/bench.rs @@ -483,7 +483,7 @@ fn rayon_benches(c: &mut Criterion) { use rayon::prelude::*; let query_norm = vector_search::compute_norm(&query); let num_cores = rayon::current_num_threads().max(1); - let chunk_size = (n + num_cores - 1) / num_cores; + let chunk_size = n.div_ceil(num_cores); let mut results: Vec<(usize, f32)> = vectors .par_chunks(chunk_size) .enumerate() @@ -537,7 +537,7 @@ fn rayon_benches(c: &mut Criterion) { use rayon::prelude::*; let query_norm = vector_search::compute_norm(&query); let num_cores = rayon::current_num_threads().max(1); - let chunk_size = (n + num_cores - 1) / num_cores; + let chunk_size = n.div_ceil(num_cores); let mut results: Vec<(usize, f32)> = vectors .par_chunks(chunk_size) .enumerate() @@ -766,12 +766,22 @@ fn adaptive_benches(c: &mut Criterion) { .map(|v| vector_search::compute_norm(v)) .collect(); let tombstones = vec![0u8; n]; + let flat: Vec = vectors.iter().flatten().copied().collect(); c.bench_function("adaptive_search_10k", |b| { let hw = HardwareCapabilities::detect(); let strat = strategy::auto_select_strategy(n, &hw); b.iter(|| { - strategy::search_with_metrics(&query, &vectors, &norms, &tombstones, 10, strat, None) + strategy::search_with_metrics( + &query, + &vectors, + &flat, + &norms, + &tombstones, + 10, + strat, + None, + ) }); }); @@ -781,6 +791,7 @@ fn adaptive_benches(c: &mut Criterion) { strategy::search_with_metrics( &query, &vectors, + &flat, &norms, &tombstones, 10, @@ -795,6 +806,7 @@ fn adaptive_benches(c: &mut Criterion) { strategy::search_with_metrics( &query, &vectors, + &flat, &norms, &tombstones, 10, @@ -809,6 +821,7 @@ fn adaptive_benches(c: &mut Criterion) { strategy::search_with_metrics( &query, &vectors, + &flat, &norms, &tombstones, 10, diff --git a/crates/clawhdf5-agent/benches/memory_bench.rs b/crates/clawhdf5-agent/benches/memory_bench.rs index 0f21d3f..b57f18a 100644 --- a/crates/clawhdf5-agent/benches/memory_bench.rs +++ b/crates/clawhdf5-agent/benches/memory_bench.rs @@ -1,6 +1,7 @@ use clawhdf5_agent::bm25::BM25Index; use clawhdf5_agent::consolidation::{ ConsolidationConfig, ConsolidationEngine, ImportanceScorer, ImportanceWeights, MemorySource, + UntrustedSource, }; use clawhdf5_agent::hybrid::{hybrid_search, rrf_hybrid_search}; use clawhdf5_agent::knowledge::KnowledgeCache; @@ -285,7 +286,12 @@ fn consolidation_benches(c: &mut Criterion) { for i in 0..n { let embedding = make_vec(&mut rng, DIM); let chunk = format!("memory record {i} with some content"); - engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64); + engine.add_memory( + chunk, + embedding, + UntrustedSource::User, + now + i as f64, + ); } engine }, @@ -307,9 +313,10 @@ fn consolidation_benches(c: &mut Criterion) { for i in 0..50usize { let embedding = make_vec(&mut rng, DIM); let chunk = format!("existing record {i}"); - engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64); + engine.add_memory(chunk, embedding, UntrustedSource::User, now + i as f64); } let records = engine.records().to_vec(); + let record_refs: Vec<&_> = records.iter().collect(); let weights = ImportanceWeights::default(); let query_embedding = make_vec(&mut rng, DIM); let sample_text = @@ -317,7 +324,7 @@ fn consolidation_benches(c: &mut Criterion) { group.bench_function("bench_importance_scoring", |b| { b.iter(|| { - let surprise = ImportanceScorer::score_surprise(&query_embedding, &records); + let surprise = ImportanceScorer::score_surprise(&query_embedding, &record_refs); let correction = ImportanceScorer::score_correction(&MemorySource::Correction); let length = ImportanceScorer::score_length(sample_text); ImportanceScorer::score_combined(surprise, correction, length, &weights) @@ -354,7 +361,7 @@ fn temporal_benches(c: &mut Criterion) { // Insert benchmark: measure time to insert 10k timestamps one by one group.bench_function("bench_temporal_insert_10k", |b| { b.iter_batched( - || TemporalIndex::new(), + TemporalIndex::new, |mut idx| { for i in 0..N { // Shuffle insertion order slightly using a simple offset pattern @@ -442,7 +449,8 @@ fn large_consolidation_benches(c: &mut Criterion) { let mut group = c.benchmark_group("consolidation_large"); group.sample_size(10); - for (label, n) in [("10k", 10_000usize)] { + { + let (label, n) = ("10k", 10_000usize); group.bench_with_input( BenchmarkId::new("bench_consolidation_cycle", label), &n, @@ -459,7 +467,12 @@ fn large_consolidation_benches(c: &mut Criterion) { for i in 0..n { let embedding = make_vec(&mut rng, DIM); let chunk = format!("memory record {i} with content"); - engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64); + engine.add_memory( + chunk, + embedding, + UntrustedSource::User, + now + i as f64, + ); } engine }, diff --git a/crates/clawhdf5-agent/src/consolidation.rs b/crates/clawhdf5-agent/src/consolidation.rs index 228e1ae..a7023b4 100644 --- a/crates/clawhdf5-agent/src/consolidation.rs +++ b/crates/clawhdf5-agent/src/consolidation.rs @@ -563,7 +563,7 @@ mod tests { #[test] fn test_importance_scorer_surprise_identical() { let emb = unit_vec(4, 0); - let existing = vec![MemoryRecord { + let existing = [MemoryRecord { id: 0, chunk: "existing".to_string(), embedding: emb.clone(), @@ -603,23 +603,20 @@ mod tests { fn test_importance_scorer_length() { assert!((ImportanceScorer::score_length("")).abs() < f32::EPSILON); // 50 words → 0.5 - let fifty_words = std::iter::repeat("word") - .take(50) + let fifty_words = std::iter::repeat_n("word", 50) .collect::>() .join(" "); let s50 = ImportanceScorer::score_length(&fifty_words); assert!((s50 - 0.5).abs() < 1e-5, "expected 0.5, got {s50}"); // 100 words → 1.0 - let hundred_words = std::iter::repeat("word") - .take(100) + let hundred_words = std::iter::repeat_n("word", 100) .collect::>() .join(" "); assert_eq!(ImportanceScorer::score_length(&hundred_words), 1.0); // 200 words → still 1.0 (clamped) - let two_hundred = std::iter::repeat("word") - .take(200) + let two_hundred = std::iter::repeat_n("word", 200) .collect::>() .join(" "); assert_eq!(ImportanceScorer::score_length(&two_hundred), 1.0); @@ -693,9 +690,11 @@ mod tests { // --------------------------------------------------------------------------- #[test] fn test_consolidate_eviction_working() { - let mut cfg = ConsolidationConfig::default(); - cfg.working_capacity = 3; - cfg.working_to_episodic_threshold = 2.0; // never promote in this test + let cfg = ConsolidationConfig { + working_capacity: 3, + working_to_episodic_threshold: 2.0, // never promote in this test + ..Default::default() + }; let mut engine = ConsolidationEngine::new(cfg); // Add 5 records; all have very low importance so none get promoted. @@ -853,9 +852,11 @@ mod tests { // --------------------------------------------------------------------------- #[test] fn test_consolidate_episodic_eviction() { - let mut cfg = ConsolidationConfig::default(); - cfg.episodic_capacity = 3; - cfg.working_to_episodic_threshold = 2.0; // never auto-promote from Working + let cfg = ConsolidationConfig { + episodic_capacity: 3, + working_to_episodic_threshold: 2.0, // never auto-promote from Working + ..Default::default() + }; let mut engine = ConsolidationEngine::new(cfg); // Seed 5 records directly in Episodic. diff --git a/crates/clawhdf5-agent/src/entity_extract.rs b/crates/clawhdf5-agent/src/entity_extract.rs index b6840ca..e334151 100644 --- a/crates/clawhdf5-agent/src/entity_extract.rs +++ b/crates/clawhdf5-agent/src/entity_extract.rs @@ -777,8 +777,10 @@ mod tests { #[test] fn test_tech_disabled() { - let mut config = ExtractorConfig::default(); - config.extract_technology = false; + let config = ExtractorConfig { + extract_technology: false, + ..Default::default() + }; let e = EntityExtractor::new(config); let entities = e.extract("We use Rust and Docker."); assert!( @@ -847,8 +849,10 @@ mod tests { #[test] fn test_date_disabled() { - let mut config = ExtractorConfig::default(); - config.extract_dates = false; + let config = ExtractorConfig { + extract_dates: false, + ..Default::default() + }; let e = EntityExtractor::new(config); let entities = e.extract("Released on 2024-03-19."); assert!( @@ -981,8 +985,10 @@ mod tests { #[test] fn test_confidence_filter() { - let mut config = ExtractorConfig::default(); - config.min_confidence = 0.95; + let config = ExtractorConfig { + min_confidence: 0.95, + ..Default::default() + }; let e = EntityExtractor::new(config); // Only dates (0.95) and techs (0.9) should survive; 0.9 < 0.95 filters techs. let entities = e.extract("We use Rust since 2024-01-01."); @@ -1002,7 +1008,7 @@ mod tests { fn test_batch_dedup() { let e = default_extractor(); let texts = ["We use Rust.", "Rust is fast.", "Also Rust for safety."]; - let entities = e.extract_batch(&texts.iter().map(|s| *s).collect::>()); + let entities = e.extract_batch(&texts); let rust_count = entities.iter().filter(|x| x.text == "Rust").count(); assert_eq!(rust_count, 1, "Rust should appear exactly once after dedup"); } @@ -1011,7 +1017,7 @@ mod tests { fn test_batch_multiple_types() { let e = default_extractor(); let texts = ["Deploy with Docker.", "We merged last week."]; - let entities = e.extract_batch(&texts.iter().map(|s| *s).collect::>()); + let entities = e.extract_batch(&texts); assert!( entities .iter() diff --git a/crates/clawhdf5-agent/src/lib.rs b/crates/clawhdf5-agent/src/lib.rs index 246b946..b7f0310 100644 --- a/crates/clawhdf5-agent/src/lib.rs +++ b/crates/clawhdf5-agent/src/lib.rs @@ -839,6 +839,199 @@ fn is_leap(y: i64) -> bool { (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 } +impl HDF5Memory { + pub fn set_strategy(&mut self, s: Box) { + self.strategy = Some(s); + } + pub fn record(&mut self, exchange: Exchange) -> Result { + let strat = self.strategy.as_ref().ok_or_else(|| { + MemoryError::Schema( + "strategy not initialized: call set_strategy() before record()".to_owned(), + ) + })?; + let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge); + let output = strat.evaluate(&exchange, &view); + for e in &output.entries { + self.cache.push( + e.chunk.clone(), + e.embedding.clone(), + e.source_channel.clone(), + e.timestamp, + e.session_id.clone(), + e.tags.clone(), + ); + } + for eu in &output.entity_updates { + let id = self.knowledge.add_entity(&eu.name, &eu.entity_type, -1); + for a in &eu.aliases { + self.knowledge.add_alias(a, id as i64); + } + } + if !output.entries.is_empty() || !output.entity_updates.is_empty() { + self.flush()?; + } + Ok(output) + } +} + +impl HDF5Memory { + pub fn tick_session(&mut self) -> Result<()> { + let d = self.config.decay_factor; + for w in self.cache.activation_weights.iter_mut() { + *w *= d; + } + self.flush()?; + if let Some(ref mut w) = self.wal { + w.truncate()?; + } + Ok(()) + } + + /// Number of pending WAL entries (0 if WAL disabled). + pub fn wal_pending_count(&self) -> usize { + self.wal.as_ref().map_or(0, |w| w.pending_count() as usize) + } + + /// Explicit WAL merge: flush .h5, truncate WAL. + pub fn flush_wal(&mut self) -> Result<()> { + self.flush()?; + if let Some(ref mut w) = self.wal { + w.truncate()?; + } + Ok(()) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Ephemeral tier integration +// ───────────────────────────────────────────────────────────────────────────── + +impl HDF5Memory { + /// Enable the ephemeral working memory tier with the given configuration. + pub fn enable_ephemeral(&mut self, config: EphemeralConfig) { + self.ephemeral = Some(EphemeralStore::new(config)); + } + + /// Return a shared reference to the ephemeral store, if enabled. + pub fn ephemeral(&self) -> Option<&EphemeralStore> { + self.ephemeral.as_ref() + } + + /// Return a mutable reference to the ephemeral store, if enabled. + pub fn ephemeral_mut(&mut self) -> Option<&mut EphemeralStore> { + self.ephemeral.as_mut() + } + + /// Promote frequently-accessed ephemeral entries into the persistent cache. + /// + /// Every entry whose `access_count >= min_access_count` is removed from the + /// ephemeral store and written to the HDF5 cache, then the file is flushed. + /// Returns the number of entries promoted. + pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { + let candidates = match &self.ephemeral { + None => return Ok(0), + Some(s) => s.promotion_candidates(min_access_count), + }; + + if candidates.is_empty() { + return Ok(0); + } + + let dim = self.config.embedding_dim; + let mut promoted = 0; + + for key in candidates { + let entry = match self + .ephemeral + .as_mut() + .and_then(|s| s.take_for_promotion(&key)) + { + Some(e) => e, + None => continue, + }; + + let chunk = entry + .text + .clone() + .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); + let embedding = entry.embedding.clone().unwrap_or_else(|| vec![0.0f32; dim]); + + self.cache.push( + chunk, + embedding, + format!("ephemeral::{key}"), + entry.created_at, + String::new(), + entry.tags.join(","), + ); + promoted += 1; + } + + if promoted > 0 { + self.flush()?; + } + Ok(promoted) + } + + /// Search both the persistent HDF5 tier and the ephemeral tier, returning + /// the top `k` results sorted by score descending. + /// + /// Ephemeral results are boosted by a factor of 1.2 to surface recent + /// in-context information above older persisted data. + pub fn unified_search( + &mut self, + query_embedding: &[f32], + query_text: &str, + k: usize, + ) -> Vec { + // Persistent tier. + let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k); + const EPHEMERAL_BOOST: f32 = 1.2; + let mut results = persistent; + + if self.ephemeral.is_none() { + return results; + } + + let eph = self.ephemeral.as_mut().unwrap(); + + // Collect (key, score) pairs from ephemeral — borrow ends before we + // access entries again below. + let eph_hits: Vec<(String, f32)> = if !query_embedding.is_empty() { + eph.search_embedding(query_embedding, k) + } else if !query_text.is_empty() { + eph.search_text(query_text, k) + } else { + Vec::new() + }; + + for (key, score) in &eph_hits { + if let Some(entry) = eph.get_entry(key) { + let chunk = entry + .text + .clone() + .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); + results.push(SearchResult { + score: score * EPHEMERAL_BOOST, + chunk, + index: usize::MAX, + timestamp: entry.created_at, + source_channel: format!("ephemeral::{key}"), + activation: 1.0, + }); + } + } + + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + results.truncate(k); + results + } +} + // --- Tests --- #[cfg(test)] @@ -1676,196 +1869,3 @@ mod tests { assert!((mem.cache.tombstone_fraction() - 0.50).abs() < 0.01); } } - -impl HDF5Memory { - pub fn set_strategy(&mut self, s: Box) { - self.strategy = Some(s); - } - pub fn record(&mut self, exchange: Exchange) -> Result { - let strat = self.strategy.as_ref().ok_or_else(|| { - MemoryError::Schema( - "strategy not initialized: call set_strategy() before record()".to_owned(), - ) - })?; - let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge); - let output = strat.evaluate(&exchange, &view); - for e in &output.entries { - self.cache.push( - e.chunk.clone(), - e.embedding.clone(), - e.source_channel.clone(), - e.timestamp, - e.session_id.clone(), - e.tags.clone(), - ); - } - for eu in &output.entity_updates { - let id = self.knowledge.add_entity(&eu.name, &eu.entity_type, -1); - for a in &eu.aliases { - self.knowledge.add_alias(a, id as i64); - } - } - if !output.entries.is_empty() || !output.entity_updates.is_empty() { - self.flush()?; - } - Ok(output) - } -} - -impl HDF5Memory { - pub fn tick_session(&mut self) -> Result<()> { - let d = self.config.decay_factor; - for w in self.cache.activation_weights.iter_mut() { - *w *= d; - } - self.flush()?; - if let Some(ref mut w) = self.wal { - w.truncate()?; - } - Ok(()) - } - - /// Number of pending WAL entries (0 if WAL disabled). - pub fn wal_pending_count(&self) -> usize { - self.wal.as_ref().map_or(0, |w| w.pending_count() as usize) - } - - /// Explicit WAL merge: flush .h5, truncate WAL. - pub fn flush_wal(&mut self) -> Result<()> { - self.flush()?; - if let Some(ref mut w) = self.wal { - w.truncate()?; - } - Ok(()) - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// Ephemeral tier integration -// ───────────────────────────────────────────────────────────────────────────── - -impl HDF5Memory { - /// Enable the ephemeral working memory tier with the given configuration. - pub fn enable_ephemeral(&mut self, config: EphemeralConfig) { - self.ephemeral = Some(EphemeralStore::new(config)); - } - - /// Return a shared reference to the ephemeral store, if enabled. - pub fn ephemeral(&self) -> Option<&EphemeralStore> { - self.ephemeral.as_ref() - } - - /// Return a mutable reference to the ephemeral store, if enabled. - pub fn ephemeral_mut(&mut self) -> Option<&mut EphemeralStore> { - self.ephemeral.as_mut() - } - - /// Promote frequently-accessed ephemeral entries into the persistent cache. - /// - /// Every entry whose `access_count >= min_access_count` is removed from the - /// ephemeral store and written to the HDF5 cache, then the file is flushed. - /// Returns the number of entries promoted. - pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { - let candidates = match &self.ephemeral { - None => return Ok(0), - Some(s) => s.promotion_candidates(min_access_count), - }; - - if candidates.is_empty() { - return Ok(0); - } - - let dim = self.config.embedding_dim; - let mut promoted = 0; - - for key in candidates { - let entry = match self - .ephemeral - .as_mut() - .and_then(|s| s.take_for_promotion(&key)) - { - Some(e) => e, - None => continue, - }; - - let chunk = entry - .text - .clone() - .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); - let embedding = entry.embedding.clone().unwrap_or_else(|| vec![0.0f32; dim]); - - self.cache.push( - chunk, - embedding, - format!("ephemeral::{key}"), - entry.created_at, - String::new(), - entry.tags.join(","), - ); - promoted += 1; - } - - if promoted > 0 { - self.flush()?; - } - Ok(promoted) - } - - /// Search both the persistent HDF5 tier and the ephemeral tier, returning - /// the top `k` results sorted by score descending. - /// - /// Ephemeral results are boosted by a factor of 1.2 to surface recent - /// in-context information above older persisted data. - pub fn unified_search( - &mut self, - query_embedding: &[f32], - query_text: &str, - k: usize, - ) -> Vec { - // Persistent tier. - let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k); - const EPHEMERAL_BOOST: f32 = 1.2; - let mut results = persistent; - - if self.ephemeral.is_none() { - return results; - } - - let eph = self.ephemeral.as_mut().unwrap(); - - // Collect (key, score) pairs from ephemeral — borrow ends before we - // access entries again below. - let eph_hits: Vec<(String, f32)> = if !query_embedding.is_empty() { - eph.search_embedding(query_embedding, k) - } else if !query_text.is_empty() { - eph.search_text(query_text, k) - } else { - Vec::new() - }; - - for (key, score) in &eph_hits { - if let Some(entry) = eph.get_entry(key) { - let chunk = entry - .text - .clone() - .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); - results.push(SearchResult { - score: score * EPHEMERAL_BOOST, - chunk, - index: usize::MAX, - timestamp: entry.created_at, - source_channel: format!("ephemeral::{key}"), - activation: 1.0, - }); - } - } - - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - results.truncate(k); - results - } -} diff --git a/crates/clawhdf5-agent/src/openclaw.rs b/crates/clawhdf5-agent/src/openclaw.rs index 770536c..8bb0acd 100644 --- a/crates/clawhdf5-agent/src/openclaw.rs +++ b/crates/clawhdf5-agent/src/openclaw.rs @@ -748,6 +748,69 @@ impl MemoryBackend for ClawhdfBackend { } } +// ───────────────────────────────────────────────────────────────────────────── +// Ephemeral tier methods on ClawhdfBackend +// ───────────────────────────────────────────────────────────────────────────── + +impl ClawhdfBackend { + /// Enable the ephemeral (in-memory only) working memory tier. + pub fn enable_ephemeral(&mut self, config: crate::ephemeral::EphemeralConfig) { + self.memory.enable_ephemeral(config); + } + + /// Store a text value in ephemeral memory. + /// + /// Returns an error string if the ephemeral tier has not been enabled. + pub fn ephemeral_set( + &mut self, + key: &str, + value: &str, + ttl_secs: Option, + ) -> Result<(), String> { + match self.memory.ephemeral_mut() { + Some(s) => { + s.set_text(key, value, ttl_secs); + Ok(()) + } + None => Err("ephemeral tier not enabled".to_string()), + } + } + + /// Retrieve a text value from ephemeral memory. + /// + /// Returns `None` if the tier is disabled, the key is absent, or the + /// entry has expired. + pub fn ephemeral_get(&mut self, key: &str) -> Option { + self.memory + .ephemeral_mut()? + .get_text(key) + .map(|s| s.to_string()) + } + + /// Delete a key from ephemeral memory. + /// + /// Returns `true` if the key existed and was removed. + pub fn ephemeral_delete(&mut self, key: &str) -> bool { + self.memory.ephemeral_mut().is_some_and(|s| s.delete(key)) + } + + /// Return a snapshot of ephemeral tier statistics, or `None` if the tier + /// is not enabled. + pub fn ephemeral_stats(&self) -> Option { + self.memory.ephemeral().map(|s| s.stats()) + } + + /// Promote frequently-accessed ephemeral entries to persistent HDF5 storage. + /// + /// Entries with `access_count >= min_access_count` are moved from the + /// ephemeral store into the persistent cache. Returns the count promoted. + pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { + self.memory + .promote_ephemeral(min_access_count) + .map_err(|e| e.to_string()) + } +} + // ───────────────────────────────────────────────────────────────────────────── // Tests // ───────────────────────────────────────────────────────────────────────────── @@ -1333,66 +1396,3 @@ mod tests { assert!(out.starts_with("# Title")); } } - -// ───────────────────────────────────────────────────────────────────────────── -// Ephemeral tier methods on ClawhdfBackend -// ───────────────────────────────────────────────────────────────────────────── - -impl ClawhdfBackend { - /// Enable the ephemeral (in-memory only) working memory tier. - pub fn enable_ephemeral(&mut self, config: crate::ephemeral::EphemeralConfig) { - self.memory.enable_ephemeral(config); - } - - /// Store a text value in ephemeral memory. - /// - /// Returns an error string if the ephemeral tier has not been enabled. - pub fn ephemeral_set( - &mut self, - key: &str, - value: &str, - ttl_secs: Option, - ) -> Result<(), String> { - match self.memory.ephemeral_mut() { - Some(s) => { - s.set_text(key, value, ttl_secs); - Ok(()) - } - None => Err("ephemeral tier not enabled".to_string()), - } - } - - /// Retrieve a text value from ephemeral memory. - /// - /// Returns `None` if the tier is disabled, the key is absent, or the - /// entry has expired. - pub fn ephemeral_get(&mut self, key: &str) -> Option { - self.memory - .ephemeral_mut()? - .get_text(key) - .map(|s| s.to_string()) - } - - /// Delete a key from ephemeral memory. - /// - /// Returns `true` if the key existed and was removed. - pub fn ephemeral_delete(&mut self, key: &str) -> bool { - self.memory.ephemeral_mut().is_some_and(|s| s.delete(key)) - } - - /// Return a snapshot of ephemeral tier statistics, or `None` if the tier - /// is not enabled. - pub fn ephemeral_stats(&self) -> Option { - self.memory.ephemeral().map(|s| s.stats()) - } - - /// Promote frequently-accessed ephemeral entries to persistent HDF5 storage. - /// - /// Entries with `access_count >= min_access_count` are moved from the - /// ephemeral store into the persistent cache. Returns the count promoted. - pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { - self.memory - .promote_ephemeral(min_access_count) - .map_err(|e| e.to_string()) - } -} diff --git a/crates/clawhdf5-agent/src/wal.rs b/crates/clawhdf5-agent/src/wal.rs index 74467d9..97ff36d 100644 --- a/crates/clawhdf5-agent/src/wal.rs +++ b/crates/clawhdf5-agent/src/wal.rs @@ -786,7 +786,7 @@ mod tests { let dir = TempDir::new().unwrap(); let wal_path = dir.path().join("test.h5.wal"); let unicode_chunk = "Hello 世界! 🌍 émojis & ünïcödé"; - let embedding = vec![0.1, -0.2, 3.14159, f32::MAX, f32::MIN_POSITIVE]; + let embedding = vec![0.1, -0.2, 3.4567, f32::MAX, f32::MIN_POSITIVE]; { let mut wal = WalFile::open(&wal_path).unwrap(); let entry = WalEntry { diff --git a/crates/clawhdf5-agent/tests/e2e_tests.rs b/crates/clawhdf5-agent/tests/e2e_tests.rs index 75504ce..d151ece 100644 --- a/crates/clawhdf5-agent/tests/e2e_tests.rs +++ b/crates/clawhdf5-agent/tests/e2e_tests.rs @@ -1048,7 +1048,7 @@ fn test_gpu_l2_fallback_works() { let tombstones = vec![0u8; 3]; let gpu = clawhdf5_agent::gpu_search::GpuSearchBackend::try_init(&vectors, &norms, 2, 1); - let results = gpu.search_l2(&vec![0.0, 0.0], &vectors, &tombstones, 3); + let results = gpu.search_l2(&[0.0, 0.0], &vectors, &tombstones, 3); assert_eq!(results.len(), 3); assert_eq!(results[0].0, 0); @@ -1099,7 +1099,7 @@ fn test_mmap_reader_direct_access() { // Open via MmapReader directly let mmap = clawhdf5_io::MmapReader::open(&path).unwrap(); - assert!(mmap.len() > 0); + assert!(!mmap.is_empty()); // Verify we can read bytes at specific offsets let bytes = mmap.read_at(0, 8); assert!(bytes.is_some()); diff --git a/crates/clawhdf5-agent/tests/memx_comparison.rs b/crates/clawhdf5-agent/tests/memx_comparison.rs index 7a9af42..2976834 100644 --- a/crates/clawhdf5-agent/tests/memx_comparison.rs +++ b/crates/clawhdf5-agent/tests/memx_comparison.rs @@ -137,10 +137,10 @@ fn bench_hit_at_1_1014_records() { 0.3, 1, ); - if let Some((top_idx, _)) = results.first() { - if *top_idx == target_indices[qi] { - hits += 1; - } + if let Some((top_idx, _)) = results.first() + && *top_idx == target_indices[qi] + { + hits += 1; } } diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index f3137f9..2c8eaca 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -472,14 +472,13 @@ mod tests { // Name padded to 8 bytes data.extend_from_slice(name); - while data.len() % 8 != 0 || data.len() == 8 { + if data.len() % 8 != 0 || data.len() == 8 { // Pad name to 8-byte boundary from start of name let name_start = 8; let name_padded = pad8(name_size); while data.len() < name_start + name_padded { data.push(0); } - break; } // Datatype padded to 8 bytes @@ -749,11 +748,11 @@ mod tests { data.extend_from_slice(name); data.extend_from_slice(&dt_bytes); data.extend_from_slice(&ds_bytes); - data.extend_from_slice(&3.14f64.to_le_bytes()); + data.extend_from_slice(&3.25f64.to_le_bytes()); let attr = AttributeMessage::parse(&data, 8).unwrap(); let vals = attr.read_as_f64().unwrap(); - assert_eq!(vals, vec![3.14]); + assert_eq!(vals, vec![3.25]); } #[test] diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 6192510..ad58aa0 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -416,6 +416,7 @@ fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 { mod tests { use super::*; + #[allow(clippy::too_many_arguments)] fn build_btree_v2_header( tree_type: u8, node_size: u32, diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 36767d7..43184a3 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -1657,9 +1657,9 @@ mod tests { let chunk_bytes = chunk_size_elems * elem_size; // full chunk allocation // Write chunk data (full chunk size, padding with zeros) - for i in start..end { + for (i, value) in values.iter().enumerate().take(end).skip(start) { let byte_offset = data_offset + (i - start) * elem_size; - file_data[byte_offset..byte_offset + 8].copy_from_slice(&values[i].to_le_bytes()); + file_data[byte_offset..byte_offset + 8].copy_from_slice(&value.to_le_bytes()); } chunk_infos.push(ChunkInfo { @@ -1837,8 +1837,8 @@ mod tests { for chunk_idx in 0..2 { let start = chunk_idx * chunk_elems; let mut chunk_bytes = Vec::new(); - for i in start..start + chunk_elems { - chunk_bytes.extend_from_slice(&values[i].to_le_bytes()); + for value in values.iter().skip(start).take(chunk_elems) { + chunk_bytes.extend_from_slice(&value.to_le_bytes()); } let compressed = compress_chunk(&chunk_bytes, &pipeline, elem_size as u32).unwrap(); diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 4d94899..a469084 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -1763,11 +1763,11 @@ mod tests { fn f16_bits(v: f32) -> u16 { // Encode a few exact values used by the test. match v { - x if x == 0.0 => 0x0000, - x if x == 1.0 => 0x3c00, - x if x == -2.0 => 0xc000, - x if x == 0.5 => 0x3800, - x if x == 65504.0 => 0x7bff, // f16 max + 0.0 => 0x0000, + 1.0 => 0x3c00, + -2.0 => 0xc000, + 0.5 => 0x3800, + 65504.0 => 0x7bff, // f16 max _ => panic!("unsupported test value {v}"), } } @@ -2186,7 +2186,7 @@ mod tests { ], }; let mut raw = Vec::new(); - raw.extend_from_slice(&3.14f64.to_le_bytes()); + raw.extend_from_slice(&3.25f64.to_le_bytes()); raw.extend_from_slice(&42i32.to_le_bytes()); let field = read_compound_field(&raw, &dt, "id").unwrap(); diff --git a/crates/clawhdf5-format/src/dataspace.rs b/crates/clawhdf5-format/src/dataspace.rs index ccdd2c6..efece6d 100644 --- a/crates/clawhdf5-format/src/dataspace.rs +++ b/crates/clawhdf5-format/src/dataspace.rs @@ -189,11 +189,7 @@ mod tests { fn build_v1_dataspace(rank: u8, flags: u8, dims: &[u64], max_dims: Option<&[u64]>) -> Vec { let length_size = 8u8; - let mut buf = Vec::new(); - buf.push(1); // version - buf.push(rank); - buf.push(flags); - buf.push(0); // reserved + let mut buf = vec![1, rank, flags, 0]; // version, rank, flags, reserved buf.extend_from_slice(&[0u8; 4]); // reserved(4) for &d in dims { buf.extend_from_slice(&d.to_le_bytes()); @@ -214,11 +210,7 @@ mod tests { dims: &[u64], max_dims: Option<&[u64]>, ) -> Vec { - let mut buf = Vec::new(); - buf.push(2); // version - buf.push(rank); - buf.push(flags); - buf.push(type_byte); + let mut buf = vec![2, rank, flags, type_byte]; // version, rank, flags, type for &d in dims { buf.extend_from_slice(&d.to_le_bytes()); } @@ -298,11 +290,7 @@ mod tests { #[test] fn v1_with_4byte_length() { - let mut buf = Vec::new(); - buf.push(1); // version - buf.push(1); // rank - buf.push(0); // flags - buf.push(0); // reserved + let mut buf = vec![1, 1, 0, 0]; // version, rank, flags, reserved buf.extend_from_slice(&[0u8; 4]); // reserved(4) buf.extend_from_slice(&10u32.to_le_bytes()); // dim with length_size=4 let ds = Dataspace::parse(&buf, 4).unwrap(); diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 8cf5439..2e99d70 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -1045,24 +1045,30 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result, FormatEr match element_size { 4 => { let nums: Vec = data - .chunks_exact(4) - .map(|b| f32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) .collect(); simple_compress(&nums, &config) .map_err(|e| FormatError::CompressionError(format!("pco: {e}"))) } 8 => { let nums: Vec = data - .chunks_exact(8) - .map(|b| f64::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|b| f64::from_le_bytes(*b)) .collect(); simple_compress(&nums, &config) .map_err(|e| FormatError::CompressionError(format!("pco: {e}"))) } _ => { let nums: Vec = data - .chunks_exact(4) - .map(|b| u32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|b| u32::from_le_bytes(*b)) .collect(); simple_compress(&nums, &config) .map_err(|e| FormatError::CompressionError(format!("pco: {e}"))) @@ -1092,11 +1098,7 @@ fn pcodec_decompress( } else { MAX_DECOMPRESS_SIZE }; - let n = if element_size != 0 { - limit_bytes / element_size - } else { - 0 - }; + let n = limit_bytes.checked_div(element_size).unwrap_or(0); match element_size { 4 => { let mut buf = vec![0f32; n]; @@ -1543,8 +1545,10 @@ mod tests { fn as_f32(bytes: &[u8]) -> Vec { bytes - .chunks_exact(4) - .map(|c| f32::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) .collect() } @@ -1578,8 +1582,10 @@ mod tests { fn as_f64(bytes: &[u8]) -> Vec { bytes - .chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect() } diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index 52f1746..dfba3f0 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -184,9 +184,7 @@ mod tests { buf.extend_from_slice(data); // Pad to 8 bytes let padded = pad8(data.len()); - for _ in data.len()..padded { - buf.push(0); - } + buf.resize(buf.len() + (padded - data.len()), 0); } // Free space marker diff --git a/crates/clawhdf5-format/src/link_message.rs b/crates/clawhdf5-format/src/link_message.rs index 9c4f97d..27044cc 100644 --- a/crates/clawhdf5-format/src/link_message.rs +++ b/crates/clawhdf5-format/src/link_message.rs @@ -413,11 +413,8 @@ mod tests { #[test] fn soft_link() { let target = "/group1/dataset"; - let mut data = Vec::new(); - data.push(1); // version - data.push(0x08); // flags: bit 3 = link type present, name size = 1 byte (bits 0-1 = 0) - data.push(1); // link type = soft - data.push(4); // name length = 4 + // version, flags (bit 3 = link type present, name size = 1 byte), link type = soft, name length = 4 + let mut data = vec![1, 0x08, 1, 4]; data.extend_from_slice(b"link"); data.extend_from_slice(&(target.len() as u16).to_le_bytes()); data.extend_from_slice(target.as_bytes()); @@ -455,12 +452,8 @@ mod tests { #[test] fn invalid_link_type() { - let mut data = Vec::new(); - data.push(1); // version - data.push(0x08); // flags: bit 3 = link type present - data.push(99); // invalid link type - data.push(1); // name length = 1 - data.push(b'x'); + // version, flags (bit 3 = link type present), invalid link type = 99, name length = 1, name = 'x' + let data = vec![1, 0x08, 99, 1, b'x']; let err = LinkMessage::parse(&data, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLinkType(99)); } diff --git a/crates/clawhdf5-format/src/selection.rs b/crates/clawhdf5-format/src/selection.rs index 79d994c..e1f98aa 100644 --- a/crates/clawhdf5-format/src/selection.rs +++ b/crates/clawhdf5-format/src/selection.rs @@ -509,7 +509,7 @@ mod tests { #[test] fn selection_slice_1d() { - let sel = Selection::slice(&[5..15]); + let sel = Selection::slice(std::slice::from_ref(&(5..15))); assert_eq!(sel.num_elements(&[100]), 10); assert_eq!(sel.output_shape(&[100]), vec![10]); } diff --git a/crates/clawhdf5-format/tests/integration_test.rs b/crates/clawhdf5-format/tests/integration_test.rs index 6d9a8bd..7aab9f0 100644 --- a/crates/clawhdf5-format/tests/integration_test.rs +++ b/crates/clawhdf5-format/tests/integration_test.rs @@ -343,7 +343,11 @@ fn attrs_h5_dataset_scale() { let scale_attr = find_attribute(&attrs, "scale").expect("scale attr not found"); let vals = scale_attr.read_as_f64().unwrap(); assert_eq!(vals.len(), 1); - assert!((vals[0] - 3.14).abs() < 1e-10); + // 3.14 here is the literal value baked into the binary fixture (fixtures/attrs.h5), + // not an arbitrary sample value, so it cannot be swapped for another constant. + #[allow(clippy::approx_constant)] + let expected = 3.14; + assert!((vals[0] - expected).abs() < 1e-10); } #[test] @@ -556,8 +560,8 @@ fn chunked_deflate_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -567,8 +571,8 @@ fn chunked_shuffle_deflate_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -578,8 +582,8 @@ fn chunked_fletcher32_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -589,11 +593,10 @@ fn chunked_2d_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix"); let values = read_as_f32(&raw, &datatype).unwrap(); assert_eq!(values.len(), 60); - for i in 0..60 { + for (i, &v) in values.iter().enumerate() { assert!( - (values[i] - i as f32).abs() < 1e-6, - "mismatch at index {i}: got {}", - values[i] + (v - i as f32).abs() < 1e-6, + "mismatch at index {i}: got {v}" ); } } @@ -604,8 +607,8 @@ fn chunked_large_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "big"); let values = read_as_i32(&raw, &datatype).unwrap(); assert_eq!(values.len(), 1000); - for i in 0..1000 { - assert_eq!(values[i], i as i32, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as i32, "mismatch at index {i}"); } } @@ -615,8 +618,8 @@ fn chunked_nofilter_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "raw"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 50); - for i in 0..50 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -646,8 +649,8 @@ fn v4_implicit_read() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -657,8 +660,8 @@ fn v4_fixed_array_read() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -871,11 +874,10 @@ fn v4_2d_fixed_array_read() { let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix"); let values = read_as_f32(&raw, &datatype).unwrap(); assert_eq!(values.len(), 60); - for i in 0..60 { + for (i, &v) in values.iter().enumerate() { assert!( - (values[i] - i as f32).abs() < 1e-6, - "mismatch at index {i}: got {}", - values[i] + (v - i as f32).abs() < 1e-6, + "mismatch at index {i}: got {v}" ); } } @@ -1272,7 +1274,7 @@ fn write_roundtrip_scalar_f64_attr() { let mut fw = FileWriter::new(); fw.create_dataset("data") .with_f64_data(&[1.0]) - .set_attr("scale", AttrValue::F64(3.14)); + .set_attr("scale", AttrValue::F64(3.25)); let bytes = fw.finish().unwrap(); let sig = find_signature(&bytes).unwrap(); @@ -1283,7 +1285,7 @@ fn write_roundtrip_scalar_f64_attr() { let scale = find_attribute(&attrs, "scale").expect("scale attr not found"); let vals = scale.read_as_f64().unwrap(); assert_eq!(vals.len(), 1); - assert!((vals[0] - 3.14).abs() < 1e-10); + assert!((vals[0] - 3.25).abs() < 1e-10); } #[test] diff --git a/crates/clawhdf5-format/tests/reference_tests.rs b/crates/clawhdf5-format/tests/reference_tests.rs index 2b1b1e6..af44b15 100644 --- a/crates/clawhdf5-format/tests/reference_tests.rs +++ b/crates/clawhdf5-format/tests/reference_tests.rs @@ -180,15 +180,20 @@ print('ok') let output = match output { Ok(o) if o.status.success() => o, _ => { + // CI sets CLAWHDF5_REQUIRE_INTEROP=1 so this can't silently skip. + assert!( + !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); eprintln!("skipping h5py_object_reference_roundtrip: python3+h5py not available"); return; } }; let stdout = String::from_utf8(output.stdout).unwrap(); - if !stdout.trim().contains("ok") { - eprintln!("skipping h5py_object_reference_roundtrip: h5py script failed"); - return; - } + assert!( + stdout.trim().contains("ok"), + "h5py reference-file generator did not report ok: {stdout}" + ); // Read the file and parse object references let file_data = std::fs::read(&path).unwrap(); diff --git a/crates/clawhdf5-migrate/src/sqlite_reader.rs b/crates/clawhdf5-migrate/src/sqlite_reader.rs index 817d040..26c04e6 100644 --- a/crates/clawhdf5-migrate/src/sqlite_reader.rs +++ b/crates/clawhdf5-migrate/src/sqlite_reader.rs @@ -185,8 +185,10 @@ fn detect_embedding_dim(conn: &Connection, config: &SchemaConfig) -> SqlResult. fn blob_to_f32(blob: &[u8]) -> Vec { - blob.chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + blob.as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) .collect() } diff --git a/crates/clawhdf5-netcdf4/tests/interop_tests.rs b/crates/clawhdf5-netcdf4/tests/interop_tests.rs index 6205885..17c7373 100644 --- a/crates/clawhdf5-netcdf4/tests/interop_tests.rs +++ b/crates/clawhdf5-netcdf4/tests/interop_tests.rs @@ -10,6 +10,12 @@ use clawhdf5_netcdf4::{AttrValue, NetCDF4File}; // Helpers // --------------------------------------------------------------------------- +/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency +/// is a test failure instead of a silent skip. +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + fn netcdf4_python_available() -> bool { Command::new("python3") .args(["-c", "import netCDF4; print(netCDF4.__version__)"]) @@ -29,6 +35,10 @@ fn xarray_available() -> bool { macro_rules! skip_if_no_netcdf4 { () => { if !netcdf4_python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with netCDF4 is not available" + ); eprintln!("SKIP: python3 with netCDF4 not available"); return; } @@ -38,6 +48,10 @@ macro_rules! skip_if_no_netcdf4 { macro_rules! skip_if_no_xarray { () => { if !xarray_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with xarray is not available" + ); eprintln!("SKIP: python3 with xarray not available"); return; } diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 5604176..d6ebcaa 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -469,13 +469,10 @@ mod tests { let ds = file.dataset("data").unwrap(); // Zero-copy should succeed for contiguous LE f64 on mmap - match ds.read_f64_zerocopy() { - Ok(slice) => { - assert_eq!(slice, &original[..]); - assert_eq!(slice, &ds.read_f64().unwrap()[..]); - } - Err(_) => {} // alignment issue, acceptable - } + if let Ok(slice) = ds.read_f64_zerocopy() { + assert_eq!(slice, &original[..]); + assert_eq!(slice, &ds.read_f64().unwrap()[..]); + } // else: alignment issue, acceptable assert_eq!(ds.read_f64().unwrap(), original); std::fs::remove_file(&path).ok(); diff --git a/crates/clawhdf5/tests/h5py_interop_tests.rs b/crates/clawhdf5/tests/h5py_interop_tests.rs index afb8dba..dc6121a 100644 --- a/crates/clawhdf5/tests/h5py_interop_tests.rs +++ b/crates/clawhdf5/tests/h5py_interop_tests.rs @@ -10,6 +10,12 @@ use clawhdf5::{AttrValue, CompoundTypeBuilder, DType, File, FileBuilder}; // Helpers // --------------------------------------------------------------------------- +/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency +/// is a test failure instead of a silent skip. +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + fn python_available() -> bool { Command::new("python3") .args(["-c", "import h5py; print(h5py.__version__)"]) @@ -21,6 +27,10 @@ fn python_available() -> bool { macro_rules! skip_if_no_python { () => { if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); eprintln!("SKIP: python3 with h5py not available"); return; } diff --git a/crates/clawhdf5/tests/integration_tests.rs b/crates/clawhdf5/tests/integration_tests.rs index 3b23292..e8ddab9 100644 --- a/crates/clawhdf5/tests/integration_tests.rs +++ b/crates/clawhdf5/tests/integration_tests.rs @@ -955,8 +955,10 @@ fn read_selection_all_matches_read_raw_on_chunked_dataset() { let via_read_f64 = ds.read_f64().unwrap(); let via_selection_bytes = ds.read_selection(&Selection::All).unwrap(); let via_selection: Vec = via_selection_bytes - .chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect(); assert_eq!(via_read_f64, data); diff --git a/crates/clawhdf5/tests/zerocopy_tests.rs b/crates/clawhdf5/tests/zerocopy_tests.rs index 7eff7e8..018363f 100644 --- a/crates/clawhdf5/tests/zerocopy_tests.rs +++ b/crates/clawhdf5/tests/zerocopy_tests.rs @@ -315,7 +315,7 @@ fn zerocopy_roundtrip_write_read() { let dir = std::env::temp_dir(); let path = dir.join("zc_roundtrip.h5"); - let original = vec![3.14, 2.718, 1.414, 1.732, 0.577]; + let original = vec![3.25, 2.75, 1.414, 1.732, 0.577]; let mut b = FileBuilder::new(); b.create_dataset("data") .with_f64_data(&original) diff --git a/docs/known-issues.md b/docs/known-issues.md index 4191157..d078acc 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -72,11 +72,38 @@ generated yet; one written with the C API (`H5T_STD_REF`) is needed. ## `clawhdf5-gpu` `gpu_tests` can hang under the default parallel test runner -**Status:** open. Observed 2026-09-18 (RTX 5060 Ti, Linux). +**Status:** fixed 2026-09-19. -**Summary:** during `cargo test --workspace`, the `gpu_tests` binary sat idle -(~1% CPU) for 25+ minutes and had to be killed. Run single-threaded it passes -in seconds (20/20): `cargo test -p clawhdf5-gpu --test gpu_tests -- --test-threads=1`. -Suspected cause: several tests creating wgpu devices concurrently (possibly -compounded by the rest of the workspace's tests loading the machine). Not yet -root-caused; workaround is `--test-threads=1` for that crate. +**Summary:** during `cargo test --workspace` the `gpu_tests` binary sat idle for +25+ minutes. Every test created its own `wgpu::Instance` + device (requesting +adapter-maximum limits) concurrently, and readback used an unbounded +`device.poll(Wait)`. + +**Fix:** tests hold a process-wide lock while they own a device, and +`GpuAccelerator` readback waits time out after 30 s with `GpuError::BufferMap`. + +## Compound datatype versions 1 and 2 are mis-parsed (default libver files) + +**Status:** fixed 2026-09-19. Found by adding a default-libver axis to the h5py +interop tests. + +**Summary:** any compound dataset written with default libver bounds (plain +`h5py.File(path, 'w')`, datatype message version 1) failed to read, typically +with `Overflow("compound member 'x': byte_offset(0) + field_size(4136977) ...")`. +Only `libver='latest'` files (version 3+) and files written by clawhdf5 itself +worked, which is why the existing tests never caught it. + +**Root cause:** `Datatype::parse` skipped 24 bytes of legacy per-member array +fields for v1 where the format has 28 (dimensionality 1 + reserved 3 + +permutation 4 + reserved 4 + 4 dimension sizes 16), and treated v2 like v1 minus +name padding, whereas v2 keeps the 8-byte name padding and has no array fields. + +## Attributes with unsupported datatypes are silently dropped + +**Status:** open. + +**Summary:** `Dataset::attrs()` / `Group::attrs()` in the `clawhdf5` facade return +only attributes convertible to `AttrValue`. An attribute with, e.g., a compound +datatype is omitted from the map with no error or indication that it exists. +Planned: surface these as an explicit `AttrValue` variant (raw bytes + datatype) +or an error, as part of the "no silent skips" robustness work. diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 00cd826..65dc9ea 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -1,9 +1,18 @@ #!/usr/bin/env bash -# CI test script — runs fmt, clippy, tests, and no_std checks. +# CI test script — runs fmt, clippy (all targets + feature matrix), tests, +# Python interop suites, bench compilation, and no_std checks. # # Usage: # ./scripts/ci-test.sh # +# Environment: +# CLAWHDF5_REQUIRE_INTEROP=1 Fail (instead of skip) when python3 with +# h5py/netCDF4/xarray is missing. CI sets this. +# Unset locally, the interop steps are skipped +# if python3+h5py is not importable. +# CLAWHDF5_FUZZ_SECONDS=N Run each cargo-fuzz target for N seconds +# (needs nightly + cargo-fuzz). Default: skip. +# # Exit codes: # 0 — all checks passed # 1 — one or more checks failed @@ -31,23 +40,72 @@ run_step() { fi } +# All steps always run so one failure doesn't hide the others; the summary +# and the exit code at the end are the verdict. + # 1. Format check run_step "cargo fmt --check" cargo fmt --check -# 2. Clippy (exclude clawhdf5-py which needs PyO3/Python) -run_step "cargo clippy" cargo clippy \ +# 2. Clippy over every target (lib, bins, tests, benches, examples). Without +# --all-targets, test and bench code is never linted. clawhdf5-py is +# excluded because it needs PyO3/Python headers. +run_step "cargo clippy --all-targets" cargo clippy \ --workspace \ --exclude clawhdf5-py \ + --all-targets \ -- -D warnings -# 3. Tests (exclude clawhdf5-py) +# 3. Clippy over clawhdf5-format's optional features, which the default +# workspace build never compiles (szip is left out: it needs libaec). +run_step "cargo clippy (format feature matrix)" cargo clippy \ + -p clawhdf5-format \ + --all-targets \ + --features parallel,lz4,zstd,pcodec,fast-checksum \ + -- -D warnings + +# 4. Tests (exclude clawhdf5-py) run_step "cargo test" cargo test \ --workspace \ --exclude clawhdf5-py -# 4. no_std check +run_step "cargo test (format feature matrix)" cargo test \ + -p clawhdf5-format \ + --features parallel,lz4,zstd,pcodec,fast-checksum + +# 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain +# `cargo test` stays hermetic; run them explicitly here. +if python3 -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then + run_step "h5py interop (format, ignored tests)" cargo test \ + -p clawhdf5-format --test writer_h5py_tests -- --include-ignored +else + echo "" + echo "==> [h5py interop] SKIPPED: python3 with h5py not available" + STEPS+=("SKIP: h5py interop (format, ignored tests)") +fi + +# 6. Benches must keep compiling (they are not run). +run_step "cargo bench --no-run" cargo bench \ + --workspace \ + --exclude clawhdf5-py \ + --no-run + +# 7. no_std check run_step "check-nostd.sh" "$SCRIPT_DIR/check-nostd.sh" +# 8. Optional fuzz smoke run +if [ -n "${CLAWHDF5_FUZZ_SECONDS:-}" ]; then + fuzz_smoke() { + local target + cd "$SCRIPT_DIR/../crates/clawhdf5-format" || return 1 + for target in $(cargo +nightly fuzz list); do + echo "--- fuzz: $target" + cargo +nightly fuzz run "$target" -- \ + -max_total_time="$CLAWHDF5_FUZZ_SECONDS" || return 1 + done + } + run_step "fuzz smoke (${CLAWHDF5_FUZZ_SECONDS}s/target)" fuzz_smoke +fi + # Summary echo "" echo "========================================"