From 926dc457e09ee375317e5e38445a4f4c661e798c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 05:36:22 -0700 Subject: [PATCH 1/4] fix(format): parse compound datatype versions 1 and 2 correctly Compound datasets written with default libver bounds (datatype message version 1, i.e. plain h5py.File(path, 'w')) could not be read: the v1 member layout has 28 bytes of legacy array fields after the byte offset (dimensionality 1, reserved 3, permutation 4, reserved 4, four sizes 16) and the parser skipped 24, so every following member was read 4 bytes off. v2 was also wrong: it keeps the 8-byte name padding and has no array fields. Found by adding a default-libver axis to the h5py-generated-file tests (HDF5 2.0 raised the default low bound to 1.8, so "default" files are a distinct format path from libver='latest'). Adds byte-level v1/v2 regression tests, a truncation test, and fuzz corpus seeds for v1 compound and native complex. Co-Authored-By: Claude Fable 5.1 --- .../fuzz_datatype/seed_complex_f64_hdf5_2_0 | Bin 0 -> 28 bytes .../seed_compound_v1_default_libver | Bin 0 -> 180 bytes crates/clawhdf5-format/src/datatype.rs | 119 +++++++++++++++--- .../tests/writer_h5py_tests.rs | 40 +++++- 4 files changed, 137 insertions(+), 22 deletions(-) create mode 100644 crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 create mode 100644 crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_compound_v1_default_libver diff --git a/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 b/crates/clawhdf5-format/fuzz/corpus/fuzz_datatype/seed_complex_f64_hdf5_2_0 new file mode 100644 index 0000000000000000000000000000000000000000..5a2bc929ce2a1edae29ad7c4039a3486037b3b44 GIT binary patch literal 28 hcma!MWMB|rU| Vec { + let f64le: [u8; 20] = [ + 0x11, 0x20, 0x3f, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x34, 0x0b, + 0x00, 0x34, 0xff, 0x03, 0x00, 0x00, + ]; + let i32le: [u8; 12] = [ + 0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + ]; + let mut b = vec![0x16, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00]; + for (name, offset, dt) in [ + (&b"x"[..], 0u32, &f64le[..]), + (&b"y"[..], 8, &f64le[..]), + (&b"id"[..], 16, &i32le[..]), + ] { + let mut padded = name.to_vec(); + padded.resize((name.len() + 1 + 7) & !7, 0); + b.extend_from_slice(&padded); + b.extend_from_slice(&offset.to_le_bytes()); + b.extend_from_slice(&[0u8; 28]); + b.extend_from_slice(dt); + } + b + } + + fn assert_xyid_compound(dt: Datatype) { + match dt { + Datatype::Compound { size, members } => { + assert_eq!(size, 20); + let got: Vec<(&str, u64, u32)> = members + .iter() + .map(|m| (m.name.as_str(), m.byte_offset, m.datatype.type_size())) + .collect(); + assert_eq!(got, vec![("x", 0, 8), ("y", 8, 8), ("id", 16, 4)]); + } + other => panic!("expected Compound, got {other:?}"), + } + } + + #[test] + fn test_compound_v1_default_libver() { + let bytes = compound_v1_bytes(); + let (dt, consumed) = Datatype::parse(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_xyid_compound(dt); + } + + #[test] + fn test_compound_v2_padded_names_no_array_fields() { + // v2 = v1 without the 28 bytes of per-member array fields; names are + // still padded to a multiple of 8 (matches libhdf5's H5O decoder). + let v1 = compound_v1_bytes(); + let mut v2 = vec![0x26, 0x03, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00]; + let mut pos = 8; + for dt_len in [20usize, 20, 12] { + v2.extend_from_slice(&v1[pos..pos + 8 + 4]); // padded name + offset + pos += 8 + 4 + 28; + v2.extend_from_slice(&v1[pos..pos + dt_len]); + pos += dt_len; + } + let (dt, consumed) = Datatype::parse(&v2).unwrap(); + assert_eq!(consumed, v2.len()); + assert_xyid_compound(dt); + } + + #[test] + fn test_compound_v1_truncated_is_error_not_panic() { + let bytes = compound_v1_bytes(); + for cut in 8..bytes.len() { + assert!(Datatype::parse(&bytes[..cut]).is_err(), "cut at {cut}"); + } + } + /// Real datatype message bytes emitted by HDF5 2.0 for the native complex /// type `H5T_COMPLEX_IEEE_F64LE`: class 11, version 5, size 16, followed by /// the base IEEE f64 datatype message. @@ -1172,7 +1254,9 @@ mod tests { // Compound { z: complex f64 @0, k: i64 @16 } as written by HDF5 2.0. // Regression guard: the complex member must consume exactly its own // bytes so the following member parses. - let mut bytes = vec![0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00]; + let mut bytes = vec![ + 0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00, + ]; bytes.extend_from_slice(&COMPLEX_F64_HDF5_2_0); bytes.extend_from_slice(&[b'k', 0x00, 0x10]); bytes.extend_from_slice(&[ @@ -1188,7 +1272,10 @@ mod tests { &members[0].datatype, Datatype::Compound { size: 16, members } if members.len() == 2 )); - assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("k", 16)); + assert_eq!( + (members[1].name.as_str(), members[1].byte_offset), + ("k", 16) + ); } other => panic!("expected Compound, got {other:?}"), } diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index 7506717..58d1660 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -235,17 +235,31 @@ fn h5py_reads_our_array_dataset() { #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_compound() { - let path = std::env::temp_dir().join("clawhdf5_h5py_compound.h5"); + check_h5py_generated_compound("latest", ", libver='latest'"); +} + +/// Same file written with h5py's default format bounds. HDF5 2.0 raised the +/// default low bound to 1.8, so "default" files exercise different on-disk +/// structures than both `libver='latest'` and pre-2.0 defaults. +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_compound_default_libver() { + check_h5py_generated_compound("default", ""); +} + +fn check_h5py_generated_compound(tag: &str, libver_kw: &str) { + let path = std::env::temp_dir().join(format!("clawhdf5_h5py_compound_{tag}.h5")); let gen_script = format!( r#" import h5py, numpy as np dt = np.dtype([('x', 'f8'), ('y', 'f8'), ('id', 'i4')]) data = np.array([(1.0, 2.0, 10), (3.0, 4.0, 20)], dtype=dt) -f = h5py.File('{}', 'w', libver='latest') +f = h5py.File('{}', 'w'{}) f.create_dataset('particles', data=data) f.close() "#, - path.display() + path.display(), + libver_kw ); h5py_read(&path, &gen_script); @@ -363,17 +377,31 @@ else: #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_enum() { - let path = std::env::temp_dir().join("clawhdf5_h5py_enum.h5"); + check_h5py_generated_enum("latest", ", libver='latest'"); +} + +/// Same file written with h5py's default format bounds. HDF5 2.0 raised the +/// default low bound to 1.8, so "default" files exercise different on-disk +/// structures than both `libver='latest'` and pre-2.0 defaults. +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_enum_default_libver() { + check_h5py_generated_enum("default", ""); +} + +fn check_h5py_generated_enum(tag: &str, libver_kw: &str) { + let path = std::env::temp_dir().join(format!("clawhdf5_h5py_enum_{tag}.h5")); let gen_script = format!( r#" import h5py, numpy as np dt = h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype=np.int32) data = np.array([1, 0, 2, 1], dtype=np.int32) -f = h5py.File('{}', 'w', libver='latest') +f = h5py.File('{}', 'w'{}) f.create_dataset('colors', data=data, dtype=dt) f.close() "#, - path.display() + path.display(), + libver_kw ); h5py_read(&path, &gen_script); From 706189c3efae0ff8d5042248bb0d543000b3465e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 05:36:22 -0700 Subject: [PATCH 2/4] fix(gpu): stop gpu_tests hanging under the parallel test runner Every test created its own wgpu instance and device (with adapter-maximum limits) concurrently, which could wedge the driver and hang the suite indefinitely. Tests now hold a process-wide lock while they own a device, and GpuAccelerator readback waits are bounded at 30s so a stuck driver surfaces as GpuError::BufferMap instead of blocking forever. Co-Authored-By: Claude Fable 5.1 --- crates/clawhdf5-gpu/src/accelerator.rs | 7 ++++- crates/clawhdf5-gpu/tests/gpu_tests.rs | 38 ++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-gpu/src/accelerator.rs b/crates/clawhdf5-gpu/src/accelerator.rs index d9ffd70..9ae7ded 100644 --- a/crates/clawhdf5-gpu/src/accelerator.rs +++ b/crates/clawhdf5-gpu/src/accelerator.rs @@ -6,6 +6,9 @@ use crate::shaders; use bytemuck::Pod; use wgpu::util::DeviceExt; +/// Upper bound on a single GPU→CPU readback wait. +const READBACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + /// GPU-accelerated vector search engine. /// /// Upload vectors once, then run many searches against them. @@ -1033,10 +1036,12 @@ impl GpuAccelerator { slice.map_async(wgpu::MapMode::Read, move |result| { let _ = tx.send(result); }); + // Bounded wait: a wedged driver must surface as an error, not hang + // the caller forever. self.device .poll(wgpu::PollType::Wait { submission_index: None, - timeout: None, + timeout: Some(READBACK_TIMEOUT), }) .map_err(|e| GpuError::BufferMap(format!("device poll failed: {e}")))?; rx.recv() diff --git a/crates/clawhdf5-gpu/tests/gpu_tests.rs b/crates/clawhdf5-gpu/tests/gpu_tests.rs index f4c8ca0..8781563 100644 --- a/crates/clawhdf5-gpu/tests/gpu_tests.rs +++ b/crates/clawhdf5-gpu/tests/gpu_tests.rs @@ -6,9 +6,41 @@ mod tests { use clawhdf5_gpu::{GpuAccelerator, GpuError}; - fn skip_if_no_gpu() -> Option { + /// Serialises GPU access across tests. The harness runs tests on many + /// threads; letting each create its own wgpu instance + device (with + /// adapter-maximum limits) at the same time can wedge the driver and hang + /// the whole suite, so every test holds this lock while it owns a device. + static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn gpu_lock() -> std::sync::MutexGuard<'static, ()> { + // A panicking test poisons the lock; the guarded state is `()`. + GPU_LOCK.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// A `GpuAccelerator` plus the lock that keeps other tests off the GPU. + /// Field order matters: the device is dropped before the lock is released. + struct LockedGpu { + gpu: GpuAccelerator, + _guard: std::sync::MutexGuard<'static, ()>, + } + + impl std::ops::Deref for LockedGpu { + type Target = GpuAccelerator; + fn deref(&self) -> &GpuAccelerator { + &self.gpu + } + } + + impl std::ops::DerefMut for LockedGpu { + fn deref_mut(&mut self) -> &mut GpuAccelerator { + &mut self.gpu + } + } + + fn skip_if_no_gpu() -> Option { + let guard = gpu_lock(); match GpuAccelerator::new() { - Ok(gpu) => Some(gpu), + Ok(gpu) => Some(LockedGpu { gpu, _guard: guard }), Err(_) => { eprintln!("SKIPPED: no GPU available"); None @@ -69,6 +101,7 @@ mod tests { #[test] fn test_gpu_availability_detection() { // Should not panic regardless of GPU presence + let _guard = gpu_lock(); let available = GpuAccelerator::is_available(); eprintln!("GPU available: {available}"); } @@ -425,6 +458,7 @@ mod tests { #[test] fn test_graceful_no_gpu_fallback() { // This test just demonstrates the pattern — it always passes + let _guard = gpu_lock(); match GpuAccelerator::new() { Ok(gpu) => { eprintln!("GPU found: {}", gpu.device_info()); From bbe1baa2082890cd21269839760d8533949c3c27 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 05:36:22 -0700 Subject: [PATCH 3/4] ci: lint all targets, run interop suites for real, compile benches - clippy --all-targets plus a clawhdf5-format feature matrix (parallel, lz4, zstd, pcodec, fast-checksum); fix the accumulated lint backlog in test, bench and feature-gated code (no behaviour changes). - Install python3 + h5py/numpy/netCDF4/xarray in the CI container and set CLAWHDF5_REQUIRE_INTEROP=1, which makes a missing interop dependency a test failure. Every h5py/netCDF4 interop test used to skip silently in CI. Run the #[ignore]d writer_h5py_tests suite explicitly. - cargo bench --no-run so benches can't rot; fix bench.rs and memory_bench.rs, which no longer compiled against the current strategy/consolidation APIs. - Optional fuzz smoke run via CLAWHDF5_FUZZ_SECONDS. - CHANGELOG and docs/known-issues.md updated. Co-Authored-By: Claude Fable 5.1 --- .gitea/workflows/ci.yml | 14 + CHANGELOG.md | 34 ++ crates/clawhdf5-agent/benches/bench.rs | 19 +- crates/clawhdf5-agent/benches/memory_bench.rs | 25 +- crates/clawhdf5-agent/src/consolidation.rs | 27 +- crates/clawhdf5-agent/src/entity_extract.rs | 22 +- crates/clawhdf5-agent/src/lib.rs | 386 +++++++++--------- crates/clawhdf5-agent/src/openclaw.rs | 126 +++--- crates/clawhdf5-agent/src/wal.rs | 2 +- crates/clawhdf5-agent/tests/e2e_tests.rs | 4 +- .../clawhdf5-agent/tests/memx_comparison.rs | 8 +- crates/clawhdf5-format/src/attribute.rs | 7 +- crates/clawhdf5-format/src/btree_v2.rs | 1 + crates/clawhdf5-format/src/chunked_read.rs | 8 +- crates/clawhdf5-format/src/data_read.rs | 12 +- crates/clawhdf5-format/src/dataspace.rs | 18 +- crates/clawhdf5-format/src/filters.rs | 36 +- crates/clawhdf5-format/src/global_heap.rs | 4 +- crates/clawhdf5-format/src/link_message.rs | 15 +- crates/clawhdf5-format/src/selection.rs | 2 +- .../clawhdf5-format/tests/integration_test.rs | 52 +-- .../clawhdf5-format/tests/reference_tests.rs | 13 +- crates/clawhdf5-migrate/src/sqlite_reader.rs | 6 +- .../clawhdf5-netcdf4/tests/interop_tests.rs | 14 + crates/clawhdf5/src/lib.rs | 11 +- crates/clawhdf5/tests/h5py_interop_tests.rs | 10 + crates/clawhdf5/tests/integration_tests.rs | 6 +- crates/clawhdf5/tests/zerocopy_tests.rs | 2 +- docs/known-issues.md | 41 +- scripts/ci-test.sh | 68 ++- 30 files changed, 588 insertions(+), 405 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 39751d6..289d7cc 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -22,5 +22,19 @@ jobs: run: rustup component add rustfmt clippy - name: Install thumbv7em-none-eabihf target run: rustup target add thumbv7em-none-eabihf + - name: Install Python interop dependencies + # The interop suites used to skip silently when python3/h5py were + # missing, so they never ran in CI. Install them and make a missing + # dependency a failure (CLAWHDF5_REQUIRE_INTEROP below). + run: | + apt-get update + apt-get install -y --no-install-recommends python3 python3-venv + python3 -m venv /opt/interop + /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray + echo "/opt/interop/bin" >> "$GITHUB_PATH" + - name: Show interop library versions + run: python3 -c "import h5py, netCDF4; print('h5py', h5py.__version__, 'HDF5', h5py.version.hdf5_version, 'netCDF4', netCDF4.__version__)" - name: Run CI script + env: + CLAWHDF5_REQUIRE_INTEROP: "1" run: bash scripts/ci-test.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b04e0..7cc517b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## Unreleased + +### Bug Fixes +- `clawhdf5-format`: compound datatypes written with **default libver bounds** + (datatype message version 1 — what plain `h5py.File(path, 'w')` produces) + were mis-parsed. The v1 member layout carries 28 bytes of legacy array + fields after the byte offset (the parser skipped 24), and v2 pads member + names to 8 bytes and has no array fields at all (the parser did neither), so + every member after the first byte offset was read from the wrong position — + typically surfacing as `Overflow("compound member ...")` on read. Found by + adding a default-libver axis to the h5py interop tests; byte-level regression + tests for v1 and v2 added. +- `clawhdf5-gpu`: `gpu_tests` could hang forever under the default parallel + test runner — every test created its own wgpu instance and device at once. + Tests now serialise GPU access, and GPU→CPU readback waits are bounded + (30 s) so a wedged driver returns `GpuError::BufferMap` instead of blocking. +- `clawhdf5-agent`: `benches/bench.rs` and `benches/memory_bench.rs` no longer + compiled against the current `strategy`/`consolidation` APIs. + +### CI / Testing +- CI now lints every target (`cargo clippy --all-targets`) plus + `clawhdf5-format`'s optional features, compiles all benches, and tests the + format feature matrix. Previously test/bench code and feature-gated modules + were never linted; the accumulated clippy backlog is fixed. +- CI installs python3 + h5py/numpy/netCDF4/xarray and sets + `CLAWHDF5_REQUIRE_INTEROP=1`, which turns a missing interop dependency into a + test **failure**. Until now every h5py/netCDF4 interop test silently skipped + in CI, which is how the HDF5 2.0 compound bug fixed in v2.2.0 reached a user. + The `#[ignore]`d `writer_h5py_tests` suite is run explicitly. +- h5py-generated-file tests now cover default libver bounds as well as + `libver='latest'` (HDF5 2.0 raised the default low bound to 1.8). +- Optional fuzz smoke run (`CLAWHDF5_FUZZ_SECONDS=N scripts/ci-test.sh`); new + datatype corpus seeds for v1 compound and native complex messages. + ## v2.2.0 (2026-09-18) ### Security diff --git a/crates/clawhdf5-agent/benches/bench.rs b/crates/clawhdf5-agent/benches/bench.rs index 60b4758..ef5606b 100644 --- a/crates/clawhdf5-agent/benches/bench.rs +++ b/crates/clawhdf5-agent/benches/bench.rs @@ -483,7 +483,7 @@ fn rayon_benches(c: &mut Criterion) { use rayon::prelude::*; let query_norm = vector_search::compute_norm(&query); let num_cores = rayon::current_num_threads().max(1); - let chunk_size = (n + num_cores - 1) / num_cores; + let chunk_size = n.div_ceil(num_cores); let mut results: Vec<(usize, f32)> = vectors .par_chunks(chunk_size) .enumerate() @@ -537,7 +537,7 @@ fn rayon_benches(c: &mut Criterion) { use rayon::prelude::*; let query_norm = vector_search::compute_norm(&query); let num_cores = rayon::current_num_threads().max(1); - let chunk_size = (n + num_cores - 1) / num_cores; + let chunk_size = n.div_ceil(num_cores); let mut results: Vec<(usize, f32)> = vectors .par_chunks(chunk_size) .enumerate() @@ -766,12 +766,22 @@ fn adaptive_benches(c: &mut Criterion) { .map(|v| vector_search::compute_norm(v)) .collect(); let tombstones = vec![0u8; n]; + let flat: Vec = vectors.iter().flatten().copied().collect(); c.bench_function("adaptive_search_10k", |b| { let hw = HardwareCapabilities::detect(); let strat = strategy::auto_select_strategy(n, &hw); b.iter(|| { - strategy::search_with_metrics(&query, &vectors, &norms, &tombstones, 10, strat, None) + strategy::search_with_metrics( + &query, + &vectors, + &flat, + &norms, + &tombstones, + 10, + strat, + None, + ) }); }); @@ -781,6 +791,7 @@ fn adaptive_benches(c: &mut Criterion) { strategy::search_with_metrics( &query, &vectors, + &flat, &norms, &tombstones, 10, @@ -795,6 +806,7 @@ fn adaptive_benches(c: &mut Criterion) { strategy::search_with_metrics( &query, &vectors, + &flat, &norms, &tombstones, 10, @@ -809,6 +821,7 @@ fn adaptive_benches(c: &mut Criterion) { strategy::search_with_metrics( &query, &vectors, + &flat, &norms, &tombstones, 10, diff --git a/crates/clawhdf5-agent/benches/memory_bench.rs b/crates/clawhdf5-agent/benches/memory_bench.rs index 0f21d3f..b57f18a 100644 --- a/crates/clawhdf5-agent/benches/memory_bench.rs +++ b/crates/clawhdf5-agent/benches/memory_bench.rs @@ -1,6 +1,7 @@ use clawhdf5_agent::bm25::BM25Index; use clawhdf5_agent::consolidation::{ ConsolidationConfig, ConsolidationEngine, ImportanceScorer, ImportanceWeights, MemorySource, + UntrustedSource, }; use clawhdf5_agent::hybrid::{hybrid_search, rrf_hybrid_search}; use clawhdf5_agent::knowledge::KnowledgeCache; @@ -285,7 +286,12 @@ fn consolidation_benches(c: &mut Criterion) { for i in 0..n { let embedding = make_vec(&mut rng, DIM); let chunk = format!("memory record {i} with some content"); - engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64); + engine.add_memory( + chunk, + embedding, + UntrustedSource::User, + now + i as f64, + ); } engine }, @@ -307,9 +313,10 @@ fn consolidation_benches(c: &mut Criterion) { for i in 0..50usize { let embedding = make_vec(&mut rng, DIM); let chunk = format!("existing record {i}"); - engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64); + engine.add_memory(chunk, embedding, UntrustedSource::User, now + i as f64); } let records = engine.records().to_vec(); + let record_refs: Vec<&_> = records.iter().collect(); let weights = ImportanceWeights::default(); let query_embedding = make_vec(&mut rng, DIM); let sample_text = @@ -317,7 +324,7 @@ fn consolidation_benches(c: &mut Criterion) { group.bench_function("bench_importance_scoring", |b| { b.iter(|| { - let surprise = ImportanceScorer::score_surprise(&query_embedding, &records); + let surprise = ImportanceScorer::score_surprise(&query_embedding, &record_refs); let correction = ImportanceScorer::score_correction(&MemorySource::Correction); let length = ImportanceScorer::score_length(sample_text); ImportanceScorer::score_combined(surprise, correction, length, &weights) @@ -354,7 +361,7 @@ fn temporal_benches(c: &mut Criterion) { // Insert benchmark: measure time to insert 10k timestamps one by one group.bench_function("bench_temporal_insert_10k", |b| { b.iter_batched( - || TemporalIndex::new(), + TemporalIndex::new, |mut idx| { for i in 0..N { // Shuffle insertion order slightly using a simple offset pattern @@ -442,7 +449,8 @@ fn large_consolidation_benches(c: &mut Criterion) { let mut group = c.benchmark_group("consolidation_large"); group.sample_size(10); - for (label, n) in [("10k", 10_000usize)] { + { + let (label, n) = ("10k", 10_000usize); group.bench_with_input( BenchmarkId::new("bench_consolidation_cycle", label), &n, @@ -459,7 +467,12 @@ fn large_consolidation_benches(c: &mut Criterion) { for i in 0..n { let embedding = make_vec(&mut rng, DIM); let chunk = format!("memory record {i} with content"); - engine.add_memory(chunk, embedding, MemorySource::User, now + i as f64); + engine.add_memory( + chunk, + embedding, + UntrustedSource::User, + now + i as f64, + ); } engine }, diff --git a/crates/clawhdf5-agent/src/consolidation.rs b/crates/clawhdf5-agent/src/consolidation.rs index 228e1ae..a7023b4 100644 --- a/crates/clawhdf5-agent/src/consolidation.rs +++ b/crates/clawhdf5-agent/src/consolidation.rs @@ -563,7 +563,7 @@ mod tests { #[test] fn test_importance_scorer_surprise_identical() { let emb = unit_vec(4, 0); - let existing = vec![MemoryRecord { + let existing = [MemoryRecord { id: 0, chunk: "existing".to_string(), embedding: emb.clone(), @@ -603,23 +603,20 @@ mod tests { fn test_importance_scorer_length() { assert!((ImportanceScorer::score_length("")).abs() < f32::EPSILON); // 50 words → 0.5 - let fifty_words = std::iter::repeat("word") - .take(50) + let fifty_words = std::iter::repeat_n("word", 50) .collect::>() .join(" "); let s50 = ImportanceScorer::score_length(&fifty_words); assert!((s50 - 0.5).abs() < 1e-5, "expected 0.5, got {s50}"); // 100 words → 1.0 - let hundred_words = std::iter::repeat("word") - .take(100) + let hundred_words = std::iter::repeat_n("word", 100) .collect::>() .join(" "); assert_eq!(ImportanceScorer::score_length(&hundred_words), 1.0); // 200 words → still 1.0 (clamped) - let two_hundred = std::iter::repeat("word") - .take(200) + let two_hundred = std::iter::repeat_n("word", 200) .collect::>() .join(" "); assert_eq!(ImportanceScorer::score_length(&two_hundred), 1.0); @@ -693,9 +690,11 @@ mod tests { // --------------------------------------------------------------------------- #[test] fn test_consolidate_eviction_working() { - let mut cfg = ConsolidationConfig::default(); - cfg.working_capacity = 3; - cfg.working_to_episodic_threshold = 2.0; // never promote in this test + let cfg = ConsolidationConfig { + working_capacity: 3, + working_to_episodic_threshold: 2.0, // never promote in this test + ..Default::default() + }; let mut engine = ConsolidationEngine::new(cfg); // Add 5 records; all have very low importance so none get promoted. @@ -853,9 +852,11 @@ mod tests { // --------------------------------------------------------------------------- #[test] fn test_consolidate_episodic_eviction() { - let mut cfg = ConsolidationConfig::default(); - cfg.episodic_capacity = 3; - cfg.working_to_episodic_threshold = 2.0; // never auto-promote from Working + let cfg = ConsolidationConfig { + episodic_capacity: 3, + working_to_episodic_threshold: 2.0, // never auto-promote from Working + ..Default::default() + }; let mut engine = ConsolidationEngine::new(cfg); // Seed 5 records directly in Episodic. diff --git a/crates/clawhdf5-agent/src/entity_extract.rs b/crates/clawhdf5-agent/src/entity_extract.rs index b6840ca..e334151 100644 --- a/crates/clawhdf5-agent/src/entity_extract.rs +++ b/crates/clawhdf5-agent/src/entity_extract.rs @@ -777,8 +777,10 @@ mod tests { #[test] fn test_tech_disabled() { - let mut config = ExtractorConfig::default(); - config.extract_technology = false; + let config = ExtractorConfig { + extract_technology: false, + ..Default::default() + }; let e = EntityExtractor::new(config); let entities = e.extract("We use Rust and Docker."); assert!( @@ -847,8 +849,10 @@ mod tests { #[test] fn test_date_disabled() { - let mut config = ExtractorConfig::default(); - config.extract_dates = false; + let config = ExtractorConfig { + extract_dates: false, + ..Default::default() + }; let e = EntityExtractor::new(config); let entities = e.extract("Released on 2024-03-19."); assert!( @@ -981,8 +985,10 @@ mod tests { #[test] fn test_confidence_filter() { - let mut config = ExtractorConfig::default(); - config.min_confidence = 0.95; + let config = ExtractorConfig { + min_confidence: 0.95, + ..Default::default() + }; let e = EntityExtractor::new(config); // Only dates (0.95) and techs (0.9) should survive; 0.9 < 0.95 filters techs. let entities = e.extract("We use Rust since 2024-01-01."); @@ -1002,7 +1008,7 @@ mod tests { fn test_batch_dedup() { let e = default_extractor(); let texts = ["We use Rust.", "Rust is fast.", "Also Rust for safety."]; - let entities = e.extract_batch(&texts.iter().map(|s| *s).collect::>()); + let entities = e.extract_batch(&texts); let rust_count = entities.iter().filter(|x| x.text == "Rust").count(); assert_eq!(rust_count, 1, "Rust should appear exactly once after dedup"); } @@ -1011,7 +1017,7 @@ mod tests { fn test_batch_multiple_types() { let e = default_extractor(); let texts = ["Deploy with Docker.", "We merged last week."]; - let entities = e.extract_batch(&texts.iter().map(|s| *s).collect::>()); + let entities = e.extract_batch(&texts); assert!( entities .iter() diff --git a/crates/clawhdf5-agent/src/lib.rs b/crates/clawhdf5-agent/src/lib.rs index 246b946..b7f0310 100644 --- a/crates/clawhdf5-agent/src/lib.rs +++ b/crates/clawhdf5-agent/src/lib.rs @@ -839,6 +839,199 @@ fn is_leap(y: i64) -> bool { (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 } +impl HDF5Memory { + pub fn set_strategy(&mut self, s: Box) { + self.strategy = Some(s); + } + pub fn record(&mut self, exchange: Exchange) -> Result { + let strat = self.strategy.as_ref().ok_or_else(|| { + MemoryError::Schema( + "strategy not initialized: call set_strategy() before record()".to_owned(), + ) + })?; + let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge); + let output = strat.evaluate(&exchange, &view); + for e in &output.entries { + self.cache.push( + e.chunk.clone(), + e.embedding.clone(), + e.source_channel.clone(), + e.timestamp, + e.session_id.clone(), + e.tags.clone(), + ); + } + for eu in &output.entity_updates { + let id = self.knowledge.add_entity(&eu.name, &eu.entity_type, -1); + for a in &eu.aliases { + self.knowledge.add_alias(a, id as i64); + } + } + if !output.entries.is_empty() || !output.entity_updates.is_empty() { + self.flush()?; + } + Ok(output) + } +} + +impl HDF5Memory { + pub fn tick_session(&mut self) -> Result<()> { + let d = self.config.decay_factor; + for w in self.cache.activation_weights.iter_mut() { + *w *= d; + } + self.flush()?; + if let Some(ref mut w) = self.wal { + w.truncate()?; + } + Ok(()) + } + + /// Number of pending WAL entries (0 if WAL disabled). + pub fn wal_pending_count(&self) -> usize { + self.wal.as_ref().map_or(0, |w| w.pending_count() as usize) + } + + /// Explicit WAL merge: flush .h5, truncate WAL. + pub fn flush_wal(&mut self) -> Result<()> { + self.flush()?; + if let Some(ref mut w) = self.wal { + w.truncate()?; + } + Ok(()) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Ephemeral tier integration +// ───────────────────────────────────────────────────────────────────────────── + +impl HDF5Memory { + /// Enable the ephemeral working memory tier with the given configuration. + pub fn enable_ephemeral(&mut self, config: EphemeralConfig) { + self.ephemeral = Some(EphemeralStore::new(config)); + } + + /// Return a shared reference to the ephemeral store, if enabled. + pub fn ephemeral(&self) -> Option<&EphemeralStore> { + self.ephemeral.as_ref() + } + + /// Return a mutable reference to the ephemeral store, if enabled. + pub fn ephemeral_mut(&mut self) -> Option<&mut EphemeralStore> { + self.ephemeral.as_mut() + } + + /// Promote frequently-accessed ephemeral entries into the persistent cache. + /// + /// Every entry whose `access_count >= min_access_count` is removed from the + /// ephemeral store and written to the HDF5 cache, then the file is flushed. + /// Returns the number of entries promoted. + pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { + let candidates = match &self.ephemeral { + None => return Ok(0), + Some(s) => s.promotion_candidates(min_access_count), + }; + + if candidates.is_empty() { + return Ok(0); + } + + let dim = self.config.embedding_dim; + let mut promoted = 0; + + for key in candidates { + let entry = match self + .ephemeral + .as_mut() + .and_then(|s| s.take_for_promotion(&key)) + { + Some(e) => e, + None => continue, + }; + + let chunk = entry + .text + .clone() + .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); + let embedding = entry.embedding.clone().unwrap_or_else(|| vec![0.0f32; dim]); + + self.cache.push( + chunk, + embedding, + format!("ephemeral::{key}"), + entry.created_at, + String::new(), + entry.tags.join(","), + ); + promoted += 1; + } + + if promoted > 0 { + self.flush()?; + } + Ok(promoted) + } + + /// Search both the persistent HDF5 tier and the ephemeral tier, returning + /// the top `k` results sorted by score descending. + /// + /// Ephemeral results are boosted by a factor of 1.2 to surface recent + /// in-context information above older persisted data. + pub fn unified_search( + &mut self, + query_embedding: &[f32], + query_text: &str, + k: usize, + ) -> Vec { + // Persistent tier. + let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k); + const EPHEMERAL_BOOST: f32 = 1.2; + let mut results = persistent; + + if self.ephemeral.is_none() { + return results; + } + + let eph = self.ephemeral.as_mut().unwrap(); + + // Collect (key, score) pairs from ephemeral — borrow ends before we + // access entries again below. + let eph_hits: Vec<(String, f32)> = if !query_embedding.is_empty() { + eph.search_embedding(query_embedding, k) + } else if !query_text.is_empty() { + eph.search_text(query_text, k) + } else { + Vec::new() + }; + + for (key, score) in &eph_hits { + if let Some(entry) = eph.get_entry(key) { + let chunk = entry + .text + .clone() + .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); + results.push(SearchResult { + score: score * EPHEMERAL_BOOST, + chunk, + index: usize::MAX, + timestamp: entry.created_at, + source_channel: format!("ephemeral::{key}"), + activation: 1.0, + }); + } + } + + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + results.truncate(k); + results + } +} + // --- Tests --- #[cfg(test)] @@ -1676,196 +1869,3 @@ mod tests { assert!((mem.cache.tombstone_fraction() - 0.50).abs() < 0.01); } } - -impl HDF5Memory { - pub fn set_strategy(&mut self, s: Box) { - self.strategy = Some(s); - } - pub fn record(&mut self, exchange: Exchange) -> Result { - let strat = self.strategy.as_ref().ok_or_else(|| { - MemoryError::Schema( - "strategy not initialized: call set_strategy() before record()".to_owned(), - ) - })?; - let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge); - let output = strat.evaluate(&exchange, &view); - for e in &output.entries { - self.cache.push( - e.chunk.clone(), - e.embedding.clone(), - e.source_channel.clone(), - e.timestamp, - e.session_id.clone(), - e.tags.clone(), - ); - } - for eu in &output.entity_updates { - let id = self.knowledge.add_entity(&eu.name, &eu.entity_type, -1); - for a in &eu.aliases { - self.knowledge.add_alias(a, id as i64); - } - } - if !output.entries.is_empty() || !output.entity_updates.is_empty() { - self.flush()?; - } - Ok(output) - } -} - -impl HDF5Memory { - pub fn tick_session(&mut self) -> Result<()> { - let d = self.config.decay_factor; - for w in self.cache.activation_weights.iter_mut() { - *w *= d; - } - self.flush()?; - if let Some(ref mut w) = self.wal { - w.truncate()?; - } - Ok(()) - } - - /// Number of pending WAL entries (0 if WAL disabled). - pub fn wal_pending_count(&self) -> usize { - self.wal.as_ref().map_or(0, |w| w.pending_count() as usize) - } - - /// Explicit WAL merge: flush .h5, truncate WAL. - pub fn flush_wal(&mut self) -> Result<()> { - self.flush()?; - if let Some(ref mut w) = self.wal { - w.truncate()?; - } - Ok(()) - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// Ephemeral tier integration -// ───────────────────────────────────────────────────────────────────────────── - -impl HDF5Memory { - /// Enable the ephemeral working memory tier with the given configuration. - pub fn enable_ephemeral(&mut self, config: EphemeralConfig) { - self.ephemeral = Some(EphemeralStore::new(config)); - } - - /// Return a shared reference to the ephemeral store, if enabled. - pub fn ephemeral(&self) -> Option<&EphemeralStore> { - self.ephemeral.as_ref() - } - - /// Return a mutable reference to the ephemeral store, if enabled. - pub fn ephemeral_mut(&mut self) -> Option<&mut EphemeralStore> { - self.ephemeral.as_mut() - } - - /// Promote frequently-accessed ephemeral entries into the persistent cache. - /// - /// Every entry whose `access_count >= min_access_count` is removed from the - /// ephemeral store and written to the HDF5 cache, then the file is flushed. - /// Returns the number of entries promoted. - pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { - let candidates = match &self.ephemeral { - None => return Ok(0), - Some(s) => s.promotion_candidates(min_access_count), - }; - - if candidates.is_empty() { - return Ok(0); - } - - let dim = self.config.embedding_dim; - let mut promoted = 0; - - for key in candidates { - let entry = match self - .ephemeral - .as_mut() - .and_then(|s| s.take_for_promotion(&key)) - { - Some(e) => e, - None => continue, - }; - - let chunk = entry - .text - .clone() - .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); - let embedding = entry.embedding.clone().unwrap_or_else(|| vec![0.0f32; dim]); - - self.cache.push( - chunk, - embedding, - format!("ephemeral::{key}"), - entry.created_at, - String::new(), - entry.tags.join(","), - ); - promoted += 1; - } - - if promoted > 0 { - self.flush()?; - } - Ok(promoted) - } - - /// Search both the persistent HDF5 tier and the ephemeral tier, returning - /// the top `k` results sorted by score descending. - /// - /// Ephemeral results are boosted by a factor of 1.2 to surface recent - /// in-context information above older persisted data. - pub fn unified_search( - &mut self, - query_embedding: &[f32], - query_text: &str, - k: usize, - ) -> Vec { - // Persistent tier. - let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k); - const EPHEMERAL_BOOST: f32 = 1.2; - let mut results = persistent; - - if self.ephemeral.is_none() { - return results; - } - - let eph = self.ephemeral.as_mut().unwrap(); - - // Collect (key, score) pairs from ephemeral — borrow ends before we - // access entries again below. - let eph_hits: Vec<(String, f32)> = if !query_embedding.is_empty() { - eph.search_embedding(query_embedding, k) - } else if !query_text.is_empty() { - eph.search_text(query_text, k) - } else { - Vec::new() - }; - - for (key, score) in &eph_hits { - if let Some(entry) = eph.get_entry(key) { - let chunk = entry - .text - .clone() - .unwrap_or_else(|| String::from_utf8_lossy(&entry.value).into_owned()); - results.push(SearchResult { - score: score * EPHEMERAL_BOOST, - chunk, - index: usize::MAX, - timestamp: entry.created_at, - source_channel: format!("ephemeral::{key}"), - activation: 1.0, - }); - } - } - - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - results.truncate(k); - results - } -} diff --git a/crates/clawhdf5-agent/src/openclaw.rs b/crates/clawhdf5-agent/src/openclaw.rs index 770536c..8bb0acd 100644 --- a/crates/clawhdf5-agent/src/openclaw.rs +++ b/crates/clawhdf5-agent/src/openclaw.rs @@ -748,6 +748,69 @@ impl MemoryBackend for ClawhdfBackend { } } +// ───────────────────────────────────────────────────────────────────────────── +// Ephemeral tier methods on ClawhdfBackend +// ───────────────────────────────────────────────────────────────────────────── + +impl ClawhdfBackend { + /// Enable the ephemeral (in-memory only) working memory tier. + pub fn enable_ephemeral(&mut self, config: crate::ephemeral::EphemeralConfig) { + self.memory.enable_ephemeral(config); + } + + /// Store a text value in ephemeral memory. + /// + /// Returns an error string if the ephemeral tier has not been enabled. + pub fn ephemeral_set( + &mut self, + key: &str, + value: &str, + ttl_secs: Option, + ) -> Result<(), String> { + match self.memory.ephemeral_mut() { + Some(s) => { + s.set_text(key, value, ttl_secs); + Ok(()) + } + None => Err("ephemeral tier not enabled".to_string()), + } + } + + /// Retrieve a text value from ephemeral memory. + /// + /// Returns `None` if the tier is disabled, the key is absent, or the + /// entry has expired. + pub fn ephemeral_get(&mut self, key: &str) -> Option { + self.memory + .ephemeral_mut()? + .get_text(key) + .map(|s| s.to_string()) + } + + /// Delete a key from ephemeral memory. + /// + /// Returns `true` if the key existed and was removed. + pub fn ephemeral_delete(&mut self, key: &str) -> bool { + self.memory.ephemeral_mut().is_some_and(|s| s.delete(key)) + } + + /// Return a snapshot of ephemeral tier statistics, or `None` if the tier + /// is not enabled. + pub fn ephemeral_stats(&self) -> Option { + self.memory.ephemeral().map(|s| s.stats()) + } + + /// Promote frequently-accessed ephemeral entries to persistent HDF5 storage. + /// + /// Entries with `access_count >= min_access_count` are moved from the + /// ephemeral store into the persistent cache. Returns the count promoted. + pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { + self.memory + .promote_ephemeral(min_access_count) + .map_err(|e| e.to_string()) + } +} + // ───────────────────────────────────────────────────────────────────────────── // Tests // ───────────────────────────────────────────────────────────────────────────── @@ -1333,66 +1396,3 @@ mod tests { assert!(out.starts_with("# Title")); } } - -// ───────────────────────────────────────────────────────────────────────────── -// Ephemeral tier methods on ClawhdfBackend -// ───────────────────────────────────────────────────────────────────────────── - -impl ClawhdfBackend { - /// Enable the ephemeral (in-memory only) working memory tier. - pub fn enable_ephemeral(&mut self, config: crate::ephemeral::EphemeralConfig) { - self.memory.enable_ephemeral(config); - } - - /// Store a text value in ephemeral memory. - /// - /// Returns an error string if the ephemeral tier has not been enabled. - pub fn ephemeral_set( - &mut self, - key: &str, - value: &str, - ttl_secs: Option, - ) -> Result<(), String> { - match self.memory.ephemeral_mut() { - Some(s) => { - s.set_text(key, value, ttl_secs); - Ok(()) - } - None => Err("ephemeral tier not enabled".to_string()), - } - } - - /// Retrieve a text value from ephemeral memory. - /// - /// Returns `None` if the tier is disabled, the key is absent, or the - /// entry has expired. - pub fn ephemeral_get(&mut self, key: &str) -> Option { - self.memory - .ephemeral_mut()? - .get_text(key) - .map(|s| s.to_string()) - } - - /// Delete a key from ephemeral memory. - /// - /// Returns `true` if the key existed and was removed. - pub fn ephemeral_delete(&mut self, key: &str) -> bool { - self.memory.ephemeral_mut().is_some_and(|s| s.delete(key)) - } - - /// Return a snapshot of ephemeral tier statistics, or `None` if the tier - /// is not enabled. - pub fn ephemeral_stats(&self) -> Option { - self.memory.ephemeral().map(|s| s.stats()) - } - - /// Promote frequently-accessed ephemeral entries to persistent HDF5 storage. - /// - /// Entries with `access_count >= min_access_count` are moved from the - /// ephemeral store into the persistent cache. Returns the count promoted. - pub fn promote_ephemeral(&mut self, min_access_count: u32) -> Result { - self.memory - .promote_ephemeral(min_access_count) - .map_err(|e| e.to_string()) - } -} diff --git a/crates/clawhdf5-agent/src/wal.rs b/crates/clawhdf5-agent/src/wal.rs index 74467d9..97ff36d 100644 --- a/crates/clawhdf5-agent/src/wal.rs +++ b/crates/clawhdf5-agent/src/wal.rs @@ -786,7 +786,7 @@ mod tests { let dir = TempDir::new().unwrap(); let wal_path = dir.path().join("test.h5.wal"); let unicode_chunk = "Hello 世界! 🌍 émojis & ünïcödé"; - let embedding = vec![0.1, -0.2, 3.14159, f32::MAX, f32::MIN_POSITIVE]; + let embedding = vec![0.1, -0.2, 3.4567, f32::MAX, f32::MIN_POSITIVE]; { let mut wal = WalFile::open(&wal_path).unwrap(); let entry = WalEntry { diff --git a/crates/clawhdf5-agent/tests/e2e_tests.rs b/crates/clawhdf5-agent/tests/e2e_tests.rs index 75504ce..d151ece 100644 --- a/crates/clawhdf5-agent/tests/e2e_tests.rs +++ b/crates/clawhdf5-agent/tests/e2e_tests.rs @@ -1048,7 +1048,7 @@ fn test_gpu_l2_fallback_works() { let tombstones = vec![0u8; 3]; let gpu = clawhdf5_agent::gpu_search::GpuSearchBackend::try_init(&vectors, &norms, 2, 1); - let results = gpu.search_l2(&vec![0.0, 0.0], &vectors, &tombstones, 3); + let results = gpu.search_l2(&[0.0, 0.0], &vectors, &tombstones, 3); assert_eq!(results.len(), 3); assert_eq!(results[0].0, 0); @@ -1099,7 +1099,7 @@ fn test_mmap_reader_direct_access() { // Open via MmapReader directly let mmap = clawhdf5_io::MmapReader::open(&path).unwrap(); - assert!(mmap.len() > 0); + assert!(!mmap.is_empty()); // Verify we can read bytes at specific offsets let bytes = mmap.read_at(0, 8); assert!(bytes.is_some()); diff --git a/crates/clawhdf5-agent/tests/memx_comparison.rs b/crates/clawhdf5-agent/tests/memx_comparison.rs index 7a9af42..2976834 100644 --- a/crates/clawhdf5-agent/tests/memx_comparison.rs +++ b/crates/clawhdf5-agent/tests/memx_comparison.rs @@ -137,10 +137,10 @@ fn bench_hit_at_1_1014_records() { 0.3, 1, ); - if let Some((top_idx, _)) = results.first() { - if *top_idx == target_indices[qi] { - hits += 1; - } + if let Some((top_idx, _)) = results.first() + && *top_idx == target_indices[qi] + { + hits += 1; } } diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index f3137f9..2c8eaca 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -472,14 +472,13 @@ mod tests { // Name padded to 8 bytes data.extend_from_slice(name); - while data.len() % 8 != 0 || data.len() == 8 { + if data.len() % 8 != 0 || data.len() == 8 { // Pad name to 8-byte boundary from start of name let name_start = 8; let name_padded = pad8(name_size); while data.len() < name_start + name_padded { data.push(0); } - break; } // Datatype padded to 8 bytes @@ -749,11 +748,11 @@ mod tests { data.extend_from_slice(name); data.extend_from_slice(&dt_bytes); data.extend_from_slice(&ds_bytes); - data.extend_from_slice(&3.14f64.to_le_bytes()); + data.extend_from_slice(&3.25f64.to_le_bytes()); let attr = AttributeMessage::parse(&data, 8).unwrap(); let vals = attr.read_as_f64().unwrap(); - assert_eq!(vals, vec![3.14]); + assert_eq!(vals, vec![3.25]); } #[test] diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 6192510..ad58aa0 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -416,6 +416,7 @@ fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 { mod tests { use super::*; + #[allow(clippy::too_many_arguments)] fn build_btree_v2_header( tree_type: u8, node_size: u32, diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index 36767d7..43184a3 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -1657,9 +1657,9 @@ mod tests { let chunk_bytes = chunk_size_elems * elem_size; // full chunk allocation // Write chunk data (full chunk size, padding with zeros) - for i in start..end { + for (i, value) in values.iter().enumerate().take(end).skip(start) { let byte_offset = data_offset + (i - start) * elem_size; - file_data[byte_offset..byte_offset + 8].copy_from_slice(&values[i].to_le_bytes()); + file_data[byte_offset..byte_offset + 8].copy_from_slice(&value.to_le_bytes()); } chunk_infos.push(ChunkInfo { @@ -1837,8 +1837,8 @@ mod tests { for chunk_idx in 0..2 { let start = chunk_idx * chunk_elems; let mut chunk_bytes = Vec::new(); - for i in start..start + chunk_elems { - chunk_bytes.extend_from_slice(&values[i].to_le_bytes()); + for value in values.iter().skip(start).take(chunk_elems) { + chunk_bytes.extend_from_slice(&value.to_le_bytes()); } let compressed = compress_chunk(&chunk_bytes, &pipeline, elem_size as u32).unwrap(); diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 4d94899..a469084 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -1763,11 +1763,11 @@ mod tests { fn f16_bits(v: f32) -> u16 { // Encode a few exact values used by the test. match v { - x if x == 0.0 => 0x0000, - x if x == 1.0 => 0x3c00, - x if x == -2.0 => 0xc000, - x if x == 0.5 => 0x3800, - x if x == 65504.0 => 0x7bff, // f16 max + 0.0 => 0x0000, + 1.0 => 0x3c00, + -2.0 => 0xc000, + 0.5 => 0x3800, + 65504.0 => 0x7bff, // f16 max _ => panic!("unsupported test value {v}"), } } @@ -2186,7 +2186,7 @@ mod tests { ], }; let mut raw = Vec::new(); - raw.extend_from_slice(&3.14f64.to_le_bytes()); + raw.extend_from_slice(&3.25f64.to_le_bytes()); raw.extend_from_slice(&42i32.to_le_bytes()); let field = read_compound_field(&raw, &dt, "id").unwrap(); diff --git a/crates/clawhdf5-format/src/dataspace.rs b/crates/clawhdf5-format/src/dataspace.rs index ccdd2c6..efece6d 100644 --- a/crates/clawhdf5-format/src/dataspace.rs +++ b/crates/clawhdf5-format/src/dataspace.rs @@ -189,11 +189,7 @@ mod tests { fn build_v1_dataspace(rank: u8, flags: u8, dims: &[u64], max_dims: Option<&[u64]>) -> Vec { let length_size = 8u8; - let mut buf = Vec::new(); - buf.push(1); // version - buf.push(rank); - buf.push(flags); - buf.push(0); // reserved + let mut buf = vec![1, rank, flags, 0]; // version, rank, flags, reserved buf.extend_from_slice(&[0u8; 4]); // reserved(4) for &d in dims { buf.extend_from_slice(&d.to_le_bytes()); @@ -214,11 +210,7 @@ mod tests { dims: &[u64], max_dims: Option<&[u64]>, ) -> Vec { - let mut buf = Vec::new(); - buf.push(2); // version - buf.push(rank); - buf.push(flags); - buf.push(type_byte); + let mut buf = vec![2, rank, flags, type_byte]; // version, rank, flags, type for &d in dims { buf.extend_from_slice(&d.to_le_bytes()); } @@ -298,11 +290,7 @@ mod tests { #[test] fn v1_with_4byte_length() { - let mut buf = Vec::new(); - buf.push(1); // version - buf.push(1); // rank - buf.push(0); // flags - buf.push(0); // reserved + let mut buf = vec![1, 1, 0, 0]; // version, rank, flags, reserved buf.extend_from_slice(&[0u8; 4]); // reserved(4) buf.extend_from_slice(&10u32.to_le_bytes()); // dim with length_size=4 let ds = Dataspace::parse(&buf, 4).unwrap(); diff --git a/crates/clawhdf5-format/src/filters.rs b/crates/clawhdf5-format/src/filters.rs index 8cf5439..2e99d70 100644 --- a/crates/clawhdf5-format/src/filters.rs +++ b/crates/clawhdf5-format/src/filters.rs @@ -1045,24 +1045,30 @@ fn pcodec_compress(data: &[u8], element_size: usize) -> Result, FormatEr match element_size { 4 => { let nums: Vec = data - .chunks_exact(4) - .map(|b| f32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) .collect(); simple_compress(&nums, &config) .map_err(|e| FormatError::CompressionError(format!("pco: {e}"))) } 8 => { let nums: Vec = data - .chunks_exact(8) - .map(|b| f64::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|b| f64::from_le_bytes(*b)) .collect(); simple_compress(&nums, &config) .map_err(|e| FormatError::CompressionError(format!("pco: {e}"))) } _ => { let nums: Vec = data - .chunks_exact(4) - .map(|b| u32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|b| u32::from_le_bytes(*b)) .collect(); simple_compress(&nums, &config) .map_err(|e| FormatError::CompressionError(format!("pco: {e}"))) @@ -1092,11 +1098,7 @@ fn pcodec_decompress( } else { MAX_DECOMPRESS_SIZE }; - let n = if element_size != 0 { - limit_bytes / element_size - } else { - 0 - }; + let n = limit_bytes.checked_div(element_size).unwrap_or(0); match element_size { 4 => { let mut buf = vec![0f32; n]; @@ -1543,8 +1545,10 @@ mod tests { fn as_f32(bytes: &[u8]) -> Vec { bytes - .chunks_exact(4) - .map(|c| f32::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) .collect() } @@ -1578,8 +1582,10 @@ mod tests { fn as_f64(bytes: &[u8]) -> Vec { bytes - .chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect() } diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index 52f1746..dfba3f0 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -184,9 +184,7 @@ mod tests { buf.extend_from_slice(data); // Pad to 8 bytes let padded = pad8(data.len()); - for _ in data.len()..padded { - buf.push(0); - } + buf.resize(buf.len() + (padded - data.len()), 0); } // Free space marker diff --git a/crates/clawhdf5-format/src/link_message.rs b/crates/clawhdf5-format/src/link_message.rs index 9c4f97d..27044cc 100644 --- a/crates/clawhdf5-format/src/link_message.rs +++ b/crates/clawhdf5-format/src/link_message.rs @@ -413,11 +413,8 @@ mod tests { #[test] fn soft_link() { let target = "/group1/dataset"; - let mut data = Vec::new(); - data.push(1); // version - data.push(0x08); // flags: bit 3 = link type present, name size = 1 byte (bits 0-1 = 0) - data.push(1); // link type = soft - data.push(4); // name length = 4 + // version, flags (bit 3 = link type present, name size = 1 byte), link type = soft, name length = 4 + let mut data = vec![1, 0x08, 1, 4]; data.extend_from_slice(b"link"); data.extend_from_slice(&(target.len() as u16).to_le_bytes()); data.extend_from_slice(target.as_bytes()); @@ -455,12 +452,8 @@ mod tests { #[test] fn invalid_link_type() { - let mut data = Vec::new(); - data.push(1); // version - data.push(0x08); // flags: bit 3 = link type present - data.push(99); // invalid link type - data.push(1); // name length = 1 - data.push(b'x'); + // version, flags (bit 3 = link type present), invalid link type = 99, name length = 1, name = 'x' + let data = vec![1, 0x08, 99, 1, b'x']; let err = LinkMessage::parse(&data, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLinkType(99)); } diff --git a/crates/clawhdf5-format/src/selection.rs b/crates/clawhdf5-format/src/selection.rs index 79d994c..e1f98aa 100644 --- a/crates/clawhdf5-format/src/selection.rs +++ b/crates/clawhdf5-format/src/selection.rs @@ -509,7 +509,7 @@ mod tests { #[test] fn selection_slice_1d() { - let sel = Selection::slice(&[5..15]); + let sel = Selection::slice(std::slice::from_ref(&(5..15))); assert_eq!(sel.num_elements(&[100]), 10); assert_eq!(sel.output_shape(&[100]), vec![10]); } diff --git a/crates/clawhdf5-format/tests/integration_test.rs b/crates/clawhdf5-format/tests/integration_test.rs index 6d9a8bd..7aab9f0 100644 --- a/crates/clawhdf5-format/tests/integration_test.rs +++ b/crates/clawhdf5-format/tests/integration_test.rs @@ -343,7 +343,11 @@ fn attrs_h5_dataset_scale() { let scale_attr = find_attribute(&attrs, "scale").expect("scale attr not found"); let vals = scale_attr.read_as_f64().unwrap(); assert_eq!(vals.len(), 1); - assert!((vals[0] - 3.14).abs() < 1e-10); + // 3.14 here is the literal value baked into the binary fixture (fixtures/attrs.h5), + // not an arbitrary sample value, so it cannot be swapped for another constant. + #[allow(clippy::approx_constant)] + let expected = 3.14; + assert!((vals[0] - expected).abs() < 1e-10); } #[test] @@ -556,8 +560,8 @@ fn chunked_deflate_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -567,8 +571,8 @@ fn chunked_shuffle_deflate_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -578,8 +582,8 @@ fn chunked_fletcher32_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -589,11 +593,10 @@ fn chunked_2d_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix"); let values = read_as_f32(&raw, &datatype).unwrap(); assert_eq!(values.len(), 60); - for i in 0..60 { + for (i, &v) in values.iter().enumerate() { assert!( - (values[i] - i as f32).abs() < 1e-6, - "mismatch at index {i}: got {}", - values[i] + (v - i as f32).abs() < 1e-6, + "mismatch at index {i}: got {v}" ); } } @@ -604,8 +607,8 @@ fn chunked_large_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "big"); let values = read_as_i32(&raw, &datatype).unwrap(); assert_eq!(values.len(), 1000); - for i in 0..1000 { - assert_eq!(values[i], i as i32, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as i32, "mismatch at index {i}"); } } @@ -615,8 +618,8 @@ fn chunked_nofilter_read_values() { let (raw, datatype, _) = read_chunked_dataset(file_data, "raw"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 50); - for i in 0..50 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -646,8 +649,8 @@ fn v4_implicit_read() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -657,8 +660,8 @@ fn v4_fixed_array_read() { let (raw, datatype, _) = read_chunked_dataset(file_data, "data"); let values = read_as_f64(&raw, &datatype).unwrap(); assert_eq!(values.len(), 100); - for i in 0..100 { - assert_eq!(values[i], i as f64, "mismatch at index {i}"); + for (i, &v) in values.iter().enumerate() { + assert_eq!(v, i as f64, "mismatch at index {i}"); } } @@ -871,11 +874,10 @@ fn v4_2d_fixed_array_read() { let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix"); let values = read_as_f32(&raw, &datatype).unwrap(); assert_eq!(values.len(), 60); - for i in 0..60 { + for (i, &v) in values.iter().enumerate() { assert!( - (values[i] - i as f32).abs() < 1e-6, - "mismatch at index {i}: got {}", - values[i] + (v - i as f32).abs() < 1e-6, + "mismatch at index {i}: got {v}" ); } } @@ -1272,7 +1274,7 @@ fn write_roundtrip_scalar_f64_attr() { let mut fw = FileWriter::new(); fw.create_dataset("data") .with_f64_data(&[1.0]) - .set_attr("scale", AttrValue::F64(3.14)); + .set_attr("scale", AttrValue::F64(3.25)); let bytes = fw.finish().unwrap(); let sig = find_signature(&bytes).unwrap(); @@ -1283,7 +1285,7 @@ fn write_roundtrip_scalar_f64_attr() { let scale = find_attribute(&attrs, "scale").expect("scale attr not found"); let vals = scale.read_as_f64().unwrap(); assert_eq!(vals.len(), 1); - assert!((vals[0] - 3.14).abs() < 1e-10); + assert!((vals[0] - 3.25).abs() < 1e-10); } #[test] diff --git a/crates/clawhdf5-format/tests/reference_tests.rs b/crates/clawhdf5-format/tests/reference_tests.rs index 2b1b1e6..af44b15 100644 --- a/crates/clawhdf5-format/tests/reference_tests.rs +++ b/crates/clawhdf5-format/tests/reference_tests.rs @@ -180,15 +180,20 @@ print('ok') let output = match output { Ok(o) if o.status.success() => o, _ => { + // CI sets CLAWHDF5_REQUIRE_INTEROP=1 so this can't silently skip. + assert!( + !std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); eprintln!("skipping h5py_object_reference_roundtrip: python3+h5py not available"); return; } }; let stdout = String::from_utf8(output.stdout).unwrap(); - if !stdout.trim().contains("ok") { - eprintln!("skipping h5py_object_reference_roundtrip: h5py script failed"); - return; - } + assert!( + stdout.trim().contains("ok"), + "h5py reference-file generator did not report ok: {stdout}" + ); // Read the file and parse object references let file_data = std::fs::read(&path).unwrap(); diff --git a/crates/clawhdf5-migrate/src/sqlite_reader.rs b/crates/clawhdf5-migrate/src/sqlite_reader.rs index 817d040..26c04e6 100644 --- a/crates/clawhdf5-migrate/src/sqlite_reader.rs +++ b/crates/clawhdf5-migrate/src/sqlite_reader.rs @@ -185,8 +185,10 @@ fn detect_embedding_dim(conn: &Connection, config: &SchemaConfig) -> SqlResult. fn blob_to_f32(blob: &[u8]) -> Vec { - blob.chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + blob.as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_le_bytes(*b)) .collect() } diff --git a/crates/clawhdf5-netcdf4/tests/interop_tests.rs b/crates/clawhdf5-netcdf4/tests/interop_tests.rs index 6205885..17c7373 100644 --- a/crates/clawhdf5-netcdf4/tests/interop_tests.rs +++ b/crates/clawhdf5-netcdf4/tests/interop_tests.rs @@ -10,6 +10,12 @@ use clawhdf5_netcdf4::{AttrValue, NetCDF4File}; // Helpers // --------------------------------------------------------------------------- +/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency +/// is a test failure instead of a silent skip. +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + fn netcdf4_python_available() -> bool { Command::new("python3") .args(["-c", "import netCDF4; print(netCDF4.__version__)"]) @@ -29,6 +35,10 @@ fn xarray_available() -> bool { macro_rules! skip_if_no_netcdf4 { () => { if !netcdf4_python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with netCDF4 is not available" + ); eprintln!("SKIP: python3 with netCDF4 not available"); return; } @@ -38,6 +48,10 @@ macro_rules! skip_if_no_netcdf4 { macro_rules! skip_if_no_xarray { () => { if !xarray_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with xarray is not available" + ); eprintln!("SKIP: python3 with xarray not available"); return; } diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 5604176..d6ebcaa 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -469,13 +469,10 @@ mod tests { let ds = file.dataset("data").unwrap(); // Zero-copy should succeed for contiguous LE f64 on mmap - match ds.read_f64_zerocopy() { - Ok(slice) => { - assert_eq!(slice, &original[..]); - assert_eq!(slice, &ds.read_f64().unwrap()[..]); - } - Err(_) => {} // alignment issue, acceptable - } + if let Ok(slice) = ds.read_f64_zerocopy() { + assert_eq!(slice, &original[..]); + assert_eq!(slice, &ds.read_f64().unwrap()[..]); + } // else: alignment issue, acceptable assert_eq!(ds.read_f64().unwrap(), original); std::fs::remove_file(&path).ok(); diff --git a/crates/clawhdf5/tests/h5py_interop_tests.rs b/crates/clawhdf5/tests/h5py_interop_tests.rs index afb8dba..dc6121a 100644 --- a/crates/clawhdf5/tests/h5py_interop_tests.rs +++ b/crates/clawhdf5/tests/h5py_interop_tests.rs @@ -10,6 +10,12 @@ use clawhdf5::{AttrValue, CompoundTypeBuilder, DType, File, FileBuilder}; // Helpers // --------------------------------------------------------------------------- +/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency +/// is a test failure instead of a silent skip. +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + fn python_available() -> bool { Command::new("python3") .args(["-c", "import h5py; print(h5py.__version__)"]) @@ -21,6 +27,10 @@ fn python_available() -> bool { macro_rules! skip_if_no_python { () => { if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); eprintln!("SKIP: python3 with h5py not available"); return; } diff --git a/crates/clawhdf5/tests/integration_tests.rs b/crates/clawhdf5/tests/integration_tests.rs index 3b23292..e8ddab9 100644 --- a/crates/clawhdf5/tests/integration_tests.rs +++ b/crates/clawhdf5/tests/integration_tests.rs @@ -955,8 +955,10 @@ fn read_selection_all_matches_read_raw_on_chunked_dataset() { let via_read_f64 = ds.read_f64().unwrap(); let via_selection_bytes = ds.read_selection(&Selection::All).unwrap(); let via_selection: Vec = via_selection_bytes - .chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect(); assert_eq!(via_read_f64, data); diff --git a/crates/clawhdf5/tests/zerocopy_tests.rs b/crates/clawhdf5/tests/zerocopy_tests.rs index 7eff7e8..018363f 100644 --- a/crates/clawhdf5/tests/zerocopy_tests.rs +++ b/crates/clawhdf5/tests/zerocopy_tests.rs @@ -315,7 +315,7 @@ fn zerocopy_roundtrip_write_read() { let dir = std::env::temp_dir(); let path = dir.join("zc_roundtrip.h5"); - let original = vec![3.14, 2.718, 1.414, 1.732, 0.577]; + let original = vec![3.25, 2.75, 1.414, 1.732, 0.577]; let mut b = FileBuilder::new(); b.create_dataset("data") .with_f64_data(&original) diff --git a/docs/known-issues.md b/docs/known-issues.md index 4191157..d078acc 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -72,11 +72,38 @@ generated yet; one written with the C API (`H5T_STD_REF`) is needed. ## `clawhdf5-gpu` `gpu_tests` can hang under the default parallel test runner -**Status:** open. Observed 2026-09-18 (RTX 5060 Ti, Linux). +**Status:** fixed 2026-09-19. -**Summary:** during `cargo test --workspace`, the `gpu_tests` binary sat idle -(~1% CPU) for 25+ minutes and had to be killed. Run single-threaded it passes -in seconds (20/20): `cargo test -p clawhdf5-gpu --test gpu_tests -- --test-threads=1`. -Suspected cause: several tests creating wgpu devices concurrently (possibly -compounded by the rest of the workspace's tests loading the machine). Not yet -root-caused; workaround is `--test-threads=1` for that crate. +**Summary:** during `cargo test --workspace` the `gpu_tests` binary sat idle for +25+ minutes. Every test created its own `wgpu::Instance` + device (requesting +adapter-maximum limits) concurrently, and readback used an unbounded +`device.poll(Wait)`. + +**Fix:** tests hold a process-wide lock while they own a device, and +`GpuAccelerator` readback waits time out after 30 s with `GpuError::BufferMap`. + +## Compound datatype versions 1 and 2 are mis-parsed (default libver files) + +**Status:** fixed 2026-09-19. Found by adding a default-libver axis to the h5py +interop tests. + +**Summary:** any compound dataset written with default libver bounds (plain +`h5py.File(path, 'w')`, datatype message version 1) failed to read, typically +with `Overflow("compound member 'x': byte_offset(0) + field_size(4136977) ...")`. +Only `libver='latest'` files (version 3+) and files written by clawhdf5 itself +worked, which is why the existing tests never caught it. + +**Root cause:** `Datatype::parse` skipped 24 bytes of legacy per-member array +fields for v1 where the format has 28 (dimensionality 1 + reserved 3 + +permutation 4 + reserved 4 + 4 dimension sizes 16), and treated v2 like v1 minus +name padding, whereas v2 keeps the 8-byte name padding and has no array fields. + +## Attributes with unsupported datatypes are silently dropped + +**Status:** open. + +**Summary:** `Dataset::attrs()` / `Group::attrs()` in the `clawhdf5` facade return +only attributes convertible to `AttrValue`. An attribute with, e.g., a compound +datatype is omitted from the map with no error or indication that it exists. +Planned: surface these as an explicit `AttrValue` variant (raw bytes + datatype) +or an error, as part of the "no silent skips" robustness work. diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index 00cd826..65dc9ea 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -1,9 +1,18 @@ #!/usr/bin/env bash -# CI test script — runs fmt, clippy, tests, and no_std checks. +# CI test script — runs fmt, clippy (all targets + feature matrix), tests, +# Python interop suites, bench compilation, and no_std checks. # # Usage: # ./scripts/ci-test.sh # +# Environment: +# CLAWHDF5_REQUIRE_INTEROP=1 Fail (instead of skip) when python3 with +# h5py/netCDF4/xarray is missing. CI sets this. +# Unset locally, the interop steps are skipped +# if python3+h5py is not importable. +# CLAWHDF5_FUZZ_SECONDS=N Run each cargo-fuzz target for N seconds +# (needs nightly + cargo-fuzz). Default: skip. +# # Exit codes: # 0 — all checks passed # 1 — one or more checks failed @@ -31,23 +40,72 @@ run_step() { fi } +# All steps always run so one failure doesn't hide the others; the summary +# and the exit code at the end are the verdict. + # 1. Format check run_step "cargo fmt --check" cargo fmt --check -# 2. Clippy (exclude clawhdf5-py which needs PyO3/Python) -run_step "cargo clippy" cargo clippy \ +# 2. Clippy over every target (lib, bins, tests, benches, examples). Without +# --all-targets, test and bench code is never linted. clawhdf5-py is +# excluded because it needs PyO3/Python headers. +run_step "cargo clippy --all-targets" cargo clippy \ --workspace \ --exclude clawhdf5-py \ + --all-targets \ -- -D warnings -# 3. Tests (exclude clawhdf5-py) +# 3. Clippy over clawhdf5-format's optional features, which the default +# workspace build never compiles (szip is left out: it needs libaec). +run_step "cargo clippy (format feature matrix)" cargo clippy \ + -p clawhdf5-format \ + --all-targets \ + --features parallel,lz4,zstd,pcodec,fast-checksum \ + -- -D warnings + +# 4. Tests (exclude clawhdf5-py) run_step "cargo test" cargo test \ --workspace \ --exclude clawhdf5-py -# 4. no_std check +run_step "cargo test (format feature matrix)" cargo test \ + -p clawhdf5-format \ + --features parallel,lz4,zstd,pcodec,fast-checksum + +# 5. Python interop suites. The h5py writer tests are #[ignore]d so a plain +# `cargo test` stays hermetic; run them explicitly here. +if python3 -c "import h5py" >/dev/null 2>&1 || [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then + run_step "h5py interop (format, ignored tests)" cargo test \ + -p clawhdf5-format --test writer_h5py_tests -- --include-ignored +else + echo "" + echo "==> [h5py interop] SKIPPED: python3 with h5py not available" + STEPS+=("SKIP: h5py interop (format, ignored tests)") +fi + +# 6. Benches must keep compiling (they are not run). +run_step "cargo bench --no-run" cargo bench \ + --workspace \ + --exclude clawhdf5-py \ + --no-run + +# 7. no_std check run_step "check-nostd.sh" "$SCRIPT_DIR/check-nostd.sh" +# 8. Optional fuzz smoke run +if [ -n "${CLAWHDF5_FUZZ_SECONDS:-}" ]; then + fuzz_smoke() { + local target + cd "$SCRIPT_DIR/../crates/clawhdf5-format" || return 1 + for target in $(cargo +nightly fuzz list); do + echo "--- fuzz: $target" + cargo +nightly fuzz run "$target" -- \ + -max_total_time="$CLAWHDF5_FUZZ_SECONDS" || return 1 + done + } + run_step "fuzz smoke (${CLAWHDF5_FUZZ_SECONDS}s/target)" fuzz_smoke +fi + # Summary echo "" echo "========================================" From a3f7c6fe893cd300697f7ff1dff7905766656ee4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 05:36:23 -0700 Subject: [PATCH 4/4] style: cargo fmt --all Formatting only. cargo fmt --check was already failing on main (accel SIMD kernels, agent, format, migrate, bench); CI now enforces it. Co-Authored-By: Claude Fable 5.1 --- crates/clawhdf5-accel/src/avx2.rs | 6 +++++- crates/clawhdf5-accel/src/avx512.rs | 6 +++++- crates/clawhdf5-accel/src/neon.rs | 6 +++++- crates/clawhdf5-accel/src/scalar.rs | 6 +++++- crates/clawhdf5-agent/src/anomaly.rs | 18 +++++++++++++++--- crates/clawhdf5-agent/src/consolidation.rs | 7 ++++++- crates/clawhdf5-agent/src/lib.rs | 5 ++++- crates/clawhdf5-agent/src/wal.rs | 3 +-- .../src/bin/consolidation_efficiency.rs | 7 ++++++- crates/clawhdf5-format/src/symbol_table.rs | 13 ++++++++----- crates/clawhdf5-migrate/src/hdf5_writer.rs | 6 +++++- crates/clawhdf5/src/lib.rs | 4 ++-- crates/clawhdf5/src/reader.rs | 1 - 13 files changed, 67 insertions(+), 21 deletions(-) diff --git a/crates/clawhdf5-accel/src/avx2.rs b/crates/clawhdf5-accel/src/avx2.rs index 754c86c..dcdbf7e 100644 --- a/crates/clawhdf5-accel/src/avx2.rs +++ b/crates/clawhdf5-accel/src/avx2.rs @@ -111,7 +111,11 @@ pub unsafe fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { } let denom = (norm_a * norm_b).sqrt(); - if denom < f32::EPSILON { 0.0 } else { dot / denom } + if denom < f32::EPSILON { + 0.0 + } else { + dot / denom + } } } diff --git a/crates/clawhdf5-accel/src/avx512.rs b/crates/clawhdf5-accel/src/avx512.rs index c794064..b98a07c 100644 --- a/crates/clawhdf5-accel/src/avx512.rs +++ b/crates/clawhdf5-accel/src/avx512.rs @@ -89,7 +89,11 @@ pub unsafe fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { } let denom = (norm_a * norm_b).sqrt(); - if denom < f32::EPSILON { 0.0 } else { dot / denom } + if denom < f32::EPSILON { + 0.0 + } else { + dot / denom + } } } diff --git a/crates/clawhdf5-accel/src/neon.rs b/crates/clawhdf5-accel/src/neon.rs index 4c1e716..495955f 100644 --- a/crates/clawhdf5-accel/src/neon.rs +++ b/crates/clawhdf5-accel/src/neon.rs @@ -94,7 +94,11 @@ pub unsafe fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { } let denom = (norm_a * norm_b).sqrt(); - if denom < f32::EPSILON { 0.0 } else { dot / denom } + if denom < f32::EPSILON { + 0.0 + } else { + dot / denom + } } /// NEON L2 distance. diff --git a/crates/clawhdf5-accel/src/scalar.rs b/crates/clawhdf5-accel/src/scalar.rs index ed9144e..3c9a507 100644 --- a/crates/clawhdf5-accel/src/scalar.rs +++ b/crates/clawhdf5-accel/src/scalar.rs @@ -21,7 +21,11 @@ pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { norm_b += y * y; } let denom = (norm_a * norm_b).sqrt(); - if denom < f32::EPSILON { 0.0 } else { dot / denom } + if denom < f32::EPSILON { + 0.0 + } else { + dot / denom + } } pub fn batch_cosine(query: &[f32], vectors: &[&[f32]], results: &mut [(usize, f32)]) { diff --git a/crates/clawhdf5-agent/src/anomaly.rs b/crates/clawhdf5-agent/src/anomaly.rs index 7474606..9564341 100644 --- a/crates/clawhdf5-agent/src/anomaly.rs +++ b/crates/clawhdf5-agent/src/anomaly.rs @@ -437,7 +437,11 @@ mod tests { fn rate_anomaly_names_offending_session() { let mut det = WriteAnomalyDetector::new(cfg()); for i in 0..11 { - det.record_write(event(1.0 + i as f64 * 0.1, "flood-session", MemorySource::User)); + det.record_write(event( + 1.0 + i as f64 * 0.1, + "flood-session", + MemorySource::User, + )); } let alert = det.check_rate_anomaly().unwrap(); assert!( @@ -454,11 +458,19 @@ mod tests { let mut det = WriteAnomalyDetector::new(cfg()); // 5 sessions with 1 write each (below any per-session limit)... for i in 0..5 { - det.record_write(event(1.0 + i as f64 * 0.1, "minor-session", MemorySource::User)); + det.record_write(event( + 1.0 + i as f64 * 0.1, + "minor-session", + MemorySource::User, + )); } // ...plus one session responsible for the majority of the flood. for i in 0..8 { - det.record_write(event(2.0 + i as f64 * 0.1, "major-session", MemorySource::User)); + det.record_write(event( + 2.0 + i as f64 * 0.1, + "major-session", + MemorySource::User, + )); } let alert = det.check_rate_anomaly().unwrap(); assert!( diff --git a/crates/clawhdf5-agent/src/consolidation.rs b/crates/clawhdf5-agent/src/consolidation.rs index a7023b4..420b7b6 100644 --- a/crates/clawhdf5-agent/src/consolidation.rs +++ b/crates/clawhdf5-agent/src/consolidation.rs @@ -799,7 +799,12 @@ mod tests { #[test] fn test_access_memory_reactivation() { let mut engine = ConsolidationEngine::new(ConsolidationConfig::default()); - let id = engine.add_memory("chunk".to_string(), unit_vec(4, 0), UntrustedSource::User, 0.0); + let id = engine.add_memory( + "chunk".to_string(), + unit_vec(4, 0), + UntrustedSource::User, + 0.0, + ); engine.access_memory(id, 5000.0); let rec = engine.get_by_id(id).unwrap(); diff --git a/crates/clawhdf5-agent/src/lib.rs b/crates/clawhdf5-agent/src/lib.rs index b7f0310..d2be326 100644 --- a/crates/clawhdf5-agent/src/lib.rs +++ b/crates/clawhdf5-agent/src/lib.rs @@ -427,7 +427,10 @@ impl HDF5Memory { if self.provenance.get(record_id as u64).is_none() { return; // nothing recorded yet this session — nothing to check } - if !self.provenance.verify_integrity(record_id as u64, current_chunk) { + if !self + .provenance + .verify_integrity(record_id as u64, current_chunk) + { self.anomaly_alerts.push(anomaly::AnomalyAlert { severity: anomaly::Severity::High, message: format!( diff --git a/crates/clawhdf5-agent/src/wal.rs b/crates/clawhdf5-agent/src/wal.rs index 97ff36d..5f1c78b 100644 --- a/crates/clawhdf5-agent/src/wal.rs +++ b/crates/clawhdf5-agent/src/wal.rs @@ -1228,8 +1228,7 @@ mod tests { drop(wal); // simulate a restart without ever truncating the WAL let mut wal2 = WalFile::open(&wal_path).unwrap(); - wal2.append_save(&make_wal_entry("second", &[2.0])) - .unwrap(); + wal2.append_save(&make_wal_entry("second", &[2.0])).unwrap(); drop(wal2); let entries = WalFile::read_entries(&wal_path).unwrap(); diff --git a/crates/clawhdf5-bench/src/bin/consolidation_efficiency.rs b/crates/clawhdf5-bench/src/bin/consolidation_efficiency.rs index 96e2bb9..629a5d6 100644 --- a/crates/clawhdf5-bench/src/bin/consolidation_efficiency.rs +++ b/crates/clawhdf5-bench/src/bin/consolidation_efficiency.rs @@ -242,7 +242,12 @@ fn run_quality_benchmark() { for i in 0..990 { let chunk = make_noise_content(i); let embedding = make_embedding(i + 100); - engine.add_trusted_memory(chunk, embedding, TrustedSource::System, now + i as f64 * 0.1); + engine.add_trusted_memory( + chunk, + embedding, + TrustedSource::System, + now + i as f64 * 0.1, + ); } println!(" → Inserted {} records total", engine.records().len()); diff --git a/crates/clawhdf5-format/src/symbol_table.rs b/crates/clawhdf5-format/src/symbol_table.rs index 0d09e23..9809c75 100644 --- a/crates/clawhdf5-format/src/symbol_table.rs +++ b/crates/clawhdf5-format/src/symbol_table.rs @@ -80,7 +80,10 @@ impl SymbolTableNode { offset_size: u8, ) -> Result { // signature(4) + version(1) + reserved(1) + number_of_symbols(2) = 8 - if offset.checked_add(8).is_none_or(|end| end > file_data.len()) { + if offset + .checked_add(8) + .is_none_or(|end| end > file_data.len()) + { return Err(FormatError::UnexpectedEof { expected: offset.saturating_add(8), available: file_data.len(), @@ -103,12 +106,12 @@ impl SymbolTableNode { // Each entry: link_name_offset(os) + obj_hdr_addr(os) + cache_type(4) + reserved(4) + scratch(16) let entry_size = os + os + 4 + 4 + 16; let entries_start = offset + 8; - let needed = entries_start - .checked_add(num_symbols * entry_size) - .ok_or(FormatError::UnexpectedEof { + let needed = entries_start.checked_add(num_symbols * entry_size).ok_or( + FormatError::UnexpectedEof { expected: usize::MAX, available: file_data.len(), - })?; + }, + )?; if needed > file_data.len() { return Err(FormatError::UnexpectedEof { expected: needed, diff --git a/crates/clawhdf5-migrate/src/hdf5_writer.rs b/crates/clawhdf5-migrate/src/hdf5_writer.rs index 3b1f8f4..8cf1f18 100644 --- a/crates/clawhdf5-migrate/src/hdf5_writer.rs +++ b/crates/clawhdf5-migrate/src/hdf5_writer.rs @@ -57,7 +57,11 @@ fn iso8601_now() -> String { .as_secs(); let days = (secs / 86_400) as i64; let time_of_day = secs % 86_400; - let (h, m, s) = (time_of_day / 3600, (time_of_day % 3600) / 60, time_of_day % 60); + let (h, m, s) = ( + time_of_day / 3600, + (time_of_day % 3600) / 60, + time_of_day % 60, + ); let (y, mo, d) = civil_from_days(days); format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z") } diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index d6ebcaa..40ae557 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -48,11 +48,11 @@ pub use clawhdf5_format::dict_encoding::{DictEncoded, DictionaryEncoder}; pub use clawhdf5_format::property_list::{ DatasetCreateProps, FileAccessProps, FileCreateProps, lib_version, }; +#[cfg(feature = "provenance")] +pub use clawhdf5_format::provenance; pub use clawhdf5_format::selection::Selection; pub use clawhdf5_format::superblock::swmr_flags; pub use clawhdf5_format::type_builders::{CompoundTypeBuilder, EnumTypeBuilder, FillTime}; -#[cfg(feature = "provenance")] -pub use clawhdf5_format::provenance; #[cfg(test)] mod tests { diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 75d107e..223664d 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -426,7 +426,6 @@ impl<'f> Dataset<'f> { Ok(data_read::read_as_strings(&raw, &dt)?) } - // ----- Selection-based read methods ----- /// Read selected elements as raw bytes.