quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: integrate HNSW into agent search, fix Python 3.14 build

Browse Source 
Resolves two gaps found in a project-state review:

1. Python build was broken: PyO3/numpy 0.23 caps at Python 3.13 but the
   environment has 3.14. Bumped to 0.28 and updated the two breaking APIs
   (PyObject -> Py<PyAny>, allow_threads -> detach). The extension module now
   imports and round-trips under Python 3.14, unblocking cargo build --workspace.

2. The "HNSW vector search over agent memories" headline was unwired:
   clawhdf5-ann had zero dependents and the agent used a linear cosine+BM25 scan.
   - clawhdf5-ann is now a live index: insert, mark_deleted (soft delete with a
     deleted bitset, traversed but never returned), compact, and a format
     version tag (v2) with backward-compatible load of v1 files.
   - clawhdf5-agent wires HNSW behind the `hnsw` feature (ON by default). The
     index mirrors the cache (node id == cache index) and self-heals: it rebuilds
     whenever hnsw_synced_len drifts from cache.len(), so unhooked pushes can't
     desync it. Non-indexable stores (no/zero-dim/mixed embeddings) and queries
     whose dim doesn't match fall back to the exact linear scan.
   - hybrid.rs gains merge_vector_keyword, shared by the linear and HNSW paths.
   - tests/hnsw_integration.rs validates recall vs a brute-force oracle plus
     insert/delete/batch behaviour.

Disable HNSW for exact search with `--no-default-features --features float16`.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-06-03 07:10:48 +00:00
co-authored by Claude Opus 4.8
parent dd3e1d0b11
commit 4137846e31
12 changed files with 781 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-py/src/dataset.rs
+10 -10
View File
@@ -65,7 +65,7 @@ fn dtype_to_numpy_str(dt: &DType) -> &'static str {
impl PyDataset {
/// The shape of the dataset as a tuple.
#[getter]
fn shape(&self, py: Python<'_>) -> PyResult<PyObject> {
fn shape(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let tuple = pyo3::types::PyTuple::new(py, self.cached_shape.iter().map(|&d| d as usize))?;
Ok(tuple.into_any().unbind())
}
@@ -88,7 +88,7 @@ impl PyDataset {
///
/// The full dataset is always read from the underlying file; the index
/// is then applied on the resulting numpy array.
fn __getitem__<'py>(&self, py: Python<'py>, key: &Bound<'py, PyAny>) -> PyResult<PyObject> {
fn __getitem__<'py>(&self, py: Python<'py>, key: &Bound<'py, PyAny>) -> PyResult<Py<PyAny>> {
let arr = self.read_as_numpy(py)?;
let indexed = arr.get_item(key)?;
Ok(indexed.unbind())
@@ -112,7 +112,7 @@ impl PyDataset {
/// Read the full dataset and return it as a numpy array (or list for strings).
///
/// For numeric types, the Rust I/O (file reading + decompression) is
/// performed inside `py.allow_threads()` so that the GIL is released
/// performed inside `py.detach()` so that the GIL is released
/// during the potentially expensive operation. The numpy array
/// construction still happens with the GIL held.
fn read_as_numpy<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
@@ -123,7 +123,7 @@ impl PyDataset {
match &self.cached_dtype {
DType::F64 => {
let data = py
.allow_threads(|| file.dataset(path).and_then(|ds| ds.read_f64()))
.detach(|| file.dataset(path).and_then(|ds| ds.read_f64()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
@@ -132,7 +132,7 @@ impl PyDataset {
}
DType::F32 => {
let data = py
.allow_threads(|| file.dataset(path).and_then(|ds| ds.read_f32()))
.detach(|| file.dataset(path).and_then(|ds| ds.read_f32()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
@@ -141,7 +141,7 @@ impl PyDataset {
}
DType::I32 => {
let data = py
.allow_threads(|| file.dataset(path).and_then(|ds| ds.read_i32()))
.detach(|| file.dataset(path).and_then(|ds| ds.read_i32()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
@@ -150,7 +150,7 @@ impl PyDataset {
}
DType::I64 => {
let data = py
.allow_threads(|| file.dataset(path).and_then(|ds| ds.read_i64()))
.detach(|| file.dataset(path).and_then(|ds| ds.read_i64()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
@@ -161,7 +161,7 @@ impl PyDataset {
// Try zero-copy first (contiguous layout), fall back to
// read_u64 + cast for chunked/compact datasets.
let data: Vec<u8> = py
.allow_threads(|| {
.detach(|| {
let ds = file.dataset(path)?;
match ds.read_u8_zerocopy() {
Ok(slice) => Ok(slice.to_vec()),
@@ -179,7 +179,7 @@ impl PyDataset {
}
DType::U64 => {
let data = py
.allow_threads(|| file.dataset(path).and_then(|ds| ds.read_u64()))
.detach(|| file.dataset(path).and_then(|ds| ds.read_u64()))
.map_err(to_py_err)?;
let nd = ArrayD::from_shape_vec(IxDyn(&shape), data)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(e.to_string()))?;
@@ -190,7 +190,7 @@ impl PyDataset {
// String reads need the GIL for PyList construction, but we
// release it during the Rust I/O portion.
let data = py
.allow_threads(|| file.dataset(path).and_then(|ds| ds.read_string()))
.detach(|| file.dataset(path).and_then(|ds| ds.read_string()))
.map_err(to_py_err)?;
let list = PyList::new(py, &data)?;
Ok(list.into_any())