Files
clawhdf5/crates/clawhdf5-py/src/attrs.rs
T
osobhandClaude Opus 4.8 8f9dbd812c feat: integrate HNSW into agent search, fix Python 3.14 build
Resolves two gaps found in a project-state review:

1. Python build was broken: PyO3/numpy 0.23 caps at Python 3.13 but the
   environment has 3.14. Bumped to 0.28 and updated the two breaking APIs
   (PyObject -> Py<PyAny>, allow_threads -> detach). The extension module now
   imports and round-trips under Python 3.14, unblocking cargo build --workspace.

2. The "HNSW vector search over agent memories" headline was unwired:
   clawhdf5-ann had zero dependents and the agent used a linear cosine+BM25 scan.
   - clawhdf5-ann is now a live index: insert, mark_deleted (soft delete with a
     deleted bitset, traversed but never returned), compact, and a format
     version tag (v2) with backward-compatible load of v1 files.
   - clawhdf5-agent wires HNSW behind the `hnsw` feature (ON by default). The
     index mirrors the cache (node id == cache index) and self-heals: it rebuilds
     whenever hnsw_synced_len drifts from cache.len(), so unhooked pushes can't
     desync it. Non-indexable stores (no/zero-dim/mixed embeddings) and queries
     whose dim doesn't match fall back to the exact linear scan.
   - hybrid.rs gains merge_vector_keyword, shared by the linear and HNSW paths.
   - tests/hnsw_integration.rs validates recall vs a brute-force oracle plus
     insert/delete/batch behaviour.

Disable HNSW for exact search with `--no-default-features --features float16`.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
2026-06-03 07:10:48 +00:00

202 lines
6.5 KiB
Rust

//! PyAttrs — dict-like access to HDF5 attributes.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use pyo3::prelude::*;
use pyo3::types::PyList;
use crate::{OwnedAttrValue, attr_value_to_py, py_to_attr_value};
/// Backing storage for attributes.
enum AttrsInner {
/// Read-only attributes from an existing HDF5 object.
Read(HashMap<String, clawhdf5_rs::AttrValue>),
/// Writable attribute list shared with a parent (PyFile or PyGroup).
Write(Arc<Mutex<Vec<(String, OwnedAttrValue)>>>),
}
/// Dict-like access to HDF5 attributes.
///
/// In read mode, provides immutable access to attribute key/value pairs.
/// In write mode, attributes set here are accumulated and written when
/// the parent file is closed.
#[pyclass(name = "Attrs")]
pub struct PyAttrs {
inner: AttrsInner,
}
impl PyAttrs {
/// Create a read-only attrs from an existing attribute map.
pub(crate) fn from_read(map: HashMap<String, clawhdf5_rs::AttrValue>) -> Self {
Self {
inner: AttrsInner::Read(map),
}
}
/// Create a writable attrs that shares storage with a parent object.
pub(crate) fn from_write(store: Arc<Mutex<Vec<(String, OwnedAttrValue)>>>) -> Self {
Self {
inner: AttrsInner::Write(store),
}
}
}
#[pymethods]
impl PyAttrs {
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
match &self.inner {
AttrsInner::Read(map) => match map.get(key) {
Some(val) => Ok(attr_value_to_py(py, val)),
None => Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(
key.to_string(),
)),
},
AttrsInner::Write(store) => {
let guard = store.lock().unwrap();
for (k, v) in guard.iter() {
if k == key {
let attr_val: clawhdf5_rs::AttrValue = v.clone().into();
return Ok(attr_value_to_py(py, &attr_val));
}
}
Err(PyErr::new::<pyo3::exceptions::PyKeyError, _>(
key.to_string(),
))
}
}
}
fn __setitem__(&self, key: &str, value: &Bound<'_, PyAny>) -> PyResult<()> {
match &self.inner {
AttrsInner::Read(_) => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
"cannot set attributes on a read-only file",
)),
AttrsInner::Write(store) => {
let owned = py_to_attr_value(value)?;
let mut guard = store.lock().unwrap();
// Replace existing key if present.
if let Some(entry) = guard.iter_mut().find(|(k, _)| k == key) {
entry.1 = owned;
} else {
guard.push((key.to_string(), owned));
}
Ok(())
}
}
}
fn __len__(&self) -> usize {
match &self.inner {
AttrsInner::Read(map) => map.len(),
AttrsInner::Write(store) => store.lock().unwrap().len(),
}
}
fn __contains__(&self, key: &str) -> bool {
match &self.inner {
AttrsInner::Read(map) => map.contains_key(key),
AttrsInner::Write(store) => store.lock().unwrap().iter().any(|(k, _)| k == key),
}
}
fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let keys = self.keys(py)?;
let iter = keys.call_method0(py, "__iter__")?;
Ok(iter)
}
fn __repr__(&self) -> String {
let n = self.__len__();
format!("<HDF5 Attrs ({n} members)>")
}
/// Return attribute names as a list.
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let names: Vec<String> = match &self.inner {
AttrsInner::Read(map) => map.keys().cloned().collect(),
AttrsInner::Write(store) => store
.lock()
.unwrap()
.iter()
.map(|(k, _)| k.clone())
.collect(),
};
let list = PyList::new(py, &names)?;
Ok(list.into_any().unbind())
}
/// Return attribute values as a list.
fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let vals: Vec<Py<PyAny>> = match &self.inner {
AttrsInner::Read(map) => map.values().map(|v| attr_value_to_py(py, v)).collect(),
AttrsInner::Write(store) => store
.lock()
.unwrap()
.iter()
.map(|(_, v)| {
let attr: clawhdf5_rs::AttrValue = v.clone().into();
attr_value_to_py(py, &attr)
})
.collect(),
};
let list = PyList::new(py, &vals)?;
Ok(list.into_any().unbind())
}
/// Return attribute (key, value) pairs as a list of tuples.
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let pairs: Vec<(String, Py<PyAny>)> = match &self.inner {
AttrsInner::Read(map) => map
.iter()
.map(|(k, v)| (k.clone(), attr_value_to_py(py, v)))
.collect(),
AttrsInner::Write(store) => store
.lock()
.unwrap()
.iter()
.map(|(k, v)| {
let attr: clawhdf5_rs::AttrValue = v.clone().into();
(k.clone(), attr_value_to_py(py, &attr))
})
.collect(),
};
let list = PyList::new(py, &pairs)?;
Ok(list.into_any().unbind())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn read_attrs_len() {
let mut map = HashMap::new();
map.insert("a".into(), clawhdf5_rs::AttrValue::I64(1));
map.insert("b".into(), clawhdf5_rs::AttrValue::F64(2.0));
let attrs = PyAttrs::from_read(map);
assert_eq!(attrs.__len__(), 2);
}
#[test]
fn read_attrs_contains() {
let mut map = HashMap::new();
map.insert("x".into(), clawhdf5_rs::AttrValue::String("hello".into()));
let attrs = PyAttrs::from_read(map);
assert!(attrs.__contains__("x"));
assert!(!attrs.__contains__("y"));
}
#[test]
fn write_attrs_len() {
let store = Arc::new(Mutex::new(Vec::new()));
store
.lock()
.unwrap()
.push(("key".into(), OwnedAttrValue::I64(99)));
let attrs = PyAttrs::from_write(store);
assert_eq!(attrs.__len__(), 1);
}
}