security: Tier 3 — Android JNI length validation, pyo3 bump, WAL caps
CI / test (push) Failing after 2s
CI / test (push) Failing after 2s
- clawhdf5-android: validate embedding_len/query_embedding_len against the handle's configured embedding_dim (and reject null pointers) before constructing a slice via from_raw_parts in edgehdf5_save and edgehdf5_hybrid_search. Strengthen the # Safety docs to state the now-enforced invariant and its limits. Add unit tests covering mismatched length and null-pointer rejection. - clawhdf5-py: bump pyo3/numpy 0.28 -> 0.29, clearing RUSTSEC-2026-0176 (OOB read in PyList/PyTuple iterator) and RUSTSEC-2026-0177 (missing Sync bound on PyCFunction::new_closure). No source changes needed; confirmed via cargo audit that both advisories no longer appear. - clawhdf5-agent/wal.rs: cap read_len_prefixed_str/read_embedding's length claims at a new MAX_WAL_FIELD_LEN (64 MiB) before allocating, so a corrupted/truncated WAL length field fails cleanly instead of attempting a huge allocation. Add regression tests for both. - BENCHMARKS.md: add a top-of-file traceability note distinguishing the dated/hardware-cited/reproducible h5bench and tank-validation sections from the older sections that don't yet meet that bar.
This commit is contained in:
@@ -6,6 +6,13 @@
|
|||||||
**Rust:** 1.96.0-nightly (2026-03-14) · `--release` profile
|
**Rust:** 1.96.0-nightly (2026-03-14) · `--release` profile
|
||||||
**Date:** 2026-07-01
|
**Date:** 2026-07-01
|
||||||
|
|
||||||
|
> **Traceability note:** the "h5bench-Equivalent I/O Benchmarks" and
|
||||||
|
> "Independent Validation: tank" sections below meet a dated,
|
||||||
|
> hardware-cited, reproducible standard (explicit date, machine spec, and a
|
||||||
|
> runnable command per result). The sections above them do not yet meet
|
||||||
|
> that bar consistently — this is a known, tracked documentation gap, not
|
||||||
|
> a claim that those numbers are wrong.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Vector Search Latency
|
## Vector Search Latency
|
||||||
|
|||||||
@@ -12,6 +12,11 @@ use crate::MemoryError;
|
|||||||
const WAL_MAGIC: [u8; 4] = [0x45, 0x48, 0x57, 0x4C]; // "EHWL"
|
const WAL_MAGIC: [u8; 4] = [0x45, 0x48, 0x57, 0x4C]; // "EHWL"
|
||||||
const WAL_VERSION: u8 = 1;
|
const WAL_VERSION: u8 = 1;
|
||||||
|
|
||||||
|
/// Upper bound on a single length-prefixed WAL field (string bytes, or
|
||||||
|
/// embedding element count), to reject a corrupted/truncated WAL length
|
||||||
|
/// claim before allocating a large buffer for it.
|
||||||
|
const MAX_WAL_FIELD_LEN: usize = 64 * 1024 * 1024;
|
||||||
|
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum WalEntryType {
|
pub enum WalEntryType {
|
||||||
@@ -349,6 +354,11 @@ fn read_len_prefixed_str(f: &mut File) -> Result<String, MemoryError> {
|
|||||||
let mut len_buf = [0u8; 4];
|
let mut len_buf = [0u8; 4];
|
||||||
f.read_exact(&mut len_buf)?;
|
f.read_exact(&mut len_buf)?;
|
||||||
let len = u32::from_le_bytes(len_buf) as usize;
|
let len = u32::from_le_bytes(len_buf) as usize;
|
||||||
|
if len > MAX_WAL_FIELD_LEN {
|
||||||
|
return Err(MemoryError::Schema(format!(
|
||||||
|
"WAL string field length {len} exceeds max {MAX_WAL_FIELD_LEN}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
let mut buf = vec![0u8; len];
|
let mut buf = vec![0u8; len];
|
||||||
f.read_exact(&mut buf)?;
|
f.read_exact(&mut buf)?;
|
||||||
String::from_utf8(buf).map_err(|e| MemoryError::Schema(format!("invalid UTF-8 in WAL: {e}")))
|
String::from_utf8(buf).map_err(|e| MemoryError::Schema(format!("invalid UTF-8 in WAL: {e}")))
|
||||||
@@ -358,6 +368,12 @@ fn read_embedding(f: &mut File) -> Result<Vec<f32>, MemoryError> {
|
|||||||
let mut len_buf = [0u8; 4];
|
let mut len_buf = [0u8; 4];
|
||||||
f.read_exact(&mut len_buf)?;
|
f.read_exact(&mut len_buf)?;
|
||||||
let count = u32::from_le_bytes(len_buf) as usize;
|
let count = u32::from_le_bytes(len_buf) as usize;
|
||||||
|
if count > MAX_WAL_FIELD_LEN / 4 {
|
||||||
|
return Err(MemoryError::Schema(format!(
|
||||||
|
"WAL embedding element count {count} exceeds max {}",
|
||||||
|
MAX_WAL_FIELD_LEN / 4
|
||||||
|
)));
|
||||||
|
}
|
||||||
let mut vals = Vec::with_capacity(count);
|
let mut vals = Vec::with_capacity(count);
|
||||||
for _ in 0..count {
|
for _ in 0..count {
|
||||||
let mut val_buf = [0u8; 4];
|
let mut val_buf = [0u8; 4];
|
||||||
@@ -427,6 +443,40 @@ mod tests {
|
|||||||
assert_eq!(entries[2].embedding, vec![5.0, 6.0]);
|
assert_eq!(entries[2].embedding, vec![5.0, 6.0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_len_prefixed_str_rejects_oversized_len_claim() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let path = dir.path().join("oversized_str.bin");
|
||||||
|
{
|
||||||
|
let mut f = File::create(&path).unwrap();
|
||||||
|
// Claim a length far beyond MAX_WAL_FIELD_LEN; no payload follows.
|
||||||
|
f.write_all(&(u32::MAX).to_le_bytes()).unwrap();
|
||||||
|
}
|
||||||
|
let mut f = File::open(&path).unwrap();
|
||||||
|
let result = read_len_prefixed_str(&mut f);
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(MemoryError::Schema(_))),
|
||||||
|
"expected a clean Schema error, got {result:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_embedding_rejects_oversized_count_claim() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let path = dir.path().join("oversized_embedding.bin");
|
||||||
|
{
|
||||||
|
let mut f = File::create(&path).unwrap();
|
||||||
|
// Claim a count far beyond MAX_WAL_FIELD_LEN / 4; no payload follows.
|
||||||
|
f.write_all(&(u32::MAX).to_le_bytes()).unwrap();
|
||||||
|
}
|
||||||
|
let mut f = File::open(&path).unwrap();
|
||||||
|
let result = read_embedding(&mut f);
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(MemoryError::Schema(_))),
|
||||||
|
"expected a clean Schema error, got {result:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_wal_truncate() {
|
fn test_wal_truncate() {
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
|
|||||||
@@ -10,3 +10,6 @@ crate-type = ["cdylib"]
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
clawhdf5-agent = { path = "../clawhdf5-agent", default-features = false }
|
clawhdf5-agent = { path = "../clawhdf5-agent", default-features = false }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tempfile = "3"
|
||||||
|
|||||||
@@ -92,11 +92,18 @@ pub unsafe extern "C" fn edgehdf5_close(handle: Handle) {
|
|||||||
|
|
||||||
/// Save a memory entry. Returns the entry index, or -1 on failure.
|
/// Save a memory entry. Returns the entry index, or -1 on failure.
|
||||||
///
|
///
|
||||||
|
/// `embedding_len` is validated against the handle's configured
|
||||||
|
/// `embedding_dim` before the input slice is constructed; a mismatch fails
|
||||||
|
/// the call with -1 rather than reading out of bounds. This is a length
|
||||||
|
/// check only — it cannot detect a same-length buffer that is otherwise
|
||||||
|
/// too short or invalid.
|
||||||
|
///
|
||||||
/// # Safety
|
/// # Safety
|
||||||
///
|
///
|
||||||
/// - `handle` must be a valid, non-null handle.
|
/// - `handle` must be a valid, non-null handle.
|
||||||
/// - All `*const c_char` arguments must be valid, null-terminated C strings.
|
/// - All `*const c_char` arguments must be valid, null-terminated C strings.
|
||||||
/// - `embedding_ptr` must point to at least `embedding_len` contiguous `f32` values.
|
/// - If `embedding_len` matches the handle's `embedding_dim`, `embedding_ptr`
|
||||||
|
/// must point to at least that many contiguous, valid `f32` values.
|
||||||
#[unsafe(no_mangle)]
|
#[unsafe(no_mangle)]
|
||||||
pub unsafe extern "C" fn edgehdf5_save(
|
pub unsafe extern "C" fn edgehdf5_save(
|
||||||
handle: Handle,
|
handle: Handle,
|
||||||
@@ -135,8 +142,14 @@ pub unsafe extern "C" fn edgehdf5_save(
|
|||||||
None => return -1,
|
None => return -1,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if embedding_ptr.is_null() || embedding_len as usize != mem.config().embedding_dim {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
let embedding =
|
let embedding =
|
||||||
// SAFETY: JNI caller guarantees embedding_ptr points to embedding_len valid f32 values.
|
// SAFETY: embedding_ptr is non-null and embedding_len matches the handle's configured
|
||||||
|
// embedding_dim (checked above); JNI caller guarantees it points to that many valid f32
|
||||||
|
// values. A mismatched-but-equal-length short buffer is not caught by this length check
|
||||||
|
// alone — the caller is still responsible for pointer validity.
|
||||||
unsafe { std::slice::from_raw_parts(embedding_ptr, embedding_len as usize) }.to_vec();
|
unsafe { std::slice::from_raw_parts(embedding_ptr, embedding_len as usize) }.to_vec();
|
||||||
|
|
||||||
let entry = MemoryEntry {
|
let entry = MemoryEntry {
|
||||||
@@ -210,11 +223,18 @@ pub unsafe extern "C" fn edgehdf5_delete(handle: Handle, index: u64) -> i32 {
|
|||||||
/// Performs hybrid search and writes up to `max_results` entries into the
|
/// Performs hybrid search and writes up to `max_results` entries into the
|
||||||
/// provided output arrays. Returns the number of results written.
|
/// provided output arrays. Returns the number of results written.
|
||||||
///
|
///
|
||||||
|
/// `query_embedding_len` is validated against the handle's configured
|
||||||
|
/// `embedding_dim` before the input slice is constructed; a mismatch fails
|
||||||
|
/// the call (returns 0) rather than reading out of bounds. This is a length
|
||||||
|
/// check only — it cannot detect a same-length buffer that is otherwise too
|
||||||
|
/// short or invalid.
|
||||||
|
///
|
||||||
/// # Safety
|
/// # Safety
|
||||||
///
|
///
|
||||||
/// - `handle` must be a valid, non-null handle.
|
/// - `handle` must be a valid, non-null handle.
|
||||||
/// - `query_text` must be a valid, null-terminated C string.
|
/// - `query_text` must be a valid, null-terminated C string.
|
||||||
/// - `query_embedding_ptr` must point to at least `query_embedding_len` `f32` values.
|
/// - If `query_embedding_len` matches the handle's `embedding_dim`,
|
||||||
|
/// `query_embedding_ptr` must point to at least that many valid `f32` values.
|
||||||
/// - `out_indices` and `out_scores` must point to arrays of at least `max_results` elements.
|
/// - `out_indices` and `out_scores` must point to arrays of at least `max_results` elements.
|
||||||
/// - `out_chunks` must be null or point to an array of at least `max_results` pointers.
|
/// - `out_chunks` must be null or point to an array of at least `max_results` pointers.
|
||||||
#[unsafe(no_mangle)]
|
#[unsafe(no_mangle)]
|
||||||
@@ -240,8 +260,14 @@ pub unsafe extern "C" fn edgehdf5_hybrid_search(
|
|||||||
Some(s) => s,
|
Some(s) => s,
|
||||||
None => return 0,
|
None => return 0,
|
||||||
};
|
};
|
||||||
|
if query_embedding_ptr.is_null() || query_embedding_len as usize != mem.config().embedding_dim {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
let query_embedding =
|
let query_embedding =
|
||||||
// SAFETY: JNI caller guarantees query_embedding_ptr points to query_embedding_len valid f32 values.
|
// SAFETY: query_embedding_ptr is non-null and query_embedding_len matches the handle's
|
||||||
|
// configured embedding_dim (checked above); JNI caller guarantees it points to that many
|
||||||
|
// valid f32 values. A mismatched-but-equal-length short buffer is not caught by this
|
||||||
|
// length check alone — the caller is still responsible for pointer validity.
|
||||||
unsafe { std::slice::from_raw_parts(query_embedding_ptr, query_embedding_len as usize) };
|
unsafe { std::slice::from_raw_parts(query_embedding_ptr, query_embedding_len as usize) };
|
||||||
|
|
||||||
let results = mem.hybrid_search(
|
let results = mem.hybrid_search(
|
||||||
@@ -456,3 +482,112 @@ unsafe fn cstr_to_string(ptr: *const c_char) -> Option<String> {
|
|||||||
.ok()
|
.ok()
|
||||||
.map(String::from)
|
.map(String::from)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const EMBEDDING_DIM: u32 = 4;
|
||||||
|
|
||||||
|
fn open_handle(dir: &tempfile::TempDir) -> Handle {
|
||||||
|
let path = CString::new(dir.path().join("mem.h5").to_str().unwrap()).unwrap();
|
||||||
|
let agent_id = CString::new("test-agent").unwrap();
|
||||||
|
// SAFETY: both C strings are valid and null-terminated.
|
||||||
|
unsafe { edgehdf5_create(path.as_ptr(), agent_id.as_ptr(), EMBEDDING_DIM) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn save_rejects_mismatched_embedding_len() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let handle = open_handle(&dir);
|
||||||
|
assert!(!handle.is_null());
|
||||||
|
|
||||||
|
let embedding = [1.0f32, 2.0, 3.0]; // len 3, dim is 4
|
||||||
|
let chunk = CString::new("hello").unwrap();
|
||||||
|
let channel = CString::new("test").unwrap();
|
||||||
|
let session = CString::new("s1").unwrap();
|
||||||
|
let tags = CString::new("").unwrap();
|
||||||
|
|
||||||
|
// SAFETY: handle is valid; all C strings are valid; embedding_len (3) intentionally
|
||||||
|
// does not match embedding_dim (4), which edgehdf5_save must reject before touching
|
||||||
|
// embedding_ptr.
|
||||||
|
let result = unsafe {
|
||||||
|
edgehdf5_save(
|
||||||
|
handle,
|
||||||
|
chunk.as_ptr(),
|
||||||
|
embedding.as_ptr(),
|
||||||
|
embedding.len() as u32,
|
||||||
|
channel.as_ptr(),
|
||||||
|
0.0,
|
||||||
|
session.as_ptr(),
|
||||||
|
tags.as_ptr(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(result, -1, "mismatched embedding_len must be rejected");
|
||||||
|
|
||||||
|
unsafe { edgehdf5_close(handle) };
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn save_rejects_null_embedding_ptr() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let handle = open_handle(&dir);
|
||||||
|
assert!(!handle.is_null());
|
||||||
|
|
||||||
|
let chunk = CString::new("hello").unwrap();
|
||||||
|
let channel = CString::new("test").unwrap();
|
||||||
|
let session = CString::new("s1").unwrap();
|
||||||
|
let tags = CString::new("").unwrap();
|
||||||
|
|
||||||
|
// SAFETY: handle and C strings are valid; embedding_ptr is intentionally null, which
|
||||||
|
// edgehdf5_save must reject before constructing a slice from it.
|
||||||
|
let result = unsafe {
|
||||||
|
edgehdf5_save(
|
||||||
|
handle,
|
||||||
|
chunk.as_ptr(),
|
||||||
|
ptr::null(),
|
||||||
|
EMBEDDING_DIM,
|
||||||
|
channel.as_ptr(),
|
||||||
|
0.0,
|
||||||
|
session.as_ptr(),
|
||||||
|
tags.as_ptr(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(result, -1, "null embedding_ptr must be rejected");
|
||||||
|
|
||||||
|
unsafe { edgehdf5_close(handle) };
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hybrid_search_rejects_mismatched_embedding_len() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let handle = open_handle(&dir);
|
||||||
|
assert!(!handle.is_null());
|
||||||
|
|
||||||
|
let query_embedding = [1.0f32, 2.0]; // len 2, dim is 4
|
||||||
|
let query_text = CString::new("hello").unwrap();
|
||||||
|
let mut out_indices = [0u64; 4];
|
||||||
|
let mut out_scores = [0.0f32; 4];
|
||||||
|
|
||||||
|
// SAFETY: handle and query_text are valid; query_embedding_len (2) intentionally does
|
||||||
|
// not match embedding_dim (4), which edgehdf5_hybrid_search must reject before touching
|
||||||
|
// query_embedding_ptr. Output buffers are sized to max_results.
|
||||||
|
let count = unsafe {
|
||||||
|
edgehdf5_hybrid_search(
|
||||||
|
handle,
|
||||||
|
query_embedding.as_ptr(),
|
||||||
|
query_embedding.len() as u32,
|
||||||
|
query_text.as_ptr(),
|
||||||
|
0.7,
|
||||||
|
0.3,
|
||||||
|
4,
|
||||||
|
out_indices.as_mut_ptr(),
|
||||||
|
out_scores.as_mut_ptr(),
|
||||||
|
ptr::null_mut(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(count, 0, "mismatched query_embedding_len must be rejected");
|
||||||
|
|
||||||
|
unsafe { edgehdf5_close(handle) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ crate-type = ["cdylib", "rlib"]
|
|||||||
[dependencies]
|
[dependencies]
|
||||||
clawhdf5_rs = { path = "../clawhdf5", version = "2.1.0", package = "clawhdf5" }
|
clawhdf5_rs = { path = "../clawhdf5", version = "2.1.0", package = "clawhdf5" }
|
||||||
clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" }
|
clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" }
|
||||||
pyo3 = "0.28"
|
pyo3 = "0.29"
|
||||||
numpy = "0.28"
|
numpy = "0.29"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
extension-module = ["pyo3/extension-module"]
|
extension-module = ["pyo3/extension-module"]
|
||||||
|
|||||||
Reference in New Issue
Block a user