security: Tier 3 — Android JNI length validation, pyo3 bump, WAL caps
CI / test (push) Failing after 2s

- clawhdf5-android: validate embedding_len/query_embedding_len against
  the handle's configured embedding_dim (and reject null pointers)
  before constructing a slice via from_raw_parts in edgehdf5_save and
  edgehdf5_hybrid_search. Strengthen the # Safety docs to state the
  now-enforced invariant and its limits. Add unit tests covering
  mismatched length and null-pointer rejection.
- clawhdf5-py: bump pyo3/numpy 0.28 -> 0.29, clearing RUSTSEC-2026-0176
  (OOB read in PyList/PyTuple iterator) and RUSTSEC-2026-0177 (missing
  Sync bound on PyCFunction::new_closure). No source changes needed;
  confirmed via cargo audit that both advisories no longer appear.
- clawhdf5-agent/wal.rs: cap read_len_prefixed_str/read_embedding's
  length claims at a new MAX_WAL_FIELD_LEN (64 MiB) before allocating,
  so a corrupted/truncated WAL length field fails cleanly instead of
  attempting a huge allocation. Add regression tests for both.
- BENCHMARKS.md: add a top-of-file traceability note distinguishing the
  dated/hardware-cited/reproducible h5bench and tank-validation sections
  from the older sections that don't yet meet that bar.
This commit is contained in:
Omar Sobh
2026-08-05 12:10:49 -07:00
parent 62595d5ac0
commit a319405ffc
5 changed files with 201 additions and 6 deletions
+7
View File
@@ -6,6 +6,13 @@
**Rust:** 1.96.0-nightly (2026-03-14) · `--release` profile **Rust:** 1.96.0-nightly (2026-03-14) · `--release` profile
**Date:** 2026-07-01 **Date:** 2026-07-01
> **Traceability note:** the "h5bench-Equivalent I/O Benchmarks" and
> "Independent Validation: tank" sections below meet a dated,
> hardware-cited, reproducible standard (explicit date, machine spec, and a
> runnable command per result). The sections above them do not yet meet
> that bar consistently — this is a known, tracked documentation gap, not
> a claim that those numbers are wrong.
--- ---
## Vector Search Latency ## Vector Search Latency
+50
View File
@@ -12,6 +12,11 @@ use crate::MemoryError;
const WAL_MAGIC: [u8; 4] = [0x45, 0x48, 0x57, 0x4C]; // "EHWL" const WAL_MAGIC: [u8; 4] = [0x45, 0x48, 0x57, 0x4C]; // "EHWL"
const WAL_VERSION: u8 = 1; const WAL_VERSION: u8 = 1;
/// Upper bound on a single length-prefixed WAL field (string bytes, or
/// embedding element count), to reject a corrupted/truncated WAL length
/// claim before allocating a large buffer for it.
const MAX_WAL_FIELD_LEN: usize = 64 * 1024 * 1024;
#[repr(u8)] #[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WalEntryType { pub enum WalEntryType {
@@ -349,6 +354,11 @@ fn read_len_prefixed_str(f: &mut File) -> Result<String, MemoryError> {
let mut len_buf = [0u8; 4]; let mut len_buf = [0u8; 4];
f.read_exact(&mut len_buf)?; f.read_exact(&mut len_buf)?;
let len = u32::from_le_bytes(len_buf) as usize; let len = u32::from_le_bytes(len_buf) as usize;
if len > MAX_WAL_FIELD_LEN {
return Err(MemoryError::Schema(format!(
"WAL string field length {len} exceeds max {MAX_WAL_FIELD_LEN}"
)));
}
let mut buf = vec![0u8; len]; let mut buf = vec![0u8; len];
f.read_exact(&mut buf)?; f.read_exact(&mut buf)?;
String::from_utf8(buf).map_err(|e| MemoryError::Schema(format!("invalid UTF-8 in WAL: {e}"))) String::from_utf8(buf).map_err(|e| MemoryError::Schema(format!("invalid UTF-8 in WAL: {e}")))
@@ -358,6 +368,12 @@ fn read_embedding(f: &mut File) -> Result<Vec<f32>, MemoryError> {
let mut len_buf = [0u8; 4]; let mut len_buf = [0u8; 4];
f.read_exact(&mut len_buf)?; f.read_exact(&mut len_buf)?;
let count = u32::from_le_bytes(len_buf) as usize; let count = u32::from_le_bytes(len_buf) as usize;
if count > MAX_WAL_FIELD_LEN / 4 {
return Err(MemoryError::Schema(format!(
"WAL embedding element count {count} exceeds max {}",
MAX_WAL_FIELD_LEN / 4
)));
}
let mut vals = Vec::with_capacity(count); let mut vals = Vec::with_capacity(count);
for _ in 0..count { for _ in 0..count {
let mut val_buf = [0u8; 4]; let mut val_buf = [0u8; 4];
@@ -427,6 +443,40 @@ mod tests {
assert_eq!(entries[2].embedding, vec![5.0, 6.0]); assert_eq!(entries[2].embedding, vec![5.0, 6.0]);
} }
#[test]
fn read_len_prefixed_str_rejects_oversized_len_claim() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("oversized_str.bin");
{
let mut f = File::create(&path).unwrap();
// Claim a length far beyond MAX_WAL_FIELD_LEN; no payload follows.
f.write_all(&(u32::MAX).to_le_bytes()).unwrap();
}
let mut f = File::open(&path).unwrap();
let result = read_len_prefixed_str(&mut f);
assert!(
matches!(result, Err(MemoryError::Schema(_))),
"expected a clean Schema error, got {result:?}"
);
}
#[test]
fn read_embedding_rejects_oversized_count_claim() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("oversized_embedding.bin");
{
let mut f = File::create(&path).unwrap();
// Claim a count far beyond MAX_WAL_FIELD_LEN / 4; no payload follows.
f.write_all(&(u32::MAX).to_le_bytes()).unwrap();
}
let mut f = File::open(&path).unwrap();
let result = read_embedding(&mut f);
assert!(
matches!(result, Err(MemoryError::Schema(_))),
"expected a clean Schema error, got {result:?}"
);
}
#[test] #[test]
fn test_wal_truncate() { fn test_wal_truncate() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();
+3
View File
@@ -10,3 +10,6 @@ crate-type = ["cdylib"]
[dependencies] [dependencies]
clawhdf5-agent = { path = "../clawhdf5-agent", default-features = false } clawhdf5-agent = { path = "../clawhdf5-agent", default-features = false }
[dev-dependencies]
tempfile = "3"
+139 -4
View File
@@ -92,11 +92,18 @@ pub unsafe extern "C" fn edgehdf5_close(handle: Handle) {
/// Save a memory entry. Returns the entry index, or -1 on failure. /// Save a memory entry. Returns the entry index, or -1 on failure.
/// ///
/// `embedding_len` is validated against the handle's configured
/// `embedding_dim` before the input slice is constructed; a mismatch fails
/// the call with -1 rather than reading out of bounds. This is a length
/// check only — it cannot detect a same-length buffer that is otherwise
/// too short or invalid.
///
/// # Safety /// # Safety
/// ///
/// - `handle` must be a valid, non-null handle. /// - `handle` must be a valid, non-null handle.
/// - All `*const c_char` arguments must be valid, null-terminated C strings. /// - All `*const c_char` arguments must be valid, null-terminated C strings.
/// - `embedding_ptr` must point to at least `embedding_len` contiguous `f32` values. /// - If `embedding_len` matches the handle's `embedding_dim`, `embedding_ptr`
/// must point to at least that many contiguous, valid `f32` values.
#[unsafe(no_mangle)] #[unsafe(no_mangle)]
pub unsafe extern "C" fn edgehdf5_save( pub unsafe extern "C" fn edgehdf5_save(
handle: Handle, handle: Handle,
@@ -135,8 +142,14 @@ pub unsafe extern "C" fn edgehdf5_save(
None => return -1, None => return -1,
}; };
if embedding_ptr.is_null() || embedding_len as usize != mem.config().embedding_dim {
return -1;
}
let embedding = let embedding =
// SAFETY: JNI caller guarantees embedding_ptr points to embedding_len valid f32 values. // SAFETY: embedding_ptr is non-null and embedding_len matches the handle's configured
// embedding_dim (checked above); JNI caller guarantees it points to that many valid f32
// values. A mismatched-but-equal-length short buffer is not caught by this length check
// alone — the caller is still responsible for pointer validity.
unsafe { std::slice::from_raw_parts(embedding_ptr, embedding_len as usize) }.to_vec(); unsafe { std::slice::from_raw_parts(embedding_ptr, embedding_len as usize) }.to_vec();
let entry = MemoryEntry { let entry = MemoryEntry {
@@ -210,11 +223,18 @@ pub unsafe extern "C" fn edgehdf5_delete(handle: Handle, index: u64) -> i32 {
/// Performs hybrid search and writes up to `max_results` entries into the /// Performs hybrid search and writes up to `max_results` entries into the
/// provided output arrays. Returns the number of results written. /// provided output arrays. Returns the number of results written.
/// ///
/// `query_embedding_len` is validated against the handle's configured
/// `embedding_dim` before the input slice is constructed; a mismatch fails
/// the call (returns 0) rather than reading out of bounds. This is a length
/// check only — it cannot detect a same-length buffer that is otherwise too
/// short or invalid.
///
/// # Safety /// # Safety
/// ///
/// - `handle` must be a valid, non-null handle. /// - `handle` must be a valid, non-null handle.
/// - `query_text` must be a valid, null-terminated C string. /// - `query_text` must be a valid, null-terminated C string.
/// - `query_embedding_ptr` must point to at least `query_embedding_len` `f32` values. /// - If `query_embedding_len` matches the handle's `embedding_dim`,
/// `query_embedding_ptr` must point to at least that many valid `f32` values.
/// - `out_indices` and `out_scores` must point to arrays of at least `max_results` elements. /// - `out_indices` and `out_scores` must point to arrays of at least `max_results` elements.
/// - `out_chunks` must be null or point to an array of at least `max_results` pointers. /// - `out_chunks` must be null or point to an array of at least `max_results` pointers.
#[unsafe(no_mangle)] #[unsafe(no_mangle)]
@@ -240,8 +260,14 @@ pub unsafe extern "C" fn edgehdf5_hybrid_search(
Some(s) => s, Some(s) => s,
None => return 0, None => return 0,
}; };
if query_embedding_ptr.is_null() || query_embedding_len as usize != mem.config().embedding_dim {
return 0;
}
let query_embedding = let query_embedding =
// SAFETY: JNI caller guarantees query_embedding_ptr points to query_embedding_len valid f32 values. // SAFETY: query_embedding_ptr is non-null and query_embedding_len matches the handle's
// configured embedding_dim (checked above); JNI caller guarantees it points to that many
// valid f32 values. A mismatched-but-equal-length short buffer is not caught by this
// length check alone — the caller is still responsible for pointer validity.
unsafe { std::slice::from_raw_parts(query_embedding_ptr, query_embedding_len as usize) }; unsafe { std::slice::from_raw_parts(query_embedding_ptr, query_embedding_len as usize) };
let results = mem.hybrid_search( let results = mem.hybrid_search(
@@ -456,3 +482,112 @@ unsafe fn cstr_to_string(ptr: *const c_char) -> Option<String> {
.ok() .ok()
.map(String::from) .map(String::from)
} }
#[cfg(test)]
mod tests {
use super::*;
const EMBEDDING_DIM: u32 = 4;
fn open_handle(dir: &tempfile::TempDir) -> Handle {
let path = CString::new(dir.path().join("mem.h5").to_str().unwrap()).unwrap();
let agent_id = CString::new("test-agent").unwrap();
// SAFETY: both C strings are valid and null-terminated.
unsafe { edgehdf5_create(path.as_ptr(), agent_id.as_ptr(), EMBEDDING_DIM) }
}
#[test]
fn save_rejects_mismatched_embedding_len() {
let dir = tempfile::tempdir().unwrap();
let handle = open_handle(&dir);
assert!(!handle.is_null());
let embedding = [1.0f32, 2.0, 3.0]; // len 3, dim is 4
let chunk = CString::new("hello").unwrap();
let channel = CString::new("test").unwrap();
let session = CString::new("s1").unwrap();
let tags = CString::new("").unwrap();
// SAFETY: handle is valid; all C strings are valid; embedding_len (3) intentionally
// does not match embedding_dim (4), which edgehdf5_save must reject before touching
// embedding_ptr.
let result = unsafe {
edgehdf5_save(
handle,
chunk.as_ptr(),
embedding.as_ptr(),
embedding.len() as u32,
channel.as_ptr(),
0.0,
session.as_ptr(),
tags.as_ptr(),
)
};
assert_eq!(result, -1, "mismatched embedding_len must be rejected");
unsafe { edgehdf5_close(handle) };
}
#[test]
fn save_rejects_null_embedding_ptr() {
let dir = tempfile::tempdir().unwrap();
let handle = open_handle(&dir);
assert!(!handle.is_null());
let chunk = CString::new("hello").unwrap();
let channel = CString::new("test").unwrap();
let session = CString::new("s1").unwrap();
let tags = CString::new("").unwrap();
// SAFETY: handle and C strings are valid; embedding_ptr is intentionally null, which
// edgehdf5_save must reject before constructing a slice from it.
let result = unsafe {
edgehdf5_save(
handle,
chunk.as_ptr(),
ptr::null(),
EMBEDDING_DIM,
channel.as_ptr(),
0.0,
session.as_ptr(),
tags.as_ptr(),
)
};
assert_eq!(result, -1, "null embedding_ptr must be rejected");
unsafe { edgehdf5_close(handle) };
}
#[test]
fn hybrid_search_rejects_mismatched_embedding_len() {
let dir = tempfile::tempdir().unwrap();
let handle = open_handle(&dir);
assert!(!handle.is_null());
let query_embedding = [1.0f32, 2.0]; // len 2, dim is 4
let query_text = CString::new("hello").unwrap();
let mut out_indices = [0u64; 4];
let mut out_scores = [0.0f32; 4];
// SAFETY: handle and query_text are valid; query_embedding_len (2) intentionally does
// not match embedding_dim (4), which edgehdf5_hybrid_search must reject before touching
// query_embedding_ptr. Output buffers are sized to max_results.
let count = unsafe {
edgehdf5_hybrid_search(
handle,
query_embedding.as_ptr(),
query_embedding.len() as u32,
query_text.as_ptr(),
0.7,
0.3,
4,
out_indices.as_mut_ptr(),
out_scores.as_mut_ptr(),
ptr::null_mut(),
)
};
assert_eq!(count, 0, "mismatched query_embedding_len must be rejected");
unsafe { edgehdf5_close(handle) };
}
}
+2 -2
View File
@@ -16,8 +16,8 @@ crate-type = ["cdylib", "rlib"]
[dependencies] [dependencies]
clawhdf5_rs = { path = "../clawhdf5", version = "2.1.0", package = "clawhdf5" } clawhdf5_rs = { path = "../clawhdf5", version = "2.1.0", package = "clawhdf5" }
clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" }
pyo3 = "0.28" pyo3 = "0.29"
numpy = "0.28" numpy = "0.29"
[features] [features]
extension-module = ["pyo3/extension-module"] extension-module = ["pyo3/extension-module"]