perf(agent): maintain a persistent flat embedding buffer for BLAS/Accelerate search
blas_cosine_batch and accelerate_cosine_batch_vecs re-flattened the entire Vec<Vec<f32>> corpus into a fresh Vec<f32> on every single query before running the batch matmul — an O(N·dim) copy paid per query when fast-math/accelerate/openblas is enabled, even though a flat fast-path (blas_cosine_batch_flat / accelerate_cosine_batch) already existed for pre-flattened input. Add MemoryCache::embeddings_flat, a contiguous [N × embedding_dim] buffer maintained incrementally in push/update/compact (O(1) amortized append, O(dim) in-place overwrite, O(n) rebuild only on compact/bulk load). schema.rs's direct-push load path calls the new rebuild_flat() explicitly. flat_embeddings() now just clones the already-maintained buffer instead of rebuilding it. Thread the flat buffer through strategy::search_with_metrics as a new vectors_flat parameter, used only by the Blas/Accelerate arms (now calling the *_flat variants); other strategies are unaffected. No current caller wires search_with_metrics into the production query path yet (only its own tests exercise it) — this fixes the identified per-query re-flatten and makes the flat buffer available for whenever that wiring lands. INT-16
This commit is contained in:
ClawHDF5 Coding Agent
@@ -167,10 +167,17 @@ pub fn auto_select_strategy(num_vectors: usize, hw: &HardwareCapabilities) -> Se
|
||||
/// This dispatches to the appropriate search implementation based on the
|
||||
/// selected strategy. For IVF-PQ, an index must be provided externally
|
||||
/// (this function uses brute-force fallback if no IVF-PQ index is available).
|
||||
///
|
||||
/// `vectors_flat` is `vectors` flattened into one contiguous `[N × dim]`
|
||||
/// row-major buffer (e.g. `MemoryCache::embeddings_flat`, maintained
|
||||
/// incrementally alongside `vectors`). It's only consulted by the
|
||||
/// `Blas`/`Accelerate` strategies, which otherwise re-flatten the whole
|
||||
/// corpus on every call — passing the already-flat buffer skips that copy.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn search_with_metrics(
|
||||
query: &[f32],
|
||||
vectors: &[Vec<f32>],
|
||||
vectors_flat: &[f32],
|
||||
norms: &[f32],
|
||||
tombstones: &[u8],
|
||||
k: usize,
|
||||
@@ -178,6 +185,10 @@ pub fn search_with_metrics(
|
||||
#[cfg(feature = "gpu")] gpu_backend: Option<&crate::gpu_search::GpuSearchBackend>,
|
||||
#[cfg(not(feature = "gpu"))] _gpu_backend: Option<&()>,
|
||||
) -> (Vec<(usize, f32)>, SearchMetrics) {
|
||||
// Only read by the Blas/Accelerate arms below, which are themselves
|
||||
// feature-gated — reference it unconditionally so a build with neither
|
||||
// feature enabled doesn't warn about an unused parameter.
|
||||
let _ = vectors_flat;
|
||||
let start = Instant::now();
|
||||
let active_count = tombstones.iter().filter(|&&t| t == 0).count();
|
||||
|
||||
@@ -197,7 +208,14 @@ pub fn search_with_metrics(
|
||||
gpu_active = false;
|
||||
#[cfg(feature = "fast-math")]
|
||||
{
|
||||
crate::blas_search::blas_cosine_batch(query, vectors, norms, tombstones, k)
|
||||
crate::blas_search::blas_cosine_batch_flat(
|
||||
query,
|
||||
vectors_flat,
|
||||
norms,
|
||||
tombstones,
|
||||
query.len(),
|
||||
k,
|
||||
)
|
||||
}
|
||||
#[cfg(not(feature = "fast-math"))]
|
||||
{
|
||||
@@ -211,8 +229,13 @@ pub fn search_with_metrics(
|
||||
gpu_active = false;
|
||||
#[cfg(any(feature = "accelerate", feature = "openblas"))]
|
||||
{
|
||||
crate::accelerate_search::accelerate_cosine_batch_vecs(
|
||||
query, vectors, norms, tombstones, k,
|
||||
crate::accelerate_search::accelerate_cosine_batch(
|
||||
query,
|
||||
vectors_flat,
|
||||
norms,
|
||||
tombstones,
|
||||
query.len(),
|
||||
k,
|
||||
)
|
||||
}
|
||||
#[cfg(not(any(feature = "accelerate", feature = "openblas")))]
|
||||
@@ -325,6 +348,10 @@ mod tests {
|
||||
(0..n).map(|_| (0..dim).map(|_| next()).collect()).collect()
|
||||
}
|
||||
|
||||
fn flatten(vectors: &[Vec<f32>]) -> Vec<f32> {
|
||||
vectors.iter().flatten().copied().collect()
|
||||
}
|
||||
|
||||
// --- auto_select_strategy tests ---
|
||||
|
||||
#[test]
|
||||
@@ -490,6 +517,7 @@ mod tests {
|
||||
let (results, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
5,
|
||||
@@ -520,6 +548,7 @@ mod tests {
|
||||
let (results, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
10,
|
||||
@@ -545,6 +574,7 @@ mod tests {
|
||||
let (_, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
10,
|
||||
@@ -570,6 +600,7 @@ mod tests {
|
||||
let (results, _) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
10,
|
||||
@@ -603,6 +634,7 @@ mod tests {
|
||||
let (results, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
100,
|
||||
@@ -647,6 +679,7 @@ mod tests {
|
||||
let (_, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
5,
|
||||
@@ -718,6 +751,7 @@ mod tests {
|
||||
let (results, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
10,
|
||||
@@ -744,6 +778,7 @@ mod tests {
|
||||
let (results, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
10,
|
||||
@@ -822,6 +857,7 @@ mod tests {
|
||||
let (results, metrics) = search_with_metrics(
|
||||
&query,
|
||||
&vectors,
|
||||
&flatten(&vectors),
|
||||
&norms,
|
||||
&tombstones,
|
||||
10,
|
||||
|
||||
Reference in New Issue
Block a user