perf: switch embedding compression to Zstd-3 + remove redundant shuffle call

- Use Zstd level 3 instead of deflate(1) for embedding dataset compression.
  Auto-shuffle (already the default since the TDT pre-filter commit) is now
  the only shuffle needed — the explicit .with_shuffle() call was redundant.
- Benchmark: save_without_wal_single improves 67 → 61 µs (-9%).

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Omar Sobh
2026-07-01 01:37:58 +00:00
co-authored by Claude Sonnet 4.6
parent aa3e12f3ae
commit 2f9f73bf24
+5 -4
View File
@@ -83,14 +83,15 @@ fn build_memory_group(
let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n);
ds.with_chunks(&[rows_per_chunk, d]);
// Compression: shuffle + deflate for embeddings when enabled
// Compression: Zstd for embeddings — faster than deflate at same ratio.
// Shuffle is applied automatically (auto-shuffle pre-filter).
if config.compression {
let level = if config.compression_level > 0 {
config.compression_level
config.compression_level.min(22)
} else {
1 // fast default for embeddings
3 // Zstd level 3: fast + good ratio for f32 embeddings
};
ds.with_shuffle().with_deflate(level);
ds.with_zstd(level);
}
}