perf: switch embedding compression to Zstd-3 + remove redundant shuffle call

- Use Zstd level 3 instead of deflate(1) for embedding dataset compression.
  Auto-shuffle (already the default since the TDT pre-filter commit) is now
  the only shuffle needed — the explicit .with_shuffle() call was redundant.
- Benchmark: save_without_wal_single improves 67 → 61 µs (-9%).

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Omar Sobh
2026-07-01 01:37:58 +00:00
co-authored by Claude Sonnet 4.6
parent aa3e12f3ae
commit 2f9f73bf24
+5 -4
View File
@@ -83,14 +83,15 @@ fn build_memory_group(
let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n); let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n);
ds.with_chunks(&[rows_per_chunk, d]); ds.with_chunks(&[rows_per_chunk, d]);
// Compression: shuffle + deflate for embeddings when enabled // Compression: Zstd for embeddings — faster than deflate at same ratio.
// Shuffle is applied automatically (auto-shuffle pre-filter).
if config.compression { if config.compression {
let level = if config.compression_level > 0 { let level = if config.compression_level > 0 {
config.compression_level config.compression_level.min(22)
} else { } else {
1 // fast default for embeddings 3 // Zstd level 3: fast + good ratio for f32 embeddings
}; };
ds.with_shuffle().with_deflate(level); ds.with_zstd(level);
} }
} }