perf: switch embedding compression to Zstd-3 + remove redundant shuffle call
- Use Zstd level 3 instead of deflate(1) for embedding dataset compression. Auto-shuffle (already the default since the TDT pre-filter commit) is now the only shuffle needed — the explicit .with_shuffle() call was redundant. - Benchmark: save_without_wal_single improves 67 → 61 µs (-9%). Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
aa3e12f3ae
commit
2f9f73bf24
@@ -83,14 +83,15 @@ fn build_memory_group(
|
||||
let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n);
|
||||
ds.with_chunks(&[rows_per_chunk, d]);
|
||||
|
||||
// Compression: shuffle + deflate for embeddings when enabled
|
||||
// Compression: Zstd for embeddings — faster than deflate at same ratio.
|
||||
// Shuffle is applied automatically (auto-shuffle pre-filter).
|
||||
if config.compression {
|
||||
let level = if config.compression_level > 0 {
|
||||
config.compression_level
|
||||
config.compression_level.min(22)
|
||||
} else {
|
||||
1 // fast default for embeddings
|
||||
3 // Zstd level 3: fast + good ratio for f32 embeddings
|
||||
};
|
||||
ds.with_shuffle().with_deflate(level);
|
||||
ds.with_zstd(level);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user