feat: add Pcodec lossless numerical compression filter (arXiv:2502.06112)

Implements Pcodec (filter ID 32023) via the `pco` 1.0.x crate as a new optional compression codec. Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64 columnar data at 1–5 GiB/s decompression speed, making it ideal for write-once/read-many embedding archives. Write throughput at 512×512: 591 MiB/s (parity with Zstd-3 at 610 MiB/s). For smaller chunks Zstd-3 remains faster due to Pcodec's fixed per-chunk distributional analysis overhead. - Add FILTER_PCODEC = 32023 constant to filter_pipeline.rs - Add pcodec_compress/pcodec_decompress using pco::standalone API - Wire into compress_chunk/decompress_chunk dispatch - Add ChunkOptions.pcodec field and DatasetBuilder.with_pcodec() method - Enable pcodec as highest-priority codec in build_pipeline() - Add pco dep (optional, feature = "pcodec") to clawhdf5-format/clawhdf5 - Add write_2d_chunked_pcodec benchmark comparing pcodec vs zstd-3 - Document results in BENCHMARKS.md Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-07-01 01:16:08 +00:00
parent e82b8f56bd
commit 5701e8045d
9 changed files with 184 additions and 7 deletions
@@ -168,6 +168,68 @@ fn bench_write_2d_chunked_zstd(c: &mut Criterion) {
    group.finish();
 }

+// ---------------------------------------------------------------------------
+// Workload: write_2d_chunked_pcodec
+// Same matrix sizes as write_2d_chunked but uses Pcodec (arXiv:2502.06112).
+// Pcodec achieves 30–94% better compression ratio than Zstd for f32/f64 at
+// 1–5 GiB/s decompression speed via a quantile-based numerical codec.
+// ---------------------------------------------------------------------------
+
+fn bench_write_2d_chunked_pcodec(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write_2d_chunked_pcodec");
+
+    let configs: &[(usize, usize, u64, u64)] = &[
+        (32, 32, 8, 32),
+        (128, 128, 32, 128),
+        (512, 512, 64, 512),
+    ];
+
+    for &(rows, cols, cr, cc) in configs {
+        let n = rows * cols;
+        let data: Vec<f32> = (0..n).map(|i| i as f32).collect();
+        let label = format!("{rows}x{cols}");
+        group.throughput(Throughput::Bytes((n * size_of::<f32>()) as u64));
+
+        group.bench_with_input(
+            BenchmarkId::new("clawhdf5/pcodec", &label),
+            &data,
+            |b, d| {
+                let tmp = TempDir::new().unwrap();
+                let path = tmp.path().join("write_2d_chunked_pcodec.h5");
+                b.iter(|| {
+                    let mut fb = FileBuilder::new();
+                    fb.create_dataset("matrix")
+                        .with_f32_data(d)
+                        .with_shape(&[rows as u64, cols as u64])
+                        .with_chunks(&[cr, cc])
+                        .with_pcodec();
+                    fb.write(&path).unwrap();
+                });
+            },
+        );
+
+        group.bench_with_input(
+            BenchmarkId::new("clawhdf5/zstd-3", &label),
+            &data,
+            |b, d| {
+                let tmp = TempDir::new().unwrap();
+                let path = tmp.path().join("write_2d_chunked_zstd.h5");
+                b.iter(|| {
+                    let mut fb = FileBuilder::new();
+                    fb.create_dataset("matrix")
+                        .with_f32_data(d)
+                        .with_shape(&[rows as u64, cols as u64])
+                        .with_chunks(&[cr, cc])
+                        .with_zstd(3);
+                    fb.write(&path).unwrap();
+                });
+            },
+        );
+    }
+
+    group.finish();
+}
+
 // ---------------------------------------------------------------------------
 // Workload: write_f64_batch
 // Write batches of f64 elements — simulates the clawhdf5-agent embedding
@@ -265,6 +327,7 @@ criterion_group!(
    bench_write_1d_contiguous,
    bench_write_2d_chunked,
    bench_write_2d_chunked_zstd,
+    bench_write_2d_chunked_pcodec,
    bench_write_f64_batch,
    bench_write_multi_dataset,
    bench_write_with_attrs,