Merge branch 'feat/p2b-blosc2' into feat/p2b-scale

# Conflicts: # CHANGELOG.md
2026-09-26 11:57:11 -05:00
parent d7f07fa5c1 3da118d2ee
commit 4b02e7d068
60 changed files with 2276 additions and 33 deletions
@@ -47,8 +47,9 @@ lzf = ["clawhdf5-format/lzf"]
 bitshuffle = ["clawhdf5-format/bitshuffle"]
 bzip2 = ["clawhdf5-format/bzip2"]
 blosc = ["clawhdf5-format/blosc"]
+blosc2 = ["clawhdf5-format/blosc2"]
 # Every plugin filter.
-plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc"]
+plugin-filters = ["lzf", "bitshuffle", "bzip2", "blosc", "blosc2"]
 # Dataset::verify_provenance() — recompute a dataset's SHA-256 and compare
 # against its stored _provenance_sha256 attribute. On by default, matching
 # clawhdf5-format's own default-on `provenance` feature.
@@ -77,7 +77,7 @@ except ImportError:
    hdf5plugin = None
 path = sys.argv[1]
 FILTERS = eval('(' + sys.argv[2] + ')')
-cases = [
+cases = eval('(' + sys.argv[3] + ')') if len(sys.argv) > 3 else [
    ('<u1', (1000,), (128,), 'ramp'),
    ('<i2', (37, 53), (10, 16), 'ramp'),
    ('<i4', (2000,), (300,), 'ramp'),
@@ -97,7 +97,10 @@ with h5py.File(path, 'w') as f:
    for label, kw in FILTERS:
        for dt, shape, chunks, kind in cases:
            n = int(np.prod(shape))
-            if kind == 'noise':
+            if kind in ('zeros', 'const', 'nan'):
+                fill = {'zeros': 0, 'const': 7, 'nan': np.nan}[kind]
+                data = np.full(n, fill, dtype=dt)
+            elif kind == 'noise':
                raw = rng.integers(0, 256, n * np.dtype(dt).itemsize, dtype=np.uint8)
                data = raw.view(dt)
                if np.dtype(dt).kind == 'f':
@@ -117,11 +120,18 @@ print(i)
 /// `(label, create_dataset kwargs)`), then check that clawhdf5 reads each
 /// filtered dataset exactly as its unfiltered twin.
 fn check_h5py_written(tag: &str, filters: &str) {
+    check_h5py_written_cases(tag, filters, None);
+}
+
+/// [`check_h5py_written`] over `cases` (a Python list of `(dtype, shape,
+/// chunks, kind)`, kind one of ramp, noise, zeros, const, nan) instead of
+/// the default ones.
+fn check_h5py_written_cases(tag: &str, filters: &str, cases: Option<&str>) {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join(format!("{tag}.h5"));
-    let n: usize = run_python(GENERATE, &[path.to_str().unwrap(), filters])
-        .parse()
-        .unwrap();
+    let mut args = vec![path.to_str().unwrap(), filters];
+    args.extend(cases);
+    let n: usize = run_python(GENERATE, &args).parse().unwrap();
    assert!(n > 0);
    let file = File::open(&path).unwrap();
    for i in 0..n {
@@ -353,6 +363,110 @@ fn blosc_written_by_clawhdf5_reads_in_hdf5plugin() {
    }
 }

+#[cfg(feature = "blosc2")]
+#[test]
+fn blosc2_written_by_hdf5plugin_reads_exactly() {
+    if !have_python("h5py, hdf5plugin") {
+        return;
+    }
+    // Every codec hdf5plugin's Blosc2 offers, each lossless filter, and
+    // levels from "store" to maximum. Truncating precision is lossy, so it
+    // is checked separately against h5py's own reading.
+    check_h5py_written(
+        "blosc2",
+        r#"[(f'{c} {f} {l}', hdf5plugin.Blosc2(cname=c, clevel=l, filters=f))
+            for c in ['blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd']
+            for f, l in [(hdf5plugin.Blosc2.NOFILTER, 5),
+                         (hdf5plugin.Blosc2.SHUFFLE, 9),
+                         (hdf5plugin.Blosc2.BITSHUFFLE, 1),
+                         (hdf5plugin.Blosc2.DELTA, 5)]]
+           + [('blosclz level 0', hdf5plugin.Blosc2(cname='blosclz', clevel=0)),
+              ('zstd level 0 bitshuffle',
+               hdf5plugin.Blosc2(cname='zstd', clevel=0, filters=hdf5plugin.Blosc2.BITSHUFFLE))]"#,
+    );
+}
+
+/// Blosc2 over more shapes and dtypes: every integer and float width,
+/// 1-D to 5-D chunks (B2ND arrays from 2-D on) with partial edge chunks and
+/// block shapes that pad the chunk, datasets of zeros, of one repeated value
+/// and of NaN (Blosc2's "special" chunks), and Fletcher32 before Blosc2
+/// (which makes hdf5-blosc2 fall back from B2ND to a plain frame).
+#[cfg(feature = "blosc2")]
+#[test]
+fn blosc2_shapes_and_special_chunks_read_exactly() {
+    if !have_python("h5py, hdf5plugin") {
+        return;
+    }
+    let cases = r#"[(dt, shape, chunks, kind)
+        for dt in ['<i1', '<u1', '<i2', '>u2', '<i4', '<u4', '<i8', '<u8', '<f4', '>f8']
+        for shape, chunks in [((777,), (100,)),
+                              ((37, 53), (10, 16)),
+                              ((9, 10, 11), (4, 5, 3)),
+                              ((6, 7, 5, 9), (3, 2, 5, 4))]
+        for kind in ['ramp', 'noise']]
+      + [('<f8', (300, 30), (64, 7), 'zeros'), ('<i4', (50, 40, 3), (7, 9, 3), 'zeros'),
+         ('<u2', (5000,), (1024,), 'const'), ('<i8', (40, 40), (16, 16), 'const'),
+         ('<f4', (100, 20), (32, 8), 'nan'), ('<f8', (3000,), (1000,), 'nan'),
+         ('<f4', (1000, 1000), (256, 512), 'ramp'),
+         ('<i2', (3, 3, 3, 3, 3), (2, 2, 2, 2, 2), 'ramp')]"#;
+    check_h5py_written_cases(
+        "blosc2_shapes",
+        r#"[('lz4 shuffle', hdf5plugin.Blosc2(cname='lz4')),
+            ('zstd bitshuffle',
+             hdf5plugin.Blosc2(cname='zstd', clevel=7, filters=hdf5plugin.Blosc2.BITSHUFFLE)),
+            ('blosclz delta', hdf5plugin.Blosc2(cname='blosclz', filters=hdf5plugin.Blosc2.DELTA)),
+            ('zlib + fletcher32', dict(**hdf5plugin.Blosc2(cname='zlib'), fletcher32=True))]"#,
+        Some(cases),
+    );
+}
+
+/// Truncating precision is lossy: clawhdf5 must read exactly what h5py
+/// (libhdf5 with hdf5plugin) reads.
+#[cfg(feature = "blosc2")]
+#[test]
+fn blosc2_truncated_precision_reads_as_h5py() {
+    if !have_python("h5py, hdf5plugin") {
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("blosc2_trunc.h5");
+    let n: usize = run_python(
+        r#"
+import sys
+import numpy as np, h5py, hdf5plugin
+rng = np.random.default_rng(3)
+i = 0
+with h5py.File(sys.argv[1], 'w') as f:
+    for dt in ['<f4', '<f8']:
+        for shape, chunks in [((1000,), (300,)), ((37, 53), (10, 16))]:
+            d = (rng.standard_normal(shape) * 1000).astype(dt)
+            ds = f.create_dataset(f'f{i}', data=d, chunks=chunks,
+                                  **hdf5plugin.Blosc2(cname='lz4',
+                                                      filters=hdf5plugin.Blosc2.TRUNC_PREC))
+            f.create_dataset(f'r{i}', data=ds[()])
+            i += 1
+print(i)
+"#,
+        &[path.to_str().unwrap()],
+    )
+    .parse()
+    .unwrap();
+    let file = File::open(&path).unwrap();
+    for i in 0..n {
+        let got = file
+            .dataset(&format!("f{i}"))
+            .unwrap()
+            .read_selection(&Selection::All)
+            .unwrap();
+        let want = file
+            .dataset(&format!("r{i}"))
+            .unwrap()
+            .read_selection(&Selection::All)
+            .unwrap();
+        assert!(got == want, "trunc_prec f{i}: data differs from h5py");
+    }
+}
+
 #[cfg(feature = "lzf")]
 #[test]
 fn lzf_written_by_h5py_reads_exactly() {
@@ -381,8 +495,9 @@ fn lzf_written_by_clawhdf5_reads_in_h5py() {
    });
 }

-/// Blosc2 and ZFP are not implemented: reading them must be a clear error
-/// naming the filter, never data.
+/// ZFP is not implemented, and Blosc2 is not in a build without the
+/// `blosc2` feature: reading them must be a clear error naming the filter,
+/// never data.
 #[test]
 fn unimplemented_filters_are_a_clear_error() {
    if !have_python("h5py, hdf5plugin") {
@@ -402,7 +517,11 @@ with h5py.File(sys.argv[1], 'w') as f:
        &[path.to_str().unwrap()],
    );
    let file = File::open(&path).unwrap();
-    for (name, id, label) in [("blosc2", 32026u16, "Blosc2"), ("zfp", 32013, "ZFP")] {
+    let mut missing = vec![("zfp", 32013u16, "ZFP")];
+    if !cfg!(feature = "blosc2") {
+        missing.push(("blosc2", 32026, "Blosc2"));
+    }
+    for (name, id, label) in missing {
        let err = file
            .dataset(name)
            .unwrap()