fix(format): read partial edge chunks stored unfiltered

Layout message v4 flag bit 0 (H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS, set with H5Pset_chunk_opts) makes libhdf5 store every chunk that extends past the dataset's extent without the filter pipeline, while its filter mask still reads 0. The parser ignored the flag, so readers tried to inflate raw bytes: libhdf5's own h5fc_edge_v3.h5 failed with "deflate: ... unknown compression method". DataLayout::Chunked gains dont_filter_partial_edge_chunks (always false for v3), and list_chunks — the one place every read path gets its chunk list from — marks such partial chunks as having skipped every filter, so the full, cached, indexed, parallel and selection readers all copy them as-is. chunked_write.rs gets `..` in one exhaustive test pattern for the new field. Regression: libhdf5_edge_chunk_fixture_reads (h5fc_edge_v3.h5 from the HDF5 tools test files, committed as a 2.5 KB fixture), and h5py_unfiltered_partial_edge_chunks_read (the flag set through h5py's bundled libhdf5 via ctypes, as h5py has no binding for it: fixed array, extensible array and B-tree v2 indexes, 1-D and 2-D, plus a hyperslab of the last chunk), and v4_chunked_dont_filter_partial_edge_chunks_flag. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-25 21:08:36 -05:00
parent e162c013fd
commit d074385944
5 changed files with 167 additions and 1 deletions
@@ -237,3 +237,109 @@ with h5py.File("{p}", "w") as f:
        grid
    );
 }
+
+// ---------------------------------------------------------------------------
+// "Don't filter partial edge chunks"
+// ---------------------------------------------------------------------------
+
+/// libhdf5's own test file for `H5Pset_chunk_opts(H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS)`:
+/// a 12x6 f32 dataset in 5x5 gzip chunks whose edge chunks are stored raw
+/// with a filter mask of 0. Reading it tried to inflate the raw edge chunks
+/// ("deflate: ... unknown compression method").
+#[test]
+fn libhdf5_edge_chunk_fixture_reads() {
+    let path = concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/tests/fixtures/h5fc_edge_v3.h5"
+    );
+    let file = File::open(path).unwrap();
+    let ds = file.dataset("DSET_EDGE").unwrap();
+    assert_eq!(ds.shape().unwrap(), vec![12, 6]);
+    assert_eq!(ds.read_f32().unwrap(), vec![100.0f32; 72]);
+}
+
+/// The same layout flag on datasets with varied contents, set through the
+/// libhdf5 that h5py ships (h5py has no binding for `H5Pset_chunk_opts`).
+#[test]
+fn h5py_unfiltered_partial_edge_chunks_read() {
+    skip_if_no_python!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("edge.h5");
+    let p = path.display().to_string();
+    let script = format!(
+        r#"
+import ctypes, glob, os, sys
+import h5py, numpy as np
+here = os.path.dirname(h5py.__file__)
+libs = glob.glob(os.path.join(here, "..", "h5py.libs", "libhdf5-*.so*"))
+libs += glob.glob(os.path.join(here, ".dylibs", "libhdf5*.dylib"))
+if not libs:
+    print("NO_LIBHDF5")
+    sys.exit(0)
+lib = ctypes.CDLL(libs[0])
+lib.H5Pset_chunk_opts.argtypes = [ctypes.c_int64, ctypes.c_uint]
+def make(f, name, data, chunk, maxshape, shuffle):
+    dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
+    dcpl.set_chunk(chunk)
+    if shuffle:
+        dcpl.set_shuffle()
+    dcpl.set_deflate(6)
+    assert lib.H5Pset_chunk_opts(dcpl.id, 0x0002) >= 0
+    space = h5py.h5s.create_simple(data.shape, maxshape)
+    d = h5py.h5d.create(f.id, name.encode(), h5py.h5t.py_create(data.dtype), space, dcpl=dcpl)
+    d.write(h5py.h5s.ALL, h5py.h5s.ALL, np.ascontiguousarray(data))
+with h5py.File("{p}", "w") as f:
+    line = np.sin(np.arange(1000) / 7.0)
+    grid = np.arange(37 * 53, dtype="<f8").reshape(37, 53) * 0.5
+    make(f, "fixed_1d", line, (64,), None, False)
+    make(f, "ea_1d", line, (64,), (h5py.h5s.UNLIMITED,), True)
+    make(f, "bt2_2d", grid, (8, 8), (h5py.h5s.UNLIMITED,) * 2, True)
+    make(f, "fixed_2d", grid, (8, 8), None, True)
+print("OK")
+"#
+    );
+    let out = Command::new(python())
+        .args(["-c", &script])
+        .output()
+        .expect("failed to run python3");
+    assert!(
+        out.status.success(),
+        "{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    if String::from_utf8_lossy(&out.stdout).contains("NO_LIBHDF5") {
+        eprintln!("SKIP: h5py's bundled libhdf5 not found (needed for H5Pset_chunk_opts)");
+        return;
+    }
+
+    let file = File::open(&path).unwrap();
+    let line: Vec<f64> = (0..1000).map(|i| (f64::from(i) / 7.0).sin()).collect();
+    for name in ["fixed_1d", "ea_1d"] {
+        let values = file.dataset(name).unwrap().read_f64().unwrap();
+        assert_eq!(values.len(), line.len(), "{name}");
+        for (i, (v, w)) in values.iter().zip(&line).enumerate() {
+            assert!((v - w).abs() < 1e-12, "{name}[{i}]: {v} vs {w}");
+        }
+    }
+    let grid: Vec<f64> = (0..37 * 53).map(|i| f64::from(i) * 0.5).collect();
+    for name in ["bt2_2d", "fixed_2d"] {
+        assert_eq!(
+            file.dataset(name).unwrap().read_f64().unwrap(),
+            grid,
+            "{name}"
+        );
+    }
+    // A selection touching only the last (partial, unfiltered) chunk.
+    let tail = file
+        .dataset("fixed_1d")
+        .unwrap()
+        .read_selection(&Selection::Hyperslab {
+            start: vec![990],
+            stride: vec![1],
+            count: vec![1],
+            block: vec![10],
+        })
+        .unwrap();
+    let want: Vec<u8> = line[990..].iter().flat_map(|v| v.to_le_bytes()).collect();
+    assert_eq!(tail, want);
+}