feat(format): apply fill values to unallocated storage on read

HDF5 allocates lazily: a chunk nobody wrote doesn't exist in the file, and a
dataset nobody wrote has no data address. Such regions must read as the
dataset's fill value. There was no Fill Value message parser at all, so:

- a sparse chunked dataset read its holes as zeros — silently wrong whenever
  the fill value isn't zero (h5py `fillvalue=-1` came back as 0);
- a dataset that was created but never written failed with NoDataAllocated /
  "no address for chunked layout" where h5py returns a filled array.

New clawhdf5_format::fill_value: parses Fill Value messages v1-v3 and the old
0x0004 message (validated against HDF5 2.0 output under default and latest
libver), builds a fully filled dataset when there is no storage, and writes the
fill value into exactly the chunk-grid cells absent from the chunk index —
never mistaking a stored zero for a hole, clipping edge chunks, any rank. It is
skipped entirely for the default (zero) fill value. The chunk index dispatch is
extracted from read_chunked_data into a reusable list_chunks.

The reader, lazy and mmap facades apply it on full reads; selection reads go
through a fill-aware full read when the fill value matters. h5py interop test
compares against h5py's own readback, including a sparse 2-D dataset and a
hyperslab straddling allocated and unallocated chunks.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/clawhdf5/src/lazy.rs

@@ -480,15 +480,27 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
         let dl = self.data_layout()?;
         let pipeline = self.filter_pipeline()?;
         let data = self.file.reader.as_bytes();
-        Ok(data_read::read_raw_data_full(
+        // Unallocated storage reads as the dataset's fill value.
+        clawhdf5_format::fill_value::read_full_with_fill(
+            &self.header.messages,
             data,
             &dl,
             &ds,
-            &dt,
-            pipeline.as_ref(),
+            dt.type_size() as usize,
             self.file.offset_size(),
             self.file.length_size(),
-        )?)
+            || {
+                Ok(data_read::read_raw_data_full(
+                    data,
+                    &dl,
+                    &ds,
+                    &dt,
+                    pipeline.as_ref(),
+                    self.file.offset_size(),
+                    self.file.length_size(),
+                )?)
+            },
+        )
     }
 }
 

crates/clawhdf5/src/mmap_file.rs

@@ -420,15 +420,27 @@ impl<'f> MmapDataset<'f> {
         let ds = self.dataspace()?;
         let dl = self.data_layout()?;
         let pipeline = self.filter_pipeline()?;
-        Ok(data_read::read_raw_data_full(
+        // Unallocated storage reads as the dataset's fill value.
+        clawhdf5_format::fill_value::read_full_with_fill(
+            &self.header.messages,
             self.file.reader.as_bytes(),
             &dl,
             &ds,
-            &dt,
-            pipeline.as_ref(),
+            dt.type_size() as usize,
             self.file.offset_size(),
             self.file.length_size(),
-        )?)
+            || {
+                Ok(data_read::read_raw_data_full(
+                    self.file.reader.as_bytes(),
+                    &dl,
+                    &ds,
+                    &dt,
+                    pipeline.as_ref(),
+                    self.file.offset_size(),
+                    self.file.length_size(),
+                )?)
+            },
+        )
     }
 }
 

crates/clawhdf5/src/reader.rs

@@ -448,6 +448,24 @@ impl<'f> Dataset<'f> {
         let ds = self.dataspace()?;
         let dl = self.data_layout()?;
         let pipeline = self.filter_pipeline()?;
+        // The selection reader knows nothing about fill values. When they
+        // matter — no storage at all, or a non-zero fill on a chunked (possibly
+        // sparse) dataset — select from a fill-aware full read instead. (The
+        // selection reader currently decodes the full dataset too, so this
+        // costs nothing extra.)
+        let fill = clawhdf5_format::fill_value::dataset_fill_value(&self.header.messages)?;
+        let fill_matters = !clawhdf5_format::fill_value::has_storage(&dl)
+            || (matches!(dl, DataLayout::Chunked { .. })
+                && !clawhdf5_format::fill_value::is_default(fill.as_deref()));
+        if fill_matters {
+            let full = self.read_raw()?;
+            return Ok(data_read::extract_selection_from_buffer(
+                &full,
+                &ds.dimensions,
+                dt.type_size() as usize,
+                selection,
+            )?);
+        }
         Ok(data_read::read_raw_data_selection(
             self.file.data.as_bytes(),
             &dl,
@@ -808,16 +826,28 @@ impl<'f> Dataset<'f> {
             )?);
         }
 
-        Ok(data_read::read_raw_data_cached(
+        // Unallocated storage reads as the dataset's fill value.
+        clawhdf5_format::fill_value::read_full_with_fill(
+            &self.header.messages,
             self.file.data.as_bytes(),
             &dl,
             &ds,
-            &dt,
-            pipeline.as_ref(),
+            dt.type_size() as usize,
             self.file.offset_size(),
             self.file.length_size(),
-            &self.file.chunk_cache,
-        )?)
+            || {
+                Ok(data_read::read_raw_data_cached(
+                    self.file.data.as_bytes(),
+                    &dl,
+                    &ds,
+                    &dt,
+                    pipeline.as_ref(),
+                    self.file.offset_size(),
+                    self.file.length_size(),
+                    &self.file.chunk_cache,
+                )?)
+            },
+        )
     }
 }
 

crates/clawhdf5/tests/h5py_interop_tests.rs

@@ -614,3 +614,88 @@ with h5py.File("{path_str}", "w"{kwargs}) as f:
         );
     }
 }
+
+// ---------------------------------------------------------------------------
+// h5py writes sparse / never-written datasets -> clawhdf5 applies fill values
+// ---------------------------------------------------------------------------
+
+/// Parse h5py's `print(arr.ravel().tolist())` output for integer data.
+fn parse_int_list(s: &str) -> Vec<i32> {
+    s.trim()
+        .trim_matches(|c| c == '[' || c == ']')
+        .split(',')
+        .filter(|t| !t.trim().is_empty())
+        .map(|t| t.trim().parse().unwrap())
+        .collect()
+}
+
+/// Storage HDF5 never allocated must read as the dataset's fill value. These
+/// used to read as zeros (silently wrong for a non-zero fill value) or fail
+/// outright (`NoDataAllocated`) for a dataset that was never written.
+#[test]
+fn h5py_fill_values_clawhdf5_reads() {
+    skip_if_no_python!();
+    let dir = tempfile::tempdir().unwrap();
+    for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
+        let path = dir.path().join(format!("fill_{tag}.h5"));
+        let path_str = path.display().to_string();
+        let script = format!(
+            r#"
+import h5py, numpy as np
+with h5py.File("{path_str}", "w"{kwargs}) as f:
+    d = f.create_dataset("partial", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=-1)
+    d[0:5] = np.arange(5)
+    f.create_dataset("never", shape=(4,), dtype="<i4", fillvalue=25)
+    f.create_dataset("never_chunked", shape=(6,), dtype="<i4", chunks=(3,), fillvalue=9)
+    f.create_dataset("default_fill", shape=(3,), dtype="<i4")
+    g = f.create_dataset("gz", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=7, compression="gzip")
+    g[10:15] = 1
+    s = f.create_dataset("sparse2d", shape=(5, 7), dtype="<i4", chunks=(2, 3), fillvalue=-3)
+    s[2:4, 3:6] = 8
+    s[4, 6] = 5
+with h5py.File("{path_str}", "r") as f:
+    for name in ["partial", "never", "never_chunked", "default_fill", "gz", "sparse2d"]:
+        print(name, f[name][...].ravel().tolist())
+    print("slab", f["sparse2d"][1:5, 2:7].ravel().tolist())
+"#
+        );
+        let expected: std::collections::HashMap<String, Vec<i32>> = run_python_output(&script)
+            .lines()
+            .map(|line| {
+                let (name, list) = line.split_once(' ').unwrap();
+                (name.to_string(), parse_int_list(list))
+            })
+            .collect();
+
+        let file = File::open(&path).unwrap();
+        for name in [
+            "partial",
+            "never",
+            "never_chunked",
+            "default_fill",
+            "gz",
+            "sparse2d",
+        ] {
+            assert_eq!(
+                file.dataset(name).unwrap().read_i32().unwrap(),
+                expected[name],
+                "{tag}/{name}"
+            );
+        }
+        // A hyperslab straddling allocated and unallocated chunks.
+        let slab = clawhdf5_format::selection::Selection::Hyperslab {
+            start: vec![1, 2],
+            stride: vec![1, 1],
+            count: vec![4, 5],
+            block: vec![1, 1],
+        };
+        assert_eq!(
+            file.dataset("sparse2d")
+                .unwrap()
+                .read_i32_selection(&slab)
+                .unwrap(),
+            expected["slab"],
+            "{tag}/sparse2d hyperslab"
+        );
+    }
+}