fix(format): resolve shared fill value messages instead of zero-filling

dataset_fill_value treated a shared Fill Value message as "no fill
value", so unwritten storage of a dataset whose fill value lives in the
file's shared-message (SOHM) heap read as zeros rather than its fill
value. libhdf5 shares fill values whenever the file has a SOHM index for
them.

- fill_value::dataset_fill_value_in follows the reference (another object
  header, or the SOHM heap); read_full_with_fill and the facade's
  selection read use it.
- dataset_fill_value, which has no file to follow a reference into, now
  returns UnresolvedSharedMessage for a shared message instead of None.
- shared_message::load_sohm_table / message_data_with_sohm load the SOHM
  table from the superblock extension on demand.
- parse_sohm_table skipped each index's leading version byte, reading
  every field one byte off; SOHM references could never resolve.

Fixture shared_fill_value.h5 (HDF5 2.0, gen_shared_fill.py): sohm_b read
[0,1,2,3,0,0,0,0] and now reads [0,1,2,3,-7,-7,-7,-7], as h5py does.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:18:12 -05:00
co-authored by Claude Opus 5.5
parent 57e938c438
commit 7c1968a34a
7 changed files with 268 additions and 12 deletions
+6 -1
View File
@@ -478,7 +478,12 @@ impl<'f> Dataset<'f> {
// sparse) dataset — select from a fill-aware full read instead. (The
// selection reader currently decodes the full dataset too, so this
// costs nothing extra.)
let fill = clawhdf5_format::fill_value::dataset_fill_value(&self.header.messages)?;
let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
self.file.data.as_bytes(),
&self.header.messages,
self.file.offset_size(),
self.file.length_size(),
)?;
let fill_matters = !clawhdf5_format::fill_value::has_storage(&dl)
|| (matches!(dl, DataLayout::Chunked { .. })
&& !clawhdf5_format::fill_value::is_default(fill.as_deref()));
@@ -0,0 +1,49 @@
//! Datasets whose Fill Value message is shared through the file's SOHM heap
//! (fixture written by HDF5 2.0, see `gen_shared_fill.py`). Their unwritten
//! storage must read as the fill value (-7), not as zeros.
use clawhdf5::File;
use clawhdf5_format::selection::Selection;
const FIXTURE: &[u8] = include_bytes!("../../clawhdf5-format/tests/fixtures/shared_fill_value.h5");
#[test]
fn shared_fill_value_applies_to_unwritten_storage() {
let file = File::from_bytes(FIXTURE.to_vec()).unwrap();
// `_a` keeps its fill value in its own header, `_b` references the SOHM
// heap; both must read the same.
for name in ["sohm_a", "sohm_b"] {
assert_eq!(
file.dataset(name).unwrap().read_i32().unwrap(),
[0, 1, 2, 3, -7, -7, -7, -7],
"{name}"
);
}
for name in ["unwritten_a", "unwritten_b"] {
assert_eq!(
file.dataset(name).unwrap().read_i32().unwrap(),
[-7, -7, -7],
"{name}"
);
}
// The selection path decides on its own whether the fill value matters.
let slab = Selection::Hyperslab {
start: vec![2],
stride: vec![1],
count: vec![4],
block: vec![1],
};
let raw = file
.dataset("sohm_b")
.unwrap()
.read_selection(&slab)
.unwrap();
let values: Vec<i32> = raw
.as_chunks::<4>()
.0
.iter()
.map(|b| i32::from_le_bytes(*b))
.collect();
assert_eq!(values, [2, 3, -7, -7]);
}