osobhandClaude Opus 5.5
7c1968a34a
dataset_fill_value treated a shared Fill Value message as "no fill value", so unwritten storage of a dataset whose fill value lives in the file's shared-message (SOHM) heap read as zeros rather than its fill value. libhdf5 shares fill values whenever the file has a SOHM index for them. - fill_value::dataset_fill_value_in follows the reference (another object header, or the SOHM heap); read_full_with_fill and the facade's selection read use it. - dataset_fill_value, which has no file to follow a reference into, now returns UnresolvedSharedMessage for a shared message instead of None. - shared_message::load_sohm_table / message_data_with_sohm load the SOHM table from the superblock extension on demand. - parse_sohm_table skipped each index's leading version byte, reading every field one byte off; SOHM references could never resolve. Fixture shared_fill_value.h5 (HDF5 2.0, gen_shared_fill.py): sohm_b read [0,1,2,3,0,0,0,0] and now reads [0,1,2,3,-7,-7,-7,-7], as h5py does. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
50 lines
2.2 KiB
Python
50 lines
2.2 KiB
Python
"""Generate shared_fill_value.h5: datasets whose Fill Value message is
|
|
*shared*, in the two ways libhdf5 can share one.
|
|
|
|
- /sohm_a, /sohm_b: the file has a shared-object-header-message (SOHM) index
|
|
for fill values, so libhdf5 stores the fill value (-7, int32) in the SOHM
|
|
heap and /sohm_b's header holds only a reference to it. Chunked, with only
|
|
the first chunk written, so the rest reads as the fill value.
|
|
- /unwritten_a, /unwritten_b: the same, never written: no storage at all,
|
|
read entirely as the fill value.
|
|
|
|
h5py has no API for SOHM indexes, so the file creation property list is
|
|
configured by calling the libhdf5 bundled in the h5py wheel through ctypes.
|
|
Written with h5py 3.16.0 / HDF5 2.0.0. Re-run only to regenerate:
|
|
|
|
python gen_shared_fill.py shared_fill_value.h5
|
|
"""
|
|
import ctypes
|
|
import glob
|
|
import os
|
|
import sys
|
|
|
|
import h5py
|
|
import numpy as np
|
|
|
|
libdir = os.path.join(os.path.dirname(os.path.dirname(h5py.__file__)), "h5py.libs")
|
|
libs = [p for p in glob.glob(os.path.join(libdir, "libhdf5*.so*")) if "_hl" not in os.path.basename(p)]
|
|
lib = ctypes.CDLL(libs[0])
|
|
lib.H5open()
|
|
|
|
H5O_SHMESG_FILL_FLAG = 1 << 0x0005
|
|
|
|
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
|
|
lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]
|
|
lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]
|
|
assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0
|
|
assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, H5O_SHMESG_FILL_FLAG, 0) >= 0
|
|
|
|
fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
|
|
fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
|
|
fid = h5py.h5f.create(sys.argv[1].encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)
|
|
with h5py.File(fid) as f:
|
|
# Chunked, with only the first chunk written: the rest reads as fill.
|
|
# libhdf5 keeps the first copy of a message in its own header; the second
|
|
# identical one (the `_b` datasets) is the SOHM reference.
|
|
for name in ("sohm_a", "sohm_b"):
|
|
d = f.create_dataset(name, shape=(8,), chunks=(4,), dtype="<i4", fillvalue=-7)
|
|
d[:4] = np.arange(4)
|
|
for name in ("unwritten_a", "unwritten_b"):
|
|
f.create_dataset(name, shape=(3,), dtype="<i4", fillvalue=-7)
|