fix(format): write fill times with libhdf5's codes; add fill values

FillTime::to_byte had the fill-time field rotated against libhdf5
(H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2): Never was written as
ALLOC, Alloc as IFSET and IfSet as NEVER, as h5py reported. The flags
byte is now late allocation plus the right code, and FillTime::from_byte
decodes it.

The default becomes IfSet, which is libhdf5's default and exactly the
byte (0x0a) every dataset was already written with, so default output
does not change; `Alloc` was documented as the C library's default but
never was. DatasetCreateProps follows.

DatasetBuilder::with_fill_value sets a user-defined fill value (one
element's stored bytes, checked against the datatype size), written as a
defined value in the fill value message. h5py reports it, and extending
the dataset in h5py fills the new elements with it.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:11:07 -05:00
co-authored by Claude Opus 5.5
parent 74fdf0582b
commit 8c3ef996ea
4 changed files with 193 additions and 35 deletions
@@ -13,7 +13,7 @@ use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::signature;
use clawhdf5_format::superblock::Superblock;
use clawhdf5_format::type_builders::make_u8_type;
use clawhdf5_format::type_builders::{FillTime, make_u8_type};
// ---- helpers ----
@@ -373,3 +373,94 @@ fn h5py_opens_paged_files() {
h5dump_ok(&path);
}
}
// ---- 4. fill time and fill value ----
fn fill_message(bytes: &[u8], path: &str) -> clawhdf5_format::object_header::HeaderMessage {
let (_, oh) = header_at(bytes, path);
oh.messages
.into_iter()
.find(|m| m.msg_type == MessageType::FillValue)
.unwrap()
}
fn fill_file() -> Vec<u8> {
let mut fw = FileWriter::new();
fw.create_dataset("never")
.with_f64_data(&[1.0, 2.0])
.fill_time(FillTime::Never);
fw.create_dataset("alloc")
.with_f64_data(&[1.0, 2.0])
.fill_time(FillTime::Alloc);
fw.create_dataset("ifset")
.with_f64_data(&[1.0, 2.0])
.fill_time(FillTime::IfSet);
fw.create_dataset("default").with_f64_data(&[1.0, 2.0]);
fw.create_dataset("filled")
.with_i32_data(&[1, 2, 3, 4])
.with_chunks(&[2])
.with_maxshape(&[u64::MAX])
.with_fill_value(&(-1i32).to_le_bytes());
fw.finish().unwrap()
}
#[test]
fn fill_time_uses_libhdf5_codes() {
// H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2, in bits 2-3. Measured:
// h5py saw our Never as ALLOC, Alloc as IFSET and IfSet as NEVER.
let bytes = fill_file();
for (path, code) in [("never", 1), ("alloc", 0), ("ifset", 2), ("default", 2)] {
let msg = fill_message(&bytes, path);
assert_eq!((msg.data[1] >> 2) & 3, code, "{path}");
assert_eq!(msg.data[1] & 3, 2, "{path}: allocation time stays late");
}
for ft in [FillTime::Never, FillTime::Alloc, FillTime::IfSet] {
assert_eq!(FillTime::from_byte(ft.to_byte()), Some(ft));
}
assert_eq!(FillTime::default(), FillTime::IfSet);
}
#[test]
fn fill_value_is_written_and_read_back() {
let bytes = fill_file();
let msg = fill_message(&bytes, "filled");
assert_eq!(
clawhdf5_format::fill_value::parse_fill_value(&msg).unwrap(),
Some((-1i32).to_le_bytes().to_vec())
);
assert_eq!(
clawhdf5_format::fill_value::parse_fill_value(&fill_message(&bytes, "ifset")).unwrap(),
None
);
// One element's bytes, no more, no less.
let mut fw = FileWriter::new();
fw.create_dataset("d")
.with_f64_data(&[1.0])
.with_fill_value(&[0; 4]);
assert!(fw.finish().is_err());
}
#[test]
#[ignore = "requires Python h5py module and h5dump"]
fn h5py_sees_our_fill_time_and_fill_value() {
let path = write_tmp("fill", &fill_file());
let out = h5py(
&path,
"from h5py import h5d\n\
f = h5py.File(path, 'r')\n\
names = {h5d.FILL_TIME_NEVER: 'never', h5d.FILL_TIME_ALLOC: 'alloc', h5d.FILL_TIME_IFSET: 'ifset'}\n\
t = [names[f[n].id.get_create_plist().get_fill_time()] for n in ('never', 'alloc', 'ifset', 'default')]\n\
print(json.dumps([t, int(f['filled'].fillvalue), f['filled'][()].tolist()]))\n\
f.close()\n\
f = h5py.File(path, 'r+')\n\
f['filled'].resize((7,))\n\
f.close()\n\
print(json.dumps(h5py.File(path, 'r')['filled'][()].tolist()))",
);
assert_eq!(
out,
"[[\"never\", \"alloc\", \"ifset\", \"ifset\"], -1, [1, 2, 3, 4]]\n[1, 2, 3, 4, -1, -1, -1]"
);
h5dump_ok(&path);
}