fix(format): write fill times with libhdf5's codes; add fill values
FillTime::to_byte had the fill-time field rotated against libhdf5 (H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2): Never was written as ALLOC, Alloc as IFSET and IfSet as NEVER, as h5py reported. The flags byte is now late allocation plus the right code, and FillTime::from_byte decodes it. The default becomes IfSet, which is libhdf5's default and exactly the byte (0x0a) every dataset was already written with, so default output does not change; `Alloc` was documented as the C library's default but never was. DatasetCreateProps follows. DatasetBuilder::with_fill_value sets a user-defined fill value (one element's stored bytes, checked against the datatype size), written as a defined value in the fill value message. h5py reports it, and extending the dataset in h5py fills the new elements with it. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -13,7 +13,7 @@ use clawhdf5_format::message_type::MessageType;
|
||||
use clawhdf5_format::object_header::ObjectHeader;
|
||||
use clawhdf5_format::signature;
|
||||
use clawhdf5_format::superblock::Superblock;
|
||||
use clawhdf5_format::type_builders::make_u8_type;
|
||||
use clawhdf5_format::type_builders::{FillTime, make_u8_type};
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
@@ -373,3 +373,94 @@ fn h5py_opens_paged_files() {
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 4. fill time and fill value ----
|
||||
|
||||
fn fill_message(bytes: &[u8], path: &str) -> clawhdf5_format::object_header::HeaderMessage {
|
||||
let (_, oh) = header_at(bytes, path);
|
||||
oh.messages
|
||||
.into_iter()
|
||||
.find(|m| m.msg_type == MessageType::FillValue)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn fill_file() -> Vec<u8> {
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("never")
|
||||
.with_f64_data(&[1.0, 2.0])
|
||||
.fill_time(FillTime::Never);
|
||||
fw.create_dataset("alloc")
|
||||
.with_f64_data(&[1.0, 2.0])
|
||||
.fill_time(FillTime::Alloc);
|
||||
fw.create_dataset("ifset")
|
||||
.with_f64_data(&[1.0, 2.0])
|
||||
.fill_time(FillTime::IfSet);
|
||||
fw.create_dataset("default").with_f64_data(&[1.0, 2.0]);
|
||||
fw.create_dataset("filled")
|
||||
.with_i32_data(&[1, 2, 3, 4])
|
||||
.with_chunks(&[2])
|
||||
.with_maxshape(&[u64::MAX])
|
||||
.with_fill_value(&(-1i32).to_le_bytes());
|
||||
fw.finish().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_time_uses_libhdf5_codes() {
|
||||
// H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2, in bits 2-3. Measured:
|
||||
// h5py saw our Never as ALLOC, Alloc as IFSET and IfSet as NEVER.
|
||||
let bytes = fill_file();
|
||||
for (path, code) in [("never", 1), ("alloc", 0), ("ifset", 2), ("default", 2)] {
|
||||
let msg = fill_message(&bytes, path);
|
||||
assert_eq!((msg.data[1] >> 2) & 3, code, "{path}");
|
||||
assert_eq!(msg.data[1] & 3, 2, "{path}: allocation time stays late");
|
||||
}
|
||||
for ft in [FillTime::Never, FillTime::Alloc, FillTime::IfSet] {
|
||||
assert_eq!(FillTime::from_byte(ft.to_byte()), Some(ft));
|
||||
}
|
||||
assert_eq!(FillTime::default(), FillTime::IfSet);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fill_value_is_written_and_read_back() {
|
||||
let bytes = fill_file();
|
||||
let msg = fill_message(&bytes, "filled");
|
||||
assert_eq!(
|
||||
clawhdf5_format::fill_value::parse_fill_value(&msg).unwrap(),
|
||||
Some((-1i32).to_le_bytes().to_vec())
|
||||
);
|
||||
assert_eq!(
|
||||
clawhdf5_format::fill_value::parse_fill_value(&fill_message(&bytes, "ifset")).unwrap(),
|
||||
None
|
||||
);
|
||||
|
||||
// One element's bytes, no more, no less.
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("d")
|
||||
.with_f64_data(&[1.0])
|
||||
.with_fill_value(&[0; 4]);
|
||||
assert!(fw.finish().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module and h5dump"]
|
||||
fn h5py_sees_our_fill_time_and_fill_value() {
|
||||
let path = write_tmp("fill", &fill_file());
|
||||
let out = h5py(
|
||||
&path,
|
||||
"from h5py import h5d\n\
|
||||
f = h5py.File(path, 'r')\n\
|
||||
names = {h5d.FILL_TIME_NEVER: 'never', h5d.FILL_TIME_ALLOC: 'alloc', h5d.FILL_TIME_IFSET: 'ifset'}\n\
|
||||
t = [names[f[n].id.get_create_plist().get_fill_time()] for n in ('never', 'alloc', 'ifset', 'default')]\n\
|
||||
print(json.dumps([t, int(f['filled'].fillvalue), f['filled'][()].tolist()]))\n\
|
||||
f.close()\n\
|
||||
f = h5py.File(path, 'r+')\n\
|
||||
f['filled'].resize((7,))\n\
|
||||
f.close()\n\
|
||||
print(json.dumps(h5py.File(path, 'r')['filled'][()].tolist()))",
|
||||
);
|
||||
assert_eq!(
|
||||
out,
|
||||
"[[\"never\", \"alloc\", \"ifset\", \"ifset\"], -1, [1, 2, 3, 4]]\n[1, 2, 3, 4, -1, -1, -1]"
|
||||
);
|
||||
h5dump_ok(&path);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user