osobhandClaude Opus 5.5
74fdf0582b
FileWriter::with_page_size wrote a "version 4" superblock with an extra
page-size field. HDF5 has no superblock version 4, so libhdf5 refused
every such file ("bad superblock version number").
A paged file is now what libhdf5 itself writes for fs_strategy="page":
a v3 superblock whose extension object header holds a File Space Info
message (strategy PAGE, the page size, free space not persisted; same
bytes and flags as HDF5 2.0), with the file padded to a whole page.
h5py opens it, reports the strategy and page size, and can modify it in
r+ mode. Page sizes outside libhdf5's 512 B..1 GiB are an error.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
376 lines
13 KiB
Rust
376 lines
13 KiB
Rust
//! Regression tests for writer metadata bugs that produced files libhdf5
|
|
//! refuses (or reads differently from us), plus the reader-side counterparts.
|
|
//!
|
|
//! The plain tests check the bytes we write with our own parser. The
|
|
//! `#[ignore]`d ones are the interop half: they open what we write in h5py
|
|
//! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it.
|
|
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType};
|
|
use clawhdf5_format::file_writer::{AttrValue, FileWriter};
|
|
use clawhdf5_format::group_v2::resolve_path_any;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
use clawhdf5_format::signature;
|
|
use clawhdf5_format::superblock::Superblock;
|
|
use clawhdf5_format::type_builders::make_u8_type;
|
|
|
|
// ---- helpers ----
|
|
|
|
fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) {
|
|
let sig = signature::find_signature(bytes).unwrap();
|
|
let sb = Superblock::parse(bytes, sig).unwrap();
|
|
let addr = if path == "/" {
|
|
sb.root_group_address
|
|
} else {
|
|
resolve_path_any(bytes, &sb, path).unwrap()
|
|
};
|
|
let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
(sb, oh)
|
|
}
|
|
|
|
fn layout_of(bytes: &[u8], path: &str) -> DataLayout {
|
|
let (sb, oh) = header_at(bytes, path);
|
|
let msg = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap();
|
|
DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap()
|
|
}
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf {
|
|
let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5"));
|
|
std::fs::write(&path, bytes).unwrap();
|
|
path
|
|
}
|
|
|
|
/// Run `script` (with `path` bound to the file) under h5py; return stdout.
|
|
fn h5py(path: &std::path::Path, script: &str) -> String {
|
|
let full = format!(
|
|
"import h5py, numpy as np, json\npath = {:?}\n{script}",
|
|
path.display().to_string()
|
|
);
|
|
let o = std::process::Command::new(python())
|
|
.args(["-c", &full])
|
|
.output()
|
|
.expect("python interpreter");
|
|
assert!(
|
|
o.status.success(),
|
|
"h5py failed: {}",
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
String::from_utf8(o.stdout).unwrap().trim().to_string()
|
|
}
|
|
|
|
/// `h5dump` must read the whole file without error.
|
|
fn h5dump_ok(path: &std::path::Path) {
|
|
let o = std::process::Command::new("h5dump")
|
|
.arg(path)
|
|
.output()
|
|
.expect("h5dump");
|
|
assert!(
|
|
o.status.success(),
|
|
"h5dump failed: {}{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
}
|
|
|
|
fn u8_ramp(n: usize) -> Vec<u8> {
|
|
(0..n).map(|i| (i % 251) as u8).collect()
|
|
}
|
|
|
|
// ---- 1. object header message size limit ----
|
|
|
|
#[test]
|
|
fn attribute_too_big_for_a_header_message_is_an_error() {
|
|
// Measured: a 70000-byte attribute was written with its message size
|
|
// wrapped to 16 bits, and libhdf5 refused the whole root group.
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr(
|
|
"a",
|
|
AttrValue::Raw {
|
|
datatype: make_u8_type(),
|
|
shape: vec![70_000],
|
|
data: u8_ramp(70_000),
|
|
},
|
|
);
|
|
assert!(fw.finish().is_err());
|
|
|
|
// 65500 bytes still fits and still works.
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr(
|
|
"a",
|
|
AttrValue::Raw {
|
|
datatype: make_u8_type(),
|
|
shape: vec![65_500],
|
|
data: u8_ramp(65_500),
|
|
},
|
|
);
|
|
let bytes = fw.finish().unwrap();
|
|
let (sb, oh) = header_at(&bytes, "/");
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap();
|
|
assert_eq!(attrs[0].raw_data, u8_ramp(65_500));
|
|
}
|
|
|
|
#[test]
|
|
fn compact_layout_falls_back_to_contiguous_past_the_message_limit() {
|
|
// Layout message = 4 bytes + data; data may be at most 65531 bytes.
|
|
for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
|
|
let bytes = fw.finish().unwrap();
|
|
match layout_of(&bytes, "d") {
|
|
DataLayout::Compact { data } => {
|
|
assert!(compact, "{n} bytes must not be compact");
|
|
assert_eq!(data, u8_ramp(n));
|
|
}
|
|
DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"),
|
|
other => panic!("unexpected layout {other:?}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_reads_compact_datasets_at_the_limit() {
|
|
for n in [65_531usize, 65_534] {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
|
|
let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap());
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r'); v = f['d'][()]\n\
|
|
print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)",
|
|
);
|
|
assert_eq!(out, format!("True {n}"));
|
|
h5dump_ok(&path);
|
|
}
|
|
}
|
|
|
|
// ---- 2. Time / BitField / Opaque / Reference datatypes ----
|
|
|
|
fn exotic_types() -> Vec<(&'static str, Datatype, Vec<u8>)> {
|
|
// Four elements each. The object references point at the root group,
|
|
// which a v3-superblock file without an extension puts at address 48.
|
|
let refs: Vec<u8> = (0..4).flat_map(|_| 48u64.to_le_bytes()).collect();
|
|
vec![
|
|
(
|
|
"bits",
|
|
Datatype::BitField {
|
|
size: 1,
|
|
byte_order: DatatypeByteOrder::LittleEndian,
|
|
bit_offset: 0,
|
|
bit_precision: 8,
|
|
},
|
|
vec![1, 2, 4, 8],
|
|
),
|
|
(
|
|
"opaque",
|
|
Datatype::Opaque {
|
|
size: 4,
|
|
tag: b"mytag".to_vec(),
|
|
},
|
|
(0..16).collect(),
|
|
),
|
|
(
|
|
"ref",
|
|
Datatype::Reference {
|
|
size: 8,
|
|
ref_type: ReferenceType::Object,
|
|
},
|
|
refs,
|
|
),
|
|
(
|
|
"time",
|
|
Datatype::Time {
|
|
size: 4,
|
|
bit_precision: 32,
|
|
},
|
|
(0..16).collect(),
|
|
),
|
|
]
|
|
}
|
|
|
|
fn exotic_file() -> Vec<u8> {
|
|
let mut fw = FileWriter::new();
|
|
for (name, dt, raw) in exotic_types() {
|
|
fw.create_dataset(name)
|
|
.with_compound_data(dt.clone(), raw.clone(), 4);
|
|
fw.set_root_attr(
|
|
name,
|
|
AttrValue::Raw {
|
|
datatype: dt,
|
|
shape: vec![4],
|
|
data: raw,
|
|
},
|
|
);
|
|
}
|
|
fw.finish().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn exotic_datatypes_are_written_not_emptied() {
|
|
let bytes = exotic_file();
|
|
let (sb, root) = header_at(&bytes, "/");
|
|
assert_eq!(sb.root_group_address, 48);
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
|
|
for (name, dt, raw) in exotic_types() {
|
|
let (_, oh) = header_at(&bytes, name);
|
|
let msg = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::Datatype)
|
|
.unwrap();
|
|
assert_eq!(msg.data, dt.serialize(), "{name}");
|
|
assert_eq!(Datatype::parse(&msg.data).unwrap().0, dt, "{name}");
|
|
let attr = attrs.iter().find(|a| a.name == name).unwrap();
|
|
assert_eq!(attr.datatype, dt, "{name}");
|
|
assert_eq!(attr.raw_data, raw, "{name}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_reads_exotic_datatypes() {
|
|
let path = write_tmp("exotic", &exotic_file());
|
|
let out = h5py(
|
|
&path,
|
|
"from h5py import h5t, h5s\n\
|
|
f = h5py.File(path, 'r')\n\
|
|
r = {}\n\
|
|
buf = np.zeros(4, dtype='V4')\n\
|
|
f['opaque'].id.read(h5s.ALL, h5s.ALL, buf, mtype=f['opaque'].id.get_type())\n\
|
|
r['bits'] = f['bits'][()].tolist(), f.attrs['bits'].tolist()\n\
|
|
r['opaque'] = (f['opaque'].id.get_type().get_tag().decode(),\n\
|
|
\x20 f.attrs.get_id('opaque').get_type().get_tag().decode(),\n\
|
|
\x20 buf.tobytes().hex())\n\
|
|
r['ref'] = [f[x].name for x in f['ref'][()]] + [f[x].name for x in f.attrs['ref']]\n\
|
|
r['time'] = (f['time'].id.get_type().get_class() == h5t.TIME,\n\
|
|
\x20 f.attrs.get_id('time').get_type().get_class() == h5t.TIME)\n\
|
|
print(json.dumps(r))",
|
|
);
|
|
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
|
|
assert_eq!(v["bits"], serde_json::json!([[1, 2, 4, 8], [1, 2, 4, 8]]));
|
|
assert_eq!(
|
|
v["opaque"],
|
|
serde_json::json!(["mytag", "mytag", "000102030405060708090a0b0c0d0e0f"])
|
|
);
|
|
assert_eq!(v["ref"], serde_json::json!(vec!["/"; 8]));
|
|
assert_eq!(v["time"], serde_json::json!([true, true]));
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn raw_attributes_copied_from_h5py_survive_a_rewrite() {
|
|
// Read Raw attributes of the exotic classes out of an h5py file and write
|
|
// them back: this used to emit empty datatype messages.
|
|
let src = std::env::temp_dir().join("clawhdf5_writer_meta_exotic_src.h5");
|
|
h5py(
|
|
&src,
|
|
"from h5py import h5t, h5s, h5a\n\
|
|
f = h5py.File(path, 'w')\n\
|
|
f.attrs['ref'] = np.array([f.ref, f.ref], dtype=h5py.ref_dtype)\n\
|
|
f.attrs.create('opaque', np.frombuffer(b'abcdefgh', dtype='V4'))\n\
|
|
t = h5t.STD_B16BE.copy()\n\
|
|
a = h5a.create(f.id, b'bits', t, h5s.create_simple((2,)))\n\
|
|
a.write(np.array([0x0102, 0x0304], dtype='>u2'), mtype=t)\n\
|
|
a.close()\n\
|
|
f.close()",
|
|
);
|
|
let src_bytes = std::fs::read(&src).unwrap();
|
|
let (sb, root) = header_at(&src_bytes, "/");
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
|
|
assert_eq!(attrs.len(), 3);
|
|
let mut fw = FileWriter::new();
|
|
for a in &attrs {
|
|
let data = if a.name == "ref" {
|
|
// Re-target the references at our root group.
|
|
48u64.to_le_bytes().repeat(2)
|
|
} else {
|
|
a.raw_data.clone()
|
|
};
|
|
fw.set_root_attr(
|
|
&a.name,
|
|
AttrValue::Raw {
|
|
datatype: a.datatype.clone(),
|
|
shape: a.dataspace.dimensions.clone(),
|
|
data,
|
|
},
|
|
);
|
|
}
|
|
let path = write_tmp("exotic_copy", &fw.finish().unwrap());
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r')\n\
|
|
print(json.dumps([[f[x].name for x in f.attrs['ref']],\n\
|
|
\x20 f.attrs['opaque'].tobytes().decode(),\n\
|
|
\x20 f.attrs.get_id('bits').get_type().get_order(),\n\
|
|
\x20 f.attrs['bits'].tolist()]))",
|
|
);
|
|
assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
// ---- 3. paged file-space strategy ----
|
|
|
|
fn paged_file(page_size: u32) -> Vec<u8> {
|
|
let mut fw = FileWriter::new();
|
|
fw.with_page_size(page_size);
|
|
fw.create_dataset("d").with_f64_data(&[1.0, 2.0, 3.0]);
|
|
fw.create_dataset("c")
|
|
.with_i32_data(&(0..100).collect::<Vec<_>>())
|
|
.with_chunks(&[10]);
|
|
fw.set_root_attr("a", AttrValue::I64(7));
|
|
let mut g = fw.create_group("g");
|
|
g.create_dataset("e").with_u8_data(&[9; 5000]);
|
|
fw.add_group(g.finish());
|
|
fw.finish().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn paged_file_has_a_real_superblock() {
|
|
// Measured: `with_page_size` wrote superblock version 4, which does not
|
|
// exist ("bad superblock version number" in libhdf5).
|
|
for ps in [512u32, 4096, 65536] {
|
|
let bytes = paged_file(ps);
|
|
let (sb, _) = header_at(&bytes, "/");
|
|
assert_eq!(sb.version, 3);
|
|
assert_eq!(bytes.len() % ps as usize, 0);
|
|
let (_, e) = header_at(&bytes, "g/e");
|
|
assert!(
|
|
e.messages
|
|
.iter()
|
|
.any(|m| m.msg_type == MessageType::Dataspace)
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_opens_paged_files() {
|
|
for ps in [512u32, 4096, 65536] {
|
|
let path = write_tmp(&format!("paged_{ps}"), &paged_file(ps));
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r')\n\
|
|
p = f.id.get_create_plist()\n\
|
|
print(json.dumps([p.get_file_space_strategy()[0], p.get_file_space_page_size(),\n\
|
|
\x20 f['d'][()].tolist(), int(f['c'][()].sum()), int(f.attrs['a']),\n\
|
|
\x20 int(f['g/e'][()].sum())]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
format!("[1, {ps}, [1.0, 2.0, 3.0], 4950, 7, 45000]"),
|
|
"page size {ps}"
|
|
);
|
|
h5dump_ok(&path);
|
|
}
|
|
}
|