osobhandClaude Opus 5.5
d102c06306
FileWriter wrote the root group plus one level of groups, and refused
path-like names. The writer now flattens its builders into a group tree
(writer_tree.rs) before layout:
- A name may be a path ("a/b/x", "/a/b/x" at the root); missing
intermediate groups are created as h5py does, and GroupBuilder gains
create_group/add_group so builders nest to any depth. A group added at a
path that already holds a group is merged into it (require_group);
any other repeated name, an empty or "." component, or an absolute path
below the root is an error.
- add_soft_link, add_hard_link and add_external_link on FileWriter,
FileBuilder and GroupBuilder. Hard-link targets are resolved to objects
at finish (through other hard links; a missing target, a soft link on the
way or a cycle of paths is an error). Objects with several hard links get
an Object Reference Count message so libhdf5 can delete one link without
freeing the object.
- track_order(true) per group, or as the file default, tracks and indexes
link creation order: Link Info flags and max order, the order in each
Link message, and a type-6 creation-order B-tree for dense groups.
- A group's link index is one B-tree leaf; more than 65535 links is an
error.
Groups are laid out depth-first from the root, datasets group by group,
and untracked groups keep writing datasets, then groups, then other links:
files with one level of groups are byte-identical to before.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
664 lines
23 KiB
Rust
664 lines
23 KiB
Rust
//! Regression tests for writer metadata bugs that produced files libhdf5
|
|
//! refuses (or reads differently from us), plus the reader-side counterparts.
|
|
//!
|
|
//! The plain tests check the bytes we write with our own parser. The
|
|
//! `#[ignore]`d ones are the interop half: they open what we write in h5py
|
|
//! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it.
|
|
|
|
use clawhdf5_format::data_layout::DataLayout;
|
|
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType};
|
|
use clawhdf5_format::file_writer::{AttrValue, FileWriter};
|
|
use clawhdf5_format::group_v2::resolve_path_any;
|
|
use clawhdf5_format::message_type::MessageType;
|
|
use clawhdf5_format::object_header::ObjectHeader;
|
|
use clawhdf5_format::signature;
|
|
use clawhdf5_format::superblock::Superblock;
|
|
use clawhdf5_format::type_builders::{FillTime, make_u8_type};
|
|
|
|
// ---- helpers ----
|
|
|
|
fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) {
|
|
let sig = signature::find_signature(bytes).unwrap();
|
|
let sb = Superblock::parse(bytes, sig).unwrap();
|
|
let addr = if path == "/" {
|
|
sb.root_group_address
|
|
} else {
|
|
resolve_path_any(bytes, &sb, path).unwrap()
|
|
};
|
|
let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
|
(sb, oh)
|
|
}
|
|
|
|
fn layout_of(bytes: &[u8], path: &str) -> DataLayout {
|
|
let (sb, oh) = header_at(bytes, path);
|
|
let msg = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::DataLayout)
|
|
.unwrap();
|
|
DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap()
|
|
}
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf {
|
|
let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5"));
|
|
std::fs::write(&path, bytes).unwrap();
|
|
path
|
|
}
|
|
|
|
/// Run `script` (with `path` bound to the file) under h5py; return stdout.
|
|
fn h5py(path: &std::path::Path, script: &str) -> String {
|
|
let full = format!(
|
|
"import h5py, numpy as np, json\npath = {:?}\n{script}",
|
|
path.display().to_string()
|
|
);
|
|
let o = std::process::Command::new(python())
|
|
.args(["-c", &full])
|
|
.output()
|
|
.expect("python interpreter");
|
|
assert!(
|
|
o.status.success(),
|
|
"h5py failed: {}",
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
String::from_utf8(o.stdout).unwrap().trim().to_string()
|
|
}
|
|
|
|
/// `h5dump` must read the whole file without error.
|
|
fn h5dump_ok(path: &std::path::Path) {
|
|
let o = std::process::Command::new("h5dump")
|
|
.arg(path)
|
|
.output()
|
|
.expect("h5dump");
|
|
assert!(
|
|
o.status.success(),
|
|
"h5dump failed: {}{}",
|
|
String::from_utf8_lossy(&o.stdout),
|
|
String::from_utf8_lossy(&o.stderr)
|
|
);
|
|
}
|
|
|
|
fn u8_ramp(n: usize) -> Vec<u8> {
|
|
(0..n).map(|i| (i % 251) as u8).collect()
|
|
}
|
|
|
|
// ---- 1. object header message size limit ----
|
|
|
|
#[test]
|
|
fn attribute_too_big_for_a_header_message_is_an_error() {
|
|
// Measured: a 70000-byte attribute was written with its message size
|
|
// wrapped to 16 bits, and libhdf5 refused the whole root group.
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr(
|
|
"a",
|
|
AttrValue::Raw {
|
|
datatype: make_u8_type(),
|
|
shape: vec![70_000],
|
|
data: u8_ramp(70_000),
|
|
},
|
|
);
|
|
assert!(fw.finish().is_err());
|
|
|
|
// 65500 bytes still fits and still works.
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr(
|
|
"a",
|
|
AttrValue::Raw {
|
|
datatype: make_u8_type(),
|
|
shape: vec![65_500],
|
|
data: u8_ramp(65_500),
|
|
},
|
|
);
|
|
let bytes = fw.finish().unwrap();
|
|
let (sb, oh) = header_at(&bytes, "/");
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap();
|
|
assert_eq!(attrs[0].raw_data, u8_ramp(65_500));
|
|
}
|
|
|
|
#[test]
|
|
fn compact_layout_falls_back_to_contiguous_past_the_message_limit() {
|
|
// Layout message = 4 bytes + data; data may be at most 65531 bytes.
|
|
for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
|
|
let bytes = fw.finish().unwrap();
|
|
match layout_of(&bytes, "d") {
|
|
DataLayout::Compact { data } => {
|
|
assert!(compact, "{n} bytes must not be compact");
|
|
assert_eq!(data, u8_ramp(n));
|
|
}
|
|
DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"),
|
|
other => panic!("unexpected layout {other:?}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_reads_compact_datasets_at_the_limit() {
|
|
for n in [65_531usize, 65_534] {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
|
|
let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap());
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r'); v = f['d'][()]\n\
|
|
print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)",
|
|
);
|
|
assert_eq!(out, format!("True {n}"));
|
|
h5dump_ok(&path);
|
|
}
|
|
}
|
|
|
|
// ---- 2. Time / BitField / Opaque / Reference datatypes ----
|
|
|
|
fn exotic_types() -> Vec<(&'static str, Datatype, Vec<u8>)> {
|
|
// Four elements each. The object references point at the root group,
|
|
// which a v3-superblock file without an extension puts at address 48.
|
|
let refs: Vec<u8> = (0..4).flat_map(|_| 48u64.to_le_bytes()).collect();
|
|
vec![
|
|
(
|
|
"bits",
|
|
Datatype::BitField {
|
|
size: 1,
|
|
byte_order: DatatypeByteOrder::LittleEndian,
|
|
bit_offset: 0,
|
|
bit_precision: 8,
|
|
},
|
|
vec![1, 2, 4, 8],
|
|
),
|
|
(
|
|
"opaque",
|
|
Datatype::Opaque {
|
|
size: 4,
|
|
tag: b"mytag".to_vec(),
|
|
},
|
|
(0..16).collect(),
|
|
),
|
|
(
|
|
"ref",
|
|
Datatype::Reference {
|
|
size: 8,
|
|
ref_type: ReferenceType::Object,
|
|
},
|
|
refs,
|
|
),
|
|
(
|
|
"time",
|
|
Datatype::Time {
|
|
size: 4,
|
|
bit_precision: 32,
|
|
},
|
|
(0..16).collect(),
|
|
),
|
|
]
|
|
}
|
|
|
|
fn exotic_file() -> Vec<u8> {
|
|
let mut fw = FileWriter::new();
|
|
for (name, dt, raw) in exotic_types() {
|
|
fw.create_dataset(name)
|
|
.with_compound_data(dt.clone(), raw.clone(), 4);
|
|
fw.set_root_attr(
|
|
name,
|
|
AttrValue::Raw {
|
|
datatype: dt,
|
|
shape: vec![4],
|
|
data: raw,
|
|
},
|
|
);
|
|
}
|
|
fw.finish().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn exotic_datatypes_are_written_not_emptied() {
|
|
let bytes = exotic_file();
|
|
let (sb, root) = header_at(&bytes, "/");
|
|
assert_eq!(sb.root_group_address, 48);
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
|
|
for (name, dt, raw) in exotic_types() {
|
|
let (_, oh) = header_at(&bytes, name);
|
|
let msg = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::Datatype)
|
|
.unwrap();
|
|
assert_eq!(msg.data, dt.serialize(), "{name}");
|
|
assert_eq!(Datatype::parse(&msg.data).unwrap().0, dt, "{name}");
|
|
let attr = attrs.iter().find(|a| a.name == name).unwrap();
|
|
assert_eq!(attr.datatype, dt, "{name}");
|
|
assert_eq!(attr.raw_data, raw, "{name}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_reads_exotic_datatypes() {
|
|
let path = write_tmp("exotic", &exotic_file());
|
|
let out = h5py(
|
|
&path,
|
|
"from h5py import h5t, h5s\n\
|
|
f = h5py.File(path, 'r')\n\
|
|
r = {}\n\
|
|
buf = np.zeros(4, dtype='V4')\n\
|
|
f['opaque'].id.read(h5s.ALL, h5s.ALL, buf, mtype=f['opaque'].id.get_type())\n\
|
|
r['bits'] = f['bits'][()].tolist(), f.attrs['bits'].tolist()\n\
|
|
r['opaque'] = (f['opaque'].id.get_type().get_tag().decode(),\n\
|
|
\x20 f.attrs.get_id('opaque').get_type().get_tag().decode(),\n\
|
|
\x20 buf.tobytes().hex())\n\
|
|
r['ref'] = [f[x].name for x in f['ref'][()]] + [f[x].name for x in f.attrs['ref']]\n\
|
|
r['time'] = (f['time'].id.get_type().get_class() == h5t.TIME,\n\
|
|
\x20 f.attrs.get_id('time').get_type().get_class() == h5t.TIME)\n\
|
|
print(json.dumps(r))",
|
|
);
|
|
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
|
|
assert_eq!(v["bits"], serde_json::json!([[1, 2, 4, 8], [1, 2, 4, 8]]));
|
|
assert_eq!(
|
|
v["opaque"],
|
|
serde_json::json!(["mytag", "mytag", "000102030405060708090a0b0c0d0e0f"])
|
|
);
|
|
assert_eq!(v["ref"], serde_json::json!(vec!["/"; 8]));
|
|
assert_eq!(v["time"], serde_json::json!([true, true]));
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn raw_attributes_copied_from_h5py_survive_a_rewrite() {
|
|
// Read Raw attributes of the exotic classes out of an h5py file and write
|
|
// them back: this used to emit empty datatype messages.
|
|
let src = std::env::temp_dir().join("clawhdf5_writer_meta_exotic_src.h5");
|
|
h5py(
|
|
&src,
|
|
"from h5py import h5t, h5s, h5a\n\
|
|
f = h5py.File(path, 'w')\n\
|
|
f.attrs['ref'] = np.array([f.ref, f.ref], dtype=h5py.ref_dtype)\n\
|
|
f.attrs.create('opaque', np.frombuffer(b'abcdefgh', dtype='V4'))\n\
|
|
t = h5t.STD_B16BE.copy()\n\
|
|
a = h5a.create(f.id, b'bits', t, h5s.create_simple((2,)))\n\
|
|
a.write(np.array([0x0102, 0x0304], dtype='>u2'), mtype=t)\n\
|
|
a.close()\n\
|
|
f.close()",
|
|
);
|
|
let src_bytes = std::fs::read(&src).unwrap();
|
|
let (sb, root) = header_at(&src_bytes, "/");
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
|
|
assert_eq!(attrs.len(), 3);
|
|
let mut fw = FileWriter::new();
|
|
for a in &attrs {
|
|
let data = if a.name == "ref" {
|
|
// Re-target the references at our root group.
|
|
48u64.to_le_bytes().repeat(2)
|
|
} else {
|
|
a.raw_data.clone()
|
|
};
|
|
fw.set_root_attr(
|
|
&a.name,
|
|
AttrValue::Raw {
|
|
datatype: a.datatype.clone(),
|
|
shape: a.dataspace.dimensions.clone(),
|
|
data,
|
|
},
|
|
);
|
|
}
|
|
let path = write_tmp("exotic_copy", &fw.finish().unwrap());
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r')\n\
|
|
print(json.dumps([[f[x].name for x in f.attrs['ref']],\n\
|
|
\x20 f.attrs['opaque'].tobytes().decode(),\n\
|
|
\x20 f.attrs.get_id('bits').get_type().get_order(),\n\
|
|
\x20 f.attrs['bits'].tolist()]))",
|
|
);
|
|
assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
// ---- 3. paged file-space strategy ----
|
|
|
|
fn paged_file(page_size: u32) -> Vec<u8> {
|
|
let mut fw = FileWriter::new();
|
|
fw.with_page_size(page_size);
|
|
fw.create_dataset("d").with_f64_data(&[1.0, 2.0, 3.0]);
|
|
fw.create_dataset("c")
|
|
.with_i32_data(&(0..100).collect::<Vec<_>>())
|
|
.with_chunks(&[10]);
|
|
fw.set_root_attr("a", AttrValue::I64(7));
|
|
let mut g = fw.create_group("g");
|
|
g.create_dataset("e").with_u8_data(&[9; 5000]);
|
|
fw.add_group(g.finish());
|
|
fw.finish().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn paged_file_has_a_real_superblock() {
|
|
// Measured: `with_page_size` wrote superblock version 4, which does not
|
|
// exist ("bad superblock version number" in libhdf5).
|
|
for ps in [512u32, 4096, 65536] {
|
|
let bytes = paged_file(ps);
|
|
let (sb, _) = header_at(&bytes, "/");
|
|
assert_eq!(sb.version, 3);
|
|
assert_eq!(bytes.len() % ps as usize, 0);
|
|
let (_, e) = header_at(&bytes, "g/e");
|
|
assert!(
|
|
e.messages
|
|
.iter()
|
|
.any(|m| m.msg_type == MessageType::Dataspace)
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_opens_paged_files() {
|
|
for ps in [512u32, 4096, 65536] {
|
|
let path = write_tmp(&format!("paged_{ps}"), &paged_file(ps));
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r')\n\
|
|
p = f.id.get_create_plist()\n\
|
|
print(json.dumps([p.get_file_space_strategy()[0], p.get_file_space_page_size(),\n\
|
|
\x20 f['d'][()].tolist(), int(f['c'][()].sum()), int(f.attrs['a']),\n\
|
|
\x20 int(f['g/e'][()].sum())]))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
format!("[1, {ps}, [1.0, 2.0, 3.0], 4950, 7, 45000]"),
|
|
"page size {ps}"
|
|
);
|
|
h5dump_ok(&path);
|
|
}
|
|
}
|
|
|
|
// ---- 4. fill time and fill value ----
|
|
|
|
fn fill_message(bytes: &[u8], path: &str) -> clawhdf5_format::object_header::HeaderMessage {
|
|
let (_, oh) = header_at(bytes, path);
|
|
oh.messages
|
|
.into_iter()
|
|
.find(|m| m.msg_type == MessageType::FillValue)
|
|
.unwrap()
|
|
}
|
|
|
|
fn fill_file() -> Vec<u8> {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("never")
|
|
.with_f64_data(&[1.0, 2.0])
|
|
.fill_time(FillTime::Never);
|
|
fw.create_dataset("alloc")
|
|
.with_f64_data(&[1.0, 2.0])
|
|
.fill_time(FillTime::Alloc);
|
|
fw.create_dataset("ifset")
|
|
.with_f64_data(&[1.0, 2.0])
|
|
.fill_time(FillTime::IfSet);
|
|
fw.create_dataset("default").with_f64_data(&[1.0, 2.0]);
|
|
fw.create_dataset("filled")
|
|
.with_i32_data(&[1, 2, 3, 4])
|
|
.with_chunks(&[2])
|
|
.with_maxshape(&[u64::MAX])
|
|
.with_fill_value(&(-1i32).to_le_bytes());
|
|
fw.finish().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn fill_time_uses_libhdf5_codes() {
|
|
// H5D_FILL_TIME_ALLOC = 0, NEVER = 1, IFSET = 2, in bits 2-3. Measured:
|
|
// h5py saw our Never as ALLOC, Alloc as IFSET and IfSet as NEVER.
|
|
let bytes = fill_file();
|
|
for (path, code) in [("never", 1), ("alloc", 0), ("ifset", 2), ("default", 2)] {
|
|
let msg = fill_message(&bytes, path);
|
|
assert_eq!((msg.data[1] >> 2) & 3, code, "{path}");
|
|
assert_eq!(msg.data[1] & 3, 2, "{path}: allocation time stays late");
|
|
}
|
|
for ft in [FillTime::Never, FillTime::Alloc, FillTime::IfSet] {
|
|
assert_eq!(FillTime::from_byte(ft.to_byte()), Some(ft));
|
|
}
|
|
assert_eq!(FillTime::default(), FillTime::IfSet);
|
|
}
|
|
|
|
#[test]
|
|
fn fill_value_is_written_and_read_back() {
|
|
let bytes = fill_file();
|
|
let msg = fill_message(&bytes, "filled");
|
|
assert_eq!(
|
|
clawhdf5_format::fill_value::parse_fill_value(&msg).unwrap(),
|
|
Some((-1i32).to_le_bytes().to_vec())
|
|
);
|
|
assert_eq!(
|
|
clawhdf5_format::fill_value::parse_fill_value(&fill_message(&bytes, "ifset")).unwrap(),
|
|
None
|
|
);
|
|
|
|
// One element's bytes, no more, no less.
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("d")
|
|
.with_f64_data(&[1.0])
|
|
.with_fill_value(&[0; 4]);
|
|
assert!(fw.finish().is_err());
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_sees_our_fill_time_and_fill_value() {
|
|
let path = write_tmp("fill", &fill_file());
|
|
let out = h5py(
|
|
&path,
|
|
"from h5py import h5d\n\
|
|
f = h5py.File(path, 'r')\n\
|
|
names = {h5d.FILL_TIME_NEVER: 'never', h5d.FILL_TIME_ALLOC: 'alloc', h5d.FILL_TIME_IFSET: 'ifset'}\n\
|
|
t = [names[f[n].id.get_create_plist().get_fill_time()] for n in ('never', 'alloc', 'ifset', 'default')]\n\
|
|
print(json.dumps([t, int(f['filled'].fillvalue), f['filled'][()].tolist()]))\n\
|
|
f.close()\n\
|
|
f = h5py.File(path, 'r+')\n\
|
|
f['filled'].resize((7,))\n\
|
|
f.close()\n\
|
|
print(json.dumps(h5py.File(path, 'r')['filled'][()].tolist()))",
|
|
);
|
|
assert_eq!(
|
|
out,
|
|
"[[\"never\", \"alloc\", \"ifset\", \"ifset\"], -1, [1, 2, 3, 4]]\n[1, 2, 3, 4, -1, -1, -1]"
|
|
);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
// ---- 5. empty string attributes ----
|
|
|
|
fn empty_string_file() -> Vec<u8> {
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr("empty", AttrValue::String(String::new()));
|
|
fw.set_root_attr("x", AttrValue::String("héllo".into()));
|
|
fw.set_root_attr(
|
|
"empties",
|
|
AttrValue::StringArray(vec![String::new(), String::new()]),
|
|
);
|
|
fw.set_root_attr("n", AttrValue::I64(3));
|
|
fw.finish().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn empty_string_attribute_has_a_one_byte_type() {
|
|
// Measured: "" got a size-0 string type, and libhdf5 then refused every
|
|
// attribute on the object ("invalid datatype size").
|
|
let bytes = empty_string_file();
|
|
let (sb, root) = header_at(&bytes, "/");
|
|
let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
|
|
for name in ["empty", "empties"] {
|
|
let a = attrs.iter().find(|a| a.name == name).unwrap();
|
|
assert_eq!(a.datatype.type_size(), 1, "{name}");
|
|
let strings = a.read_as_strings().unwrap();
|
|
assert!(strings.iter().all(String::is_empty), "{name}: {strings:?}");
|
|
}
|
|
|
|
// A size-0 string type handed in directly is refused, not written.
|
|
let mut fw = FileWriter::new();
|
|
fw.set_root_attr(
|
|
"raw",
|
|
AttrValue::Raw {
|
|
datatype: Datatype::String {
|
|
size: 0,
|
|
padding: clawhdf5_format::datatype::StringPadding::NullPad,
|
|
charset: clawhdf5_format::datatype::CharacterSet::Ascii,
|
|
},
|
|
shape: vec![],
|
|
data: vec![],
|
|
},
|
|
);
|
|
assert!(fw.finish().is_err());
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "requires Python h5py module and h5dump"]
|
|
fn h5py_reads_all_attributes_next_to_an_empty_string() {
|
|
let path = write_tmp("empty_str", &empty_string_file());
|
|
let out = h5py(
|
|
&path,
|
|
"f = h5py.File(path, 'r')\n\
|
|
d = lambda v: v.decode() if isinstance(v, bytes) else v\n\
|
|
print(json.dumps([d(f.attrs['empty']), d(f.attrs['x']),\n\
|
|
\x20 [d(s) for s in f.attrs['empties']], int(f.attrs['n'])], ensure_ascii=False))",
|
|
);
|
|
assert_eq!(out, r#"["", "héllo", ["", ""], 3]"#);
|
|
h5dump_ok(&path);
|
|
}
|
|
|
|
// ---- 6. path-like names ----
|
|
|
|
#[test]
|
|
fn path_names_create_nested_groups() {
|
|
// create_group("a/b") used to write one link literally named "a/b",
|
|
// which h5py cannot reach ("component not found"); then such names were
|
|
// refused. Now a path creates its missing intermediate groups, as h5py
|
|
// does.
|
|
let mut fw = FileWriter::new();
|
|
let mut g = fw.create_group("a/b");
|
|
g.create_dataset("c").with_f64_data(&[1.0]);
|
|
fw.add_group(g.finish());
|
|
fw.create_dataset("x/y").with_f64_data(&[2.0]);
|
|
fw.create_dataset("/a/b/z").with_f64_data(&[3.0]);
|
|
let mut g = fw.create_group("g");
|
|
g.create_dataset("x/y").with_f64_data(&[4.0]);
|
|
fw.add_group(g.finish());
|
|
let bytes = fw.finish().unwrap();
|
|
for path in ["a", "a/b", "a/b/c", "a/b/z", "x", "x/y", "g/x", "g/x/y"] {
|
|
header_at(&bytes, path);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn names_that_are_not_valid_link_names_are_errors() {
|
|
for bad in ["", ".", "a//b", "a/", "a/./b", "/"] {
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset(bad).with_f64_data(&[1.0]);
|
|
assert!(fw.finish().is_err(), "{bad:?}");
|
|
}
|
|
// An absolute path inside a group, and a name used twice.
|
|
let mut fw = FileWriter::new();
|
|
let mut g = fw.create_group("g");
|
|
g.create_dataset("/x").with_f64_data(&[1.0]);
|
|
fw.add_group(g.finish());
|
|
assert!(fw.finish().is_err());
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("x").with_f64_data(&[1.0]);
|
|
fw.create_dataset("x").with_f64_data(&[1.0]);
|
|
assert!(fw.finish().is_err());
|
|
// A dataset in the way of a path.
|
|
let mut fw = FileWriter::new();
|
|
fw.create_dataset("x").with_f64_data(&[1.0]);
|
|
fw.create_dataset("x/y").with_f64_data(&[1.0]);
|
|
assert!(fw.finish().is_err());
|
|
|
|
// '/' stays legal in attribute names.
|
|
let mut fw = FileWriter::new();
|
|
let mut g = fw.create_group("g");
|
|
g.create_dataset("c").with_f64_data(&[1.0]);
|
|
g.set_attr("m/s", AttrValue::I64(1));
|
|
fw.add_group(g.finish());
|
|
let bytes = fw.finish().unwrap();
|
|
header_at(&bytes, "g/c");
|
|
}
|
|
|
|
// ---- 7. unknown-message flags on read ----
|
|
|
|
#[test]
|
|
fn unknown_message_flags_follow_libhdf5_on_tbogus() {
|
|
// libhdf5's own test file (test/testfiles/tbogus.h5): datasets carrying
|
|
// an unknown message with various flags. libhdf5 (read-only) opens
|
|
// Dataset1, 2, 4 and 5 and refuses Dataset3 ("unknown message with 'fail
|
|
// if unknown' flag found"). We used to refuse Dataset2 (bit 3, which only
|
|
// applies when writing) and open Dataset3 (bit 7, fail always).
|
|
let bytes = include_bytes!("fixtures/tbogus.h5");
|
|
let sig = signature::find_signature(bytes).unwrap();
|
|
let sb = Superblock::parse(bytes, sig).unwrap();
|
|
for (name, readable) in [
|
|
("Dataset1", true),
|
|
("Dataset2", true),
|
|
("Dataset3", false),
|
|
("Dataset4", true),
|
|
("Dataset5", true),
|
|
] {
|
|
let addr = resolve_path_any(bytes, &sb, name).unwrap();
|
|
let parsed = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size);
|
|
match parsed {
|
|
Ok(_) => assert!(readable, "{name} must be refused"),
|
|
Err(e) => {
|
|
assert!(!readable, "{name} must be readable, got {e:?}");
|
|
assert!(matches!(
|
|
e,
|
|
clawhdf5_format::error::FormatError::UnsupportedMessage(_)
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- 8. shared fill value messages ----
|
|
|
|
#[test]
|
|
fn shared_fill_value_is_resolved_not_zero() {
|
|
// gen_shared_fill.py: HDF5 2.0 with a SOHM index for fill values, so each
|
|
// dataset's fill value message is a reference into the SOHM heap. It
|
|
// used to be read as "no fill value" (zeros) instead of -7.
|
|
let bytes = include_bytes!("fixtures/shared_fill_value.h5");
|
|
for (name, shared) in [
|
|
("sohm_a", false),
|
|
("sohm_b", true),
|
|
("unwritten_a", false),
|
|
("unwritten_b", true),
|
|
] {
|
|
let (sb, oh) = header_at(bytes, name);
|
|
let msg = oh
|
|
.messages
|
|
.iter()
|
|
.find(|m| m.msg_type == MessageType::FillValue)
|
|
.unwrap();
|
|
assert_eq!(
|
|
clawhdf5_format::shared_message::is_shared(msg.flags),
|
|
shared,
|
|
"{name}: fixture layout"
|
|
);
|
|
if shared {
|
|
// Without the file the reference cannot be followed: an error,
|
|
// never a silent default.
|
|
assert_eq!(
|
|
clawhdf5_format::fill_value::dataset_fill_value(&oh.messages),
|
|
Err(clawhdf5_format::error::FormatError::UnresolvedSharedMessage)
|
|
);
|
|
}
|
|
assert_eq!(
|
|
clawhdf5_format::fill_value::dataset_fill_value_in(
|
|
bytes,
|
|
&oh.messages,
|
|
sb.offset_size,
|
|
sb.length_size
|
|
)
|
|
.unwrap(),
|
|
Some((-7i32).to_le_bytes().to_vec()),
|
|
"{name}"
|
|
);
|
|
}
|
|
}
|