fix(format): give empty string attributes a 1-byte type

An empty AttrValue::String (or a StringArray of empty strings) was
written with a size-0 fixed-length string type. libhdf5 rejects that
("invalid datatype size"), and the failure takes every attribute on the
object with it. Strings are now at least 1 byte, NUL-padded, which is
how h5py stores "" and reads back as "" in both h5py and our reader.
check_encodable also refuses a size-0 string type passed in directly.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:12:12 -05:00
co-authored by Claude Opus 5.5
parent 8c3ef996ea
commit 14876b8ae5
3 changed files with 71 additions and 3 deletions
+3
View File
@@ -842,6 +842,9 @@ impl Datatype {
opaque_tag_text(tag).len()
)))
}
Datatype::String { size: 0, .. } => Err(FormatError::SerializationError(
"fixed-length string datatype of size 0 (libhdf5 requires at least 1 byte)".into(),
)),
Datatype::Compound { members, .. } => members
.iter()
.try_for_each(|m| m.datatype.check_encodable()),
+8 -3
View File
@@ -384,7 +384,11 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess
raw_data: data.clone(),
},
AttrValue::String(s) => {
let bytes = s.as_bytes();
// A fixed-length string type must be at least 1 byte: libhdf5
// rejects size 0 ("invalid datatype size") and with it every
// attribute on the object. h5py stores "" as one NUL byte.
let mut bytes = s.as_bytes().to_vec();
bytes.resize(bytes.len().max(1), 0);
AttributeMessage {
name: name.to_string(),
datatype: Datatype::String {
@@ -393,11 +397,12 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess
charset: CharacterSet::Utf8,
},
dataspace: scalar_ds(),
raw_data: bytes.to_vec(),
raw_data: bytes,
}
}
AttrValue::StringArray(arr) => {
let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0);
// At least 1 byte per element, as for a single string.
let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0).max(1);
let mut raw = Vec::new();
for s in arr {
let mut b = s.as_bytes().to_vec();
@@ -464,3 +464,63 @@ fn h5py_sees_our_fill_time_and_fill_value() {
);
h5dump_ok(&path);
}
// ---- 5. empty string attributes ----
fn empty_string_file() -> Vec<u8> {
let mut fw = FileWriter::new();
fw.set_root_attr("empty", AttrValue::String(String::new()));
fw.set_root_attr("x", AttrValue::String("héllo".into()));
fw.set_root_attr(
"empties",
AttrValue::StringArray(vec![String::new(), String::new()]),
);
fw.set_root_attr("n", AttrValue::I64(3));
fw.finish().unwrap()
}
#[test]
fn empty_string_attribute_has_a_one_byte_type() {
// Measured: "" got a size-0 string type, and libhdf5 then refused every
// attribute on the object ("invalid datatype size").
let bytes = empty_string_file();
let (sb, root) = header_at(&bytes, "/");
let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
for name in ["empty", "empties"] {
let a = attrs.iter().find(|a| a.name == name).unwrap();
assert_eq!(a.datatype.type_size(), 1, "{name}");
let strings = a.read_as_strings().unwrap();
assert!(strings.iter().all(String::is_empty), "{name}: {strings:?}");
}
// A size-0 string type handed in directly is refused, not written.
let mut fw = FileWriter::new();
fw.set_root_attr(
"raw",
AttrValue::Raw {
datatype: Datatype::String {
size: 0,
padding: clawhdf5_format::datatype::StringPadding::NullPad,
charset: clawhdf5_format::datatype::CharacterSet::Ascii,
},
shape: vec![],
data: vec![],
},
);
assert!(fw.finish().is_err());
}
#[test]
#[ignore = "requires Python h5py module and h5dump"]
fn h5py_reads_all_attributes_next_to_an_empty_string() {
let path = write_tmp("empty_str", &empty_string_file());
let out = h5py(
&path,
"f = h5py.File(path, 'r')\n\
d = lambda v: v.decode() if isinstance(v, bytes) else v\n\
print(json.dumps([d(f.attrs['empty']), d(f.attrs['x']),\n\
\x20 [d(s) for s in f.attrs['empties']], int(f.attrs['n'])], ensure_ascii=False))",
);
assert_eq!(out, r#"["", "héllo", ["", ""], 3]"#);
h5dump_ok(&path);
}