fix(format): encode Time, BitField, Opaque and Reference datatypes

Datatype::serialize returned an empty message for these four classes, so
any dataset or attribute of them (including a Raw attribute copied from
another file) was unreadable by libhdf5 ("ran off end of input buffer
while decoding"). They now encode exactly as libhdf5 does: legacy object
and region references as datatype version 1, H5T_STD_REF kinds as version
4 with their encoding version, opaque tags NUL-padded to 8 bytes.

Parsing an opaque tag now stops at its first NUL, so libhdf5's padding
no longer becomes part of the tag. Datatype::check_encodable rejects
what has no encoding (an opaque tag over 248 bytes); FileWriter::finish
calls it for every dataset and attribute type.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/tests/writer_meta_tests.rs

@@ -6,6 +6,7 @@
 //! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it.
 
 use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType};
 use clawhdf5_format::file_writer::{AttrValue, FileWriter};
 use clawhdf5_format::group_v2::resolve_path_any;
 use clawhdf5_format::message_type::MessageType;
@@ -151,3 +152,168 @@ fn h5py_reads_compact_datasets_at_the_limit() {
         h5dump_ok(&path);
     }
 }
+
+// ---- 2. Time / BitField / Opaque / Reference datatypes ----
+
+fn exotic_types() -> Vec<(&'static str, Datatype, Vec<u8>)> {
+    // Four elements each. The object references point at the root group,
+    // which a v3-superblock file without an extension puts at address 48.
+    let refs: Vec<u8> = (0..4).flat_map(|_| 48u64.to_le_bytes()).collect();
+    vec![
+        (
+            "bits",
+            Datatype::BitField {
+                size: 1,
+                byte_order: DatatypeByteOrder::LittleEndian,
+                bit_offset: 0,
+                bit_precision: 8,
+            },
+            vec![1, 2, 4, 8],
+        ),
+        (
+            "opaque",
+            Datatype::Opaque {
+                size: 4,
+                tag: b"mytag".to_vec(),
+            },
+            (0..16).collect(),
+        ),
+        (
+            "ref",
+            Datatype::Reference {
+                size: 8,
+                ref_type: ReferenceType::Object,
+            },
+            refs,
+        ),
+        (
+            "time",
+            Datatype::Time {
+                size: 4,
+                bit_precision: 32,
+            },
+            (0..16).collect(),
+        ),
+    ]
+}
+
+fn exotic_file() -> Vec<u8> {
+    let mut fw = FileWriter::new();
+    for (name, dt, raw) in exotic_types() {
+        fw.create_dataset(name)
+            .with_compound_data(dt.clone(), raw.clone(), 4);
+        fw.set_root_attr(
+            name,
+            AttrValue::Raw {
+                datatype: dt,
+                shape: vec![4],
+                data: raw,
+            },
+        );
+    }
+    fw.finish().unwrap()
+}
+
+#[test]
+fn exotic_datatypes_are_written_not_emptied() {
+    let bytes = exotic_file();
+    let (sb, root) = header_at(&bytes, "/");
+    assert_eq!(sb.root_group_address, 48);
+    let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
+    for (name, dt, raw) in exotic_types() {
+        let (_, oh) = header_at(&bytes, name);
+        let msg = oh
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::Datatype)
+            .unwrap();
+        assert_eq!(msg.data, dt.serialize(), "{name}");
+        assert_eq!(Datatype::parse(&msg.data).unwrap().0, dt, "{name}");
+        let attr = attrs.iter().find(|a| a.name == name).unwrap();
+        assert_eq!(attr.datatype, dt, "{name}");
+        assert_eq!(attr.raw_data, raw, "{name}");
+    }
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_reads_exotic_datatypes() {
+    let path = write_tmp("exotic", &exotic_file());
+    let out = h5py(
+        &path,
+        "from h5py import h5t, h5s\n\
+         f = h5py.File(path, 'r')\n\
+         r = {}\n\
+         buf = np.zeros(4, dtype='V4')\n\
+         f['opaque'].id.read(h5s.ALL, h5s.ALL, buf, mtype=f['opaque'].id.get_type())\n\
+         r['bits'] = f['bits'][()].tolist(), f.attrs['bits'].tolist()\n\
+         r['opaque'] = (f['opaque'].id.get_type().get_tag().decode(),\n\
+         \x20              f.attrs.get_id('opaque').get_type().get_tag().decode(),\n\
+         \x20              buf.tobytes().hex())\n\
+         r['ref'] = [f[x].name for x in f['ref'][()]] + [f[x].name for x in f.attrs['ref']]\n\
+         r['time'] = (f['time'].id.get_type().get_class() == h5t.TIME,\n\
+         \x20            f.attrs.get_id('time').get_type().get_class() == h5t.TIME)\n\
+         print(json.dumps(r))",
+    );
+    let v: serde_json::Value = serde_json::from_str(&out).unwrap();
+    assert_eq!(v["bits"], serde_json::json!([[1, 2, 4, 8], [1, 2, 4, 8]]));
+    assert_eq!(
+        v["opaque"],
+        serde_json::json!(["mytag", "mytag", "000102030405060708090a0b0c0d0e0f"])
+    );
+    assert_eq!(v["ref"], serde_json::json!(vec!["/"; 8]));
+    assert_eq!(v["time"], serde_json::json!([true, true]));
+    h5dump_ok(&path);
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn raw_attributes_copied_from_h5py_survive_a_rewrite() {
+    // Read Raw attributes of the exotic classes out of an h5py file and write
+    // them back: this used to emit empty datatype messages.
+    let src = std::env::temp_dir().join("clawhdf5_writer_meta_exotic_src.h5");
+    h5py(
+        &src,
+        "from h5py import h5t, h5s, h5a\n\
+         f = h5py.File(path, 'w')\n\
+         f.attrs['ref'] = np.array([f.ref, f.ref], dtype=h5py.ref_dtype)\n\
+         f.attrs.create('opaque', np.frombuffer(b'abcdefgh', dtype='V4'))\n\
+         t = h5t.STD_B16BE.copy()\n\
+         a = h5a.create(f.id, b'bits', t, h5s.create_simple((2,)))\n\
+         a.write(np.array([0x0102, 0x0304], dtype='>u2'), mtype=t)\n\
+         a.close()\n\
+         f.close()",
+    );
+    let src_bytes = std::fs::read(&src).unwrap();
+    let (sb, root) = header_at(&src_bytes, "/");
+    let attrs = clawhdf5_format::attribute::extract_attributes(&root, sb.length_size).unwrap();
+    assert_eq!(attrs.len(), 3);
+    let mut fw = FileWriter::new();
+    for a in &attrs {
+        let data = if a.name == "ref" {
+            // Re-target the references at our root group.
+            48u64.to_le_bytes().repeat(2)
+        } else {
+            a.raw_data.clone()
+        };
+        fw.set_root_attr(
+            &a.name,
+            AttrValue::Raw {
+                datatype: a.datatype.clone(),
+                shape: a.dataspace.dimensions.clone(),
+                data,
+            },
+        );
+    }
+    let path = write_tmp("exotic_copy", &fw.finish().unwrap());
+    let out = h5py(
+        &path,
+        "f = h5py.File(path, 'r')\n\
+         print(json.dumps([[f[x].name for x in f.attrs['ref']],\n\
+         \x20                f.attrs['opaque'].tobytes().decode(),\n\
+         \x20                f.attrs.get_id('bits').get_type().get_order(),\n\
+         \x20                f.attrs['bits'].tolist()]))",
+    );
+    assert_eq!(out, r#"[["/", "/"], "abcdefgh", 1, [258, 772]]"#);
+    h5dump_ok(&path);
+}