fix(format): refuse object header messages over 64 KiB

A v2 object header message has a 2-byte size field. The writer truncated
larger sizes to 16 bits, so an attribute over ~64 KiB (or a compact
dataset of 65532-65535 bytes, whose layout message adds 4 bytes) produced
a file libhdf5 rejects ("message of unshareable class flagged as
shareable", "bad flag combination").

ObjectHeaderWriter::serialize now returns a Result and fails on any message
over MAX_MESSAGE_SIZE; FileWriter::finish propagates it. Compact storage
falls back to contiguous above 65531 bytes, the real limit. Dense storage
for large attributes remains future work.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-format/tests/writer_meta_tests.rs

@@ -0,0 +1,153 @@
+//! Regression tests for writer metadata bugs that produced files libhdf5
+//! refuses (or reads differently from us), plus the reader-side counterparts.
+//!
+//! The plain tests check the bytes we write with our own parser. The
+//! `#[ignore]`d ones are the interop half: they open what we write in h5py
+//! (`CLAWHDF5_PYTHON`, as in `writer_h5py_tests.rs`) and run `h5dump` over it.
+
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::file_writer::{AttrValue, FileWriter};
+use clawhdf5_format::group_v2::resolve_path_any;
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::signature;
+use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::type_builders::make_u8_type;
+
+// ---- helpers ----
+
+fn header_at(bytes: &[u8], path: &str) -> (Superblock, ObjectHeader) {
+    let sig = signature::find_signature(bytes).unwrap();
+    let sb = Superblock::parse(bytes, sig).unwrap();
+    let addr = if path == "/" {
+        sb.root_group_address
+    } else {
+        resolve_path_any(bytes, &sb, path).unwrap()
+    };
+    let oh = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
+    (sb, oh)
+}
+
+fn layout_of(bytes: &[u8], path: &str) -> DataLayout {
+    let (sb, oh) = header_at(bytes, path);
+    let msg = oh
+        .messages
+        .iter()
+        .find(|m| m.msg_type == MessageType::DataLayout)
+        .unwrap();
+    DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap()
+}
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn write_tmp(name: &str, bytes: &[u8]) -> std::path::PathBuf {
+    let path = std::env::temp_dir().join(format!("clawhdf5_writer_meta_{name}.h5"));
+    std::fs::write(&path, bytes).unwrap();
+    path
+}
+
+/// Run `script` (with `path` bound to the file) under h5py; return stdout.
+fn h5py(path: &std::path::Path, script: &str) -> String {
+    let full = format!(
+        "import h5py, numpy as np, json\npath = {:?}\n{script}",
+        path.display().to_string()
+    );
+    let o = std::process::Command::new(python())
+        .args(["-c", &full])
+        .output()
+        .expect("python interpreter");
+    assert!(
+        o.status.success(),
+        "h5py failed: {}",
+        String::from_utf8_lossy(&o.stderr)
+    );
+    String::from_utf8(o.stdout).unwrap().trim().to_string()
+}
+
+/// `h5dump` must read the whole file without error.
+fn h5dump_ok(path: &std::path::Path) {
+    let o = std::process::Command::new("h5dump")
+        .arg(path)
+        .output()
+        .expect("h5dump");
+    assert!(
+        o.status.success(),
+        "h5dump failed: {}{}",
+        String::from_utf8_lossy(&o.stdout),
+        String::from_utf8_lossy(&o.stderr)
+    );
+}
+
+fn u8_ramp(n: usize) -> Vec<u8> {
+    (0..n).map(|i| (i % 251) as u8).collect()
+}
+
+// ---- 1. object header message size limit ----
+
+#[test]
+fn attribute_too_big_for_a_header_message_is_an_error() {
+    // Measured: a 70000-byte attribute was written with its message size
+    // wrapped to 16 bits, and libhdf5 refused the whole root group.
+    let mut fw = FileWriter::new();
+    fw.set_root_attr(
+        "a",
+        AttrValue::Raw {
+            datatype: make_u8_type(),
+            shape: vec![70_000],
+            data: u8_ramp(70_000),
+        },
+    );
+    assert!(fw.finish().is_err());
+
+    // 65500 bytes still fits and still works.
+    let mut fw = FileWriter::new();
+    fw.set_root_attr(
+        "a",
+        AttrValue::Raw {
+            datatype: make_u8_type(),
+            shape: vec![65_500],
+            data: u8_ramp(65_500),
+        },
+    );
+    let bytes = fw.finish().unwrap();
+    let (sb, oh) = header_at(&bytes, "/");
+    let attrs = clawhdf5_format::attribute::extract_attributes(&oh, sb.length_size).unwrap();
+    assert_eq!(attrs[0].raw_data, u8_ramp(65_500));
+}
+
+#[test]
+fn compact_layout_falls_back_to_contiguous_past_the_message_limit() {
+    // Layout message = 4 bytes + data; data may be at most 65531 bytes.
+    for (n, compact) in [(65_531, true), (65_532, false), (65_534, false)] {
+        let mut fw = FileWriter::new();
+        fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
+        let bytes = fw.finish().unwrap();
+        match layout_of(&bytes, "d") {
+            DataLayout::Compact { data } => {
+                assert!(compact, "{n} bytes must not be compact");
+                assert_eq!(data, u8_ramp(n));
+            }
+            DataLayout::Contiguous { .. } => assert!(!compact, "{n} bytes should be compact"),
+            other => panic!("unexpected layout {other:?}"),
+        }
+    }
+}
+
+#[test]
+#[ignore = "requires Python h5py module and h5dump"]
+fn h5py_reads_compact_datasets_at_the_limit() {
+    for n in [65_531usize, 65_534] {
+        let mut fw = FileWriter::new();
+        fw.create_dataset("d").with_u8_data(&u8_ramp(n)).compact();
+        let path = write_tmp(&format!("compact_{n}"), &fw.finish().unwrap());
+        let out = h5py(
+            &path,
+            "f = h5py.File(path, 'r'); v = f['d'][()]\n\
+             print(bool((v == (np.arange(v.size) % 251).astype(np.uint8)).all()), v.size)",
+        );
+        assert_eq!(out, format!("True {n}"));
+        h5dump_ok(&path);
+    }
+}