fix(format): read VL values in files with 4-byte offsets

In a file with sizeof_addr = 4, a VL string attribute came back as
AttrValue::Raw, a compound's VL member failed with
GlobalHeapObjectNotFound and VL datasets failed with a size mismatch.

Two bugs: Datatype::type_size() said 16 for every VL type, while the
element is 4 + offset size + 4 bytes (12 here); and the global heap was
parsed without the padding libhdf5 puts after its collection and object
headers (both round up to 8), so with 4-byte lengths every object was
looked up 4 bytes early. Datatype::VariableLength now carries the size
its datatype message stores, and writes it back.

Checked against h5py in tests/vl_offset4_interop.rs (fails with either
fix reverted). Conformance unchanged at 575 of 697; in cve-2024-32608 a
VL attribute whose datatype claims 524304-byte elements is now an error
(h5py cannot iterate those attributes at all).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5/tests/vl_offset4_interop.rs

@@ -0,0 +1,126 @@
+//! Variable-length values in files with 4-byte offsets and lengths
+//! (`sizeof_addr = 4`), checked against h5py/libhdf5 through the
+//! `clawhdf5_format` decoders.
+//!
+//! These failed with `GlobalHeapObjectNotFound` or came back as
+//! `AttrValue::Raw`: the VL datatype claimed 16-byte elements whatever the
+//! file's offset size, and the global heap was read without the padding
+//! libhdf5 puts after its collection and object headers. Skipped when
+//! python3 with h5py is unavailable, unless `CLAWHDF5_REQUIRE_INTEROP=1`.
+
+use std::process::Command;
+
+use clawhdf5::{AttrValue, File, Selection};
+use clawhdf5_format::data_read::{read_as_i64, read_compound_fields};
+use clawhdf5_format::vl_data::{read_vl_bytes, read_vl_strings};
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+fn python_available() -> bool {
+    Command::new(python())
+        .args(["-c", "import h5py, numpy"])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+#[test]
+fn vl_values_in_a_file_with_4_byte_offsets_read_like_h5py() {
+    if !python_available() {
+        assert!(
+            !interop_required(),
+            "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
+        );
+        eprintln!("SKIP: python3 with h5py not available");
+        return;
+    }
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("offset4.h5");
+    let script = format!(
+        r#"
+import h5py, numpy as np
+S = h5py.string_dtype()
+fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(4, 4)
+with h5py.File(h5py.h5f.create({path:?}.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl)) as f:
+    f.attrs['vlattr'] = 'attr-value'
+    f.attrs.create('vlattr_arr', np.array(['p', 'qq', ''], dtype=object), dtype=S)
+    ct = np.dtype([('id', '<i4'), ('name', S), ('v', '<f8')])
+    arr = np.zeros(3, dtype=ct); arr['id'] = [1, 2, 3]; arr['name'] = ['one', '', 'three']
+    f.create_dataset('compound', data=arr)
+    f.attrs.create('vlen_attr', np.array([np.array([1, 2], dtype='<i8'), np.array([3], dtype='<i8')],
+                   dtype=object), dtype=h5py.vlen_dtype(np.dtype('<i8')))
+with h5py.File({path:?}, 'r') as f:
+    assert f.id.get_create_plist().get_sizes() == (4, 4)
+    print(f.attrs['vlattr'])
+    print(','.join(f.attrs['vlattr_arr']))
+    print(','.join(s.decode() for s in f['compound']['name']))
+    print(';'.join(' '.join(str(x) for x in s) for s in f.attrs['vlen_attr']))
+"#,
+        path = path.display().to_string()
+    );
+    let output = Command::new(python())
+        .args(["-c", &script])
+        .output()
+        .expect("failed to run python");
+    assert!(
+        output.status.success(),
+        "python failed:\n{}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    let lines: Vec<&str> = stdout.lines().collect();
+    let (vlattr, vlattr_arr, names, seqs) = (lines[0], lines[1], lines[2], lines[3]);
+
+    let file = File::open(&path).unwrap();
+    let sb = file.superblock();
+    assert_eq!((sb.offset_size, sb.length_size), (4, 4));
+
+    let attrs = file.root().attrs().unwrap();
+    match &attrs["vlattr"] {
+        AttrValue::String(s) => assert_eq!(s, vlattr),
+        other => panic!("vlattr: {other:?}"),
+    }
+    match &attrs["vlattr_arr"] {
+        AttrValue::StringArray(s) => assert_eq!(s.join(","), vlattr_arr),
+        other => panic!("vlattr_arr: {other:?}"),
+    }
+
+    // The compound's VL string member.
+    let ds = file.dataset("compound").unwrap();
+    let dt = ds.raw_datatype().unwrap();
+    assert_eq!(dt.type_size(), 24, "4 + 12-byte VL element + 8");
+    let raw = ds.read_selection(&Selection::All).unwrap();
+    let fields = read_compound_fields(&raw, &dt).unwrap();
+    let name = fields.iter().find(|f| f.name == "name").unwrap();
+    assert_eq!(name.datatype.type_size(), 12);
+    let got = read_vl_strings(file.as_bytes(), &name.raw_data, 3, 4, 4).unwrap();
+    assert_eq!(got.join(","), names);
+    let id = fields.iter().find(|f| f.name == "id").unwrap();
+    assert_eq!(
+        read_as_i64(&id.raw_data, &id.datatype).unwrap(),
+        vec![1, 2, 3]
+    );
+
+    // A VL sequence attribute.
+    let AttrValue::Raw { datatype, data, .. } = &attrs["vlen_attr"] else {
+        panic!("vlen_attr is Raw");
+    };
+    let clawhdf5_format::datatype::Datatype::VariableLength { base_type, .. } = datatype else {
+        panic!("vlen_attr is VL");
+    };
+    let got: Vec<String> = read_vl_bytes(file.as_bytes(), data, 2, 4, 4)
+        .unwrap()
+        .iter()
+        .map(|b| {
+            let v = read_as_i64(b, base_type).unwrap();
+            v.iter().map(i64::to_string).collect::<Vec<_>>().join(" ")
+        })
+        .collect();
+    assert_eq!(got.join(";"), seqs);
+}