fix(format): a VL element at the undefined heap address is an error

libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.

Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-tools/tests/gen_vl_files.py

@@ -6,7 +6,8 @@ For 8-byte (`vl8`) and 4-byte (`vl4`) offsets, writes OUTDIR/vl8.h5 and
 OUTDIR/vl4.h5, which libhdf5 reads in full, and OUTDIR/bad8.h5 and
 OUTDIR/bad4.h5, whose `bad` and `badseq` elements 0 have a length that
 disagrees with their global heap object (libhdf5: "Expected global heap
-object size does not match"). h5py cannot write a VL string with a NUL in
+object size does not match"), and whose `undef` element 1 has length 0 and
+the undefined heap address (libhdf5: "addr undefined"). h5py cannot write a VL string with a NUL in
 it or a null element in a contiguous dataset, so those are patched in.
 
 Prints one JSON object: for each file, each dataset's values as h5py reads
@@ -76,11 +77,17 @@ def bad(path, sizes):
         s = f.create_dataset("badseq", shape=(2,), dtype=I4)
         s[0] = [1, 2, 3]
         s[1] = [4]
+        f.create_dataset("undef", data=np.array(["x", "", "yz"], dtype=object), dtype=S)
         off, soff = f["bad"].id.get_offset(), f["badseq"].id.get_offset()
+        uoff = f["undef"].id.get_offset()
     b = bytearray(open(path, "rb").read())
     gcol = int.from_bytes(b[off + 4 : off + 4 + os_], "little")
     struct.pack_into("<I", b, off, 3)  # "cdefgh": length 6 -> 3
     struct.pack_into("<I", b, soff, 2)  # [1, 2, 3]: length 3 -> 2
+    # "": length 0 at the undefined address (all 0xff), which libhdf5 fails
+    # to read ("addr undefined"); it writes a null element as address 0.
+    es = 8 + os_
+    b[uoff + es : uoff + 2 * es] = element(0, (1 << (8 * os_)) - 1, 1, os_)
     open(path, "wb").write(bytes(b))
     return gcol
 
@@ -116,6 +123,6 @@ for tag, sizes in (("8", None), ("4", (4, 4))):
         result[f"vl{tag}"] = {n: read(f[n]) for n in ("d", "u", "seq", "sequ", "cmp")}
         result[f"vl{tag}"]["va"] = [value(s) for s in f.attrs["va"]]
     with h5py.File(x, "r") as f:
-        result[f"bad{tag}"] = {n: read(f[n]) for n in ("bad", "badseq")}
+        result[f"bad{tag}"] = {n: read(f[n]) for n in ("bad", "badseq", "undef")}
         result[f"bad{tag}"]["gcol"] = gcol
 json.dump(result, sys.stdout)

crates/clawhdf5-tools/tests/h5rs_interop.rs

@@ -828,7 +828,8 @@ fn dump_prints_vl_data_like_h5dump() {
 
 /// `dump --json` gives the values h5py reads, element by element; and an
 /// element whose heap object is not its length × base size is an error, as
-/// in h5py, not a truncated value (it printed "cde" and (1, 2)).
+/// in h5py, not a truncated value (it printed "cde" and (1, 2)); so is a
+/// length-0 element at the undefined heap address (it printed "").
 #[test]
 fn dump_json_vl_values_match_h5py() {
     let Some(f) = generate_vl() else { return };
@@ -860,7 +861,12 @@ fn dump_json_vl_values_match_h5py() {
                     let e = g["error"]
                         .as_str()
                         .unwrap_or_else(|| panic!("{bad}: {path}: {g}"));
-                    assert!(e.contains("holds"), "{bad}: {path}: {e}");
+                    let why = if path == "/undef" {
+                        "undefined"
+                    } else {
+                        "holds"
+                    };
+                    assert!(e.contains(why), "{bad}: {path}: {e}");
                 } else {
                     assert_eq!(g, w, "{bad}: {path}");
                 }
@@ -871,7 +877,8 @@ fn dump_json_vl_values_match_h5py() {
 
 /// `check --data` holds VL elements to libhdf5's rule: a heap object whose
 /// size is not exactly the element's length × base size is a problem (it
-/// only caught objects shorter than the element).
+/// only caught objects shorter than the element), and so is an element at
+/// the undefined heap address.
 #[test]
 fn check_data_flags_mis_sized_vl_heap_objects() {
     let Some(f) = generate_vl() else { return };
@@ -893,5 +900,11 @@ fn check_data_flags_mis_sized_vl_heap_objects() {
             assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}");
             assert!(s.contains(what), "bad{tag}: {s}");
         }
+        // A length-0 element at the undefined heap address: libhdf5 fails
+        // to read it; check skipped it.
+        let undef: u64 = if tag == "8" { u64::MAX } else { 0xffff_ffff };
+        let want = format!("problem: {undef:#x} /undef: variable-length data: global heap:");
+        assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}");
+        assert!(s.contains("undefined global heap address"), "bad{tag}: {s}");
     }
 }