fix(format): resolve VL elements as libhdf5 does

Checked with h5py on a patched file:
- a VL string with an embedded NUL reads up to the NUL (libhdf5 converts
  VL strings to C strings); read_vl_strings returned "a\0b";
- an element whose global heap object is not length x base size bytes is
  an error ("Expected global heap object size does not match"); we
  returned the object cut to the length;
- a heap address of 0 is a null element whatever its length.

vl_data::VlResolver does this, caching each parsed heap collection:
read_vl_strings parsed the whole collection again for every element.
read_vl_strings and read_vl_bytes use it; check_element_size refuses a VL
type whose stored element size is not 4 + offset size + 4. The
conformance probe resolves VL values through VlResolver instead of its
own lenient copy (575 of 697, unchanged).

The new unit tests fail against the old read_vl_strings.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

conformance/probe/src/main.rs

@@ -19,9 +19,8 @@
 //! with its message, location and the clawhdf5 frames of its backtrace.
 
 use std::cell::RefCell;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::panic::{self, AssertUnwindSafe};
-use std::rc::Rc;
 
 use clawhdf5_format::attribute::extract_attributes_full;
 use clawhdf5_format::data_layout::DataLayout;
@@ -29,7 +28,6 @@ use clawhdf5_format::data_read;
 use clawhdf5_format::dataspace::{Dataspace, DataspaceType};
 use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
 use clawhdf5_format::filter_pipeline::FilterPipeline;
-use clawhdf5_format::global_heap::GlobalHeapCollection;
 use clawhdf5_format::group_v1::{self, GroupEntry};
 use clawhdf5_format::group_v2;
 use clawhdf5_format::message_type::MessageType;
@@ -37,6 +35,7 @@ use clawhdf5_format::object_header::ObjectHeader;
 use clawhdf5_format::signature;
 use clawhdf5_format::superblock::Superblock;
 use clawhdf5_format::symbol_table::SymbolTableMessage;
+use clawhdf5_format::vl_data::{VlResolver, check_element_size};
 use serde_json::{Map, Value, json};
 use sha2::{Digest, Sha256};
 
@@ -111,7 +110,10 @@ struct Ctx<'a> {
     os: u8,
     ls: u8,
     base_dir: std::path::PathBuf,
-    heaps: RefCell<HashMap<u64, Result<Rc<GlobalHeapCollection>, String>>>,
+    /// Resolves variable-length elements as the library does (null
+    /// elements, strings cut at a NUL, heap objects of the wrong size
+    /// refused), caching each heap collection.
+    vl: RefCell<VlResolver<'a>>,
 }
 
 impl<'a> Ctx<'a> {
@@ -130,33 +132,6 @@ impl<'a> Ctx<'a> {
         }
     }
 
-    fn heap_obj(&self, addr: u64, idx: u32) -> Result<Vec<u8>, String> {
-        let coll = {
-            let mut cache = self.heaps.borrow_mut();
-            cache
-                .entry(addr)
-                .or_insert_with(|| {
-                    GlobalHeapCollection::parse(self.data, addr as usize, self.ls)
-                        .map(Rc::new)
-                        .map_err(e)
-                })
-                .clone()?
-        };
-        coll.get_object(idx as u16)
-            .map(|o| o.data.clone())
-            .ok_or_else(|| {
-                format!("GlobalHeapObjectNotFound {{ collection_address: {addr}, index: {idx} }}")
-            })
-    }
-
-    fn read_offset(&self, b: &[u8]) -> u64 {
-        let mut v = 0u64;
-        for (i, x) in b.iter().take(self.os as usize).enumerate() {
-            v |= (*x as u64) << (8 * i);
-        }
-        v
-    }
-
     fn canon(&self, dt: &Datatype, b: &[u8], out: &mut Vec<u8>) -> Result<(), String> {
         let size = dt.type_size() as usize;
         if b.len() < size {
@@ -204,41 +179,27 @@ impl<'a> Ctx<'a> {
                 }
             }
             Datatype::VariableLength {
+                size: vl_size,
                 is_string,
                 base_type,
                 ..
             } => {
-                let len = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize;
-                let addr = self.read_offset(&b[4..]);
-                let idx_off = 4 + self.os as usize;
-                let idx = u32::from_le_bytes([
-                    b[idx_off],
-                    b[idx_off + 1],
-                    b[idx_off + 2],
-                    b[idx_off + 3],
-                ]);
-                let obj = if len == 0 || addr == 0 || addr == u64::MAX >> (64 - 8 * self.os as u32)
-                {
-                    Vec::new()
-                } else {
-                    self.heap_obj(addr, idx)?
-                };
+                check_element_size(*vl_size, self.os).map_err(e)?;
+                let el = &b[..size];
                 if *is_string {
-                    let l = len.min(obj.len());
-                    canon_str(&obj[..l], out);
+                    let s = self.vl.borrow_mut().string_bytes(el).map_err(e)?;
+                    canon_str(&s[0], out);
                 } else {
                     let bs = base_type.type_size() as usize;
-                    if bs == 0 {
-                        return Err("canon: VL base size 0".into());
-                    }
-                    let need = len.checked_mul(bs).ok_or("canon: VL overflow")?;
-                    if len > 0 && obj.len() < need {
-                        return Err(format!("canon: VL object {} < {need}", obj.len()));
-                    }
+                    // The borrow ends here: the base type may itself be
+                    // variable-length.
+                    let seq = self.vl.borrow_mut().sequences(el, bs).map_err(e)?;
+                    let seq = &seq[0];
+                    let len = seq.len() / bs;
                     out.push(b'V');
                     out.extend_from_slice(&(len as u32).to_le_bytes());
                     for i in 0..len {
-                        self.canon(base_type, &obj[i * bs..], out)?;
+                        self.canon(base_type, &seq[i * bs..], out)?;
                     }
                 }
             }
@@ -744,7 +705,7 @@ fn main() {
             .parent()
             .map(|p| p.to_path_buf())
             .unwrap_or_default(),
-        heaps: RefCell::new(HashMap::new()),
+        vl: RefCell::new(VlResolver::new(hdf5, sb.offset_size, sb.length_size)),
     };
     let mut objects: Vec<Value> = Vec::new();
     let mut visited = HashSet::new();