conformance/probe/src/main.rs

@@ -286,6 +286,29 @@ impl<'a> Ctx<'a> {
         Ok(())
     }
 
+    /// VDS source files resolve next to the virtual file; like the library,
+    /// refuse absolute paths and `..`.
+    fn vds_resolver(
+        &self,
+    ) -> impl Fn(&str) -> Result<Option<Vec<u8>>, clawhdf5_format::error::FormatError> + use<> {
+        let base = self.base_dir.clone();
+        move |name: &str| {
+            use clawhdf5_format::error::FormatError;
+            let p = std::path::Path::new(name);
+            if p.is_absolute()
+                || p.components()
+                    .any(|c| matches!(c, std::path::Component::ParentDir))
+            {
+                return Err(FormatError::ChunkedReadError(format!("refused {name}")));
+            }
+            match std::fs::read(base.join(p)) {
+                Ok(b) => Ok(Some(b)),
+                Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
+                Err(err) => Err(FormatError::ChunkedReadError(err.to_string())),
+            }
+        }
+    }
+
     fn read_dataset(&self, h: &ObjectHeader, rec: &mut Map<String, Value>) -> Result<(), String> {
         let dtb = self
             .payload(h, MessageType::Datatype)?
@@ -295,7 +318,27 @@ impl<'a> Ctx<'a> {
         let dsb = self
             .payload(h, MessageType::Dataspace)?
             .ok_or("MissingMessage(Dataspace)")?;
-        let ds = Dataspace::parse(&dsb, self.ls).map_err(e)?;
+        let mut ds = Dataspace::parse(&dsb, self.ls).map_err(e)?;
+        // A virtual dataset's extent can come from its sources (unlimited /
+        // printf mappings), as h5py reports it, rather than the stored one.
+        if let Some(lm) = h
+            .messages
+            .iter()
+            .find(|m| m.msg_type == MessageType::DataLayout)
+            && let Ok(dl @ DataLayout::Virtual { .. }) =
+                DataLayout::parse(&lm.data, self.os, self.ls)
+        {
+            let resolver = self.vds_resolver();
+            ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent(
+                self.data,
+                &dl,
+                &ds,
+                self.os,
+                self.ls,
+                Some(&resolver),
+            )
+            .map_err(e)?;
+        }
         let (shape, n) = Self::shape(&ds);
         rec.insert("shape".into(), shape);
         if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES {
@@ -331,28 +374,26 @@ impl<'a> Ctx<'a> {
             );
         }
         let raw = if matches!(dl, DataLayout::Virtual { .. }) {
-            let base = self.base_dir.clone();
-            let resolver = move |name: &str| -> Option<Vec<u8>> {
-                let p = std::path::Path::new(name);
-                if p.is_absolute()
-                    || p.components()
-                        .any(|c| matches!(c, std::path::Component::ParentDir))
-                {
-                    return None;
-                }
-                std::fs::read(base.join(p)).ok()
-            };
-            data_read::read_raw_data_full_with_resolver(
+            let resolver = self.vds_resolver();
+            let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
+                self.data,
+                &h.messages,
+                self.os,
+                self.ls,
+            )
+            .map_err(e)?;
+            clawhdf5_format::vds::read_virtual_dataset(
                 self.data,
                 &dl,
                 &ds,
                 &dt,
-                pipeline.as_ref(),
+                fill.as_deref(),
                 self.os,
                 self.ls,
                 Some(&resolver),
             )
             .map_err(e)?
+            .data
         } else {
             let cache = clawhdf5_format::chunk_cache::ChunkCache::new();
             clawhdf5_format::fill_value::read_full_with_fill::<clawhdf5_format::error::FormatError>(
@@ -660,10 +701,13 @@ fn main() {
             return;
         }
     };
-    let sb = guarded(|| {
-        let off = signature::find_signature(&data).map_err(e)?;
-        Superblock::parse(&data, off).map_err(e)
-    });
+    // Every address is relative to the superblock: look at the file from
+    // there on (past any user block), as libhdf5 does.
+    let hdf5: &[u8] = match signature::find_signature(&data) {
+        Ok(off) => &data[off..],
+        Err(_) => &data,
+    };
+    let sb = guarded(|| Superblock::parse(hdf5, 0).map_err(e));
     let sb = match sb {
         Ok(sb) => sb,
         Err(msg) => {
@@ -674,7 +718,7 @@ fn main() {
     };
     top.insert("superblock_version".into(), json!(sb.version));
     let ctx = Ctx {
-        data: &data,
+        data: hdf5,
         os: sb.offset_size,
         ls: sb.length_size,
         base_dir: std::path::Path::new(&path)