feat: read external-file Virtual Datasets via a source resolver

VDS sources living in other files were previously unsupported because the
pure-byte read API has no filesystem. Add a resolver seam and wire a default.

clawhdf5-format:
- Add VdsSourceResolver (Fn(&str) -> Option<Vec<u8>>) and
  read_raw_data_full_with_resolver. read_virtual_data uses the resolver to
  fetch an external source file's bytes by its stored name, then reads the
  named source dataset from those bytes and scatters as usual. A resolver
  returning None leaves the region at fill (HDF5's missing-source behavior);
  an external source with no resolver at all is a clean error. read_raw_data_full
  is unchanged (delegates with no resolver).

clawhdf5:
- File now records the directory it was opened from and, for virtual layouts,
  reads through a default resolver that loads sibling source files relative to
  that directory. So File::open(virt).dataset(d).read_*() transparently
  assembles cross-file VDS. In-memory files (from_bytes) have no directory, so
  only same-file VDS resolves there.

Tests: format-layer external read with an injected resolver (and the
no-resolver error path), plus facade tests that drop both files in a temp dir
and read through File::open — covering successful resolution and the
missing-source-is-fill case.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

CHANGELOG.md

@@ -15,6 +15,17 @@
   float members all read end-to-end, validated against HDF5 2.0.
 
 ### New Features
+- `clawhdf5` / `clawhdf5-format`: read **external-file Virtual Datasets (VDS)**.
+  The format layer gains `read_raw_data_full_with_resolver` and a
+  `VdsSourceResolver` callback (`Fn(&str) -> Option<Vec<u8>>`) that maps a
+  stored source file name to its bytes, so the pure-byte reader can pull in
+  external sources without a filesystem of its own. The `clawhdf5` `File` API
+  wires a default resolver that reads sibling source files relative to the
+  opened file's directory, so `File::open(...).dataset(...).read_*()` now
+  transparently assembles cross-file VDS. A source file the resolver cannot
+  supply leaves its region at the fill value (matching HDF5); an external
+  source with no resolver at all is a clean error. In-memory files
+  (`File::from_bytes`) have no directory, so only same-file VDS resolves there.
 - `clawhdf5-format`: assemble **same-file Virtual Datasets (VDS)** of any rank.
   Previously a virtual layout returned `UnsupportedVersion`. The reader now
   decodes the global-heap mapping block (reverse-engineered against HDF5 2.0:

crates/clawhdf5-format/src/data_read.rs

@@ -73,6 +73,16 @@ pub fn read_raw_data(
     read_raw_data_full(file_data, layout, dataspace, datatype, None, 8, 8)
 }
 
+/// Resolves a Virtual Dataset source **file name** (as stored in the mapping,
+/// e.g. `"ext_src.h5"`) to that file's raw bytes.
+///
+/// The pure-byte read API has no filesystem of its own, so external-file VDS
+/// sources are read through a caller-supplied resolver. The std file API wires
+/// one that reads relative to the virtual file's directory; callers can supply
+/// their own (e.g. an in-memory map) in `no_std` builds. Returning `None` means
+/// the source file is unavailable and the mapping is skipped.
+pub type VdsSourceResolver<'a> = dyn Fn(&str) -> Option<Vec<u8>> + 'a;
+
 /// Read raw bytes with full parameters including filter pipeline and sizes.
 pub fn read_raw_data_full(
     file_data: &[u8],
@@ -82,6 +92,40 @@ pub fn read_raw_data_full(
     pipeline: Option<&FilterPipeline>,
     offset_size: u8,
     length_size: u8,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_full_impl(
+        file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, None,
+    )
+}
+
+/// Like [`read_raw_data_full`], but with a resolver for external-file Virtual
+/// Dataset sources. For non-virtual layouts the resolver is ignored.
+#[allow(clippy::too_many_arguments)]
+pub fn read_raw_data_full_with_resolver(
+    file_data: &[u8],
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    resolver: Option<&VdsSourceResolver>,
+) -> Result<Vec<u8>, FormatError> {
+    read_raw_data_full_impl(
+        file_data, layout, dataspace, datatype, pipeline, offset_size, length_size, resolver,
+    )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn read_raw_data_full_impl(
+    file_data: &[u8],
+    layout: &DataLayout,
+    dataspace: &Dataspace,
+    datatype: &Datatype,
+    pipeline: Option<&FilterPipeline>,
+    offset_size: u8,
+    length_size: u8,
+    resolver: Option<&VdsSourceResolver>,
 ) -> Result<Vec<u8>, FormatError> {
     let num_elements = dataspace.num_elements() as usize;
     let elem_size = datatype.type_size() as usize;
@@ -140,6 +184,7 @@ pub fn read_raw_data_full(
             datatype,
             offset_size,
             length_size,
+            resolver,
         ),
     }
 }
@@ -385,14 +430,17 @@ pub fn read_raw_data_selection(
 
 /// Assemble a **Virtual Dataset (VDS)** from its source mappings.
 ///
-/// Supports **same-file** virtual datasets of any rank: each mapping's source
+/// Supports virtual datasets of any rank. Same-file sources are read directly;
-/// dataset is read from the same file and its selected elements are scattered
+/// **external-file** sources are read through the caller-supplied `resolver`,
-/// into the virtual buffer at the positions given by the virtual selection
+/// which maps a stored source file name to that file's bytes. Each mapping's
-/// (both enumerated in row-major order, as HDF5 pairs them). Unmapped regions
+/// selected source elements are scattered into the virtual buffer at the
-/// are left at the zero fill value.
+/// positions given by the virtual selection (both enumerated in row-major
+/// order, as HDF5 pairs them). Unmapped regions are left at the zero fill value.
 ///
-/// External-file sources are reported as unsupported rather than silently
+/// A mapping whose external source file the resolver cannot supply (`None`) is
-/// producing wrong data.
+/// skipped, leaving its region at fill — matching HDF5's tolerance of missing
+/// sources. An external source with no resolver at all is a hard error.
+#[allow(clippy::too_many_arguments)]
 fn read_virtual_data(
     file_data: &[u8],
     global_heap_address: Option<u64>,
@@ -401,6 +449,7 @@ fn read_virtual_data(
     datatype: &Datatype,
     offset_size: u8,
     length_size: u8,
+    resolver: Option<&VdsSourceResolver>,
 ) -> Result<Vec<u8>, FormatError> {
     use crate::data_layout::parse_vds_mappings;
     use crate::global_heap::GlobalHeapCollection;
@@ -425,18 +474,33 @@ fn read_virtual_data(
     let mappings = parse_vds_mappings(&obj.data, length_size)?;
 
     for m in &mappings {
-        // Only same-file sources are reachable through the pure-byte read API.
+        let same_file = m.source_file.is_empty() || m.source_file == ".";
-        if !(m.source_file.is_empty() || m.source_file == ".") {
+
-            return Err(FormatError::ChunkedReadError(
+        // Resolve the bytes of the file holding this source dataset.
-                "external-file virtual dataset sources are not supported".into(),
+        let external;
-            ));
+        let src_file_data: &[u8] = if same_file {
+            file_data
+        } else {
+            let r = resolver.ok_or_else(|| {
+                FormatError::ChunkedReadError(
+                    "external-file virtual dataset sources require a file resolver".into(),
+                )
+            })?;
+            match r(&m.source_file) {
+                Some(bytes) => {
+                    external = bytes;
+                    &external
                 }
+                // Source file unavailable: leave this region at fill value.
+                None => continue,
+            }
+        };
 
         let (vsel, _) = Selection::decode_serialized(&m.virtual_selection)?;
         let (ssel, _) = Selection::decode_serialized(&m.source_selection)?;
 
         let (src_raw, src_dims) =
-            read_named_dataset_raw(file_data, &m.source_dataset, offset_size, length_size)?;
+            read_named_dataset_raw(src_file_data, &m.source_dataset, offset_size, length_size)?;
 
         let vidx = vsel.iter_linear(virtual_dims)?;
         let sidx = ssel.iter_linear(&src_dims)?;

crates/clawhdf5-format/tests/integration_test.rs

@@ -695,6 +695,62 @@ fn v4_virtual_dataset_2d_same_file_read() {
     );
 }
 
+#[test]
+fn v4_virtual_dataset_external_file_read() {
+    use clawhdf5_format::data_read::read_raw_data_full_with_resolver;
+    // The virtual file maps virt[0:8] <- (external) ext_src.h5:/data = [10..17].
+    let virt = include_bytes!("fixtures/vds_external_virt.h5");
+    let src = include_bytes!("fixtures/vds_external_src.h5").to_vec();
+
+    let sig = find_signature(virt).unwrap();
+    let sb = Superblock::parse(virt, sig).unwrap();
+    let addr = resolve_path_any(virt, &sb, "virt").unwrap();
+    let hdr = ObjectHeader::parse(virt, addr as usize, sb.offset_size, sb.length_size).unwrap();
+    let ds = Dataspace::parse(
+        &hdr.messages.iter().find(|m| m.msg_type == MessageType::Dataspace).unwrap().data,
+        sb.length_size,
+    )
+    .unwrap();
+    let (dt, _) = Datatype::parse(
+        &hdr.messages.iter().find(|m| m.msg_type == MessageType::Datatype).unwrap().data,
+    )
+    .unwrap();
+    let layout = DataLayout::parse(
+        &hdr.messages.iter().find(|m| m.msg_type == MessageType::DataLayout).unwrap().data,
+        sb.offset_size,
+        sb.length_size,
+    )
+    .unwrap();
+
+    // Resolver supplies the external source file's bytes by its stored name.
+    let resolver = |name: &str| -> Option<Vec<u8>> {
+        if name == "ext_src.h5" {
+            Some(src.clone())
+        } else {
+            None
+        }
+    };
+    let raw = read_raw_data_full_with_resolver(
+        virt,
+        &layout,
+        &ds,
+        &dt,
+        None,
+        sb.offset_size,
+        sb.length_size,
+        Some(&resolver),
+    )
+    .unwrap();
+    let values = read_as_i32(&raw, &dt).unwrap();
+    assert_eq!(values, vec![10, 11, 12, 13, 14, 15, 16, 17]);
+
+    // With no resolver, an external source is a clean error (not wrong data).
+    let no_resolver = read_raw_data_full_with_resolver(
+        virt, &layout, &ds, &dt, None, sb.offset_size, sb.length_size, None,
+    );
+    assert!(no_resolver.is_err());
+}
+
 #[test]
 fn v4_paged_fixed_array_read() {
     // 1025 chunks of 16 int32s, gzip-filtered => Fixed Array index whose data

crates/clawhdf5/src/reader.rs

@@ -66,6 +66,9 @@ pub struct File {
     superblock: Superblock,
     /// Per-file chunk cache shared across all dataset reads.
     chunk_cache: ChunkCache,
+    /// Directory the file was opened from, used to resolve external Virtual
+    /// Dataset source files relative to this file. `None` for in-memory files.
+    base_dir: Option<std::path::PathBuf>,
 }
 
 impl File {
@@ -74,6 +77,7 @@ impl File {
     /// When the `mmap` feature is enabled (default), this uses memory-mapped
     /// I/O.  Otherwise it reads the entire file into a `Vec<u8>`.
     pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self, Error> {
+        let base_dir = path.as_ref().parent().map(|p| p.to_path_buf());
         #[cfg(feature = "mmap")]
         {
             let reader = clawhdf5_io::MmapReader::open(path).map_err(Error::Io)?;
@@ -84,12 +88,15 @@ impl File {
                 data: FileData::Mmap(reader),
                 superblock,
                 chunk_cache: ChunkCache::new(),
+                base_dir,
             })
         }
         #[cfg(not(feature = "mmap"))]
         {
             let bytes = std::fs::read(path.as_ref()).map_err(Error::Io)?;
-            Self::from_bytes(bytes)
+            let mut f = Self::from_bytes(bytes)?;
+            f.base_dir = base_dir;
+            Ok(f)
         }
     }
 
@@ -99,10 +106,15 @@ impl File {
     /// undesirable (e.g. network filesystems, very small files, etc.).
     pub fn open_buffered<P: AsRef<std::path::Path>>(path: P) -> Result<Self, Error> {
         let bytes = std::fs::read(path.as_ref()).map_err(Error::Io)?;
-        Self::from_bytes(bytes)
+        let mut f = Self::from_bytes(bytes)?;
+        f.base_dir = path.as_ref().parent().map(|p| p.to_path_buf());
+        Ok(f)
     }
 
     /// Open an HDF5 file from an in-memory byte vector.
+    ///
+    /// In-memory files have no directory, so external Virtual Dataset sources
+    /// cannot be resolved automatically (same-file VDS still works).
     pub fn from_bytes(data: Vec<u8>) -> Result<Self, Error> {
         let sig_offset = signature::find_signature(&data)?;
         let superblock = Superblock::parse(&data, sig_offset)?;
@@ -110,6 +122,7 @@ impl File {
             data: FileData::Owned(data),
             superblock,
             chunk_cache: ChunkCache::new(),
+            base_dir: None,
         })
     }
 
@@ -710,6 +723,28 @@ impl<'f> Dataset<'f> {
         let ds = self.dataspace()?;
         let dl = self.data_layout()?;
         let pipeline = self.filter_pipeline();
+
+        // Virtual datasets are assembled from source datasets; the per-file
+        // chunk cache does not apply. Route them through the resolver path so
+        // external sibling files resolve relative to this file's directory.
+        if matches!(dl, DataLayout::Virtual { .. }) {
+            let base_dir = self.file.base_dir.clone();
+            let resolver = move |name: &str| -> Option<Vec<u8>> {
+                let dir = base_dir.as_ref()?;
+                std::fs::read(dir.join(name)).ok()
+            };
+            return Ok(data_read::read_raw_data_full_with_resolver(
+                self.file.data.as_bytes(),
+                &dl,
+                &ds,
+                &dt,
+                pipeline.as_ref(),
+                self.file.offset_size(),
+                self.file.length_size(),
+                Some(&resolver),
+            )?);
+        }
+
         Ok(data_read::read_raw_data_cached(
             self.file.data.as_bytes(),
             &dl,

crates/clawhdf5/tests/integration_tests.rs

@@ -706,6 +706,38 @@ fn fletcher32_roundtrip() {
 // 15. Multiple groups with same-named datasets
 // ---------------------------------------------------------------------------
 
+#[test]
+fn virtual_dataset_external_file_auto_resolved() {
+    // The facade resolves external Virtual Dataset sources relative to the
+    // opened file's directory automatically. Drop both files side by side in a
+    // temp dir and open the virtual one through the public File API.
+    let virt = include_bytes!("../../clawhdf5-format/tests/fixtures/vds_external_virt.h5");
+    let src = include_bytes!("../../clawhdf5-format/tests/fixtures/vds_external_src.h5");
+
+    let dir = tempfile::tempdir().unwrap();
+    std::fs::write(dir.path().join("ext_src.h5"), src).unwrap();
+    let virt_path = dir.path().join("ext_virt.h5");
+    std::fs::write(&virt_path, virt).unwrap();
+
+    let file = File::open(&virt_path).unwrap();
+    let values = file.dataset("virt").unwrap().read_i32().unwrap();
+    assert_eq!(values, vec![10, 11, 12, 13, 14, 15, 16, 17]);
+}
+
+#[test]
+fn virtual_dataset_external_missing_source_is_fill() {
+    // If the external source file is absent, its region reads as the zero fill
+    // value rather than erroring.
+    let virt = include_bytes!("../../clawhdf5-format/tests/fixtures/vds_external_virt.h5");
+    let dir = tempfile::tempdir().unwrap();
+    let virt_path = dir.path().join("ext_virt.h5");
+    std::fs::write(&virt_path, virt).unwrap();
+
+    let file = File::open(&virt_path).unwrap();
+    let values = file.dataset("virt").unwrap().read_i32().unwrap();
+    assert_eq!(values, vec![0; 8]);
+}
+
 #[test]
 fn same_dataset_name_in_different_groups() {
     let mut b = FileBuilder::new();