diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 3591dc5..1298cf0 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -286,6 +286,29 @@ impl<'a> Ctx<'a> { Ok(()) } + /// VDS source files resolve next to the virtual file; like the library, + /// refuse absolute paths and `..`. + fn vds_resolver( + &self, + ) -> impl Fn(&str) -> Result>, clawhdf5_format::error::FormatError> + use<> { + let base = self.base_dir.clone(); + move |name: &str| { + use clawhdf5_format::error::FormatError; + let p = std::path::Path::new(name); + if p.is_absolute() + || p.components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(FormatError::ChunkedReadError(format!("refused {name}"))); + } + match std::fs::read(base.join(p)) { + Ok(b) => Ok(Some(b)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(FormatError::ChunkedReadError(err.to_string())), + } + } + } + fn read_dataset(&self, h: &ObjectHeader, rec: &mut Map) -> Result<(), String> { let dtb = self .payload(h, MessageType::Datatype)? @@ -295,7 +318,27 @@ impl<'a> Ctx<'a> { let dsb = self .payload(h, MessageType::Dataspace)? .ok_or("MissingMessage(Dataspace)")?; - let ds = Dataspace::parse(&dsb, self.ls).map_err(e)?; + let mut ds = Dataspace::parse(&dsb, self.ls).map_err(e)?; + // A virtual dataset's extent can come from its sources (unlimited / + // printf mappings), as h5py reports it, rather than the stored one. + if let Some(lm) = h + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + && let Ok(dl @ DataLayout::Virtual { .. }) = + DataLayout::parse(&lm.data, self.os, self.ls) + { + let resolver = self.vds_resolver(); + ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent( + self.data, + &dl, + &ds, + self.os, + self.ls, + Some(&resolver), + ) + .map_err(e)?; + } let (shape, n) = Self::shape(&ds); rec.insert("shape".into(), shape); if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES { @@ -331,28 +374,26 @@ impl<'a> Ctx<'a> { ); } let raw = if matches!(dl, DataLayout::Virtual { .. }) { - let base = self.base_dir.clone(); - let resolver = move |name: &str| -> Option> { - let p = std::path::Path::new(name); - if p.is_absolute() - || p.components() - .any(|c| matches!(c, std::path::Component::ParentDir)) - { - return None; - } - std::fs::read(base.join(p)).ok() - }; - data_read::read_raw_data_full_with_resolver( + let resolver = self.vds_resolver(); + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + self.data, + &h.messages, + self.os, + self.ls, + ) + .map_err(e)?; + clawhdf5_format::vds::read_virtual_dataset( self.data, &dl, &ds, &dt, - pipeline.as_ref(), + fill.as_deref(), self.os, self.ls, Some(&resolver), ) .map_err(e)? + .data } else { let cache = clawhdf5_format::chunk_cache::ChunkCache::new(); clawhdf5_format::fill_value::read_full_with_fill::( @@ -660,10 +701,13 @@ fn main() { return; } }; - let sb = guarded(|| { - let off = signature::find_signature(&data).map_err(e)?; - Superblock::parse(&data, off).map_err(e) - }); + // Every address is relative to the superblock: look at the file from + // there on (past any user block), as libhdf5 does. + let hdf5: &[u8] = match signature::find_signature(&data) { + Ok(off) => &data[off..], + Err(_) => &data, + }; + let sb = guarded(|| Superblock::parse(hdf5, 0).map_err(e)); let sb = match sb { Ok(sb) => sb, Err(msg) => { @@ -674,7 +718,7 @@ fn main() { }; top.insert("superblock_version".into(), json!(sb.version)); let ctx = Ctx { - data: &data, + data: hdf5, os: sb.offset_size, ls: sb.length_size, base_dir: std::path::Path::new(&path)