fix(format): read datasets and attributes that use committed datatypes
A dataset created from a committed (named) datatype stores only a shared-
message reference to it. The facade parsed those reference bytes as the
datatype itself, producing `Time { size: 0 }` and unreadable data, and an
attribute using a committed datatype was silently dropped.
- shared_message::parse_shared_ref had the encoding wrong: it skipped six
reserved bytes for version 2 (only version 1 has them) and had the version 3
types inverted (1 is the SOHM heap, 2 is "committed, in another object
header"). Verified against h5py 3.16 / HDF5 2.0, which writes
`02 02 <address>` under both default and latest libver bounds. Resolution
now dispatches on which field the reference carries.
- New shared_message::message_data resolves a header message through the
indirection; the reader, lazy and mmap facades use it for datatype,
dataspace and filter-pipeline messages.
- AttributeMessage honours the v2/v3 flags (bit 0 datatype shared, bit 1
dataspace shared) via the new parse_in_file, used everywhere file data is
available. Parsing a shared attribute without file access is now
FormatError::UnresolvedSharedMessage instead of a garbage datatype.
- h5py interop test covering both libver settings.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
0eca8574f5
commit
81e8294048
@@ -416,15 +416,43 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
|
||||
))
|
||||
}
|
||||
|
||||
/// A header message's payload, resolved through the shared-message
|
||||
/// indirection when needed (e.g. a committed datatype). See
|
||||
/// [`clawhdf5_format::shared_message::message_data`].
|
||||
fn message_payload(
|
||||
&self,
|
||||
msg_type: MessageType,
|
||||
) -> Result<Option<std::borrow::Cow<'_, [u8]>>, Error> {
|
||||
self.header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == msg_type)
|
||||
.map(|msg| {
|
||||
clawhdf5_format::shared_message::message_data(
|
||||
self.file.as_bytes(),
|
||||
msg,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
.map_err(Error::Format)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn required_payload(&self, msg_type: MessageType) -> Result<std::borrow::Cow<'_, [u8]>, Error> {
|
||||
self.message_payload(msg_type)?
|
||||
.ok_or(Error::MissingMessage(msg_type))
|
||||
}
|
||||
|
||||
fn datatype(&self) -> Result<Datatype, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&msg.data)?;
|
||||
let data = self.required_payload(MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&data)?;
|
||||
Ok(dt)
|
||||
}
|
||||
|
||||
fn dataspace(&self) -> Result<Dataspace, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Dataspace)?;
|
||||
Ok(Dataspace::parse(&msg.data, self.file.length_size())?)
|
||||
let data = self.required_payload(MessageType::Dataspace)?;
|
||||
Ok(Dataspace::parse(&data, self.file.length_size())?)
|
||||
}
|
||||
|
||||
fn data_layout(&self) -> Result<DataLayout, Error> {
|
||||
@@ -441,11 +469,8 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> {
|
||||
/// filters" would hand the caller the still-compressed bytes as if they
|
||||
/// were the data.
|
||||
fn filter_pipeline(&self) -> Result<Option<FilterPipeline>, Error> {
|
||||
self.header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::FilterPipeline)
|
||||
.map(|msg| FilterPipeline::parse(&msg.data).map_err(Error::Format))
|
||||
self.message_payload(MessageType::FilterPipeline)?
|
||||
.map(|data| FilterPipeline::parse(&data).map_err(Error::Format))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
|
||||
@@ -357,15 +357,43 @@ impl<'f> MmapDataset<'f> {
|
||||
))
|
||||
}
|
||||
|
||||
/// A header message's payload, resolved through the shared-message
|
||||
/// indirection when needed (e.g. a committed datatype). See
|
||||
/// [`clawhdf5_format::shared_message::message_data`].
|
||||
fn message_payload(
|
||||
&self,
|
||||
msg_type: MessageType,
|
||||
) -> Result<Option<std::borrow::Cow<'_, [u8]>>, Error> {
|
||||
self.header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == msg_type)
|
||||
.map(|msg| {
|
||||
clawhdf5_format::shared_message::message_data(
|
||||
self.file.as_bytes(),
|
||||
msg,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
.map_err(Error::Format)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn required_payload(&self, msg_type: MessageType) -> Result<std::borrow::Cow<'_, [u8]>, Error> {
|
||||
self.message_payload(msg_type)?
|
||||
.ok_or(Error::MissingMessage(msg_type))
|
||||
}
|
||||
|
||||
fn datatype(&self) -> Result<Datatype, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&msg.data)?;
|
||||
let data = self.required_payload(MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&data)?;
|
||||
Ok(dt)
|
||||
}
|
||||
|
||||
fn dataspace(&self) -> Result<Dataspace, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Dataspace)?;
|
||||
Ok(Dataspace::parse(&msg.data, self.file.length_size())?)
|
||||
let data = self.required_payload(MessageType::Dataspace)?;
|
||||
Ok(Dataspace::parse(&data, self.file.length_size())?)
|
||||
}
|
||||
|
||||
fn data_layout(&self) -> Result<DataLayout, Error> {
|
||||
@@ -382,11 +410,8 @@ impl<'f> MmapDataset<'f> {
|
||||
/// filters" would hand the caller the still-compressed bytes as if they
|
||||
/// were the data.
|
||||
fn filter_pipeline(&self) -> Result<Option<FilterPipeline>, Error> {
|
||||
self.header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::FilterPipeline)
|
||||
.map(|msg| FilterPipeline::parse(&msg.data).map_err(Error::Format))
|
||||
self.message_payload(MessageType::FilterPipeline)?
|
||||
.map(|data| FilterPipeline::parse(&data).map_err(Error::Format))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
|
||||
@@ -723,15 +723,43 @@ impl<'f> Dataset<'f> {
|
||||
)?)
|
||||
}
|
||||
|
||||
/// A header message's payload, resolved through the shared-message
|
||||
/// indirection when needed (e.g. a committed datatype). See
|
||||
/// [`clawhdf5_format::shared_message::message_data`].
|
||||
fn message_payload(
|
||||
&self,
|
||||
msg_type: MessageType,
|
||||
) -> Result<Option<std::borrow::Cow<'_, [u8]>>, Error> {
|
||||
self.header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == msg_type)
|
||||
.map(|msg| {
|
||||
clawhdf5_format::shared_message::message_data(
|
||||
self.file.as_bytes(),
|
||||
msg,
|
||||
self.file.offset_size(),
|
||||
self.file.length_size(),
|
||||
)
|
||||
.map_err(Error::Format)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn required_payload(&self, msg_type: MessageType) -> Result<std::borrow::Cow<'_, [u8]>, Error> {
|
||||
self.message_payload(msg_type)?
|
||||
.ok_or(Error::MissingMessage(msg_type))
|
||||
}
|
||||
|
||||
fn datatype(&self) -> Result<Datatype, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&msg.data)?;
|
||||
let data = self.required_payload(MessageType::Datatype)?;
|
||||
let (dt, _) = Datatype::parse(&data)?;
|
||||
Ok(dt)
|
||||
}
|
||||
|
||||
fn dataspace(&self) -> Result<Dataspace, Error> {
|
||||
let msg = find_message(&self.header, MessageType::Dataspace)?;
|
||||
Ok(Dataspace::parse(&msg.data, self.file.length_size())?)
|
||||
let data = self.required_payload(MessageType::Dataspace)?;
|
||||
Ok(Dataspace::parse(&data, self.file.length_size())?)
|
||||
}
|
||||
|
||||
fn data_layout(&self) -> Result<DataLayout, Error> {
|
||||
@@ -748,11 +776,8 @@ impl<'f> Dataset<'f> {
|
||||
/// filters" would hand the caller the still-compressed bytes as if they
|
||||
/// were the data.
|
||||
fn filter_pipeline(&self) -> Result<Option<FilterPipeline>, Error> {
|
||||
self.header
|
||||
.messages
|
||||
.iter()
|
||||
.find(|m| m.msg_type == MessageType::FilterPipeline)
|
||||
.map(|msg| FilterPipeline::parse(&msg.data).map_err(Error::Format))
|
||||
self.message_payload(MessageType::FilterPipeline)?
|
||||
.map(|data| FilterPipeline::parse(&data).map_err(Error::Format))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
|
||||
@@ -565,3 +565,52 @@ print("OK")
|
||||
);
|
||||
assert_eq!(run_python_output(&script), "OK");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// h5py uses committed (named) datatypes -> clawhdf5 reads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A dataset or attribute created from a committed datatype stores only a
|
||||
/// *shared message* reference to it. These used to be parsed as the datatype
|
||||
/// itself (yielding `Time { size: 0 }` and unreadable data) and the attribute
|
||||
/// was silently dropped.
|
||||
#[test]
|
||||
fn h5py_committed_datatypes_clawhdf5_reads() {
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
||||
let path = dir.path().join(format!("committed_{tag}.h5"));
|
||||
let path_str = path.display().to_string();
|
||||
let script = format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
||||
f["f8type"] = np.dtype("<f8")
|
||||
f["cmpd"] = np.dtype([("a", "<i4"), ("b", "<f8")])
|
||||
f.create_dataset("d", data=np.arange(6, dtype="<f8"), dtype=f["f8type"])
|
||||
f.create_dataset("c", data=np.array([(1, 2.5), (3, 4.5)], dtype=f["cmpd"].dtype), dtype=f["cmpd"])
|
||||
f["d"].attrs.create("att", 7.0, dtype=f["f8type"])
|
||||
"#
|
||||
);
|
||||
run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
let d = file.dataset("d").unwrap();
|
||||
assert_eq!(d.dtype().unwrap(), DType::F64, "{tag}");
|
||||
assert_eq!(
|
||||
d.read_f64().unwrap(),
|
||||
vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
|
||||
"{tag}"
|
||||
);
|
||||
assert!(
|
||||
matches!(d.attrs().unwrap().get("att"), Some(AttrValue::F64(v)) if *v == 7.0),
|
||||
"{tag}: attribute with a committed datatype"
|
||||
);
|
||||
let c = file.dataset("c").unwrap();
|
||||
assert_eq!(
|
||||
c.dtype().unwrap(),
|
||||
DType::Compound(vec![("a".into(), DType::I32), ("b".into(), DType::F64)]),
|
||||
"{tag}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user