Chunked reads beat an h5py process pool; unlimited writer B-trees; Blosc2; 599/697 conformance #16
@@ -31,7 +31,7 @@ use clawhdf5_format::filter_pipeline::FilterPipeline;
|
|||||||
use clawhdf5_format::group_v1::{self, GroupEntry};
|
use clawhdf5_format::group_v1::{self, GroupEntry};
|
||||||
use clawhdf5_format::group_v2;
|
use clawhdf5_format::group_v2;
|
||||||
use clawhdf5_format::message_type::MessageType;
|
use clawhdf5_format::message_type::MessageType;
|
||||||
use clawhdf5_format::object_header::ObjectHeader;
|
use clawhdf5_format::object_header::{ObjectClass, ObjectHeader};
|
||||||
use clawhdf5_format::signature;
|
use clawhdf5_format::signature;
|
||||||
use clawhdf5_format::superblock::Superblock;
|
use clawhdf5_format::superblock::Superblock;
|
||||||
use clawhdf5_format::symbol_table::SymbolTableMessage;
|
use clawhdf5_format::symbol_table::SymbolTableMessage;
|
||||||
@@ -657,6 +657,20 @@ fn is_group(h: &ObjectHeader) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The probe's kind for an object header: libhdf5's object class
|
||||||
|
/// ([`ObjectHeader::object_class`]: group, then dataset — a datatype *and* a
|
||||||
|
/// dataspace — then named datatype), which is what h5py opens the object as.
|
||||||
|
/// The root group, and a header with only link messages, count as groups.
|
||||||
|
fn kind_of(h: &ObjectHeader, is_root: bool) -> &'static str {
|
||||||
|
match h.object_class() {
|
||||||
|
Some(ObjectClass::Group) => "group",
|
||||||
|
Some(ObjectClass::Dataset) => "dataset",
|
||||||
|
_ if is_root || is_group(h) => "group",
|
||||||
|
Some(ObjectClass::NamedDatatype) => "datatype",
|
||||||
|
None => "unknown",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
install_hook();
|
install_hook();
|
||||||
let path = std::env::args().nth(1).expect("usage: probe <file>");
|
let path = std::env::args().nth(1).expect("usage: probe <file>");
|
||||||
@@ -735,23 +749,7 @@ fn main() {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let is_ds = h
|
let kind = kind_of(&h, addr == sb.root_group_address);
|
||||||
.messages
|
|
||||||
.iter()
|
|
||||||
.any(|m| m.msg_type == MessageType::DataLayout);
|
|
||||||
let kind = if is_ds {
|
|
||||||
"dataset"
|
|
||||||
} else if is_group(&h) || addr == sb.root_group_address {
|
|
||||||
"group"
|
|
||||||
} else if h
|
|
||||||
.messages
|
|
||||||
.iter()
|
|
||||||
.any(|m| m.msg_type == MessageType::Datatype)
|
|
||||||
{
|
|
||||||
"datatype"
|
|
||||||
} else {
|
|
||||||
"unknown"
|
|
||||||
};
|
|
||||||
rec.insert("kind".into(), Value::String(kind.into()));
|
rec.insert("kind".into(), Value::String(kind.into()));
|
||||||
if kind == "dataset"
|
if kind == "dataset"
|
||||||
&& let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec))
|
&& let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec))
|
||||||
@@ -867,6 +865,42 @@ mod tests {
|
|||||||
assert!(ieee_layout(&f32le));
|
assert!(ieee_layout(&f32le));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn kind_follows_libhdf5_object_class() {
|
||||||
|
use clawhdf5_format::object_header::HeaderMessage;
|
||||||
|
let header = |types: &[MessageType]| ObjectHeader {
|
||||||
|
version: 2,
|
||||||
|
messages: types
|
||||||
|
.iter()
|
||||||
|
.map(|&msg_type| HeaderMessage {
|
||||||
|
msg_type,
|
||||||
|
size: 0,
|
||||||
|
flags: 0,
|
||||||
|
creation_order: None,
|
||||||
|
data: Vec::new(),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
reference_count: None,
|
||||||
|
flags: 0,
|
||||||
|
access_time: None,
|
||||||
|
modification_time: None,
|
||||||
|
change_time: None,
|
||||||
|
birth_time: None,
|
||||||
|
};
|
||||||
|
use MessageType::*;
|
||||||
|
// cve-2024-33874 `/Dset1`: a datatype and a layout but no dataspace
|
||||||
|
// is a named datatype to libhdf5 (h5py opens it as one).
|
||||||
|
assert_eq!(kind_of(&header(&[Datatype, DataLayout]), false), "datatype");
|
||||||
|
assert_eq!(
|
||||||
|
kind_of(&header(&[Datatype, Dataspace, DataLayout]), false),
|
||||||
|
"dataset"
|
||||||
|
);
|
||||||
|
assert_eq!(kind_of(&header(&[SymbolTable]), false), "group");
|
||||||
|
assert_eq!(kind_of(&header(&[Link]), false), "group");
|
||||||
|
assert_eq!(kind_of(&header(&[]), true), "group");
|
||||||
|
assert_eq!(kind_of(&header(&[]), false), "unknown");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn partial_precision_int_is_shifted_and_sign_extended() {
|
fn partial_precision_int_is_shifted_and_sign_extended() {
|
||||||
let dt = Datatype::FixedPoint {
|
let dt = Datatype::FixedPoint {
|
||||||
|
|||||||
@@ -75,7 +75,40 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The kind of object an object header describes.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ObjectClass {
|
||||||
|
/// A group: the header has a Symbol Table or a Link Info message.
|
||||||
|
Group,
|
||||||
|
/// A dataset: the header has a Datatype and a Dataspace message.
|
||||||
|
Dataset,
|
||||||
|
/// A committed (named) datatype: a Datatype message, no Dataspace.
|
||||||
|
NamedDatatype,
|
||||||
|
}
|
||||||
|
|
||||||
impl ObjectHeader {
|
impl ObjectHeader {
|
||||||
|
/// The kind of object this header describes, decided as libhdf5 decides
|
||||||
|
/// it (`H5O__obj_class_real`): group first (a Symbol Table or Link Info
|
||||||
|
/// message), then dataset (a Datatype *and* a Dataspace message — not a
|
||||||
|
/// Data Layout message), then named datatype (a Datatype message).
|
||||||
|
/// `None` when none applies; libhdf5 then cannot open the object
|
||||||
|
/// ("unable to determine object type").
|
||||||
|
///
|
||||||
|
/// A header with a Datatype and a Data Layout message but no Dataspace
|
||||||
|
/// is a named datatype to libhdf5, not a dataset.
|
||||||
|
pub fn object_class(&self) -> Option<ObjectClass> {
|
||||||
|
let has = |t: MessageType| self.messages.iter().any(|m| m.msg_type == t);
|
||||||
|
if has(MessageType::SymbolTable) || has(MessageType::LinkInfo) {
|
||||||
|
Some(ObjectClass::Group)
|
||||||
|
} else if has(MessageType::Datatype) && has(MessageType::Dataspace) {
|
||||||
|
Some(ObjectClass::Dataset)
|
||||||
|
} else if has(MessageType::Datatype) {
|
||||||
|
Some(ObjectClass::NamedDatatype)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Parse an object header at the given offset in the data buffer.
|
/// Parse an object header at the given offset in the data buffer.
|
||||||
///
|
///
|
||||||
/// `offset_size` and `length_size` come from the superblock.
|
/// `offset_size` and `length_size` come from the superblock.
|
||||||
@@ -662,6 +695,54 @@ fn check_message(
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
fn header_with(types: &[MessageType]) -> ObjectHeader {
|
||||||
|
ObjectHeader {
|
||||||
|
version: 2,
|
||||||
|
messages: types
|
||||||
|
.iter()
|
||||||
|
.map(|&msg_type| HeaderMessage {
|
||||||
|
msg_type,
|
||||||
|
size: 0,
|
||||||
|
flags: 0,
|
||||||
|
creation_order: None,
|
||||||
|
data: Vec::new(),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
reference_count: None,
|
||||||
|
flags: 0,
|
||||||
|
access_time: None,
|
||||||
|
modification_time: None,
|
||||||
|
change_time: None,
|
||||||
|
birth_time: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn object_class_follows_libhdf5() {
|
||||||
|
use MessageType::*;
|
||||||
|
let class = |t: &[MessageType]| header_with(t).object_class();
|
||||||
|
assert_eq!(
|
||||||
|
class(&[Datatype, Dataspace, DataLayout]),
|
||||||
|
Some(ObjectClass::Dataset)
|
||||||
|
);
|
||||||
|
// A Data Layout message does not make a dataset without a dataspace
|
||||||
|
// (cve-2024-33874 `/Dset1`: h5py opens it as a named datatype).
|
||||||
|
assert_eq!(
|
||||||
|
class(&[Datatype, DataLayout]),
|
||||||
|
Some(ObjectClass::NamedDatatype)
|
||||||
|
);
|
||||||
|
assert_eq!(class(&[Datatype]), Some(ObjectClass::NamedDatatype));
|
||||||
|
// Group messages win over dataset messages.
|
||||||
|
assert_eq!(
|
||||||
|
class(&[Datatype, Dataspace, SymbolTable]),
|
||||||
|
Some(ObjectClass::Group)
|
||||||
|
);
|
||||||
|
assert_eq!(class(&[LinkInfo]), Some(ObjectClass::Group));
|
||||||
|
// Link messages alone are not a group; nothing is not an object.
|
||||||
|
assert_eq!(class(&[Link]), None);
|
||||||
|
assert_eq!(class(&[]), None);
|
||||||
|
}
|
||||||
|
|
||||||
// Helper: build a v1 object header with given messages
|
// Helper: build a v1 object header with given messages
|
||||||
fn build_v1_header(
|
fn build_v1_header(
|
||||||
messages: &[(u16, &[u8], u8)], // (type, data, flags)
|
messages: &[(u16, &[u8], u8)], // (type, data, flags)
|
||||||
|
|||||||
Reference in New Issue
Block a user