Chunked reads beat an h5py process pool; unlimited writer B-trees; Blosc2; 599/697 conformance #16

Merged
osobh merged 48 commits from feat/p2b-scale into main 2026-09-26 17:42:16 +00:00
2 changed files with 133 additions and 18 deletions
Showing only changes of commit 1207df5189 - Show all commits
+52 -18
View File
@@ -31,7 +31,7 @@ use clawhdf5_format::filter_pipeline::FilterPipeline;
use clawhdf5_format::group_v1::{self, GroupEntry};
use clawhdf5_format::group_v2;
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::object_header::{ObjectClass, ObjectHeader};
use clawhdf5_format::signature;
use clawhdf5_format::superblock::Superblock;
use clawhdf5_format::symbol_table::SymbolTableMessage;
@@ -657,6 +657,20 @@ fn is_group(h: &ObjectHeader) -> bool {
})
}
/// The probe's kind for an object header: libhdf5's object class
/// ([`ObjectHeader::object_class`]: group, then dataset — a datatype *and* a
/// dataspace — then named datatype), which is what h5py opens the object as.
/// The root group, and a header with only link messages, count as groups.
fn kind_of(h: &ObjectHeader, is_root: bool) -> &'static str {
match h.object_class() {
Some(ObjectClass::Group) => "group",
Some(ObjectClass::Dataset) => "dataset",
_ if is_root || is_group(h) => "group",
Some(ObjectClass::NamedDatatype) => "datatype",
None => "unknown",
}
}
fn main() {
install_hook();
let path = std::env::args().nth(1).expect("usage: probe <file>");
@@ -735,23 +749,7 @@ fn main() {
continue;
}
};
let is_ds = h
.messages
.iter()
.any(|m| m.msg_type == MessageType::DataLayout);
let kind = if is_ds {
"dataset"
} else if is_group(&h) || addr == sb.root_group_address {
"group"
} else if h
.messages
.iter()
.any(|m| m.msg_type == MessageType::Datatype)
{
"datatype"
} else {
"unknown"
};
let kind = kind_of(&h, addr == sb.root_group_address);
rec.insert("kind".into(), Value::String(kind.into()));
if kind == "dataset"
&& let Err(msg) = guarded(|| ctx.read_dataset(&h, &mut rec))
@@ -867,6 +865,42 @@ mod tests {
assert!(ieee_layout(&f32le));
}
#[test]
fn kind_follows_libhdf5_object_class() {
use clawhdf5_format::object_header::HeaderMessage;
let header = |types: &[MessageType]| ObjectHeader {
version: 2,
messages: types
.iter()
.map(|&msg_type| HeaderMessage {
msg_type,
size: 0,
flags: 0,
creation_order: None,
data: Vec::new(),
})
.collect(),
reference_count: None,
flags: 0,
access_time: None,
modification_time: None,
change_time: None,
birth_time: None,
};
use MessageType::*;
// cve-2024-33874 `/Dset1`: a datatype and a layout but no dataspace
// is a named datatype to libhdf5 (h5py opens it as one).
assert_eq!(kind_of(&header(&[Datatype, DataLayout]), false), "datatype");
assert_eq!(
kind_of(&header(&[Datatype, Dataspace, DataLayout]), false),
"dataset"
);
assert_eq!(kind_of(&header(&[SymbolTable]), false), "group");
assert_eq!(kind_of(&header(&[Link]), false), "group");
assert_eq!(kind_of(&header(&[]), true), "group");
assert_eq!(kind_of(&header(&[]), false), "unknown");
}
#[test]
fn partial_precision_int_is_shifted_and_sign_extended() {
let dt = Datatype::FixedPoint {
@@ -75,7 +75,40 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
})
}
/// The kind of object an object header describes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ObjectClass {
/// A group: the header has a Symbol Table or a Link Info message.
Group,
/// A dataset: the header has a Datatype and a Dataspace message.
Dataset,
/// A committed (named) datatype: a Datatype message, no Dataspace.
NamedDatatype,
}
impl ObjectHeader {
/// The kind of object this header describes, decided as libhdf5 decides
/// it (`H5O__obj_class_real`): group first (a Symbol Table or Link Info
/// message), then dataset (a Datatype *and* a Dataspace message — not a
/// Data Layout message), then named datatype (a Datatype message).
/// `None` when none applies; libhdf5 then cannot open the object
/// ("unable to determine object type").
///
/// A header with a Datatype and a Data Layout message but no Dataspace
/// is a named datatype to libhdf5, not a dataset.
pub fn object_class(&self) -> Option<ObjectClass> {
let has = |t: MessageType| self.messages.iter().any(|m| m.msg_type == t);
if has(MessageType::SymbolTable) || has(MessageType::LinkInfo) {
Some(ObjectClass::Group)
} else if has(MessageType::Datatype) && has(MessageType::Dataspace) {
Some(ObjectClass::Dataset)
} else if has(MessageType::Datatype) {
Some(ObjectClass::NamedDatatype)
} else {
None
}
}
/// Parse an object header at the given offset in the data buffer.
///
/// `offset_size` and `length_size` come from the superblock.
@@ -662,6 +695,54 @@ fn check_message(
mod tests {
use super::*;
fn header_with(types: &[MessageType]) -> ObjectHeader {
ObjectHeader {
version: 2,
messages: types
.iter()
.map(|&msg_type| HeaderMessage {
msg_type,
size: 0,
flags: 0,
creation_order: None,
data: Vec::new(),
})
.collect(),
reference_count: None,
flags: 0,
access_time: None,
modification_time: None,
change_time: None,
birth_time: None,
}
}
#[test]
fn object_class_follows_libhdf5() {
use MessageType::*;
let class = |t: &[MessageType]| header_with(t).object_class();
assert_eq!(
class(&[Datatype, Dataspace, DataLayout]),
Some(ObjectClass::Dataset)
);
// A Data Layout message does not make a dataset without a dataspace
// (cve-2024-33874 `/Dset1`: h5py opens it as a named datatype).
assert_eq!(
class(&[Datatype, DataLayout]),
Some(ObjectClass::NamedDatatype)
);
assert_eq!(class(&[Datatype]), Some(ObjectClass::NamedDatatype));
// Group messages win over dataset messages.
assert_eq!(
class(&[Datatype, Dataspace, SymbolTable]),
Some(ObjectClass::Group)
);
assert_eq!(class(&[LinkInfo]), Some(ObjectClass::Group));
// Link messages alone are not a group; nothing is not an object.
assert_eq!(class(&[Link]), None);
assert_eq!(class(&[]), None);
}
// Helper: build a v1 object header with given messages
fn build_v1_header(
messages: &[(u16, &[u8], u8)], // (type, data, flags)