Chunked reads beat an h5py process pool; unlimited writer B-trees; Blosc2; 599/697 conformance #16

Merged
osobh merged 48 commits from feat/p2b-scale into main 2026-09-26 17:42:16 +00:00
3 changed files with 50 additions and 4 deletions
Showing only changes of commit 16b7359485 - Show all commits
+6
View File
@@ -223,6 +223,9 @@ pub enum FormatError {
/// The file's actual length in bytes.
actual_len: u64,
},
/// A link libhdf5 refuses to list: a symbol-table entry with an empty
/// name ("invalid link name"). Listing the group fails, as in libhdf5.
InvalidLinkName,
}
impl fmt::Display for FormatError {
@@ -494,6 +497,9 @@ impl fmt::Display for FormatError {
but the file is {actual_len} bytes"
)
}
FormatError::InvalidLinkName => {
write!(f, "invalid link name: a group entry has an empty name")
}
}
}
}
+41 -3
View File
@@ -21,12 +21,35 @@ pub struct GroupEntry {
pub cache_type: u32,
}
/// Given a SymbolTableMessage, resolve all group children.
/// Given a SymbolTableMessage, resolve all group children: the group's
/// listing.
///
/// An entry with an empty name fails the listing with
/// [`FormatError::InvalidLinkName`], as it fails libhdf5's link iteration
/// (`H5G__ent_to_link`: "invalid link name"). Looking a name up
/// ([`resolve_path`], and the path resolution in
/// [`crate::group_v2::resolve_path_any`]) still works in such a group, as it
/// does in libhdf5.
pub fn resolve_v1_group_entries(
file_data: &[u8],
sym_table_msg: &SymbolTableMessage,
offset_size: u8,
length_size: u8,
) -> Result<Vec<GroupEntry>, FormatError> {
let entries = v1_group_entries(file_data, sym_table_msg, offset_size, length_size)?;
if entries.iter().any(|e| e.name.is_empty()) {
return Err(FormatError::InvalidLinkName);
}
Ok(entries)
}
/// Every entry of a v1 group, empty names included — for looking a name up,
/// which never matches an empty name.
pub(crate) fn v1_group_entries(
file_data: &[u8],
sym_table_msg: &SymbolTableMessage,
offset_size: u8,
length_size: u8,
) -> Result<Vec<GroupEntry>, FormatError> {
// Parse local heap
let heap = LocalHeap::parse(
@@ -207,8 +230,7 @@ pub fn resolve_path(
let mut current_sym_table = root_sym_table.clone();
for (i, component) in components.iter().enumerate() {
let entries =
resolve_v1_group_entries(file_data, &current_sym_table, offset_size, length_size)?;
let entries = v1_group_entries(file_data, &current_sym_table, offset_size, length_size)?;
let found = entries.iter().find(|e| e.name == *component);
match found {
@@ -425,6 +447,22 @@ mod tests {
assert_eq!(entries[1].object_header_address, 0x2000);
}
/// cve-2021-46244 `/BAG_root`: a symbol-table entry with an empty name.
/// libhdf5 fails the group's listing ("invalid link name"); a lookup of
/// the other names still works.
#[test]
fn empty_entry_name_fails_the_listing_not_a_lookup() {
let (file, msg) = build_synthetic_group(&[("", 0x1000, 0), ("elevation", 0x2000, 0)], 8, 8);
assert_eq!(
resolve_v1_group_entries(&file, &msg, 8, 8).unwrap_err(),
FormatError::InvalidLinkName
);
assert_eq!(
resolve_path(&file, &msg, "elevation", 8, 8).unwrap(),
0x2000
);
}
#[test]
fn resolve_path_single_level() {
let (file, msg) =
+3 -1
View File
@@ -450,7 +450,9 @@ fn resolve_group_entries(
.find(|m| m.msg_type == MessageType::SymbolTable)
.ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?;
let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?;
group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size)
// A lookup: an entry with an empty name (which fails a listing) is
// skipped by the name comparison, as in libhdf5.
group_v1::v1_group_entries(file_data, &stm, offset_size, length_size)
} else if is_v2_group(object_header) {
resolve_v2_group_entries(file_data, object_header, offset_size, length_size)
} else {