quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p3-storage-trait' into feat/p3-range-zfp-edit

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/btree_v1.rs
#	crates/clawhdf5-format/src/data_layout.rs
#	crates/clawhdf5-format/src/extensible_array.rs
#	crates/clawhdf5-format/src/fixed_array.rs
#	crates/clawhdf5-format/src/fractal_heap.rs
#	crates/clawhdf5-format/src/local_heap.rs
#	crates/clawhdf5-format/src/shared_message.rs
This commit is contained in:
osobh
2026-09-26 14:51:51 -05:00
parent 234dd3e36c 895c79a2fe
commit 437e81cfff
26 changed files with 3480 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/attribute.rs
+158 -33
View File
@@ -17,6 +17,7 @@ use crate::fractal_heap::FractalHeapHeader;
use crate::message_type::MessageType;
use crate::object_header::ObjectHeader;
use crate::shared_message;
use crate::storage::{Storage, require_contiguous};
use crate::vl_data;
/// A parsed HDF5 attribute message.
@@ -52,7 +53,7 @@ impl AttributeMessage {
///
/// `length_size` is needed for dataspace dimension parsing.
pub fn parse(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
Self::parse_impl(data, length_size, None)
Self::parse_impl(data, length_size, None::<(&[u8], u8)>)
}
/// [`AttributeMessage::parse`] with access to the rest of the file, which
@@ -67,13 +68,24 @@ impl AttributeMessage {
offset_size: u8,
length_size: u8,
) -> Result<AttributeMessage, FormatError> {
Self::parse_impl(data, length_size, Some((file_data, offset_size)))
Self::parse_in_storage(data, file_data, offset_size, length_size)
}
fn parse_impl(
/// [`AttributeMessage::parse_in_file`] with the file behind any
/// [`Storage`].
pub fn parse_in_storage<S: Storage + ?Sized>(
data: &[u8],
file: &S,
offset_size: u8,
length_size: u8,
) -> Result<AttributeMessage, FormatError> {
Self::parse_impl(data, length_size, Some((file, offset_size)))
}
fn parse_impl<S: Storage + ?Sized>(
data: &[u8],
length_size: u8,
file: Option<(&[u8], u8)>,
file: Option<(&S, u8)>,
) -> Result<AttributeMessage, FormatError> {
ensure_len(data, 0, 2)?;
let version = data[0];
@@ -88,19 +100,19 @@ impl AttributeMessage {
/// The bytes of an embedded datatype/dataspace message, following the
/// shared-message reference when `shared` is set.
fn embedded_message<'a>(
fn embedded_message<'a, S: Storage + ?Sized>(
bytes: &'a [u8],
shared: bool,
msg_type: MessageType,
length_size: u8,
file: Option<(&[u8], u8)>,
file: Option<(&S, u8)>,
) -> Result<Cow<'a, [u8]>, FormatError> {
if !shared {
return Ok(Cow::Borrowed(bytes));
}
let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
let shared_ref = shared_message::parse_shared_ref_sized(bytes, offset_size, length_size)?;
shared_message::resolve_shared_message(
shared_message::resolve_shared_message_in(
file_data,
&shared_ref,
msg_type,
@@ -145,10 +157,10 @@ impl AttributeMessage {
})
}
fn parse_v2(
fn parse_v2<S: Storage + ?Sized>(
data: &[u8],
length_size: u8,
file: Option<(&[u8], u8)>,
file: Option<(&S, u8)>,
) -> Result<AttributeMessage, FormatError> {
// Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
let flags = data.get(1).copied().unwrap_or(0);
@@ -199,10 +211,10 @@ impl AttributeMessage {
})
}
fn parse_v3(
fn parse_v3<S: Storage + ?Sized>(
data: &[u8],
length_size: u8,
file: Option<(&[u8], u8)>,
file: Option<(&S, u8)>,
) -> Result<AttributeMessage, FormatError> {
// Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
let flags = data.get(1).copied().unwrap_or(0);
@@ -418,7 +430,20 @@ pub fn extract_attributes_full(
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
extract_attributes_with(file_data, header, offset_size, length_size, &mut Err)
extract_attributes_full_in(file_data, header, offset_size, length_size)
}
/// [`extract_attributes_full`] over any [`Storage`]. Dense attribute
/// storage is indexed by a v2 B-tree, which is not read over [`Storage`]
/// yet: on a backend without the whole file in memory an object with dense
/// attributes is [`FormatError::ContiguousStorageRequired`].
pub fn extract_attributes_full_in<S: Storage + ?Sized>(
file: &S,
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<Vec<AttributeMessage>, FormatError> {
extract_attributes_with(file, header, offset_size, length_size, &mut Err)
}
/// Like [`extract_attributes_full`], but an attribute that cannot be read
@@ -434,6 +459,17 @@ pub fn extract_attributes_tolerant(
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
extract_attributes_tolerant_in(file_data, header, offset_size, length_size)
}
/// [`extract_attributes_tolerant`] over any [`Storage`] (see
/// [`extract_attributes_full_in`] for dense storage).
pub fn extract_attributes_tolerant_in<S: Storage + ?Sized>(
file_data: &S,
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
let mut errors = Vec::new();
let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
@@ -445,8 +481,8 @@ pub fn extract_attributes_tolerant(
/// Read every attribute; each one that fails goes to `on_error`, which
/// either stops the read (returns the error) or skips that attribute.
fn extract_attributes_with(
file_data: &[u8],
fn extract_attributes_with<S: Storage + ?Sized>(
file_data: &S,
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
@@ -514,6 +550,19 @@ pub fn find_attribute_in_file(
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<AttributeMessage>, FormatError> {
find_attribute_in(file_data, header, name, offset_size, length_size)
}
/// [`find_attribute_in_file`] over any [`Storage`] (see
/// [`extract_attributes_full_in`] for dense storage, whose name index still
/// needs the whole file in memory).
pub fn find_attribute_in<S: Storage + ?Sized>(
file_data: &S,
header: &ObjectHeader,
name: &str,
offset_size: u8,
length_size: u8,
) -> Result<Option<AttributeMessage>, FormatError> {
let attr_info = find_attribute_info(header, offset_size)?;
let dense = attr_info
@@ -523,18 +572,19 @@ pub fn find_attribute_in_file(
// Compact only (or dense storage without a name index, which a
// listing reports): as a listing finds it.
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
extract_attributes_tolerant_in(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
);
};
let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?;
let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
return Ok(
extract_attributes_tolerant(file_data, header, offset_size, length_size)?
extract_attributes_tolerant_in(file_data, header, offset_size, length_size)?
.0
.into_iter()
.find(|a| a.name == name),
@@ -560,7 +610,7 @@ pub fn find_attribute_in_file(
// hash is the last field.
let hash = jenkins_lookup3(name.as_bytes());
let hash_at = usize::from(btree_hdr.record_size) - 4;
let records = find_btree_v2_records(file_data, &btree_hdr, offset_size, &mut |r| match r
let records = find_btree_v2_records(contiguous, &btree_hdr, offset_size, &mut |r| match r
.get(hash_at..hash_at + 4)
{
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
@@ -572,8 +622,10 @@ pub fn find_attribute_in_file(
continue;
};
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.and_then(|d| AttributeMessage::parse_in_file(&d, file_data, offset_size, length_size));
.read_managed_object_in(file_data, id_bytes, offset_size)
.and_then(|d| {
AttributeMessage::parse_in_storage(&d, file_data, offset_size, length_size)
});
// One that cannot be read is left out, as from a listing.
if let Ok(attr) = attr
&& attr.name == name
@@ -586,8 +638,8 @@ pub fn find_attribute_in_file(
/// The attributes stored in the object header itself (compact storage), and
/// each one's creation order into `orders`.
fn extract_compact_attributes(
file_data: &[u8],
fn extract_compact_attributes<S: Storage + ?Sized>(
file_data: &S,
header: &ObjectHeader,
offset_size: u8,
length_size: u8,
@@ -601,7 +653,7 @@ fn extract_compact_attributes(
// Shared attribute: resolve the reference to get actual attribute data
shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
.and_then(|shared_ref| {
shared_message::resolve_shared_message(
shared_message::resolve_shared_message_in(
file_data,
&shared_ref,
MessageType::Attribute,
@@ -610,7 +662,7 @@ fn extract_compact_attributes(
)
})
.and_then(|resolved| {
AttributeMessage::parse_in_file(
AttributeMessage::parse_in_storage(
&resolved,
file_data,
offset_size,
@@ -618,7 +670,7 @@ fn extract_compact_attributes(
)
})
} else {
AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size)
AttributeMessage::parse_in_storage(&msg.data, file_data, offset_size, length_size)
};
let attr = attr.and_then(|a| check_in_header(a, header));
match attr {
@@ -650,8 +702,8 @@ fn find_attribute_info(
/// Extract attributes from dense storage (fractal heap + B-tree v2), and
/// each one's creation order into `orders`.
#[allow(clippy::too_many_arguments)]
fn extract_dense_attributes(
file_data: &[u8],
fn extract_dense_attributes<S: Storage + ?Sized>(
file_data: &S,
attr_info: &AttributeInfoMessage,
fh_addr: u64,
offset_size: u8,
@@ -661,7 +713,7 @@ fn extract_dense_attributes(
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
) -> Result<(), FormatError> {
// Parse fractal heap
let fh = FractalHeapHeader::parse(file_data, to_usize(fh_addr)?, offset_size, length_size)?;
let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
// Parse B-tree v2 for name index (type 8)
let btree_addr = attr_info
@@ -670,9 +722,10 @@ fn extract_dense_attributes(
expected: 1,
available: 0,
})?;
let contiguous = require_contiguous(file_data, "dense attribute storage (a v2 B-tree)")?;
let btree_hdr =
BTreeV2Header::parse(file_data, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?;
BTreeV2Header::parse(contiguous, to_usize(btree_addr)?, offset_size, length_size)?;
let records = collect_btree_v2_records(contiguous, &btree_hdr, offset_size, length_size)?;
for record in &records {
// Per HDF5 spec, both type 8 and type 9 records start with heap_id:
@@ -689,9 +742,9 @@ fn extract_dense_attributes(
// The data in the heap is a complete attribute message
let attr = fh
.read_managed_object(file_data, id_bytes, offset_size)
.read_managed_object_in(file_data, id_bytes, offset_size)
.and_then(|attr_data| {
AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)
AttributeMessage::parse_in_storage(&attr_data, file_data, offset_size, length_size)
});
match attr {
Ok(attr) => {
@@ -1101,4 +1154,76 @@ mod tests {
let strs = attr.read_as_strings().unwrap();
assert_eq!(strs, vec!["abcd", "EFGH"]);
}
/// Every object's attributes in h5py-written files read identically
/// through a read_at-only CountingStorage — compact ones, shared ones
/// and those behind an Attribute Info message — except dense storage,
/// whose v2 B-tree index is not read over Storage yet: that is the clean
/// ContiguousStorageRequired error, never a partial list. Through a
/// slice as Storage every object matches.
#[test]
fn storage_reads_match_slice_reads() {
use crate::storage::CountingStorage;
let files: [(&str, &[u8]); 5] = [
("attrs", include_bytes!("../tests/fixtures/attrs.h5")),
(
"mixed_attrs",
include_bytes!("../tests/fixtures/mixed_attrs.h5"),
),
(
"dense_attrs",
include_bytes!("../tests/fixtures/dense_attrs.h5"),
),
(
"dense_attrs_root",
include_bytes!("../tests/fixtures/dense_attrs_root.h5"),
),
(
"shared_fill_value",
include_bytes!("../tests/fixtures/shared_fill_value.h5"),
),
];
let (mut same, mut dense, mut attrs) = (0, 0, 0);
for (name, file) in files {
let sb = crate::superblock::Superblock::parse(file, 0).unwrap();
let (os, ls) = (sb.offset_size, sb.length_size);
let mut addrs = vec![sb.root_group_address];
addrs.extend(
crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address)
.unwrap()
.iter()
.map(|e| e.object_header_address),
);
let storage = CountingStorage::new(file.to_vec());
for addr in addrs {
let header = ObjectHeader::parse(file, addr as usize, os, ls).unwrap();
let want = extract_attributes_full(file, &header, os, ls);
let slice_storage = extract_attributes_full_in(&file, &header, os, ls);
assert_eq!(format!("{slice_storage:?}"), format!("{want:?}"));
let got = extract_attributes_full_in(&storage, &header, os, ls);
let got_t = extract_attributes_tolerant_in(&storage, &header, os, ls);
let is_dense = find_attribute_info(&header, os)
.unwrap()
.is_some_and(|i| i.fractal_heap_address.is_some());
if is_dense {
let e = FormatError::ContiguousStorageRequired(
"dense attribute storage (a v2 B-tree)",
);
assert_eq!(got.unwrap_err(), e, "{name}");
assert_eq!(got_t.unwrap_err(), e, "{name}");
dense += 1;
} else {
attrs += want.as_ref().map_or(0, Vec::len);
assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}");
let want_t = extract_attributes_tolerant(file, &header, os, ls);
assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}");
same += 1;
}
}
}
assert!(
same >= 5 && dense >= 2 && attrs >= 5,
"{same} {dense} {attrs}"
);
}
}