osobhandClaude Opus 5.5
2e906ffe3a
read_string, read_string_bytes, read_string_selection, read_vlen and read_vlen_selection now retry as a whole, the global-heap decoding after the read included; so do File::decode_strings / decode_string_bytes / decode_vlen, Group::datasets / groups / attrs / attr, Dataset::attrs / attr, the typed full reads (read_f64 ...), the header messages behind shape() and dtype() (a shared message is read from another header), and verify_provenance. Before, a transient failure on those reached the caller. Attribute reads leave out an attribute they cannot read (or return a variable-length string one as AttrValue::Raw) instead of failing, which hid a transient error as a missing or raw attribute: on a live file such an error of a retried kind now runs the read again too, and after the last attempt the last result is returned as before. The format crate gains find_attribute_reporting_in, which returns the errors find_attribute_in skips (dense name-index lookups dropped them). The zero-copy reads need the file in memory, which a live file never is, so they have nothing to retry. Test: a storage that fails one read with a checksum mismatch; for 14 read paths over a new fixture (tests/fixtures/swmr_strings_attrs.h5, an h5py copy with the SWMR-write flag: vlen strings, vlen int32, dense attributes), each read the path makes fails once in turn and the path must return the same result with exactly one retry. It fails on the previous commit (root.attrs, read 0). Dataset::attr on dense attributes returned None before find_attribute_reporting_in. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1309 lines
46 KiB
Rust
1309 lines
46 KiB
Rust
//! HDF5 Attribute message parsing (message type 0x000C).
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{borrow::Cow, string::String, vec::Vec};
|
||
#[cfg(feature = "std")]
|
||
use std::borrow::Cow;
|
||
|
||
use crate::addr::to_usize;
|
||
use crate::attribute_info::AttributeInfoMessage;
|
||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records_in, find_btree_v2_records_in};
|
||
use crate::checksum::jenkins_lookup3;
|
||
use crate::data_read;
|
||
use crate::dataspace::Dataspace;
|
||
use crate::datatype::Datatype;
|
||
use crate::error::FormatError;
|
||
use crate::fractal_heap::FractalHeapHeader;
|
||
use crate::message_type::MessageType;
|
||
use crate::object_header::ObjectHeader;
|
||
use crate::shared_message;
|
||
use crate::storage::Storage;
|
||
use crate::vl_data;
|
||
|
||
/// A parsed HDF5 attribute message.
|
||
#[derive(Debug, Clone)]
|
||
pub struct AttributeMessage {
|
||
/// Attribute name.
|
||
pub name: String,
|
||
/// Attribute datatype.
|
||
pub datatype: Datatype,
|
||
/// Attribute dataspace.
|
||
pub dataspace: Dataspace,
|
||
/// Raw attribute value data.
|
||
pub raw_data: Vec<u8>,
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
match offset.checked_add(needed) {
|
||
Some(end) if end <= data.len() => Ok(()),
|
||
_ => Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
}),
|
||
}
|
||
}
|
||
|
||
/// Round up to the next multiple of 8.
|
||
fn pad8(x: usize) -> usize {
|
||
(x + 7) & !7
|
||
}
|
||
|
||
impl AttributeMessage {
|
||
/// Parse an attribute message from raw message bytes.
|
||
///
|
||
/// `length_size` is needed for dataspace dimension parsing.
|
||
pub fn parse(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
|
||
Self::parse_impl(data, length_size, None::<(&[u8], u8)>)
|
||
}
|
||
|
||
/// [`AttributeMessage::parse`] with access to the rest of the file, which
|
||
/// is needed when the attribute's datatype or dataspace is *shared* (v2/v3
|
||
/// flag bits 0/1) — e.g. an attribute created with a committed datatype.
|
||
/// In that case the embedded bytes are a reference to the real message,
|
||
/// not the message. Without file access such an attribute is an error
|
||
/// rather than a garbage datatype.
|
||
pub fn parse_in_file(
|
||
data: &[u8],
|
||
file_data: &[u8],
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<AttributeMessage, FormatError> {
|
||
Self::parse_in_storage(data, file_data, offset_size, length_size)
|
||
}
|
||
|
||
/// [`AttributeMessage::parse_in_file`] with the file behind any
|
||
/// [`Storage`].
|
||
pub fn parse_in_storage<S: Storage + ?Sized>(
|
||
data: &[u8],
|
||
file: &S,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<AttributeMessage, FormatError> {
|
||
Self::parse_impl(data, length_size, Some((file, offset_size)))
|
||
}
|
||
|
||
fn parse_impl<S: Storage + ?Sized>(
|
||
data: &[u8],
|
||
length_size: u8,
|
||
file: Option<(&S, u8)>,
|
||
) -> Result<AttributeMessage, FormatError> {
|
||
ensure_len(data, 0, 2)?;
|
||
let version = data[0];
|
||
|
||
match version {
|
||
1 => Self::parse_v1(data, length_size),
|
||
2 => Self::parse_v2(data, length_size, file),
|
||
3 => Self::parse_v3(data, length_size, file),
|
||
_ => Err(FormatError::InvalidAttributeVersion(version)),
|
||
}
|
||
}
|
||
|
||
/// The bytes of an embedded datatype/dataspace message, following the
|
||
/// shared-message reference when `shared` is set.
|
||
fn embedded_message<'a, S: Storage + ?Sized>(
|
||
bytes: &'a [u8],
|
||
shared: bool,
|
||
msg_type: MessageType,
|
||
length_size: u8,
|
||
file: Option<(&S, u8)>,
|
||
) -> Result<Cow<'a, [u8]>, FormatError> {
|
||
if !shared {
|
||
return Ok(Cow::Borrowed(bytes));
|
||
}
|
||
let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?;
|
||
let shared_ref = shared_message::parse_shared_ref_sized(bytes, offset_size, length_size)?;
|
||
shared_message::resolve_shared_message_in(
|
||
file_data,
|
||
&shared_ref,
|
||
msg_type,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
.map(Cow::Owned)
|
||
}
|
||
|
||
fn parse_v1(data: &[u8], length_size: u8) -> Result<AttributeMessage, FormatError> {
|
||
// version(1) + reserved(1) + name_size(2) + datatype_size(2) + dataspace_size(2) = 8
|
||
ensure_len(data, 0, 8)?;
|
||
let name_size = u16::from_le_bytes([data[2], data[3]]) as usize;
|
||
let datatype_size = u16::from_le_bytes([data[4], data[5]]) as usize;
|
||
let dataspace_size = u16::from_le_bytes([data[6], data[7]]) as usize;
|
||
|
||
let mut pos = 8;
|
||
|
||
// Name (padded to 8-byte boundary)
|
||
ensure_len(data, pos, name_size)?;
|
||
let name = extract_name(&data[pos..pos + name_size]);
|
||
pos += pad8(name_size);
|
||
|
||
// Datatype (padded to 8-byte boundary)
|
||
ensure_len(data, pos, datatype_size)?;
|
||
let (datatype, _) = Datatype::parse(&data[pos..pos + datatype_size])?;
|
||
pos += pad8(datatype_size);
|
||
|
||
// Dataspace (padded to 8-byte boundary)
|
||
ensure_len(data, pos, dataspace_size)?;
|
||
let dataspace = Dataspace::parse(&data[pos..pos + dataspace_size], length_size)?;
|
||
pos += pad8(dataspace_size);
|
||
|
||
// Raw data: num_elements × type_size bytes
|
||
let raw_data = compute_raw_data(data, pos, &dataspace, &datatype);
|
||
|
||
Ok(AttributeMessage {
|
||
name,
|
||
datatype,
|
||
dataspace,
|
||
raw_data,
|
||
})
|
||
}
|
||
|
||
fn parse_v2<S: Storage + ?Sized>(
|
||
data: &[u8],
|
||
length_size: u8,
|
||
file: Option<(&S, u8)>,
|
||
) -> Result<AttributeMessage, FormatError> {
|
||
// Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
|
||
let flags = data.get(1).copied().unwrap_or(0);
|
||
// version(1) + flags(1) + name_size(2) + datatype_size(2) + dataspace_size(2) = 8
|
||
ensure_len(data, 0, 8)?;
|
||
let name_size = u16::from_le_bytes([data[2], data[3]]) as usize;
|
||
let datatype_size = u16::from_le_bytes([data[4], data[5]]) as usize;
|
||
let dataspace_size = u16::from_le_bytes([data[6], data[7]]) as usize;
|
||
|
||
let mut pos = 8;
|
||
|
||
// Name (NO padding)
|
||
ensure_len(data, pos, name_size)?;
|
||
let name = extract_name(&data[pos..pos + name_size]);
|
||
pos += name_size;
|
||
|
||
// Datatype (NO padding)
|
||
ensure_len(data, pos, datatype_size)?;
|
||
let dt_bytes = Self::embedded_message(
|
||
&data[pos..pos + datatype_size],
|
||
flags & 0x01 != 0,
|
||
MessageType::Datatype,
|
||
length_size,
|
||
file,
|
||
)?;
|
||
let (datatype, _) = Datatype::parse(&dt_bytes)?;
|
||
pos += datatype_size;
|
||
|
||
// Dataspace (NO padding)
|
||
ensure_len(data, pos, dataspace_size)?;
|
||
let ds_bytes = Self::embedded_message(
|
||
&data[pos..pos + dataspace_size],
|
||
flags & 0x02 != 0,
|
||
MessageType::Dataspace,
|
||
length_size,
|
||
file,
|
||
)?;
|
||
let dataspace = Dataspace::parse(&ds_bytes, length_size)?;
|
||
pos += dataspace_size;
|
||
|
||
let raw_data = compute_raw_data(data, pos, &dataspace, &datatype);
|
||
|
||
Ok(AttributeMessage {
|
||
name,
|
||
datatype,
|
||
dataspace,
|
||
raw_data,
|
||
})
|
||
}
|
||
|
||
fn parse_v3<S: Storage + ?Sized>(
|
||
data: &[u8],
|
||
length_size: u8,
|
||
file: Option<(&S, u8)>,
|
||
) -> Result<AttributeMessage, FormatError> {
|
||
// Flags: bit 0 = datatype is shared, bit 1 = dataspace is shared.
|
||
let flags = data.get(1).copied().unwrap_or(0);
|
||
// version(1) + flags(1) + name_size(2) + datatype_size(2) + dataspace_size(2) + encoding(1) = 9
|
||
ensure_len(data, 0, 9)?;
|
||
let name_size = u16::from_le_bytes([data[2], data[3]]) as usize;
|
||
let datatype_size = u16::from_le_bytes([data[4], data[5]]) as usize;
|
||
let dataspace_size = u16::from_le_bytes([data[6], data[7]]) as usize;
|
||
let _encoding = data[8]; // 0=ASCII, 1=UTF-8
|
||
|
||
let mut pos = 9;
|
||
|
||
// Name (NO padding)
|
||
ensure_len(data, pos, name_size)?;
|
||
let name = extract_name(&data[pos..pos + name_size]);
|
||
pos += name_size;
|
||
|
||
// Datatype (NO padding)
|
||
ensure_len(data, pos, datatype_size)?;
|
||
let dt_bytes = Self::embedded_message(
|
||
&data[pos..pos + datatype_size],
|
||
flags & 0x01 != 0,
|
||
MessageType::Datatype,
|
||
length_size,
|
||
file,
|
||
)?;
|
||
let (datatype, _) = Datatype::parse(&dt_bytes)?;
|
||
pos += datatype_size;
|
||
|
||
// Dataspace (NO padding)
|
||
ensure_len(data, pos, dataspace_size)?;
|
||
let ds_bytes = Self::embedded_message(
|
||
&data[pos..pos + dataspace_size],
|
||
flags & 0x02 != 0,
|
||
MessageType::Dataspace,
|
||
length_size,
|
||
file,
|
||
)?;
|
||
let dataspace = Dataspace::parse(&ds_bytes, length_size)?;
|
||
pos += dataspace_size;
|
||
|
||
let raw_data = compute_raw_data(data, pos, &dataspace, &datatype);
|
||
|
||
Ok(AttributeMessage {
|
||
name,
|
||
datatype,
|
||
dataspace,
|
||
raw_data,
|
||
})
|
||
}
|
||
|
||
/// Serialize attribute message (v2 format, no padding).
|
||
pub fn serialize(&self, length_size: u8) -> Vec<u8> {
|
||
self.serialize_version(2, length_size)
|
||
}
|
||
|
||
/// Serialize attribute message as v3 (adds character set encoding byte).
|
||
pub fn serialize_v3(&self, length_size: u8) -> Vec<u8> {
|
||
self.serialize_version(3, length_size)
|
||
}
|
||
|
||
fn serialize_version(&self, version: u8, length_size: u8) -> Vec<u8> {
|
||
let name_bytes = {
|
||
let mut n = self.name.as_bytes().to_vec();
|
||
n.push(0); // null terminator
|
||
n
|
||
};
|
||
let dt_bytes = self.datatype.serialize();
|
||
let ds_bytes = self.dataspace.serialize(length_size);
|
||
|
||
let mut buf = Vec::new();
|
||
buf.push(version);
|
||
buf.push(0); // flags
|
||
buf.extend_from_slice(&(name_bytes.len() as u16).to_le_bytes());
|
||
buf.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
buf.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
if version >= 3 {
|
||
buf.push(0x00); // character set encoding: ASCII
|
||
}
|
||
buf.extend_from_slice(&name_bytes);
|
||
buf.extend_from_slice(&dt_bytes);
|
||
buf.extend_from_slice(&ds_bytes);
|
||
buf.extend_from_slice(&self.raw_data);
|
||
buf
|
||
}
|
||
|
||
/// Read attribute value as f64 values.
|
||
pub fn read_as_f64(&self) -> Result<Vec<f64>, FormatError> {
|
||
data_read::read_as_f64(&self.raw_data, &self.datatype)
|
||
}
|
||
|
||
/// Read attribute value as i64 values.
|
||
pub fn read_as_i64(&self) -> Result<Vec<i64>, FormatError> {
|
||
data_read::read_as_i64(&self.raw_data, &self.datatype)
|
||
}
|
||
|
||
/// Read attribute value as u64 values.
|
||
pub fn read_as_u64(&self) -> Result<Vec<u64>, FormatError> {
|
||
data_read::read_as_u64(&self.raw_data, &self.datatype)
|
||
}
|
||
|
||
/// Read attribute value as a single string (first element).
|
||
pub fn read_as_string(&self) -> Result<String, FormatError> {
|
||
let strings = data_read::read_as_strings(&self.raw_data, &self.datatype)?;
|
||
Ok(strings.into_iter().next().unwrap_or_default())
|
||
}
|
||
|
||
/// Read attribute value as a vector of fixed-length strings.
|
||
pub fn read_as_strings(&self) -> Result<Vec<String>, FormatError> {
|
||
data_read::read_as_strings(&self.raw_data, &self.datatype)
|
||
}
|
||
|
||
/// Read variable-length string attribute values.
|
||
///
|
||
/// Needs the full file data and offset/length sizes from the superblock
|
||
/// because VL strings store their data in the global heap.
|
||
pub fn read_vl_strings(
|
||
&self,
|
||
file_data: &[u8],
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<String>, FormatError> {
|
||
self.read_vl_strings_in(file_data, offset_size, length_size)
|
||
}
|
||
|
||
/// [`Self::read_vl_strings`] over any [`Storage`].
|
||
pub fn read_vl_strings_in<S: Storage + ?Sized>(
|
||
&self,
|
||
file_data: &S,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<String>, FormatError> {
|
||
let num_elements = self.dataspace.num_elements();
|
||
vl_data::read_vl_strings_in(
|
||
file_data,
|
||
&self.raw_data,
|
||
num_elements,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
}
|
||
}
|
||
|
||
/// Compute raw data size based on dataspace and datatype, then extract from message bytes.
|
||
fn compute_raw_data(
|
||
data: &[u8],
|
||
pos: usize,
|
||
dataspace: &Dataspace,
|
||
datatype: &Datatype,
|
||
) -> Vec<u8> {
|
||
// Saturating, like the product: the size is capped at what is there.
|
||
let num_elements = usize::try_from(dataspace.num_elements()).unwrap_or(usize::MAX);
|
||
let elem_size = datatype.type_size() as usize;
|
||
let expected_size = num_elements.saturating_mul(elem_size);
|
||
let available = data.len().saturating_sub(pos);
|
||
let take = expected_size.min(available);
|
||
if take > 0 {
|
||
data[pos..pos + take].to_vec()
|
||
} else if available > 0 {
|
||
// Fallback: take whatever is available (e.g., for VL types where type_size may not match)
|
||
data[pos..].to_vec()
|
||
} else {
|
||
Vec::new()
|
||
}
|
||
}
|
||
|
||
/// Extract a name from raw bytes, stripping null terminator.
|
||
fn extract_name(bytes: &[u8]) -> String {
|
||
let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
|
||
String::from_utf8_lossy(&bytes[..end]).into_owned()
|
||
}
|
||
|
||
/// An attribute's datatype gets libhdf5's extra check for a header without
|
||
/// a checksum (see [`Datatype::check_unused_bits`]).
|
||
fn check_in_header(
|
||
attr: AttributeMessage,
|
||
header: &ObjectHeader,
|
||
) -> Result<AttributeMessage, FormatError> {
|
||
if header.version == 1 {
|
||
attr.datatype.check_unused_bits()?;
|
||
}
|
||
Ok(attr)
|
||
}
|
||
|
||
/// Extract all attribute messages from an object header.
|
||
pub fn extract_attributes(
|
||
header: &ObjectHeader,
|
||
length_size: u8,
|
||
) -> Result<Vec<AttributeMessage>, FormatError> {
|
||
let mut attrs = Vec::new();
|
||
for msg in &header.messages {
|
||
if msg.msg_type == MessageType::Attribute {
|
||
let attr = AttributeMessage::parse(&msg.data, length_size)?;
|
||
attrs.push(check_in_header(attr, header)?);
|
||
}
|
||
}
|
||
Ok(attrs)
|
||
}
|
||
|
||
/// Find a specific attribute by name.
|
||
pub fn find_attribute<'a>(
|
||
attrs: &'a [AttributeMessage],
|
||
name: &str,
|
||
) -> Option<&'a AttributeMessage> {
|
||
attrs.iter().find(|a| a.name == name)
|
||
}
|
||
|
||
/// Extract all attributes from an object header, supporting both compact and dense storage.
|
||
///
|
||
/// This function handles:
|
||
/// - Compact attributes: inline Attribute messages (0x000C) in the object header
|
||
/// - Dense attributes: AttributeInfo message (0x0015) pointing to fractal heap + B-tree v2
|
||
/// - Shared messages: resolves shared datatype references for attribute messages
|
||
///
|
||
/// Use this instead of `extract_attributes` when reading files that may use dense storage
|
||
/// (e.g., objects with many attributes, typically >8).
|
||
///
|
||
/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`]
|
||
/// to read the others.
|
||
pub fn extract_attributes_full(
|
||
file_data: &[u8],
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<AttributeMessage>, FormatError> {
|
||
extract_attributes_full_in(file_data, header, offset_size, length_size)
|
||
}
|
||
|
||
/// [`extract_attributes_full`] over any [`Storage`]. Dense attribute
|
||
/// storage is indexed by a v2 B-tree, which is not read over [`Storage`]
|
||
/// yet: on a backend without the whole file in memory an object with dense
|
||
/// attributes is [`FormatError::ContiguousStorageRequired`].
|
||
pub fn extract_attributes_full_in<S: Storage + ?Sized>(
|
||
file: &S,
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<AttributeMessage>, FormatError> {
|
||
extract_attributes_with(file, header, offset_size, length_size, &mut Err)
|
||
}
|
||
|
||
/// Like [`extract_attributes_full`], but an attribute that cannot be read
|
||
/// (a corrupt or unsupported attribute message, or a heap object that cannot
|
||
/// be located) is left out and its error returned alongside the attributes
|
||
/// that could be read, instead of failing them all.
|
||
///
|
||
/// Errors in the structures that index the attributes (the Attribute Info
|
||
/// message, the dense-storage heap header or B-tree) still fail the call:
|
||
/// then it is unknown which attributes exist at all.
|
||
pub fn extract_attributes_tolerant(
|
||
file_data: &[u8],
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
|
||
extract_attributes_tolerant_core(file_data, header, offset_size, length_size)
|
||
}
|
||
|
||
/// [`extract_attributes_tolerant`] over any [`Storage`] (see
|
||
/// [`extract_attributes_full_in`] for dense storage). One with the whole
|
||
/// file in memory is read as the slice, by code compiled in this crate (see
|
||
/// [`crate::storage`], "Slice entry points").
|
||
#[inline]
|
||
pub fn extract_attributes_tolerant_in<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
|
||
match file_data.as_contiguous() {
|
||
Some(all) => extract_attributes_tolerant(all, header, offset_size, length_size),
|
||
None => extract_attributes_tolerant_core(file_data, header, offset_size, length_size),
|
||
}
|
||
}
|
||
|
||
fn extract_attributes_tolerant_core<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<(Vec<AttributeMessage>, Vec<FormatError>), FormatError> {
|
||
let mut errors = Vec::new();
|
||
let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| {
|
||
errors.push(e);
|
||
Ok(())
|
||
})?;
|
||
Ok((attrs, errors))
|
||
}
|
||
|
||
/// Read every attribute; each one that fails goes to `on_error`, which
|
||
/// either stops the read (returns the error) or skips that attribute.
|
||
fn extract_attributes_with<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
|
||
) -> Result<Vec<AttributeMessage>, FormatError> {
|
||
let mut attrs = Vec::new();
|
||
// Each attribute's creation order, where the file records one.
|
||
let mut orders: Vec<u32> = Vec::new();
|
||
|
||
extract_compact_attributes(
|
||
file_data,
|
||
header,
|
||
offset_size,
|
||
length_size,
|
||
&mut attrs,
|
||
&mut orders,
|
||
on_error,
|
||
)?;
|
||
|
||
// Check for dense attributes via AttributeInfo message
|
||
let attr_info = find_attribute_info(header, offset_size)?;
|
||
if let Some(info) = &attr_info
|
||
&& let Some(fh_addr) = info.fractal_heap_address
|
||
{
|
||
extract_dense_attributes(
|
||
file_data,
|
||
info,
|
||
fh_addr,
|
||
offset_size,
|
||
length_size,
|
||
&mut attrs,
|
||
&mut orders,
|
||
on_error,
|
||
)?;
|
||
}
|
||
|
||
// An object that tracks attribute creation order lists its attributes
|
||
// in that order (h5py's `track_order=True`), as libhdf5 does; otherwise
|
||
// they come in storage order.
|
||
if attr_info.is_some_and(|i| i.max_creation_index.is_some()) {
|
||
let mut paired: Vec<(u32, AttributeMessage)> = orders.into_iter().zip(attrs).collect();
|
||
paired.sort_by_key(|(o, _)| *o);
|
||
attrs = paired.into_iter().map(|(_, a)| a).collect();
|
||
}
|
||
|
||
Ok(attrs)
|
||
}
|
||
|
||
/// B-tree v2 record type of dense attribute storage's name index.
|
||
const ATTRIBUTE_NAME_INDEX: u8 = 8;
|
||
|
||
/// The attribute called `name` on the object with header `header`: the
|
||
/// first one [`extract_attributes_tolerant`] returns under that name, or
|
||
/// `None` if it returns none (an attribute that cannot be read is not
|
||
/// returned there either).
|
||
///
|
||
/// Compact attributes are in the header and are scanned. Dense attributes
|
||
/// are found through the name index (a v2 B-tree of lookup3 name hashes,
|
||
/// record type 8): only the attributes whose names hash like `name` are read
|
||
/// from the heap, O(log n) instead of all of them. Errors in the structures
|
||
/// that index the attributes fail the call, as they fail a listing.
|
||
pub fn find_attribute_in_file(
|
||
file_data: &[u8],
|
||
header: &ObjectHeader,
|
||
name: &str,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Option<AttributeMessage>, FormatError> {
|
||
find_attribute_core(
|
||
file_data,
|
||
header,
|
||
name,
|
||
offset_size,
|
||
length_size,
|
||
&mut Vec::new(),
|
||
)
|
||
}
|
||
|
||
/// [`find_attribute_in_file`] over any [`Storage`] (see
|
||
/// [`extract_attributes_full_in`] for dense storage, whose name index still
|
||
/// needs the whole file in memory). One with the whole file in memory is
|
||
/// read as the slice, by code compiled in this crate (see
|
||
/// [`crate::storage`], "Slice entry points").
|
||
#[inline]
|
||
pub fn find_attribute_in<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
name: &str,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Option<AttributeMessage>, FormatError> {
|
||
match file_data.as_contiguous() {
|
||
Some(all) => find_attribute_in_file(all, header, name, offset_size, length_size),
|
||
None => find_attribute_core(
|
||
file_data,
|
||
header,
|
||
name,
|
||
offset_size,
|
||
length_size,
|
||
&mut Vec::new(),
|
||
),
|
||
}
|
||
}
|
||
|
||
/// [`find_attribute_in`], also returning the errors of the attributes it
|
||
/// could not read on the way (which it leaves out rather than failing
|
||
/// the call): the attribute asked for may be one of them. A reader of a
|
||
/// file that is being written uses them to tell a read that raced the
|
||
/// writer from an absent attribute.
|
||
pub fn find_attribute_reporting_in<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
name: &str,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<(Option<AttributeMessage>, Vec<FormatError>), FormatError> {
|
||
let mut errors = Vec::new();
|
||
let found = find_attribute_core(
|
||
file_data,
|
||
header,
|
||
name,
|
||
offset_size,
|
||
length_size,
|
||
&mut errors,
|
||
)?;
|
||
Ok((found, errors))
|
||
}
|
||
|
||
fn find_attribute_core<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
name: &str,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
errors: &mut Vec<FormatError>,
|
||
) -> Result<Option<AttributeMessage>, FormatError> {
|
||
let attr_info = find_attribute_info(header, offset_size)?;
|
||
let dense = attr_info
|
||
.as_ref()
|
||
.and_then(|i| Some((i.fractal_heap_address?, i.btree_name_index_address?)));
|
||
let Some((fh_addr, btree_addr)) = dense else {
|
||
// Compact only (or dense storage without a name index, which a
|
||
// listing reports): as a listing finds it.
|
||
let (attrs, errs) =
|
||
extract_attributes_tolerant_in(file_data, header, offset_size, length_size)?;
|
||
errors.extend(errs);
|
||
return Ok(attrs.into_iter().find(|a| a.name == name));
|
||
};
|
||
let btree_hdr = BTreeV2Header::parse_in(
|
||
file_data,
|
||
to_usize(btree_addr)? as u64,
|
||
offset_size,
|
||
length_size,
|
||
)?;
|
||
let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
|
||
if btree_hdr.tree_type != ATTRIBUTE_NAME_INDEX || btree_hdr.record_size < 4 {
|
||
let (attrs, errs) =
|
||
extract_attributes_tolerant_in(file_data, header, offset_size, length_size)?;
|
||
errors.extend(errs);
|
||
return Ok(attrs.into_iter().find(|a| a.name == name));
|
||
}
|
||
|
||
// A listing has the compact attributes first.
|
||
let mut compact = Vec::new();
|
||
extract_compact_attributes(
|
||
file_data,
|
||
header,
|
||
offset_size,
|
||
length_size,
|
||
&mut compact,
|
||
&mut Vec::new(),
|
||
&mut |_| Ok(()),
|
||
)?;
|
||
if let Some(a) = compact.into_iter().find(|a| a.name == name) {
|
||
return Ok(Some(a));
|
||
}
|
||
|
||
// Record: heap ID + message flags(1) + creation order(4) + hash(4); the
|
||
// hash is the last field.
|
||
let hash = jenkins_lookup3(name.as_bytes());
|
||
let hash_at = usize::from(btree_hdr.record_size) - 4;
|
||
let records = find_btree_v2_records_in(file_data, &btree_hdr, offset_size, &mut |r| match r
|
||
.get(hash_at..hash_at + 4)
|
||
{
|
||
Some(h) => u32::from_le_bytes([h[0], h[1], h[2], h[3]]).cmp(&hash),
|
||
None => core::cmp::Ordering::Less,
|
||
})?;
|
||
let id_len = usize::from(fh.heap_id_length);
|
||
for record in &records {
|
||
let Some(id_bytes) = record.data.get(..id_len) else {
|
||
continue;
|
||
};
|
||
let attr = fh
|
||
.read_managed_object_in(file_data, id_bytes, offset_size)
|
||
.and_then(|d| {
|
||
AttributeMessage::parse_in_storage(&d, file_data, offset_size, length_size)
|
||
});
|
||
// One that cannot be read is left out, as from a listing.
|
||
match attr {
|
||
Ok(attr) if attr.name == name => return Ok(Some(attr)),
|
||
Ok(_) => {}
|
||
Err(e) => errors.push(e),
|
||
}
|
||
}
|
||
Ok(None)
|
||
}
|
||
|
||
/// The attributes stored in the object header itself (compact storage), and
|
||
/// each one's creation order into `orders`.
|
||
fn extract_compact_attributes<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
attrs: &mut Vec<AttributeMessage>,
|
||
orders: &mut Vec<u32>,
|
||
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
|
||
) -> Result<(), FormatError> {
|
||
for msg in &header.messages {
|
||
if msg.msg_type == MessageType::Attribute {
|
||
let attr = if shared_message::is_shared(msg.flags) {
|
||
// Shared attribute: resolve the reference to get actual attribute data
|
||
shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)
|
||
.and_then(|shared_ref| {
|
||
shared_message::resolve_shared_message_in(
|
||
file_data,
|
||
&shared_ref,
|
||
MessageType::Attribute,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
})
|
||
.and_then(|resolved| {
|
||
AttributeMessage::parse_in_storage(
|
||
&resolved,
|
||
file_data,
|
||
offset_size,
|
||
length_size,
|
||
)
|
||
})
|
||
} else {
|
||
AttributeMessage::parse_in_storage(&msg.data, file_data, offset_size, length_size)
|
||
};
|
||
let attr = attr.and_then(|a| check_in_header(a, header));
|
||
match attr {
|
||
Ok(attr) => {
|
||
attrs.push(attr);
|
||
orders.push(msg.creation_order.map_or(0, u32::from));
|
||
}
|
||
Err(e) => on_error(e)?,
|
||
}
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Find and parse the Attribute Info message from an object header.
|
||
fn find_attribute_info(
|
||
header: &ObjectHeader,
|
||
offset_size: u8,
|
||
) -> Result<Option<AttributeInfoMessage>, FormatError> {
|
||
for msg in &header.messages {
|
||
if msg.msg_type == MessageType::AttributeInfo {
|
||
let info = AttributeInfoMessage::parse(&msg.data, offset_size)?;
|
||
return Ok(Some(info));
|
||
}
|
||
}
|
||
Ok(None)
|
||
}
|
||
|
||
/// Extract attributes from dense storage (fractal heap + B-tree v2), and
|
||
/// each one's creation order into `orders`.
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn extract_dense_attributes<S: Storage + ?Sized>(
|
||
file_data: &S,
|
||
attr_info: &AttributeInfoMessage,
|
||
fh_addr: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
attrs: &mut Vec<AttributeMessage>,
|
||
orders: &mut Vec<u32>,
|
||
on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>,
|
||
) -> Result<(), FormatError> {
|
||
// Parse fractal heap
|
||
let fh = FractalHeapHeader::parse_in(file_data, fh_addr, offset_size, length_size)?;
|
||
|
||
// Parse B-tree v2 for name index (type 8)
|
||
let btree_addr = attr_info
|
||
.btree_name_index_address
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: 1,
|
||
available: 0,
|
||
})?;
|
||
let btree_hdr = BTreeV2Header::parse_in(
|
||
file_data,
|
||
to_usize(btree_addr)? as u64,
|
||
offset_size,
|
||
length_size,
|
||
)?;
|
||
let records = collect_btree_v2_records_in(file_data, &btree_hdr, offset_size, length_size)?;
|
||
|
||
for record in &records {
|
||
// Per HDF5 spec, both type 8 and type 9 records start with heap_id:
|
||
// Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4)
|
||
// Type 9: heap_id(8) + msg_flags(1) + creation_order(4)
|
||
let id_len = fh.heap_id_length as usize;
|
||
let Some(id_bytes) = record.data.get(..id_len) else {
|
||
on_error(FormatError::UnexpectedEof {
|
||
expected: id_len,
|
||
available: record.data.len(),
|
||
})?;
|
||
continue;
|
||
};
|
||
|
||
// The data in the heap is a complete attribute message
|
||
let attr = fh
|
||
.read_managed_object_in(file_data, id_bytes, offset_size)
|
||
.and_then(|attr_data| {
|
||
AttributeMessage::parse_in_storage(&attr_data, file_data, offset_size, length_size)
|
||
});
|
||
match attr {
|
||
Ok(attr) => {
|
||
attrs.push(attr);
|
||
let order = record
|
||
.data
|
||
.get(id_len + 1..id_len + 5)
|
||
.map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
|
||
orders.push(order);
|
||
}
|
||
Err(e) => on_error(e)?,
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Build a datatype header for testing (8 bytes).
|
||
fn build_dt_header(class: u8, version: u8, bf: [u8; 3], size: u32) -> Vec<u8> {
|
||
let mut buf = vec![0u8; 8];
|
||
buf[0] = (class & 0x0F) | ((version & 0x0F) << 4);
|
||
buf[1] = bf[0];
|
||
buf[2] = bf[1];
|
||
buf[3] = bf[2];
|
||
buf[4..8].copy_from_slice(&size.to_le_bytes());
|
||
buf
|
||
}
|
||
|
||
/// Build an f64 LE datatype message.
|
||
fn build_f64_dt() -> Vec<u8> {
|
||
// Sign bit 63 (bits 8-15 of the class bits).
|
||
let mut buf = build_dt_header(1, 1, [0x20, 63, 0x00], 8);
|
||
let mut props = [0u8; 12];
|
||
props[2..4].copy_from_slice(&64u16.to_le_bytes()); // bit_precision
|
||
props[4] = 52; // exp_location
|
||
props[5] = 11; // exp_size
|
||
props[6] = 0; // mant_location
|
||
props[7] = 52; // mant_size
|
||
props[8..12].copy_from_slice(&1023u32.to_le_bytes()); // exp_bias
|
||
buf.extend_from_slice(&props);
|
||
buf
|
||
}
|
||
|
||
/// Build a scalar dataspace (v2).
|
||
fn build_scalar_ds() -> Vec<u8> {
|
||
vec![2, 0, 0, 0] // version=2, rank=0, flags=0, type=0(scalar)
|
||
}
|
||
|
||
/// Build a simple 1D dataspace (v1).
|
||
fn build_simple_ds_v1(dim: u64) -> Vec<u8> {
|
||
let mut buf = vec![1u8, 1, 0, 0, 0, 0, 0, 0]; // version=1, rank=1, flags=0, reserved(5)
|
||
buf.extend_from_slice(&dim.to_le_bytes());
|
||
buf
|
||
}
|
||
|
||
/// Build a fixed-length string datatype.
|
||
fn build_string_dt(size: u32) -> Vec<u8> {
|
||
// class=3, version=1, padding=NullPad(1), charset=ASCII(0) → bf0=0x01
|
||
build_dt_header(3, 1, [0x01, 0, 0], size)
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v1_attribute_f64_scalar() {
|
||
let name = b"temp\0";
|
||
let dt_bytes = build_f64_dt();
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let name_size = name.len();
|
||
let dt_size = dt_bytes.len();
|
||
let ds_size = ds_bytes.len();
|
||
|
||
let mut data = Vec::new();
|
||
data.push(1); // version
|
||
data.push(0); // reserved
|
||
data.extend_from_slice(&(name_size as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_size as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_size as u16).to_le_bytes());
|
||
|
||
// Name padded to 8 bytes
|
||
data.extend_from_slice(name);
|
||
if data.len() % 8 != 0 || data.len() == 8 {
|
||
// Pad name to 8-byte boundary from start of name
|
||
let name_start = 8;
|
||
let name_padded = pad8(name_size);
|
||
while data.len() < name_start + name_padded {
|
||
data.push(0);
|
||
}
|
||
}
|
||
|
||
// Datatype padded to 8 bytes
|
||
let dt_start = data.len();
|
||
data.extend_from_slice(&dt_bytes);
|
||
let dt_padded = pad8(dt_size);
|
||
while data.len() < dt_start + dt_padded {
|
||
data.push(0);
|
||
}
|
||
|
||
// Dataspace padded to 8 bytes
|
||
let ds_start = data.len();
|
||
data.extend_from_slice(&ds_bytes);
|
||
let ds_padded = pad8(ds_size);
|
||
while data.len() < ds_start + ds_padded {
|
||
data.push(0);
|
||
}
|
||
|
||
// Raw data: f64 value 98.6
|
||
data.extend_from_slice(&98.6f64.to_le_bytes());
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.name, "temp");
|
||
assert_eq!(attr.dataspace.num_elements(), 1);
|
||
let vals = attr.read_as_f64().unwrap();
|
||
assert_eq!(vals.len(), 1);
|
||
assert!((vals[0] - 98.6).abs() < 1e-10);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v2_attribute_fixed_string() {
|
||
let name = b"label\0";
|
||
let dt_bytes = build_string_dt(5);
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let mut data = Vec::new();
|
||
data.push(2); // version
|
||
data.push(0); // flags
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
|
||
// No padding in v2
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
|
||
// Raw data: "hello"
|
||
data.extend_from_slice(b"hello");
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.name, "label");
|
||
let s = attr.read_as_string().unwrap();
|
||
assert_eq!(s, "hello");
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v3_attribute_utf8() {
|
||
let name = b"note\0";
|
||
let dt_bytes = build_string_dt(3);
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let mut data = Vec::new();
|
||
data.push(3); // version
|
||
data.push(0); // flags
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
data.push(1); // encoding = UTF-8
|
||
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
data.extend_from_slice(b"abc");
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.name, "note");
|
||
let s = attr.read_as_string().unwrap();
|
||
assert_eq!(s, "abc");
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v2_attribute_1d_array() {
|
||
let name = b"vals\0";
|
||
let dt_bytes = build_f64_dt();
|
||
let ds_bytes = build_simple_ds_v1(3);
|
||
|
||
let mut data = Vec::new();
|
||
data.push(2); // version
|
||
data.push(0); // flags
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
|
||
// 3 f64 values
|
||
data.extend_from_slice(&1.0f64.to_le_bytes());
|
||
data.extend_from_slice(&2.0f64.to_le_bytes());
|
||
data.extend_from_slice(&3.0f64.to_le_bytes());
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.name, "vals");
|
||
let vals = attr.read_as_f64().unwrap();
|
||
assert_eq!(vals, vec![1.0, 2.0, 3.0]);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v1_padding_alignment() {
|
||
// Verify v1 pads name, dt, ds each to 8 bytes
|
||
let name = b"x\0"; // 2 bytes → pad to 8
|
||
let dt_bytes = build_f64_dt(); // 20 bytes → pad to 24
|
||
let ds_bytes = build_scalar_ds(); // 4 bytes → pad to 8
|
||
|
||
let mut data = Vec::new();
|
||
data.push(1); // version
|
||
data.push(0); // reserved
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
|
||
// Name padded to 8
|
||
data.extend_from_slice(name);
|
||
data.resize(8 + pad8(name.len()), 0);
|
||
|
||
// DT padded to 8
|
||
let dt_start = data.len();
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.resize(dt_start + pad8(dt_bytes.len()), 0);
|
||
|
||
// DS padded to 8
|
||
let ds_start = data.len();
|
||
data.extend_from_slice(&ds_bytes);
|
||
data.resize(ds_start + pad8(ds_bytes.len()), 0);
|
||
|
||
// raw data
|
||
data.extend_from_slice(&42.0f64.to_le_bytes());
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.name, "x");
|
||
let vals = attr.read_as_f64().unwrap();
|
||
assert_eq!(vals, vec![42.0]);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_v2_no_padding() {
|
||
// Same as parse_v2_attribute_fixed_string but verifying no padding
|
||
let name = b"ab\0"; // 3 bytes, no padding
|
||
let dt_bytes = build_string_dt(2); // 8 bytes, no padding
|
||
let ds_bytes = build_scalar_ds(); // 4 bytes, no padding
|
||
|
||
let mut data = Vec::new();
|
||
data.push(2);
|
||
data.push(0);
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
data.extend_from_slice(b"hi");
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.name, "ab");
|
||
assert_eq!(attr.read_as_string().unwrap(), "hi");
|
||
}
|
||
|
||
#[test]
|
||
fn truncated_attribute_error() {
|
||
let data = [1u8]; // too short
|
||
let err = AttributeMessage::parse(&data, 8).unwrap_err();
|
||
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
|
||
}
|
||
|
||
#[test]
|
||
fn invalid_version_error() {
|
||
let data = [5u8, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||
let err = AttributeMessage::parse(&data, 8).unwrap_err();
|
||
assert_eq!(err, FormatError::InvalidAttributeVersion(5));
|
||
}
|
||
|
||
#[test]
|
||
fn extract_attributes_from_header() {
|
||
// Build a fake ObjectHeader with 3 attribute messages
|
||
let mut msgs = Vec::new();
|
||
for i in 0..3 {
|
||
let name = format!("attr{}\0", i);
|
||
let dt_bytes = build_f64_dt();
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let mut attr_data = Vec::new();
|
||
attr_data.push(2); // version
|
||
attr_data.push(0);
|
||
attr_data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
attr_data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
attr_data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
attr_data.extend_from_slice(name.as_bytes());
|
||
attr_data.extend_from_slice(&dt_bytes);
|
||
attr_data.extend_from_slice(&ds_bytes);
|
||
attr_data.extend_from_slice(&((i as f64) * 1.0).to_le_bytes());
|
||
|
||
msgs.push(crate::object_header::HeaderMessage {
|
||
msg_type: MessageType::Attribute,
|
||
size: attr_data.len(),
|
||
flags: 0,
|
||
creation_order: None,
|
||
data: attr_data,
|
||
});
|
||
}
|
||
|
||
let header = ObjectHeader {
|
||
version: 2,
|
||
messages: msgs,
|
||
reference_count: None,
|
||
flags: 0,
|
||
access_time: None,
|
||
modification_time: None,
|
||
change_time: None,
|
||
birth_time: None,
|
||
};
|
||
|
||
let attrs = extract_attributes(&header, 8).unwrap();
|
||
assert_eq!(attrs.len(), 3);
|
||
assert_eq!(attrs[0].name, "attr0");
|
||
assert_eq!(attrs[1].name, "attr1");
|
||
assert_eq!(attrs[2].name, "attr2");
|
||
}
|
||
|
||
#[test]
|
||
fn find_attribute_by_name() {
|
||
let name = b"target\0";
|
||
let dt_bytes = build_f64_dt();
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let mut attr_data = Vec::new();
|
||
attr_data.push(2);
|
||
attr_data.push(0);
|
||
attr_data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
attr_data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
attr_data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
attr_data.extend_from_slice(name);
|
||
attr_data.extend_from_slice(&dt_bytes);
|
||
attr_data.extend_from_slice(&ds_bytes);
|
||
attr_data.extend_from_slice(&99.0f64.to_le_bytes());
|
||
|
||
let attr = AttributeMessage::parse(&attr_data, 8).unwrap();
|
||
let attrs = vec![attr];
|
||
|
||
assert!(find_attribute(&attrs, "target").is_some());
|
||
assert!(find_attribute(&attrs, "missing").is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn read_as_f64_scalar() {
|
||
let name = b"v\0";
|
||
let dt_bytes = build_f64_dt();
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let mut data = Vec::new();
|
||
data.push(2);
|
||
data.push(0);
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
data.extend_from_slice(&3.25f64.to_le_bytes());
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
let vals = attr.read_as_f64().unwrap();
|
||
assert_eq!(vals, vec![3.25]);
|
||
}
|
||
|
||
#[test]
|
||
fn read_as_string_fixed() {
|
||
let name = b"s\0";
|
||
let dt_bytes = build_string_dt(5);
|
||
let ds_bytes = build_scalar_ds();
|
||
|
||
let mut data = Vec::new();
|
||
data.push(2);
|
||
data.push(0);
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
data.extend_from_slice(b"world");
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
assert_eq!(attr.read_as_string().unwrap(), "world");
|
||
}
|
||
|
||
#[test]
|
||
fn read_as_strings_array() {
|
||
let name = b"arr\0";
|
||
let dt_bytes = build_string_dt(4);
|
||
let ds_bytes = build_simple_ds_v1(2);
|
||
|
||
let mut data = Vec::new();
|
||
data.push(2);
|
||
data.push(0);
|
||
data.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(dt_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(&(ds_bytes.len() as u16).to_le_bytes());
|
||
data.extend_from_slice(name);
|
||
data.extend_from_slice(&dt_bytes);
|
||
data.extend_from_slice(&ds_bytes);
|
||
data.extend_from_slice(b"abcdEFGH");
|
||
|
||
let attr = AttributeMessage::parse(&data, 8).unwrap();
|
||
let strs = attr.read_as_strings().unwrap();
|
||
assert_eq!(strs, vec!["abcd", "EFGH"]);
|
||
}
|
||
|
||
/// Every object's attributes in h5py-written files read identically
|
||
/// through a read_at-only CountingStorage — compact ones, shared ones,
|
||
/// those behind an Attribute Info message and dense storage (its v2
|
||
/// B-tree name index included) — and through a slice as Storage.
|
||
#[test]
|
||
fn storage_reads_match_slice_reads() {
|
||
use crate::storage::CountingStorage;
|
||
let files: [(&str, &[u8]); 5] = [
|
||
("attrs", include_bytes!("../tests/fixtures/attrs.h5")),
|
||
(
|
||
"mixed_attrs",
|
||
include_bytes!("../tests/fixtures/mixed_attrs.h5"),
|
||
),
|
||
(
|
||
"dense_attrs",
|
||
include_bytes!("../tests/fixtures/dense_attrs.h5"),
|
||
),
|
||
(
|
||
"dense_attrs_root",
|
||
include_bytes!("../tests/fixtures/dense_attrs_root.h5"),
|
||
),
|
||
(
|
||
"shared_fill_value",
|
||
include_bytes!("../tests/fixtures/shared_fill_value.h5"),
|
||
),
|
||
];
|
||
let (mut same, mut dense, mut attrs) = (0, 0, 0);
|
||
for (name, file) in files {
|
||
let sb = crate::superblock::Superblock::parse(file, 0).unwrap();
|
||
let (os, ls) = (sb.offset_size, sb.length_size);
|
||
let mut addrs = vec![sb.root_group_address];
|
||
addrs.extend(
|
||
crate::group_v2::resolve_group_children(file, &sb, sb.root_group_address)
|
||
.unwrap()
|
||
.iter()
|
||
.map(|e| e.object_header_address),
|
||
);
|
||
let storage = CountingStorage::new(file.to_vec());
|
||
for addr in addrs {
|
||
let header = ObjectHeader::parse(file, addr as usize, os, ls).unwrap();
|
||
let want = extract_attributes_full(file, &header, os, ls);
|
||
let slice_storage = extract_attributes_full_in(&file, &header, os, ls);
|
||
assert_eq!(format!("{slice_storage:?}"), format!("{want:?}"));
|
||
let got = extract_attributes_full_in(&storage, &header, os, ls);
|
||
let got_t = extract_attributes_tolerant_in(&storage, &header, os, ls);
|
||
let is_dense = find_attribute_info(&header, os)
|
||
.unwrap()
|
||
.is_some_and(|i| i.fractal_heap_address.is_some());
|
||
if is_dense {
|
||
dense += 1;
|
||
}
|
||
attrs += want.as_ref().map_or(0, Vec::len);
|
||
assert_eq!(format!("{got:?}"), format!("{want:?}"), "{name}");
|
||
let want_t = extract_attributes_tolerant(file, &header, os, ls);
|
||
assert_eq!(format!("{got_t:?}"), format!("{want_t:?}"), "{name}");
|
||
same += 1;
|
||
for a in want.iter().flatten() {
|
||
let one = find_attribute_in(&storage, &header, &a.name, os, ls);
|
||
let want_one = find_attribute_in_file(file, &header, &a.name, os, ls);
|
||
assert_eq!(format!("{one:?}"), format!("{want_one:?}"), "{name}");
|
||
}
|
||
}
|
||
}
|
||
assert!(
|
||
same >= 5 && dense >= 2 && attrs >= 5,
|
||
"{same} {dense} {attrs}"
|
||
);
|
||
}
|
||
}
|