quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p0-writer-meta' into fix/phase0-correctness

Browse Source
This commit is contained in:
osobh
2026-09-25 21:20:59 -05:00
parent e1115bc92a 7c1968a34a
commit c5302e587e
15 changed files with 1457 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/datatype.rs
+203 -3
View File
@@ -4,7 +4,7 @@
//! for compound, enumeration, variable-length, and array types.
#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, string::String, vec, vec::Vec};
use alloc::{boxed::Box, format, string::String, vec, vec::Vec};
use byteorder::{ByteOrder, LittleEndian};
@@ -137,6 +137,17 @@ pub enum Datatype {
},
}
/// Longest opaque tag that can be stored: its NUL-padded length must fit
/// the 8-bit length in the datatype's class bits.
pub const MAX_OPAQUE_TAG_LEN: usize = 248;
/// An opaque tag up to (not including) its first NUL.
fn opaque_tag_text(tag: &[u8]) -> &[u8] {
tag.iter()
.position(|&b| b == 0)
.map_or(tag, |end| &tag[..end])
}
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
match offset.checked_add(needed) {
Some(end) if end <= data.len() => Ok(()),
@@ -361,7 +372,10 @@ impl Datatype {
// Opaque
let tag_len = bf0 as usize;
ensure_len(data, pos, tag_len)?;
let tag = data[pos..pos + tag_len].to_vec();
// The stored tag is NUL-padded to a multiple of 8 bytes; the
// tag itself ends at the first NUL (libhdf5 reads it with
// `strndup`).
let tag = opaque_tag_text(&data[pos..pos + tag_len]).to_vec();
// Tags are padded to multiple of 8 bytes
let padded = (tag_len + 7) & !7;
let pos = 8 + padded; // from start of properties
@@ -767,7 +781,77 @@ impl Datatype {
buf.extend_from_slice(&base_type.serialize());
buf
}
_ => Vec::new(),
Datatype::Time {
size,
bit_precision,
} => {
// Byte order is not modelled for time types; write little-endian.
let mut buf = Self::build_header(2, 1, [0, 0, 0], *size);
buf.extend_from_slice(&bit_precision.to_le_bytes());
buf
}
Datatype::BitField {
size,
byte_order,
bit_offset,
bit_precision,
} => {
let bf0 = u8::from(matches!(byte_order, DatatypeByteOrder::BigEndian));
let mut buf = Self::build_header(4, 1, [bf0, 0, 0], *size);
buf.extend_from_slice(&bit_offset.to_le_bytes());
buf.extend_from_slice(&bit_precision.to_le_bytes());
buf
}
Datatype::Opaque { size, tag } => {
// The tag is stored NUL-padded to a multiple of 8 bytes and the
// padded length goes in the class bits, as libhdf5 writes it.
// A tag longer than MAX_OPAQUE_TAG_LEN cannot be encoded;
// `check_encodable` rejects it before a file is written.
let tag = opaque_tag_text(tag);
let tag = &tag[..tag.len().min(MAX_OPAQUE_TAG_LEN)];
let padded = tag.len().div_ceil(8) * 8;
let mut buf = Self::build_header(5, 1, [padded as u8, 0, 0], *size);
buf.extend_from_slice(tag);
buf.resize(8 + padded, 0);
buf
}
Datatype::Reference { size, ref_type } => {
// Legacy references are datatype version 1; the H5T_STD_REF
// kinds only exist from version 4, which also carries their
// encoding version (1) in the high nibble.
let (version, bf0) = match ref_type {
ReferenceType::Object => (1, 0),
ReferenceType::DatasetRegion => (1, 1),
ReferenceType::Object2 => (4, 0x12),
ReferenceType::DatasetRegion2 => (4, 0x13),
ReferenceType::Attribute => (4, 0x14),
};
Self::build_header(7, version, [bf0, 0, 0], *size)
}
}
}
/// Check that this datatype can be written: every part of it has an
/// on-disk encoding. [`Self::serialize`] cannot report errors, so the
/// writer calls this first.
pub fn check_encodable(&self) -> Result<(), FormatError> {
match self {
Datatype::Opaque { tag, .. } if opaque_tag_text(tag).len() > MAX_OPAQUE_TAG_LEN => {
Err(FormatError::SerializationError(format!(
"opaque tag is {} bytes; at most {MAX_OPAQUE_TAG_LEN} can be stored",
opaque_tag_text(tag).len()
)))
}
Datatype::String { size: 0, .. } => Err(FormatError::SerializationError(
"fixed-length string datatype of size 0 (libhdf5 requires at least 1 byte)".into(),
)),
Datatype::Compound { members, .. } => members
.iter()
.try_for_each(|m| m.datatype.check_encodable()),
Datatype::Enumeration { base_type, .. }
| Datatype::VariableLength { base_type, .. }
| Datatype::Array { base_type, .. } => base_type.check_encodable(),
_ => Ok(()),
}
}
@@ -1625,6 +1709,122 @@ mod tests {
);
}
fn hex(s: &str) -> Vec<u8> {
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
.collect()
}
/// `serialize` used to return an empty message for these four classes,
/// which libhdf5 rejects ("ran off end of input buffer while decoding").
/// Expected bytes are libhdf5's own encoding (HDF5 2.0 `H5Tencode`, or the
/// datatype message of an HDF5 2.0 file for `H5T_STD_REF`).
#[test]
fn serialize_matches_libhdf5_for_time_bitfield_opaque_reference() {
let cases = [
(
Datatype::Reference {
size: 8,
ref_type: ReferenceType::Object,
},
"1700000008000000",
),
(
Datatype::Reference {
size: 12,
ref_type: ReferenceType::DatasetRegion,
},
"170100000c000000",
),
(
Datatype::Reference {
size: 18,
ref_type: ReferenceType::Object2,
},
"4712000012000000",
),
(
Datatype::BitField {
size: 1,
byte_order: DatatypeByteOrder::LittleEndian,
bit_offset: 0,
bit_precision: 8,
},
"140000000100000000000800",
),
(
Datatype::BitField {
size: 2,
byte_order: DatatypeByteOrder::BigEndian,
bit_offset: 0,
bit_precision: 16,
},
"140100000200000000001000",
),
(
Datatype::Opaque {
size: 4,
tag: b"mytag".to_vec(),
},
"15080000040000006d79746167000000",
),
(
Datatype::Opaque {
size: 4,
tag: b"12345678".to_vec(),
},
"15080000040000003132333435363738",
),
(
Datatype::Opaque {
size: 4,
tag: vec![],
},
"1500000004000000",
),
(
Datatype::Time {
size: 4,
bit_precision: 32,
},
"12000000040000002000",
),
];
for (dt, expected) in cases {
let bytes = dt.serialize();
assert_eq!(bytes, hex(expected), "{dt:?}");
let (parsed, consumed) = Datatype::parse(&bytes).unwrap();
assert_eq!(parsed, dt);
assert_eq!(consumed, bytes.len());
}
}
#[test]
fn opaque_tag_padding_is_not_part_of_the_tag() {
// libhdf5 pads "mytag" to 8 bytes; parsing must not return the NULs,
// or copying the type would grow the tag.
let (dt, _) = Datatype::parse(&hex("15080000040000006d79746167000000")).unwrap();
assert_eq!(
dt,
Datatype::Opaque {
size: 4,
tag: b"mytag".to_vec()
}
);
let long = Datatype::Opaque {
size: 1,
tag: vec![b'x'; MAX_OPAQUE_TAG_LEN + 1],
};
assert!(long.check_encodable().is_err());
let ok = Datatype::Opaque {
size: 1,
tag: vec![b'x'; MAX_OPAQUE_TAG_LEN],
};
assert!(ok.check_encodable().is_ok());
assert_eq!(Datatype::parse(&ok.serialize()).unwrap().0, ok);
}
#[test]
fn test_error_invalid_reference_type() {
let buf = build_dt_header(7, 1, [5, 0, 0], 8);
crates/clawhdf5-format/src/file_writer.rs
+181 -56
View File
@@ -4,7 +4,7 @@
//! link messages, contiguous datasets, inline and dense attributes.
#[cfg(not(feature = "std"))]
use alloc::{string::String, string::ToString, vec, vec::Vec};
use alloc::{format, string::String, string::ToString, vec, vec::Vec};
use crate::attribute::AttributeMessage;
use crate::chunked_write::{
@@ -19,7 +19,7 @@ use crate::metadata_index::{DatasetMetadata, MetadataBlock, MetadataIndex};
use crate::object_header_writer::ObjectHeaderWriter;
use crate::superblock::Superblock;
use crate::type_builders::{
DatasetBuilder, FillTime, FinishedGroup, GroupBuilder, build_attr_message,
DatasetBuilder, FinishedGroup, GroupBuilder, build_attr_message, fill_value_message,
};
// Re-export public types that moved to type_builders for API compatibility.
@@ -33,6 +33,49 @@ pub(crate) const OFFSET_SIZE: u8 = 8;
pub(crate) const LENGTH_SIZE: u8 = 8;
const SUPERBLOCK_SIZE: usize = 48;
/// Largest raw data a compact dataset can hold: the layout message (version,
/// class, 2-byte size, data) must fit an object header message, whose size
/// field is 2 bytes. Bigger "compact" requests fall back to contiguous storage.
const MAX_COMPACT_DATA_SIZE: usize = crate::object_header_writer::MAX_MESSAGE_SIZE - 4;
/// libhdf5's bounds on a file space page size (`H5F_FILE_SPACE_PAGE_SIZE_MIN`
/// and `_MAX`).
const MIN_FILE_SPACE_PAGE_SIZE: u32 = 512;
const MAX_FILE_SPACE_PAGE_SIZE: u32 = 1024 * 1024 * 1024;
/// Superblock extension object header for a file using the paged file-space
/// strategy: a single File Space Info message (0x0017), as libhdf5 writes it
/// for `fs_strategy="page"` without persisted free space.
fn build_paged_superblock_extension(page_size: u32) -> Result<Vec<u8>, FormatError> {
let mut fsinfo = Vec::new();
fsinfo.push(1); // version
fsinfo.push(1); // strategy: H5F_FSPACE_STRATEGY_PAGE
fsinfo.push(0); // persisting free space: no
write_length(&mut fsinfo, 1, LENGTH_SIZE); // free-space section threshold
write_length(&mut fsinfo, u64::from(page_size), LENGTH_SIZE);
fsinfo.extend_from_slice(&0u16.to_le_bytes()); // page end metadata threshold
write_undef_offset(&mut fsinfo, OFFSET_SIZE); // EOA before free-space info
let mut w = ObjectHeaderWriter::new();
// Flags as libhdf5 sets them: bit 2 (never share) and bit 4 (mark if
// unknown). Not constant: libhdf5 rewrites the message when it closes a
// file it opened for writing.
w.add_message_with_flags(MessageType::Unknown(0x0017), fsinfo, 0x14);
w.serialize()
}
/// A group or dataset name must be one path component: not empty, not ".",
/// and without '/'. `FileWriter` writes a root group plus one level of
/// groups, and cannot create intermediate groups for a path.
fn check_link_name(name: &str) -> Result<(), FormatError> {
if name.is_empty() || name == "." || name.contains('/') {
return Err(FormatError::SerializationError(format!(
"invalid object name {name:?}: names must be a single path component \
(FileWriter does not create nested groups)"
)));
}
Ok(())
}
/// Threshold for switching from compact (inline) to dense attribute storage.
const DENSE_ATTR_THRESHOLD: usize = 8;
@@ -50,12 +93,12 @@ pub(crate) fn build_chunked_dataset_oh(
pipeline_message: Option<&[u8]>,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
fill_time: FillTime,
) -> Vec<u8> {
fill_message: &[u8],
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
w.add_message(MessageType::DataLayout, layout_message.to_vec());
if let Some(pm) = pipeline_message {
w.add_message(MessageType::FilterPipeline, pm.to_vec());
@@ -77,12 +120,12 @@ pub(crate) fn build_dataset_oh(
data_size: u64,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
fill_time: FillTime,
) -> Vec<u8> {
fill_message: &[u8],
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
let mut dl = Vec::new();
dl.push(4); // version
dl.push(1); // class = contiguous
@@ -112,12 +155,12 @@ pub(crate) fn build_compact_dataset_oh(
data: &[u8],
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
fill_time: FillTime,
) -> Vec<u8> {
fill_message: &[u8],
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
// Compact layout message: version=4, class=0, u16 size, inline data
let mut dl = Vec::new();
dl.push(4); // version
@@ -140,7 +183,7 @@ pub(crate) fn build_group_oh(
dense_link_info: Option<&[u8]>,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
) -> Vec<u8> {
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
if let Some(li) = dense_link_info {
// Dense link storage: a LinkInfo pointing at the fractal heap + name
@@ -902,12 +945,12 @@ pub(crate) fn build_vds_dataset_oh(
global_heap_addr: u64,
attrs: &[AttributeMessage],
dense_blob: Option<&DenseAttrBlob>,
fill_time: FillTime,
) -> Vec<u8> {
fill_message: &[u8],
) -> Result<Vec<u8>, FormatError> {
let mut w = ObjectHeaderWriter::new();
w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01);
w.add_message(MessageType::Dataspace, ds.serialize(LENGTH_SIZE));
w.add_message_with_flags(MessageType::FillValue, vec![3, fill_time.to_byte()], 0x01);
w.add_message_with_flags(MessageType::FillValue, fill_message.to_vec(), 0x01);
// VDS layout message: version=4, class=3, global_heap_address(8), global_heap_index=1(4)
let mut dl = Vec::new();
dl.push(4u8); // version
@@ -956,7 +999,9 @@ pub struct FileWriter {
alignment_threshold: usize,
/// Global alignment boundary in bytes (0 = disabled).
alignment_bytes: usize,
/// Page size for page-buffer mode. When set, a v4 superblock is written.
/// File space page size. When set, the file uses libhdf5's paged
/// file-space strategy (a File Space Info message in the superblock
/// extension).
page_size: Option<u32>,
}
@@ -988,9 +1033,16 @@ impl FileWriter {
self
}
/// Enable page-buffer mode with the given page size. Writing this causes
/// the file to be written with a v4 superblock (page_size field) instead
/// of the default v3.
/// Write the file with libhdf5's *paged* file-space strategy and the given
/// page size, as `H5Pset_file_space_strategy(H5F_FSPACE_STRATEGY_PAGE)` +
/// `H5Pset_file_space_page_size` (h5py: `fs_strategy="page"`,
/// `fs_page_size=...`) do: a v3 superblock with an extension holding a
/// File Space Info message, and the file padded to a whole number of
/// pages. Readers with a page buffer can then fetch metadata page by page.
///
/// `page_size` must be between 512 bytes and 1 GiB (libhdf5's limits);
/// [`Self::finish`] fails otherwise. This used to write a "version 4"
/// superblock, which does not exist and no HDF5 library can open.
pub fn with_page_size(&mut self, page_size: u32) -> &mut Self {
self.page_size = Some(page_size);
self
@@ -1015,6 +1067,14 @@ impl FileWriter {
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
let page_size = self.page_size;
if let Some(ps) = page_size
&& !(MIN_FILE_SPACE_PAGE_SIZE..=MAX_FILE_SPACE_PAGE_SIZE).contains(&ps)
{
return Err(FormatError::SerializationError(format!(
"file space page size {ps} is outside libhdf5's \
{MIN_FILE_SPACE_PAGE_SIZE}..={MAX_FILE_SPACE_PAGE_SIZE} bytes"
)));
}
struct DsFlat {
name: String,
dt: Datatype,
@@ -1023,7 +1083,8 @@ impl FileWriter {
attrs: Vec<AttributeMessage>,
chunk_options: ChunkOptions,
maxshape: Option<Vec<u64>>,
fill_time: FillTime,
/// Serialized Fill Value message.
fill_message: Vec<u8>,
compact: bool,
alignment: usize,
/// VDS source mappings (set for Virtual datasets).
@@ -1073,6 +1134,7 @@ impl FileWriter {
};
attrs.extend(p.build_attrs(&raw));
}
let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?;
Ok(DsFlat {
name: db.name,
dt,
@@ -1081,13 +1143,26 @@ impl FileWriter {
attrs,
chunk_options: db.chunk_options,
maxshape: db.maxshape,
fill_time: db.fill_time,
fill_message,
compact: db.compact,
alignment: db.alignment,
virtual_sources: db.virtual_sources,
})
};
// Every name becomes a single link in its parent group. The writer
// has no nested groups, so a path like "a/b" would be stored as one
// link literally named "a/b" — which no HDF5 reader can resolve.
let root_names = self.root_datasets.iter().map(|d| d.name.as_str());
let group_names = self.groups.iter().flat_map(|g| {
core::iter::once(g.name.as_str())
.chain(g.datasets.iter().map(|d| d.name.as_str()))
.chain(g.external_links.iter().map(|l| l.0.as_str()))
});
for name in root_names.chain(group_names) {
check_link_name(name)?;
}
let mut all_ds: Vec<DsFlat> = Vec::new();
let mut groups: Vec<GrpFlat> = Vec::new();
let mut root_ds_indices: Vec<usize> = Vec::new();
@@ -1120,6 +1195,17 @@ impl FileWriter {
root_attrs.push(build_attr_message(n, v));
}
// Every datatype must have an on-disk encoding before anything is laid
// out: `Datatype::serialize` itself cannot report a failure.
let group_attrs = groups.iter().flat_map(|g| &g.attrs);
let ds_attrs = all_ds.iter().flat_map(|d| &d.attrs);
for a in root_attrs.iter().chain(group_attrs).chain(ds_attrs) {
a.datatype.check_encodable()?;
}
for d in &all_ds {
d.dt.check_encodable()?;
}
let is_vds: Vec<bool> = all_ds.iter().map(|d| d.virtual_sources.is_some()).collect();
let is_chunked: Vec<bool> = all_ds
.iter()
@@ -1135,7 +1221,9 @@ impl FileWriter {
let is_compact: Vec<bool> = all_ds
.iter()
.enumerate()
.map(|(i, d)| !is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= 65535)
.map(|(i, d)| {
!is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= MAX_COMPACT_DATA_SIZE
})
.collect();
let root_dense = root_attrs.len() > DENSE_ATTR_THRESHOLD;
let group_dense: Vec<bool> = groups
@@ -1174,9 +1262,9 @@ impl FileWriter {
}
let attr_blob = group_dense[gi].then(|| build_dense_attrs(&g.attrs, 0));
let dl = group_links_dense[gi].then_some(dummy_link_info.as_slice());
build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).len()
build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).map(|oh| oh.len())
})
.collect();
.collect::<Result<_, _>>()?;
let root_dummy_links: Vec<LinkMessage> = {
let mut links = Vec::new();
@@ -1191,7 +1279,7 @@ impl FileWriter {
let root_oh_size = {
let attr_blob = root_dense.then(|| build_dense_attrs(&root_attrs, 0));
let dl = root_links_dense.then_some(dummy_link_info.as_slice());
build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref()).len()
build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref())?.len()
};
struct DataBlob {
@@ -1219,8 +1307,8 @@ impl FileWriter {
0, // dummy address
&d.attrs,
dense_blob.as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
// Global heap blob size is address-independent; compute it now
// so pass 2 can place it correctly.
let vds_mappings = d.virtual_sources.as_deref().unwrap_or(&[]);
@@ -1263,8 +1351,8 @@ impl FileWriter {
result.pipeline_message.as_deref(),
&d.attrs,
dense_blob.as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
dummy_blobs.push(DataBlob {
data: result.data_bytes,
oh_bytes: oh,
@@ -1282,8 +1370,8 @@ impl FileWriter {
&d.raw,
&d.attrs,
dense_blob.as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
dummy_blobs.push(DataBlob {
data: vec![],
oh_bytes: oh,
@@ -1302,8 +1390,8 @@ impl FileWriter {
d.raw.len() as u64,
&d.attrs,
dense_blob.as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
dummy_blobs.push(DataBlob {
data: d.raw.clone(),
oh_bytes: oh,
@@ -1315,12 +1403,12 @@ impl FileWriter {
let actual_ds_oh_sizes: Vec<usize> = dummy_blobs.iter().map(|b| b.oh_bytes.len()).collect();
// Pass 2: compute real addresses
// v4 superblocks add a 4-byte page_size field before the checksum.
let superblock_size = if page_size.is_some() {
SUPERBLOCK_SIZE + 4
} else {
SUPERBLOCK_SIZE
};
// A paged file carries its File Space Info in a superblock extension
// object header, placed right after the superblock.
let sb_ext = page_size
.map(build_paged_superblock_extension)
.transpose()?;
let superblock_size = SUPERBLOCK_SIZE + sb_ext.as_ref().map_or(0, Vec::len);
let root_group_addr = superblock_size as u64;
let mut cursor2 = superblock_size + root_oh_size;
@@ -1411,8 +1499,8 @@ impl FileWriter {
heap_addr,
&d.attrs,
ds_dense_blobs[i].as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
ds_blobs2.push(DataBlob {
data: gcol_bytes.clone(),
oh_bytes: oh,
@@ -1438,8 +1526,8 @@ impl FileWriter {
result.pipeline_message.as_deref(),
&d.attrs,
ds_dense_blobs[i].as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
ds_blobs2.push(DataBlob {
data: result.data_bytes,
oh_bytes: oh,
@@ -1453,8 +1541,8 @@ impl FileWriter {
&d.raw,
&d.attrs,
ds_dense_blobs[i].as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
ds_blobs2.push(DataBlob {
data: vec![],
oh_bytes: oh,
@@ -1478,8 +1566,8 @@ impl FileWriter {
d.raw.len() as u64,
&d.attrs,
ds_dense_blobs[i].as_ref(),
d.fill_time,
);
&d.fill_message,
)?;
let mut data = vec![0u8; padding];
data.extend_from_slice(&d.raw);
cursor2 += d.raw.len();
@@ -1494,11 +1582,16 @@ impl FileWriter {
let actual_ds_oh_sizes2: Vec<usize> = ds_blobs2.iter().map(|b| b.oh_bytes.len()).collect();
debug_assert_eq!(actual_ds_oh_sizes, actual_ds_oh_sizes2);
// libhdf5 ends a paged file on a page boundary.
let data_end = cursor2;
if let Some(ps) = page_size {
cursor2 = cursor2.next_multiple_of(ps as usize);
}
let eof_addr2 = cursor2 as u64;
let mut buf = Vec::with_capacity(cursor2);
let sb = Superblock {
version: if page_size.is_some() { 4 } else { 3 },
version: 3,
offset_size: OFFSET_SIZE,
length_size: LENGTH_SIZE,
base_address: 0,
@@ -1510,11 +1603,18 @@ impl FileWriter {
free_space_address: None,
driver_info_address: None,
consistency_flags: 0,
superblock_extension_address: Some(u64::MAX),
superblock_extension_address: Some(if sb_ext.is_some() {
SUPERBLOCK_SIZE as u64
} else {
u64::MAX
}),
checksum: None,
page_size,
page_size: None,
};
buf.extend_from_slice(&sb.serialize());
if let Some(ref ext) = sb_ext {
buf.extend_from_slice(ext);
}
// Root group OH
let mut root_links: Vec<LinkMessage> = Vec::new();
@@ -1535,7 +1635,7 @@ impl FileWriter {
root_dl,
&root_attrs,
root_dense_blob.as_ref(),
));
)?);
if let Some(ref b) = root_link_blob {
buf.extend_from_slice(&b.blob);
}
@@ -1560,7 +1660,7 @@ impl FileWriter {
dl,
&g.attrs,
group_dense_blobs[gi].as_ref(),
));
)?);
if let Some(ref b) = link_blob {
buf.extend_from_slice(&b.blob);
}
@@ -1582,7 +1682,8 @@ impl FileWriter {
buf.extend_from_slice(&blob.data);
}
debug_assert_eq!(buf.len(), cursor2);
debug_assert_eq!(buf.len(), data_end);
buf.resize(cursor2, 0);
Ok(buf)
}
}
@@ -2156,7 +2257,8 @@ mod tests {
}
#[test]
fn file_writer_v4_superblock() {
fn file_writer_paged_file_uses_v3_superblock_and_fsinfo_extension() {
// This used to write superblock "version 4", which does not exist.
let mut fw = FileWriter::new();
fw.with_page_size(4096);
fw.create_dataset("data").with_f64_data(&[1.0, 2.0]);
@@ -2164,8 +2266,31 @@ mod tests {
let sig = signature::find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
assert_eq!(sb.version, 4, "expected superblock v4");
assert_eq!(sb.page_size, Some(4096));
assert_eq!(sb.version, 3);
assert_eq!(sb.superblock_extension_address, Some(48));
assert_eq!(bytes.len() % 4096, 0);
assert_eq!(sb.eof_address, bytes.len() as u64);
let ext = ObjectHeader::parse(&bytes, 48, 8, 8).unwrap();
let fsinfo = &ext.messages[0];
assert_eq!(fsinfo.msg_type, MessageType::Unknown(0x0017));
// Byte-for-byte what HDF5 2.0 writes for fs_strategy="page",
// fs_page_size=4096.
let mut expected = vec![1u8, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0];
expected.extend_from_slice(&4096u64.to_le_bytes());
expected.extend_from_slice(&[0, 0]);
expected.extend_from_slice(&[0xff; 8]);
assert_eq!(fsinfo.data, expected);
assert_eq!(fsinfo.flags, 0x14);
assert_eq!(read_dataset_f64(&bytes, "data"), vec![1.0, 2.0]);
}
#[test]
fn file_writer_rejects_page_sizes_libhdf5_would() {
for ps in [0u32, 511, MAX_FILE_SPACE_PAGE_SIZE + 1] {
let mut fw = FileWriter::new();
fw.with_page_size(ps);
assert!(fw.finish().is_err(), "page size {ps}");
}
}
#[test]
crates/clawhdf5-format/src/fill_value.rs
+42 -7
View File
@@ -98,15 +98,50 @@ pub fn parse_fill_value(msg: &HeaderMessage) -> Result<Option<Vec<u8>>, FormatEr
/// The fill value that applies to a dataset given its header messages. The new
/// message wins over the old one when both are present.
///
/// A *shared* fill value message holds only a reference to the real message,
/// which cannot be followed without the file: this returns
/// [`FormatError::UnresolvedSharedMessage`] for one (it used to answer "zeros").
/// Use [`dataset_fill_value_in`] when the file bytes are at hand.
pub fn dataset_fill_value(messages: &[HeaderMessage]) -> Result<Option<Vec<u8>>, FormatError> {
fill_value_from(messages, |_| Err(FormatError::UnresolvedSharedMessage))
}
/// [`dataset_fill_value`] for a dataset in `file_data`, following a shared
/// fill value message to where it lives: another object header, or the
/// file's shared-message (SOHM) heap, as libhdf5 writes it when the file has
/// a SOHM index for fill values.
pub fn dataset_fill_value_in(
file_data: &[u8],
messages: &[HeaderMessage],
offset_size: u8,
length_size: u8,
) -> Result<Option<Vec<u8>>, FormatError> {
fill_value_from(messages, |msg| {
crate::shared_message::message_data_with_sohm(file_data, msg, offset_size, length_size)
.map(|data| data.into_owned())
})
}
fn fill_value_from(
messages: &[HeaderMessage],
resolve_shared: impl Fn(&HeaderMessage) -> Result<Vec<u8>, FormatError>,
) -> Result<Option<Vec<u8>>, FormatError> {
for wanted in [MessageType::FillValue, MessageType::FillValueOld] {
if let Some(msg) = messages.iter().find(|m| m.msg_type == wanted) {
if crate::shared_message::is_shared(msg.flags) {
// A shared fill value is legal but vanishingly rare; treat it
// as the default rather than misparsing the reference.
return Ok(None);
}
if let Some(value) = parse_fill_value(msg)? {
let value = if crate::shared_message::is_shared(msg.flags) {
let data = resolve_shared(msg)?;
parse_fill_value(&HeaderMessage {
msg_type: msg.msg_type,
size: data.len(),
flags: msg.flags & !0x02,
creation_order: msg.creation_order,
data,
})?
} else {
parse_fill_value(msg)?
};
if let Some(value) = value {
return Ok(Some(value));
}
}
@@ -174,7 +209,7 @@ pub fn read_full_with_fill<E: From<FormatError>>(
{
return Err(FormatError::ExternalDataFilesUnsupported.into());
}
let fill = dataset_fill_value(messages)?;
let fill = dataset_fill_value_in(file_data, messages, offset_size, length_size)?;
if !has_storage(layout) {
return Ok(filled_dataset(dataspace, elem_size, fill.as_deref())?);
}
crates/clawhdf5-format/src/object_header.rs
+48 -19
View File
@@ -146,12 +146,7 @@ impl ObjectHeader {
ensure_len(data, pos, msg_data_size)?;
let msg_type = MessageType::from_u16(msg_type_raw);
// Check if unknown + must-understand (bit 3 of msg_flags)
if let MessageType::Unknown(id) = msg_type
&& msg_flags & 0x08 != 0
{
return Err(FormatError::UnsupportedMessage(id));
}
check_unknown_message(msg_type, msg_flags)?;
if msg_type != MessageType::Nil {
messages.push(HeaderMessage {
@@ -229,11 +224,7 @@ impl ObjectHeader {
let msg_type = MessageType::from_u16(msg_type_raw);
if let MessageType::Unknown(id) = msg_type
&& msg_flags & 0x08 != 0
{
return Err(FormatError::UnsupportedMessage(id));
}
check_unknown_message(msg_type, msg_flags)?;
if msg_type != MessageType::Nil {
messages.push(HeaderMessage {
@@ -424,11 +415,7 @@ impl ObjectHeader {
let msg_type = MessageType::from_u16(msg_type_raw);
if let MessageType::Unknown(id) = msg_type
&& msg_flags & 0x08 != 0
{
return Err(FormatError::UnsupportedMessage(id));
}
check_unknown_message(msg_type, msg_flags)?;
let msg_data = data[pos..pos + msg_data_size].to_vec();
@@ -509,6 +496,24 @@ impl ObjectHeader {
}
}
/// Header message flag bit 7: fail if the message is unknown, always.
const MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS: u8 = 0x80;
/// Refuse an unknown message the file says no reader may skip.
///
/// The parser only ever reads, so bit 3 (fail only when opened for writing)
/// is ignored, as libhdf5 ignores it for a read-only open; bit 7 fails
/// regardless of access mode. This had the two the wrong way round, failing
/// objects libhdf5 reads and reading ones it refuses (`tbogus.h5`).
fn check_unknown_message(msg_type: MessageType, msg_flags: u8) -> Result<(), FormatError> {
match msg_type {
MessageType::Unknown(id) if msg_flags & MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS != 0 => {
Err(FormatError::UnsupportedMessage(id))
}
_ => Ok(()),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -632,14 +637,38 @@ mod tests {
}
#[test]
fn parse_v1_unknown_must_understand_errors() {
// Bit 3 of msg_flags = must understand
let messages = [(0x00FFu16, &[0xAA][..], 0x08u8)];
fn parse_v1_unknown_fail_always_errors() {
// Bit 7 of msg_flags = fail if unknown, whatever the access mode.
let messages = [(0x00FFu16, &[0xAA][..], 0x80u8)];
let data = build_v1_header(&messages, 8, 8);
let err = ObjectHeader::parse(&data, 0, 8, 8).unwrap_err();
assert_eq!(err, FormatError::UnsupportedMessage(0x00FF));
}
#[test]
fn parse_v1_unknown_fail_on_write_is_ignored_when_reading() {
// Bit 3 = fail if unknown *and the file is opened for writing*. This
// parser only reads, so libhdf5 (read-only) opens such an object and
// so must we. Bits 4/5 (mark if unknown / was unknown) never fail.
for flags in [0x08u8, 0x10, 0x20, 0x38] {
let messages = [(0x00FFu16, &[0xAA][..], flags)];
let data = build_v1_header(&messages, 8, 8);
let hdr = ObjectHeader::parse(&data, 0, 8, 8).unwrap();
assert_eq!(hdr.messages[0].msg_type, MessageType::Unknown(0x00FF));
}
}
#[test]
fn parse_v2_unknown_message_flags() {
let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x08)], None);
assert!(ObjectHeader::parse(&data, 0, 8, 8).is_ok());
let data = build_v2_header(0x00, &[(0xF0, &[1, 2], 0x80)], None);
assert_eq!(
ObjectHeader::parse(&data, 0, 8, 8).unwrap_err(),
FormatError::UnsupportedMessage(0xF0)
);
}
#[test]
fn parse_v2_no_timestamps_one_message() {
let data = build_v2_header(0x00, &[(0x01, &[10, 20], 0)], None);
crates/clawhdf5-format/src/object_header_writer.rs
+59 -13
View File
@@ -1,11 +1,17 @@
//! Object header writer for v2 format.
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
use alloc::{format, vec::Vec};
use crate::checksum::jenkins_lookup3;
use crate::error::FormatError;
use crate::message_type::MessageType;
/// Largest message payload a v2 object header can describe: the per-message
/// size field is 2 bytes. A bigger message cannot be encoded at all — writing
/// its size truncated to 16 bits produced files libhdf5 refuses.
pub const MAX_MESSAGE_SIZE: usize = u16::MAX as usize;
/// Writer for v2 object headers with proper checksums.
pub struct ObjectHeaderWriter {
messages: Vec<(MessageType, Vec<u8>, u8)>, // (type, data, msg_flags)
@@ -30,7 +36,22 @@ impl ObjectHeaderWriter {
}
/// Serialize the complete v2 object header (OHDR + messages + checksum).
pub fn serialize(&self) -> Vec<u8> {
///
/// Fails with [`FormatError::SerializationError`] when a message is larger
/// than [`MAX_MESSAGE_SIZE`] (e.g. an attribute over ~64 KiB, which would
/// need dense attribute storage), rather than writing a corrupt header.
pub fn serialize(&self) -> Result<Vec<u8>, FormatError> {
if let Some((msg_type, data, _)) = self
.messages
.iter()
.find(|(_, data, _)| data.len() > MAX_MESSAGE_SIZE)
{
return Err(FormatError::SerializationError(format!(
"{msg_type:?} message is {} bytes; an object header message holds at most \
{MAX_MESSAGE_SIZE} bytes",
data.len()
)));
}
// Calculate total message bytes: each message has type(1) + size(2) + flags(1) + data
let msg_bytes_total: usize = self
.messages
@@ -80,7 +101,7 @@ impl ObjectHeaderWriter {
let checksum = jenkins_lookup3(&buf);
buf.extend_from_slice(&checksum.to_le_bytes());
buf
Ok(buf)
}
}
@@ -125,15 +146,22 @@ impl BatchObjectHeaderWriter {
/// Compute the serialized size of each header without actually serializing.
/// Returns sizes in the same order as headers were added.
pub fn compute_sizes(&self) -> Vec<usize> {
self.headers.iter().map(|h| h.serialize().len()).collect()
pub fn compute_sizes(&self) -> Result<Vec<usize>, FormatError> {
self.headers
.iter()
.map(|h| h.serialize().map(|b| b.len()))
.collect()
}
/// Serialize all headers into a single contiguous buffer.
/// Returns `(combined_bytes, offsets)` where `offsets[i]` is the byte
/// offset of header `i` within the combined buffer.
pub fn serialize_all(&self) -> (Vec<u8>, Vec<usize>) {
let serialized: Vec<Vec<u8>> = self.headers.iter().map(|h| h.serialize()).collect();
pub fn serialize_all(&self) -> Result<(Vec<u8>, Vec<usize>), FormatError> {
let serialized: Vec<Vec<u8>> = self
.headers
.iter()
.map(|h| h.serialize())
.collect::<Result<_, _>>()?;
let total: usize = serialized.iter().map(|s| s.len()).sum();
let mut buf = Vec::with_capacity(total);
let mut offsets = Vec::with_capacity(serialized.len());
@@ -141,7 +169,7 @@ impl BatchObjectHeaderWriter {
offsets.push(buf.len());
buf.extend_from_slice(s);
}
(buf, offsets)
Ok((buf, offsets))
}
}
@@ -159,7 +187,7 @@ mod tests {
#[test]
fn empty_header_roundtrip() {
let writer = ObjectHeaderWriter::new();
let bytes = writer.serialize();
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.version, 2);
assert_eq!(hdr.messages.len(), 0);
@@ -170,7 +198,7 @@ mod tests {
let mut writer = ObjectHeaderWriter::new();
writer.add_message(MessageType::Dataspace, vec![1, 2, 3, 4]);
writer.add_message(MessageType::Datatype, vec![5, 6]);
let bytes = writer.serialize();
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 2);
assert_eq!(hdr.messages[0].msg_type, MessageType::Dataspace);
@@ -184,12 +212,30 @@ mod tests {
let mut writer = ObjectHeaderWriter::new();
// Add a message with >255 bytes of payload
writer.add_message(MessageType::Datatype, vec![0xAA; 300]);
let bytes = writer.serialize();
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages.len(), 1);
assert_eq!(hdr.messages[0].data.len(), 300);
}
#[test]
fn oversized_message_is_an_error_not_a_truncated_size() {
// 65535 bytes is the largest encodable payload.
let mut writer = ObjectHeaderWriter::new();
writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE]);
let bytes = writer.serialize().unwrap();
let hdr = ObjectHeader::parse(&bytes, 0, 8, 8).unwrap();
assert_eq!(hdr.messages[0].data.len(), MAX_MESSAGE_SIZE);
// One byte more used to be written with its size wrapped to 0.
let mut writer = ObjectHeaderWriter::new();
writer.add_message(MessageType::Attribute, vec![0; MAX_MESSAGE_SIZE + 1]);
assert!(matches!(
writer.serialize(),
Err(FormatError::SerializationError(_))
));
}
#[test]
fn batch_writer_serialize_all() {
let mut batch = BatchObjectHeaderWriter::new();
@@ -204,7 +250,7 @@ mod tests {
batch.add(w2);
assert_eq!(batch.len(), 2);
let (buf, offsets) = batch.serialize_all();
let (buf, offsets) = batch.serialize_all().unwrap();
assert_eq!(offsets.len(), 2);
assert_eq!(offsets[0], 0);
@@ -222,7 +268,7 @@ mod tests {
fn batch_writer_empty() {
let batch = BatchObjectHeaderWriter::new();
assert!(batch.is_empty());
let (buf, offsets) = batch.serialize_all();
let (buf, offsets) = batch.serialize_all().unwrap();
assert!(buf.is_empty());
assert!(offsets.is_empty());
}
crates/clawhdf5-format/src/property_list.rs
+2 -2
View File
@@ -43,7 +43,7 @@ impl Default for DatasetCreateProps {
fletcher32: false,
lz4: false,
zstd_level: None,
fill_time: FillTime::Alloc,
fill_time: FillTime::IfSet,
compact: false,
alignment: 0,
}
@@ -335,7 +335,7 @@ mod tests {
fn dcpl_defaults() {
let dcpl = DatasetCreateProps::new();
assert!(dcpl.chunk_dims.is_none());
assert_eq!(dcpl.fill_time, FillTime::Alloc);
assert_eq!(dcpl.fill_time, FillTime::IfSet);
assert!(!dcpl.compact);
}
crates/clawhdf5-format/src/shared_message.rs
+75 -4
View File
@@ -225,9 +225,12 @@ pub fn parse_sohm_table_message(
/// Parse the SOHM table structure (signature "SMTB") from the file.
///
/// Each index entry: index_type(1) + mesg_types(2) + min_mesg_size(4) +
/// list_max(2) + btree_min(2) + num_messages(2) + index_addr(offset_size) +
/// heap_addr(offset_size)
/// Each index entry: version(1) + index_type(1) + mesg_types(2) +
/// min_mesg_size(4) + list_max(2) + btree_min(2) + num_messages(2) +
/// index_addr(offset_size) + heap_addr(offset_size)
///
/// The leading per-index version byte (0) was missing here, so every field
/// after it was read one byte off — verified against an HDF5 2.0 file.
pub fn parse_sohm_table(
file_data: &[u8],
table_addr: usize,
@@ -240,11 +243,16 @@ pub fn parse_sohm_table(
}
let mut pos = table_addr + 4;
let os = offset_size as usize;
let entry_size = 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 13 + 2*offset_size
let entry_size = 1 + 1 + 2 + 4 + 2 + 2 + 2 + os + os; // 14 + 2*offset_size
let mut indexes = Vec::with_capacity(nindexes as usize);
for _ in 0..nindexes {
ensure_len(file_data, pos, entry_size)?;
let version = file_data[pos];
if version != 0 {
return Err(FormatError::InvalidSohmTableVersion(version));
}
pos += 1;
let index_type = file_data[pos];
pos += 1;
let mesg_types = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -381,6 +389,68 @@ pub fn parse_sohm_btree_entries(
// ---- SOHM resolution ----
/// Find the SOHM index that handles the given message type.
/// Load a file's SOHM table: superblock → superblock extension → Shared
/// Message Table message → SMTB. `Ok(None)` when the file has no superblock
/// extension or no shared-message table.
pub fn load_sohm_table(
file_data: &[u8],
offset_size: u8,
length_size: u8,
) -> Result<Option<SohmTable>, FormatError> {
let sig = crate::signature::find_signature(file_data)?;
let sb = crate::superblock::Superblock::parse(file_data, sig)?;
let Some(ext_addr) = sb
.superblock_extension_address
.filter(|&a| !is_undefined(a, offset_size))
else {
return Ok(None);
};
let ext = ObjectHeader::parse(file_data, ext_addr as usize, offset_size, length_size)?;
let Some(msg) = ext
.messages
.iter()
.find(|m| m.msg_type == MessageType::SharedMessageTable)
else {
return Ok(None);
};
let table_msg = parse_sohm_table_message(&msg.data, offset_size)?;
parse_sohm_table(
file_data,
table_msg.table_address as usize,
table_msg.nindexes,
offset_size,
)
.map(Some)
}
/// Like [`message_data`], but also follows references into the file's SOHM
/// heap (shared object header messages), loading the SOHM table on demand.
pub fn message_data_with_sohm<'a>(
file_data: &[u8],
msg: &'a crate::object_header::HeaderMessage,
offset_size: u8,
length_size: u8,
) -> Result<Cow<'a, [u8]>, FormatError> {
if !is_shared(msg.flags) {
return Ok(Cow::Borrowed(&msg.data));
}
let shared_ref = parse_shared_ref(&msg.data, offset_size)?;
let table = if shared_ref.heap_id.is_some() {
load_sohm_table(file_data, offset_size, length_size)?
} else {
None
};
resolve_shared_message_with_sohm(
file_data,
&shared_ref,
msg.msg_type,
offset_size,
length_size,
table.as_ref(),
)
.map(Cow::Owned)
}
fn find_index_for_msg_type(table: &SohmTable, msg_type: MessageType) -> Option<&SohmIndex> {
let type_bit = 1u16 << msg_type.to_u16();
table
@@ -707,6 +777,7 @@ mod tests {
let mut buf = Vec::new();
buf.extend_from_slice(b"SMTB");
for idx in indexes {
buf.push(0); // version
buf.push(idx.index_type);
buf.extend_from_slice(&idx.mesg_types.to_le_bytes());
buf.extend_from_slice(&idx.min_mesg_size.to_le_bytes());
crates/clawhdf5-format/src/superblock.rs
+10 -3
View File
@@ -39,7 +39,13 @@ pub struct Superblock {
pub superblock_extension_address: Option<u64>,
/// CRC32C checksum (v2/v3 only).
pub checksum: Option<u32>,
/// Page size for page-buffer mode (v4 only). `None` for v0–v3.
/// Page size of the non-standard "version 4" superblock layout (v4 only).
/// `None` for v0–v3.
///
/// HDF5 has no superblock version 4 — libhdf5 refuses it. A real paged
/// file is a v2/v3 superblock whose extension holds a File Space Info
/// message (what `FileWriter::with_page_size` writes). This field is kept
/// only so such files written by older clawhdf5 versions still parse.
pub page_size: Option<u32>,
}
@@ -127,8 +133,9 @@ impl Superblock {
/// Serialize this superblock to bytes.
///
/// Writes v2/v3 format, or v4 (with `page_size`) when `self.version == 4`.
/// Computes and appends Jenkins lookup3 checksum.
/// Writes v2/v3 format, or the non-standard v4 (with `page_size`) when
/// `self.version == 4` — which no HDF5 library opens; see
/// [`Self::page_size`]. Computes and appends Jenkins lookup3 checksum.
pub fn serialize(&self) -> Vec<u8> {
let mut buf = Vec::with_capacity(48);
buf.extend_from_slice(&HDF5_SIGNATURE);
crates/clawhdf5-format/src/type_builders.rs
+87 -17
View File
@@ -15,29 +15,81 @@ use crate::datatype::{
/// Controls when fill values are written to dataset storage.
///
/// Corresponds to the HDF5 fill value message's "fill time" field.
/// Corresponds to the HDF5 fill value message's "fill time" field
/// (`H5D_fill_time_t`).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum FillTime {
/// Never write fill values (0x02). Avoids initialization overhead
/// for datasets that will be fully written before any read.
/// Never write fill values (`H5D_FILL_TIME_NEVER`). Avoids
/// initialization overhead for datasets that will be fully written
/// before any read.
Never,
/// Write fill values at allocation time (0x0a). This is the default
/// and matches the HDF5 C library's behavior.
#[default]
/// Write fill values when storage is allocated (`H5D_FILL_TIME_ALLOC`).
Alloc,
/// Write fill values only when the fill value has been explicitly set (0x06).
/// Write fill values at allocation only if one was set explicitly
/// (`H5D_FILL_TIME_IFSET`). The default, as in the HDF5 C library.
#[default]
IfSet,
}
/// Space allocation time written with every fill value message: late
/// (`H5D_ALLOC_TIME_LATE`), bits 0-1 of the flags byte.
const ALLOC_TIME_LATE: u8 = 2;
impl FillTime {
/// Serialize to the byte used in the fill value message (version 3).
/// Serialize to the flags byte of a version 3 fill value message: the
/// space allocation time (late) in bits 0-1 and the fill time in bits
/// 2-3 (`H5D_FILL_TIME_ALLOC` = 0, `NEVER` = 1, `IFSET` = 2).
///
/// This used to put `Never` in the ALLOC slot, `Alloc` in IFSET and
/// `IfSet` in NEVER, so libhdf5 saw every choice as a different one.
pub fn to_byte(self) -> u8 {
match self {
FillTime::Never => 0x02,
FillTime::Alloc => 0x0a,
FillTime::IfSet => 0x06,
ALLOC_TIME_LATE | (self.code() << 2)
}
/// Decode the fill time from a version 3 fill value message's flags.
pub fn from_byte(flags: u8) -> Option<FillTime> {
match (flags >> 2) & 0x03 {
0 => Some(FillTime::Alloc),
1 => Some(FillTime::Never),
2 => Some(FillTime::IfSet),
_ => None,
}
}
fn code(self) -> u8 {
match self {
FillTime::Alloc => 0,
FillTime::Never => 1,
FillTime::IfSet => 2,
}
}
}
/// Serialize a version 3 Fill Value message for a dataset of `dt`: the fill
/// time, and the user-defined fill value if there is one (bit 5).
pub(crate) fn fill_value_message(
fill_time: FillTime,
value: Option<&[u8]>,
dt: &Datatype,
) -> Result<Vec<u8>, crate::error::FormatError> {
let mut msg = vec![3, fill_time.to_byte()];
if let Some(value) = value {
if matches!(dt, Datatype::VariableLength { .. }) {
return Err(crate::error::FormatError::SerializationError(
"a fill value for a variable-length datatype is not supported".into(),
));
}
if value.len() != dt.type_size() as usize {
return Err(crate::error::FormatError::DataSizeMismatch {
expected: dt.type_size() as usize,
actual: value.len(),
});
}
msg[1] |= 0x20; // fill value defined
msg.extend_from_slice(&(value.len() as u32).to_le_bytes());
msg.extend_from_slice(value);
}
Ok(msg)
}
// ---- Datatype constructors ----
@@ -332,7 +384,11 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess
raw_data: data.clone(),
},
AttrValue::String(s) => {
let bytes = s.as_bytes();
// A fixed-length string type must be at least 1 byte: libhdf5
// rejects size 0 ("invalid datatype size") and with it every
// attribute on the object. h5py stores "" as one NUL byte.
let mut bytes = s.as_bytes().to_vec();
bytes.resize(bytes.len().max(1), 0);
AttributeMessage {
name: name.to_string(),
datatype: Datatype::String {
@@ -341,11 +397,12 @@ pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMess
charset: CharacterSet::Utf8,
},
dataspace: scalar_ds(),
raw_data: bytes.to_vec(),
raw_data: bytes,
}
}
AttrValue::StringArray(arr) => {
let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0);
// At least 1 byte per element, as for a single string.
let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0).max(1);
let mut raw = Vec::new();
for s in arr {
let mut b = s.as_bytes().to_vec();
@@ -431,8 +488,10 @@ pub struct DatasetBuilder {
pub(crate) data: Option<Vec<u8>>,
pub(crate) attrs: Vec<(String, AttrValue)>,
pub(crate) chunk_options: ChunkOptions,
/// Controls when fill values are written. Default is `FillTime::Alloc`.
/// Controls when fill values are written. Default is `FillTime::IfSet`.
pub(crate) fill_time: FillTime,
/// User-defined fill value: one element's bytes, as stored.
pub(crate) fill_value: Option<Vec<u8>>,
/// Use compact (inline) storage: data is stored in the object header.
/// Only valid when raw data is <= 65536 bytes and dataset is not chunked.
pub(crate) compact: bool,
@@ -459,6 +518,7 @@ impl DatasetBuilder {
attrs: Vec::new(),
chunk_options: ChunkOptions::default(),
fill_time: FillTime::default(),
fill_value: None,
compact: false,
alignment: 0,
virtual_sources: None,
@@ -715,10 +775,20 @@ impl DatasetBuilder {
self
}
/// Set the dataset's fill value: what readers return for storage that
/// was never written (e.g. after the dataset is extended). `value` is one
/// element's bytes as stored — the dataset datatype's size and byte order
/// (`(-1i32).to_le_bytes()` for an `i32` dataset). A size mismatch, or a
/// variable-length datatype, makes `finish` fail.
pub fn with_fill_value(&mut self, value: &[u8]) -> &mut Self {
self.fill_value = Some(value.to_vec());
self
}
/// Use compact (inline) storage for this dataset.
///
/// The raw data is stored directly in the dataset's object header rather
/// than as a separate data blob. Only effective when raw data <= 65536 bytes
/// than as a separate data blob. Only effective when raw data <= 65531 bytes
/// and the dataset is not chunked.
pub fn compact(&mut self) -> &mut Self {
self.compact = true;