Read HDF5 1.6-era files, user blocks, VDS, dense attributes and large groups #13
@@ -310,6 +310,18 @@
|
||||
estimate instead of libhdf5's per-depth record capacities, and the
|
||||
listing failed. The same B-tree code indexes dense attributes, shared
|
||||
messages and chunks.
|
||||
- Fractal-heap "huge" objects (larger than the heap's managed-object
|
||||
limit, 4 KiB by default — e.g. an 8 KiB dense attribute or a link with a
|
||||
very long name) and "tiny" objects are now read; the ID type was taken
|
||||
from the wrong bits (6-7, the version, instead of 4-5), so a huge object
|
||||
failed and took every attribute on its object down with it (NetCDF-4
|
||||
files such as netcdf4-python's `issue671.nc`). Huge objects are found
|
||||
directly from the ID or through the huge-object v2 B-tree, filtered or
|
||||
not.
|
||||
- Heaps with an I/O filter pipeline (a group created with a filter on its
|
||||
creation property list compresses its link heap) are now read: the
|
||||
header's pipeline was skipped with the wrong size, so its checksum was
|
||||
looked for in the wrong place, and filtered direct blocks were read raw.
|
||||
- `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:**
|
||||
- Extensible Array (one unlimited dimension): chunks from index 244 on were
|
||||
written but never indexed and read as 0, by libhdf5 and by us.
|
||||
|
||||
@@ -1947,6 +1947,12 @@ mod tests {
|
||||
root_block_address: 0,
|
||||
current_rows_in_root_indirect_block: 0,
|
||||
managed_objects_count: 0,
|
||||
huge_btree_address: u64::MAX,
|
||||
filter_pipeline: None,
|
||||
root_direct_block_filtered_size: 0,
|
||||
root_direct_block_filter_mask: 0,
|
||||
offset_size: 8,
|
||||
length_size: 8,
|
||||
};
|
||||
let (off, len) = fh.decode_managed_id(&id).unwrap();
|
||||
assert_eq!(off, 100);
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
//! HDF5 Fractal Heap parsing for v2 group link storage.
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
use alloc::{format, vec::Vec};
|
||||
|
||||
#[cfg(feature = "checksum")]
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
|
||||
use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
|
||||
use crate::error::FormatError;
|
||||
use crate::filter_pipeline::FilterPipeline;
|
||||
|
||||
/// Parsed fractal heap header (signature "FRHP").
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -33,6 +35,23 @@ pub struct FractalHeapHeader {
|
||||
pub current_rows_in_root_indirect_block: u16,
|
||||
/// Total number of managed objects.
|
||||
pub managed_objects_count: u64,
|
||||
/// Address of the v2 B-tree indexing "huge" objects (undefined address
|
||||
/// when the heap has none). Huge objects are those larger than
|
||||
/// `max_managed_object_size`; they live outside the heap's blocks.
|
||||
pub huge_btree_address: u64,
|
||||
/// The heap's I/O filter pipeline, if it has one. It applies to managed
|
||||
/// direct blocks and to huge objects.
|
||||
pub filter_pipeline: Option<FilterPipeline>,
|
||||
/// Stored (filtered) size of the root direct block; meaningful only when
|
||||
/// the heap is filtered and its root is a direct block.
|
||||
pub root_direct_block_filtered_size: u64,
|
||||
/// Filter mask of the root direct block (bit *i* set = filter *i*
|
||||
/// skipped); meaningful only when the heap is filtered.
|
||||
pub root_direct_block_filter_mask: u32,
|
||||
/// Size of addresses in the file ("Size of Offsets").
|
||||
pub offset_size: u8,
|
||||
/// Size of lengths in the file ("Size of Lengths").
|
||||
pub length_size: u8,
|
||||
}
|
||||
|
||||
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
|
||||
@@ -79,6 +98,38 @@ fn is_undefined(val: u64, offset_size: u8) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Little-endian unsigned integer of up to 8 bytes.
|
||||
fn le_uint(bytes: &[u8]) -> u64 {
|
||||
bytes
|
||||
.iter()
|
||||
.take(8)
|
||||
.enumerate()
|
||||
.fold(0u64, |acc, (i, &b)| acc | (u64::from(b) << (i * 8)))
|
||||
}
|
||||
|
||||
fn heap_error(msg: &str) -> FormatError {
|
||||
FormatError::ChunkedReadError(format!("fractal heap: {msg}"))
|
||||
}
|
||||
|
||||
/// Heap ID type, from bits 4-5 of an ID's first byte (libhdf5's
|
||||
/// `H5HF_ID_TYPE_MASK`, 0x30); bits 6-7 are the ID version, which must be 0.
|
||||
const HEAP_ID_MANAGED: u8 = 0;
|
||||
const HEAP_ID_HUGE: u8 = 1;
|
||||
const HEAP_ID_TINY: u8 = 2;
|
||||
|
||||
/// The type (0 managed, 1 huge, 2 tiny) of a heap ID from its first byte,
|
||||
/// refusing an ID version other than 0.
|
||||
fn heap_id_type(first: u8) -> Result<u8, FormatError> {
|
||||
if first >> 6 != 0 {
|
||||
return Err(heap_error("unsupported heap ID version"));
|
||||
}
|
||||
Ok((first >> 4) & 0x03)
|
||||
}
|
||||
|
||||
/// v2 B-tree record types indexing a heap's huge objects.
|
||||
const BTREE_HUGE_INDIRECT: u8 = 1;
|
||||
const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2;
|
||||
|
||||
impl FractalHeapHeader {
|
||||
/// Parse a fractal heap header at the given offset.
|
||||
pub fn parse(
|
||||
@@ -122,11 +173,17 @@ impl FractalHeapHeader {
|
||||
]);
|
||||
pos += 4;
|
||||
|
||||
// Skip several fixed fields: next_huge_object_id(ls), btree_huge_objects_address(os),
|
||||
// free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
|
||||
// next_huge_object_id (length_size)
|
||||
ensure_len(file_data, pos, ls)?;
|
||||
pos += ls;
|
||||
// btree_huge_objects_address (offset_size)
|
||||
let huge_btree_address = read_offset(file_data, pos, offset_size)?;
|
||||
pos += os;
|
||||
|
||||
// Skip: free_space_managed_blocks(ls), managed_block_free_space_manager_address(os),
|
||||
// managed_space_in_heap(ls), allocated_managed_space_in_heap(ls),
|
||||
// direct_block_allocation_iterator_offset(ls)
|
||||
let skip_size = 5 * ls + 2 * os;
|
||||
let skip_size = 4 * ls + os;
|
||||
ensure_len(file_data, pos, skip_size)?;
|
||||
pos += skip_size;
|
||||
|
||||
@@ -134,14 +191,9 @@ impl FractalHeapHeader {
|
||||
let managed_objects_count = read_offset(file_data, pos, length_size)?;
|
||||
pos += ls;
|
||||
|
||||
// huge_objects_size (length_size)
|
||||
pos += ls;
|
||||
// huge_objects_count (length_size)
|
||||
pos += ls;
|
||||
// tiny_objects_size (length_size)
|
||||
pos += ls;
|
||||
// tiny_objects_count (length_size)
|
||||
pos += ls;
|
||||
// huge_objects_size, huge_objects_count, tiny_objects_size,
|
||||
// tiny_objects_count (length_size each)
|
||||
pos += 4 * ls;
|
||||
|
||||
// table_width (2)
|
||||
ensure_len(file_data, pos, 2)?;
|
||||
@@ -175,16 +227,28 @@ impl FractalHeapHeader {
|
||||
ensure_len(file_data, pos, 2)?;
|
||||
let current_rows_in_root_indirect_block =
|
||||
u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
|
||||
#[allow(unused_variables, unused_mut, unused_assignments)]
|
||||
let mut pos = pos + 2;
|
||||
pos += 2;
|
||||
|
||||
// Skip IO filter encoded info if present
|
||||
// With I/O filters: root direct block's filtered size (length_size),
|
||||
// its filter mask (4), then the encoded filter pipeline message.
|
||||
let mut filter_pipeline = None;
|
||||
let mut root_direct_block_filtered_size = 0;
|
||||
let mut root_direct_block_filter_mask = 0;
|
||||
if io_filter_encoded_length > 0 {
|
||||
// root_block_filter_info_size (length_size) + filter_mask (4)
|
||||
#[allow(unused_assignments)]
|
||||
{
|
||||
pos += ls + 4;
|
||||
}
|
||||
root_direct_block_filtered_size = read_offset(file_data, pos, length_size)?;
|
||||
pos += ls;
|
||||
ensure_len(file_data, pos, 4)?;
|
||||
root_direct_block_filter_mask = u32::from_le_bytes([
|
||||
file_data[pos],
|
||||
file_data[pos + 1],
|
||||
file_data[pos + 2],
|
||||
file_data[pos + 3],
|
||||
]);
|
||||
pos += 4;
|
||||
let n = io_filter_encoded_length as usize;
|
||||
ensure_len(file_data, pos, n)?;
|
||||
filter_pipeline = Some(FilterPipeline::parse(&file_data[pos..pos + n])?);
|
||||
pos += n;
|
||||
}
|
||||
|
||||
// Validate header checksum
|
||||
@@ -200,6 +264,8 @@ impl FractalHeapHeader {
|
||||
});
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "checksum"))]
|
||||
let _ = pos;
|
||||
|
||||
Ok(FractalHeapHeader {
|
||||
heap_id_length,
|
||||
@@ -213,13 +279,19 @@ impl FractalHeapHeader {
|
||||
root_block_address,
|
||||
current_rows_in_root_indirect_block,
|
||||
managed_objects_count,
|
||||
huge_btree_address,
|
||||
filter_pipeline,
|
||||
root_direct_block_filtered_size,
|
||||
root_direct_block_filter_mask,
|
||||
offset_size,
|
||||
length_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode a managed heap ID into (offset_in_heap, object_length).
|
||||
///
|
||||
/// The heap ID layout for managed objects (type 0):
|
||||
/// - Byte 0: bits 6-7 = type (0), bits 4-5 = version (0), bits 0-3 = reserved
|
||||
/// - Byte 0: bits 6-7 = version (0), bits 4-5 = type (0), bits 0-3 = reserved
|
||||
/// - Bytes 1+: offset (max_heap_size bits, LE) then length (remaining bits, LE)
|
||||
pub fn decode_managed_id(&self, id_bytes: &[u8]) -> Result<(u64, u64), FormatError> {
|
||||
if id_bytes.is_empty() {
|
||||
@@ -229,8 +301,8 @@ impl FractalHeapHeader {
|
||||
});
|
||||
}
|
||||
|
||||
let id_type = (id_bytes[0] >> 6) & 0x03;
|
||||
if id_type != 0 {
|
||||
let id_type = heap_id_type(id_bytes[0])?;
|
||||
if id_type != HEAP_ID_MANAGED {
|
||||
return Err(FormatError::InvalidHeapIdType(id_type));
|
||||
}
|
||||
|
||||
@@ -269,12 +341,183 @@ impl FractalHeapHeader {
|
||||
Ok((heap_offset, length_val))
|
||||
}
|
||||
|
||||
/// Read a managed object from the heap given its raw heap ID bytes.
|
||||
/// Read any object from the heap given its raw heap ID bytes: managed
|
||||
/// (stored in the heap's blocks), huge (stored outside them, found
|
||||
/// directly from the ID or through the huge-object v2 B-tree, optionally
|
||||
/// filtered) or tiny (stored in the ID itself).
|
||||
///
|
||||
/// Despite its name this accepts every ID type; `offset_size` must match
|
||||
/// the one the header was parsed with.
|
||||
pub fn read_managed_object(
|
||||
&self,
|
||||
file_data: &[u8],
|
||||
id_bytes: &[u8],
|
||||
offset_size: u8,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let Some(&first) = id_bytes.first() else {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: 1,
|
||||
available: 0,
|
||||
});
|
||||
};
|
||||
match heap_id_type(first)? {
|
||||
HEAP_ID_MANAGED => self.read_heap_managed(file_data, id_bytes, offset_size),
|
||||
HEAP_ID_HUGE => self.read_huge_object(file_data, id_bytes),
|
||||
HEAP_ID_TINY => self.read_tiny_object(id_bytes),
|
||||
other => Err(FormatError::InvalidHeapIdType(other)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a huge object's ID holds its address and length directly
|
||||
/// (libhdf5 does this when they fit in the ID), rather than a key into
|
||||
/// the huge-object B-tree.
|
||||
fn huge_ids_direct(&self) -> bool {
|
||||
let room = usize::from(self.heap_id_length).saturating_sub(1);
|
||||
let os = usize::from(self.offset_size);
|
||||
let ls = usize::from(self.length_size);
|
||||
if self.filter_pipeline.is_some() {
|
||||
room >= os + ls + 4 + ls
|
||||
} else {
|
||||
room >= os + ls
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a huge object (heap ID type 1).
|
||||
fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
|
||||
let os = usize::from(self.offset_size);
|
||||
let ls = usize::from(self.length_size);
|
||||
// (address, stored length, filter mask, decoded length); the last two
|
||||
// only matter for a filtered heap.
|
||||
let (addr, stored_len, mask, mem_len) = if self.huge_ids_direct() {
|
||||
let body = &id[1..];
|
||||
let need = if self.filter_pipeline.is_some() {
|
||||
os + ls + 4 + ls
|
||||
} else {
|
||||
os + ls
|
||||
};
|
||||
ensure_len(body, 0, need)?;
|
||||
let addr = le_uint(&body[..os]);
|
||||
let len = le_uint(&body[os..os + ls]);
|
||||
if self.filter_pipeline.is_some() {
|
||||
let mask = u32::from_le_bytes([
|
||||
body[os + ls],
|
||||
body[os + ls + 1],
|
||||
body[os + ls + 2],
|
||||
body[os + ls + 3],
|
||||
]);
|
||||
let mem = le_uint(&body[os + ls + 4..os + ls + 4 + ls]);
|
||||
(addr, len, mask, mem)
|
||||
} else {
|
||||
(addr, len, 0, len)
|
||||
}
|
||||
} else {
|
||||
let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
|
||||
ensure_len(id, 1, key_len)?;
|
||||
let key = le_uint(&id[1..1 + key_len]);
|
||||
self.find_huge_record(file_data, key)?
|
||||
};
|
||||
|
||||
let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
|
||||
let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
|
||||
ensure_len(file_data, start, len)?;
|
||||
let stored = &file_data[start..start + len];
|
||||
match &self.filter_pipeline {
|
||||
None => Ok(stored.to_vec()),
|
||||
Some(pipeline) => {
|
||||
let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
|
||||
let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
|
||||
if out.len() != mem {
|
||||
return Err(heap_error("filtered huge object decoded to the wrong size"));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up huge object `key` in the huge-object v2 B-tree, returning
|
||||
/// (address, stored length, filter mask, decoded length).
|
||||
fn find_huge_record(
|
||||
&self,
|
||||
file_data: &[u8],
|
||||
key: u64,
|
||||
) -> Result<(u64, u64, u32, u64), FormatError> {
|
||||
if is_undefined(self.huge_btree_address, self.offset_size) {
|
||||
return Err(heap_error(
|
||||
"huge object ID but the heap has no huge-object index",
|
||||
));
|
||||
}
|
||||
let hdr = BTreeV2Header::parse(
|
||||
file_data,
|
||||
self.huge_btree_address as usize,
|
||||
self.offset_size,
|
||||
self.length_size,
|
||||
)?;
|
||||
let os = usize::from(self.offset_size);
|
||||
let ls = usize::from(self.length_size);
|
||||
let filtered = self.filter_pipeline.is_some();
|
||||
let (expected_type, rec_len) = if filtered {
|
||||
(BTREE_HUGE_INDIRECT_FILTERED, os + ls + 4 + ls + ls)
|
||||
} else {
|
||||
(BTREE_HUGE_INDIRECT, os + ls + ls)
|
||||
};
|
||||
if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len {
|
||||
return Err(heap_error("unexpected huge-object B-tree record type"));
|
||||
}
|
||||
let records =
|
||||
collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?;
|
||||
for rec in &records {
|
||||
let d = &rec.data;
|
||||
if d.len() < rec_len {
|
||||
continue;
|
||||
}
|
||||
let addr = le_uint(&d[..os]);
|
||||
let len = le_uint(&d[os..os + ls]);
|
||||
if filtered {
|
||||
let mask = u32::from_le_bytes([
|
||||
d[os + ls],
|
||||
d[os + ls + 1],
|
||||
d[os + ls + 2],
|
||||
d[os + ls + 3],
|
||||
]);
|
||||
let mem = le_uint(&d[os + ls + 4..os + 2 * ls + 4]);
|
||||
let id = le_uint(&d[os + 2 * ls + 4..os + 3 * ls + 4]);
|
||||
if id == key {
|
||||
return Ok((addr, len, mask, mem));
|
||||
}
|
||||
} else {
|
||||
let id = le_uint(&d[os + ls..os + 2 * ls]);
|
||||
if id == key {
|
||||
return Ok((addr, len, 0, len));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(heap_error("huge object not found in its B-tree"))
|
||||
}
|
||||
|
||||
/// Read a tiny object (heap ID type 2), stored in the ID itself.
|
||||
fn read_tiny_object(&self, id: &[u8]) -> Result<Vec<u8>, FormatError> {
|
||||
// libhdf5 uses a one-byte length (low 4 bits of byte 0) unless the ID
|
||||
// is long enough to need 12 bits, which then borrow byte 1.
|
||||
let extended = usize::from(self.heap_id_length).saturating_sub(1) > 17;
|
||||
let (len, start) = if extended {
|
||||
ensure_len(id, 0, 2)?;
|
||||
(
|
||||
((usize::from(id[0] & 0x0F)) << 8 | usize::from(id[1])) + 1,
|
||||
2,
|
||||
)
|
||||
} else {
|
||||
(usize::from(id[0] & 0x0F) + 1, 1)
|
||||
};
|
||||
ensure_len(id, start, len)?;
|
||||
Ok(id[start..start + len].to_vec())
|
||||
}
|
||||
|
||||
/// Read a managed object (heap ID type 0).
|
||||
fn read_heap_managed(
|
||||
&self,
|
||||
file_data: &[u8],
|
||||
id_bytes: &[u8],
|
||||
offset_size: u8,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
let (heap_offset, obj_len) = self.decode_managed_id(id_bytes)?;
|
||||
|
||||
@@ -289,12 +532,15 @@ impl FractalHeapHeader {
|
||||
// Root is a direct block
|
||||
self.read_from_direct_block(
|
||||
file_data,
|
||||
self.root_block_address as usize,
|
||||
self.starting_block_size,
|
||||
0, // block offset in heap = 0 for root
|
||||
DirectBlock {
|
||||
addr: self.root_block_address as usize,
|
||||
size: self.starting_block_size,
|
||||
heap_offset: 0,
|
||||
filtered_size: self.root_direct_block_filtered_size,
|
||||
filter_mask: self.root_direct_block_filter_mask,
|
||||
},
|
||||
heap_offset,
|
||||
obj_len as usize,
|
||||
offset_size,
|
||||
)
|
||||
} else {
|
||||
// Root is an indirect block — limit recursion to 64 levels
|
||||
@@ -313,27 +559,41 @@ impl FractalHeapHeader {
|
||||
|
||||
/// Read an object from a direct block.
|
||||
///
|
||||
/// The heap offset is relative to the start of the block (including its header),
|
||||
/// so we just add it to the block address minus the block's heap offset.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// The heap offset is relative to the start of the block (including its
|
||||
/// header), so we just add it to the block address minus the block's heap
|
||||
/// offset. A filtered heap stores each direct block (header included)
|
||||
/// through its filter pipeline, so the block is decoded first.
|
||||
fn read_from_direct_block(
|
||||
&self,
|
||||
file_data: &[u8],
|
||||
block_addr: usize,
|
||||
_block_size: u64,
|
||||
block_heap_offset: u64,
|
||||
block: DirectBlock,
|
||||
target_offset: u64,
|
||||
length: usize,
|
||||
_offset_size: u8,
|
||||
) -> Result<Vec<u8>, FormatError> {
|
||||
if target_offset < block_heap_offset {
|
||||
if target_offset < block.heap_offset {
|
||||
return Err(FormatError::UnexpectedEof {
|
||||
expected: block_heap_offset as usize,
|
||||
expected: block.heap_offset as usize,
|
||||
available: target_offset as usize,
|
||||
});
|
||||
}
|
||||
let local_offset = (target_offset - block_heap_offset) as usize;
|
||||
let pos = block_addr
|
||||
let local_offset = (target_offset - block.heap_offset) as usize;
|
||||
if let Some(pipeline) = &self.filter_pipeline {
|
||||
let stored_len = usize::try_from(block.filtered_size)
|
||||
.map_err(|_| heap_error("direct block size"))?;
|
||||
let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
|
||||
ensure_len(file_data, block.addr, stored_len)?;
|
||||
let decoded = crate::filters::decompress_chunk_masked(
|
||||
&file_data[block.addr..block.addr + stored_len],
|
||||
pipeline,
|
||||
size,
|
||||
1,
|
||||
block.filter_mask,
|
||||
)?;
|
||||
ensure_len(&decoded, local_offset, length)?;
|
||||
return Ok(decoded[local_offset..local_offset + length].to_vec());
|
||||
}
|
||||
let pos = block
|
||||
.addr
|
||||
.checked_add(local_offset)
|
||||
.ok_or(FormatError::UnexpectedEof {
|
||||
expected: usize::MAX,
|
||||
@@ -371,19 +631,13 @@ impl FractalHeapHeader {
|
||||
let iblock_header = 5 + offset_size as usize + block_offset_bytes;
|
||||
let mut pos = iblock_addr + iblock_header;
|
||||
|
||||
// Compute block sizes for each row using the doubling table
|
||||
let tw = self.table_width as u64;
|
||||
|
||||
let nrows_usize = nrows as usize;
|
||||
|
||||
// Build table of (block_size, heap_offset) for each child entry
|
||||
let mut current_heap_offset = iblock_heap_offset;
|
||||
|
||||
// Rows below max_direct_rows hold direct blocks; rows at/above hold
|
||||
// child indirect blocks. (NOT the FRHP "starting rows" field.)
|
||||
let start_indirect = self.max_direct_rows();
|
||||
|
||||
// Read child addresses for direct block rows
|
||||
let max_direct_rows = nrows_usize.min(start_indirect);
|
||||
|
||||
for row in 0..max_direct_rows {
|
||||
@@ -393,35 +647,49 @@ impl FractalHeapHeader {
|
||||
let child_addr = read_offset(file_data, pos, offset_size)?;
|
||||
pos += offset_size as usize;
|
||||
|
||||
if self.io_filter_encoded_length > 0 {
|
||||
// filtered_size(length_size) + filter_mask(4)
|
||||
// Skip for now - we don't handle filtered direct blocks in fractal heaps
|
||||
pos += 4; // filter_mask - simplified
|
||||
}
|
||||
// A filtered heap stores each direct block's filtered size
|
||||
// (length_size) and filter mask (4) after its address.
|
||||
let (filtered_size, filter_mask) = if self.filter_pipeline.is_some() {
|
||||
let size = read_offset(file_data, pos, self.length_size)?;
|
||||
pos += usize::from(self.length_size);
|
||||
ensure_len(file_data, pos, 4)?;
|
||||
let mask = u32::from_le_bytes([
|
||||
file_data[pos],
|
||||
file_data[pos + 1],
|
||||
file_data[pos + 2],
|
||||
file_data[pos + 3],
|
||||
]);
|
||||
pos += 4;
|
||||
(size, mask)
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
|
||||
if !is_undefined(child_addr, offset_size) {
|
||||
let block_end = current_heap_offset + block_size;
|
||||
if target_offset >= current_heap_offset && target_offset < block_end {
|
||||
return self.read_from_direct_block(
|
||||
file_data,
|
||||
child_addr as usize,
|
||||
block_size,
|
||||
current_heap_offset,
|
||||
target_offset,
|
||||
length,
|
||||
offset_size,
|
||||
);
|
||||
}
|
||||
let block_end = current_heap_offset.saturating_add(block_size);
|
||||
if !is_undefined(child_addr, offset_size)
|
||||
&& target_offset >= current_heap_offset
|
||||
&& target_offset < block_end
|
||||
{
|
||||
return self.read_from_direct_block(
|
||||
file_data,
|
||||
DirectBlock {
|
||||
addr: child_addr as usize,
|
||||
size: block_size,
|
||||
heap_offset: current_heap_offset,
|
||||
filtered_size,
|
||||
filter_mask,
|
||||
},
|
||||
target_offset,
|
||||
length,
|
||||
);
|
||||
}
|
||||
current_heap_offset += block_size;
|
||||
current_heap_offset = block_end;
|
||||
}
|
||||
}
|
||||
|
||||
// If we have indirect block rows
|
||||
// A child indirect block in row r spans exactly that row's block size
|
||||
// of heap space, so it has as many rows as a table of that total size
|
||||
// needs (not `row - start_indirect + 1`, which undercounts and makes
|
||||
// every object past the root's direct rows unreachable).
|
||||
// Rows at and above `start_indirect` hold child indirect blocks. A
|
||||
// child in row r spans exactly that row's block size of heap space,
|
||||
// so it has as many rows as a table of that total size needs.
|
||||
for row in start_indirect..nrows_usize {
|
||||
let child_space = self.block_size_for_row(row);
|
||||
let child_nrows = self.rows_for_size(child_space);
|
||||
@@ -495,6 +763,16 @@ impl FractalHeapHeader {
|
||||
}
|
||||
}
|
||||
|
||||
/// A managed direct block's location, extent and (for a filtered heap) its
|
||||
/// stored size and filter mask.
|
||||
struct DirectBlock {
|
||||
addr: usize,
|
||||
size: u64,
|
||||
heap_offset: u64,
|
||||
filtered_size: u64,
|
||||
filter_mask: u32,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -640,7 +918,7 @@ mod tests {
|
||||
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
|
||||
|
||||
// Build a managed heap ID:
|
||||
// byte 0: type=0 (bits 6-7 = 00), version=0 (bits 4-5), reserved (bits 0-3)
|
||||
// byte 0: version=0 (bits 6-7), type=0 (bits 4-5), reserved (bits 0-3)
|
||||
// bytes 1-6: offset (max_heap_size=16 bits) then length (remaining bits)
|
||||
// For offset=0, length=13:
|
||||
// payload = offset | (length << 16) = 0 | (13 << 16) = 0x000D0000
|
||||
@@ -704,9 +982,46 @@ mod tests {
|
||||
fn invalid_heap_id_type() {
|
||||
let (file_data, _) = build_simple_heap(8, 8);
|
||||
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
|
||||
// Type = 1 (tiny) in bits 6-7
|
||||
let id = vec![0x40u8, 0, 0, 0, 0, 0, 0]; // bit 6 set = type 1
|
||||
// Type = 1 (huge) in bits 4-5 is not a managed ID
|
||||
let id = vec![0x10u8, 0, 0, 0, 0, 0, 0];
|
||||
let err = hdr.decode_managed_id(&id).unwrap_err();
|
||||
assert_eq!(err, FormatError::InvalidHeapIdType(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiny_object_is_read_from_the_id() {
|
||||
let (file_data, _) = build_simple_heap(8, 8);
|
||||
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
|
||||
// Type 2 (0x20), length - 1 in the low 4 bits, data after.
|
||||
let id = [0x20 | 2, b'a', b'b', b'c', 0, 0, 0];
|
||||
assert_eq!(hdr.read_managed_object(&file_data, &id, 8).unwrap(), b"abc");
|
||||
// A length running past the ID is an error, not a short read.
|
||||
let id = [0x20 | 9, b'a', b'b', b'c', 0, 0, 0];
|
||||
assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huge_object_with_a_direct_id() {
|
||||
// With IDs long enough for an address and a length, libhdf5 stores
|
||||
// huge objects' location in the ID instead of the huge-object B-tree.
|
||||
let (mut file_data, _) = build_simple_heap(8, 8);
|
||||
let mut hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
|
||||
hdr.heap_id_length = 17;
|
||||
file_data[900..905].copy_from_slice(b"huge!");
|
||||
let mut id = vec![0x10u8];
|
||||
id.extend_from_slice(&900u64.to_le_bytes());
|
||||
id.extend_from_slice(&5u64.to_le_bytes());
|
||||
assert_eq!(
|
||||
hdr.read_managed_object(&file_data, &id, 8).unwrap(),
|
||||
b"huge!"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_heap_id_version_is_refused() {
|
||||
let (file_data, _) = build_simple_heap(8, 8);
|
||||
let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap();
|
||||
let id = [0x40u8, 0, 0, 0, 0, 0, 0];
|
||||
assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
use clawhdf5::{AttrValue, File};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
@@ -163,3 +163,113 @@ fn dense_group_with_a_three_level_name_index() {
|
||||
vec![1.0]
|
||||
);
|
||||
}
|
||||
|
||||
/// The attribute names h5py reports for `obj`, sorted.
|
||||
fn h5py_attr_names(path: &str, obj: &str) -> Vec<String> {
|
||||
let out = run_python(&format!(
|
||||
"import h5py\n\
|
||||
with h5py.File(r'{path}', 'r') as f:\n\
|
||||
\x20 print('\\x1f'.join(sorted(f[{obj:?}].attrs.keys())))\n"
|
||||
));
|
||||
out.split('\x1f')
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_attribute_stored_as_a_huge_heap_object() {
|
||||
skip_if_no_python!();
|
||||
// More than 8 attributes puts them in dense storage; one larger than the
|
||||
// heap's 4 KiB managed-object limit is stored as a "huge" object, outside
|
||||
// the heap blocks and found through the huge-object v2 B-tree. Its heap ID
|
||||
// (type bits 4-5 = 1) was misread as a managed ID, and the error made
|
||||
// every attribute on the object unreadable. NetCDF-4 files hit this
|
||||
// (netcdf-c's issue671.nc / issue672.nc).
|
||||
let (_dir, path) = h5py_file(
|
||||
"d = f.create_dataset('d', data=[1.0])\n\
|
||||
for i in range(10):\n\
|
||||
\x20 d.attrs['a%d' % i] = i\n\
|
||||
d.attrs['big'] = np.arange(1024, dtype='f8')\n\
|
||||
d.attrs['bigger'] = np.arange(20000, dtype='i8') * 3\n",
|
||||
);
|
||||
let f = File::open(&path).unwrap();
|
||||
let attrs = f.dataset("d").unwrap().attrs().unwrap();
|
||||
let mut names: Vec<String> = attrs.keys().cloned().collect();
|
||||
names.sort();
|
||||
assert_eq!(names, h5py_attr_names(&path, "d"));
|
||||
for i in 0..10 {
|
||||
assert!(
|
||||
matches!(attrs[&format!("a{i}")], AttrValue::I64(v) if v == i),
|
||||
"a{i}: {:?}",
|
||||
attrs[&format!("a{i}")]
|
||||
);
|
||||
}
|
||||
let big: Vec<f64> = (0..1024).map(f64::from).collect();
|
||||
assert!(matches!(&attrs["big"], AttrValue::F64Array(v) if *v == big));
|
||||
let bigger: Vec<i64> = (0..20000).map(|v| v * 3).collect();
|
||||
assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger));
|
||||
}
|
||||
|
||||
/// A group whose link heap has a deflate I/O filter (set on the group
|
||||
/// creation property list), with 3 000 links and one link whose message is
|
||||
/// larger than the heap's managed-object limit, so it is a huge object.
|
||||
fn huge_link_group(filtered: bool) -> (tempfile::TempDir, String) {
|
||||
let filter = if filtered {
|
||||
"import ctypes, glob, os\n\
|
||||
lib = ctypes.CDLL(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))[0])\n\
|
||||
lib.H5Pset_deflate.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\
|
||||
assert lib.H5Pset_deflate(gcpl.id, 6) >= 0\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
h5py_file(&format!(
|
||||
"t = f.create_dataset('t', data=[1.0])\n\
|
||||
gcpl = h5py.h5p.create(h5py.h5p.GROUP_CREATE)\n\
|
||||
{filter}\
|
||||
h5py.h5g.create(f.id, b'g', gcpl=gcpl)\n\
|
||||
g = f['g']\n\
|
||||
for i in range(3000):\n\
|
||||
\x20 g['l%05d' % i] = t\n\
|
||||
g['L' * 5000] = t\n"
|
||||
))
|
||||
}
|
||||
|
||||
fn check_huge_link_group(filtered: bool) {
|
||||
let (_dir, path) = huge_link_group(filtered);
|
||||
assert_same_listing(&path, "g");
|
||||
let f = File::open(&path).unwrap();
|
||||
let huge = format!("g/{}", "L".repeat(5000));
|
||||
assert_eq!(f.dataset(&huge).unwrap().read_f64().unwrap(), vec![1.0]);
|
||||
assert_eq!(
|
||||
f.dataset("g/l02999").unwrap().read_f64().unwrap(),
|
||||
vec![1.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_group_with_a_huge_link() {
|
||||
skip_if_no_python!();
|
||||
check_huge_link_group(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_group_with_a_filtered_link_heap() {
|
||||
skip_if_no_python!();
|
||||
// libhdf5 applies a group's filter pipeline to its link heap: direct
|
||||
// blocks and huge objects are stored deflated, and the heap header
|
||||
// carries the pipeline. The header's checksum was looked for in the
|
||||
// wrong place, and filtered blocks were read raw.
|
||||
if run_python(
|
||||
"import h5py, glob, os\nprint(len(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))))",
|
||||
) == "0"
|
||||
{
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found"
|
||||
);
|
||||
eprintln!("SKIP: h5py's bundled libhdf5 not found (needed to set the filter)");
|
||||
return;
|
||||
}
|
||||
check_huge_link_group(true);
|
||||
}
|
||||
|
||||
@@ -83,7 +83,8 @@ the VDS item, which is marked.
|
||||
- Soft links are left out of `datasets()`.
|
||||
- **Dense attributes:** a large attribute stored as a fractal-heap "huge"
|
||||
object makes every attribute on the object fail. This affects real NetCDF
|
||||
files (`issue671.nc`).
|
||||
files (`issue671.nc`). **Fixed 2026-09-25:** huge and tiny heap objects,
|
||||
and filtered heaps, are read.
|
||||
- **Other readers:**
|
||||
- VL-string datasets are not readable through `File`.
|
||||
- Metadata cache images are not supported.
|
||||
|
||||
Reference in New Issue
Block a user