osobhandClaude Opus 5.5
5a202f3791
libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.
Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
739 lines
28 KiB
Rust
739 lines
28 KiB
Rust
//! Variable-length data reading (VL strings & VL sequences).
|
||
//!
|
||
//! VL data elements in HDF5 store their values in the global heap.
|
||
//! The raw data for each element contains a global heap ID:
|
||
//! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`.
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
|
||
#[cfg(feature = "std")]
|
||
use std::collections::BTreeMap;
|
||
|
||
use crate::error::FormatError;
|
||
use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex};
|
||
|
||
/// A parsed variable-length element reference (global heap ID).
|
||
#[derive(Debug, Clone)]
|
||
pub struct VlElement {
|
||
/// Length of the VL data.
|
||
pub length: u32,
|
||
/// Address of the global heap collection containing the data.
|
||
pub collection_address: u64,
|
||
/// Index of the object within the collection.
|
||
pub object_index: u32,
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
match offset.checked_add(needed) {
|
||
Some(end) if end <= data.len() => Ok(()),
|
||
_ => Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
}),
|
||
}
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, offset_size: u8) -> Result<u64, FormatError> {
|
||
let s = offset_size as usize;
|
||
ensure_len(data, pos, s)?;
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match offset_size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(offset_size)),
|
||
})
|
||
}
|
||
|
||
/// Parse VL global heap references from raw attribute/dataset data.
|
||
pub fn parse_vl_references(
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
) -> Result<Vec<VlElement>, FormatError> {
|
||
let elem_size = 4 + offset_size as usize + 4; // length + address + index
|
||
let total =
|
||
(num_elements as usize)
|
||
.checked_mul(elem_size)
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: raw_data.len(),
|
||
})?;
|
||
if raw_data.len() < total {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: total,
|
||
available: raw_data.len(),
|
||
});
|
||
}
|
||
|
||
let mut elements = Vec::with_capacity(num_elements as usize);
|
||
let mut pos = 0;
|
||
|
||
for _ in 0..num_elements {
|
||
let length = u32::from_le_bytes([
|
||
raw_data[pos],
|
||
raw_data[pos + 1],
|
||
raw_data[pos + 2],
|
||
raw_data[pos + 3],
|
||
]);
|
||
pos += 4;
|
||
|
||
let collection_address = read_offset(raw_data, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
|
||
let object_index = u32::from_le_bytes([
|
||
raw_data[pos],
|
||
raw_data[pos + 1],
|
||
raw_data[pos + 2],
|
||
raw_data[pos + 3],
|
||
]);
|
||
pos += 4;
|
||
|
||
elements.push(VlElement {
|
||
length,
|
||
collection_address,
|
||
object_index,
|
||
});
|
||
}
|
||
|
||
Ok(elements)
|
||
}
|
||
|
||
/// Check if an address represents an undefined/null address.
|
||
fn is_undefined_address(addr: u64, offset_size: u8) -> bool {
|
||
match offset_size {
|
||
2 => addr == 0xFFFF,
|
||
4 => addr == 0xFFFF_FFFF,
|
||
8 => addr == 0xFFFF_FFFF_FFFF_FFFF,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
/// The size of one variable-length element in a file with `offset_size`-byte
|
||
/// addresses: a sequence length (4), a global heap collection address and an
|
||
/// object index (4). libhdf5 computes it this way rather than trusting the
|
||
/// datatype message (`H5T_set_loc`).
|
||
pub fn element_size(offset_size: u8) -> usize {
|
||
4 + offset_size as usize + 4
|
||
}
|
||
|
||
/// Refuse a variable-length datatype whose stored element size is not the
|
||
/// one this file's offset size implies. Its elements would be laid out with
|
||
/// a stride libhdf5 does not use, so every value after the first would be
|
||
/// read from the wrong place.
|
||
pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), FormatError> {
|
||
let expected = element_size(offset_size);
|
||
if stored_size as usize != expected {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"variable-length datatype stores {stored_size}-byte elements; a file with \
|
||
{offset_size}-byte offsets uses {expected}"
|
||
)));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// A collection's objects, located in the file data but not copied:
|
||
/// `(index, offset, size)` of the first object with each index, sorted by
|
||
/// index.
|
||
struct CachedCollection {
|
||
objects: Vec<(u16, usize, usize)>,
|
||
}
|
||
|
||
impl CachedCollection {
|
||
fn new(index: GlobalHeapIndex) -> Self {
|
||
let mut objects: Vec<(u16, usize, usize)> = index
|
||
.objects
|
||
.iter()
|
||
.map(|o| (o.index, o.offset, o.size))
|
||
.collect();
|
||
// Stable, so the first object with a repeated index is kept.
|
||
objects.sort_by_key(|o| o.0);
|
||
objects.dedup_by_key(|o| o.0);
|
||
Self { objects }
|
||
}
|
||
|
||
/// What this entry costs to keep, in bytes (roughly).
|
||
fn cost(&self) -> usize {
|
||
64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>()
|
||
}
|
||
|
||
fn get(&self, index: u32) -> Option<(usize, usize)> {
|
||
let index = u16::try_from(index).ok()?;
|
||
let i = self.objects.binary_search_by_key(&index, |o| o.0).ok()?;
|
||
Some((self.objects[i].1, self.objects[i].2))
|
||
}
|
||
}
|
||
|
||
/// How many bytes of collection indexes a [`VlResolver`] keeps before it
|
||
/// drops them and starts again. Values are never copied into the cache, so
|
||
/// this bounds what a read retains however many collections it visits.
|
||
const CACHE_BUDGET: usize = 32 << 20;
|
||
|
||
/// Resolves variable-length elements against a file's global heap, parsing
|
||
/// each heap collection once however many elements point into it.
|
||
///
|
||
/// Values follow libhdf5: an element whose heap address is 0 is null (an
|
||
/// empty string or sequence), and an element whose heap object is not
|
||
/// exactly `length × base size` bytes is an error ("Expected global heap
|
||
/// object size does not match"), not a truncated or padded value.
|
||
///
|
||
/// Memory stays bounded on hostile files: the cache holds where each
|
||
/// object lies, not a copy of it, up to a fixed budget; and collections
|
||
/// that overlap one another are refused (libhdf5 never writes them), so a
|
||
/// file cannot make the resolver parse the same bytes as the objects of
|
||
/// many collections.
|
||
pub struct VlResolver<'a> {
|
||
file_data: &'a [u8],
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
cache: BTreeMap<u64, CachedCollection>,
|
||
cached_bytes: usize,
|
||
budget: usize,
|
||
/// Start → end of every collection parsed so far (kept when the cache
|
||
/// is dropped, to check overlaps).
|
||
extents: BTreeMap<usize, usize>,
|
||
}
|
||
|
||
impl<'a> VlResolver<'a> {
|
||
/// A resolver over `file_data` (the file from its superblock on), with
|
||
/// the superblock's offset and length sizes.
|
||
pub fn new(file_data: &'a [u8], offset_size: u8, length_size: u8) -> Self {
|
||
Self {
|
||
file_data,
|
||
offset_size,
|
||
length_size,
|
||
cache: BTreeMap::new(),
|
||
cached_bytes: 0,
|
||
budget: CACHE_BUDGET,
|
||
extents: BTreeMap::new(),
|
||
}
|
||
}
|
||
|
||
/// The size of one element in this file (see [`element_size`]).
|
||
pub fn element_size(&self) -> usize {
|
||
element_size(self.offset_size)
|
||
}
|
||
|
||
/// Split `raw` into elements; its length must be a whole number of them.
|
||
fn elements(&self, raw: &[u8]) -> Result<Vec<VlElement>, FormatError> {
|
||
let size = self.element_size();
|
||
if !raw.len().is_multiple_of(size) {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"{} bytes is not a whole number of {size}-byte variable-length elements",
|
||
raw.len()
|
||
)));
|
||
}
|
||
parse_vl_references(raw, (raw.len() / size) as u64, self.offset_size)
|
||
}
|
||
|
||
/// The bytes of one element: `length × base_size` bytes from the heap,
|
||
/// or `None` for a null element.
|
||
fn resolve(
|
||
&mut self,
|
||
vl: &VlElement,
|
||
base_size: usize,
|
||
) -> Result<Option<&'a [u8]>, FormatError> {
|
||
let addr = vl.collection_address;
|
||
if addr == 0 {
|
||
return Ok(None);
|
||
}
|
||
let data = self.object(vl)?;
|
||
let expected = (vl.length as usize)
|
||
.checked_mul(base_size)
|
||
.ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?;
|
||
if data.len() != expected {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"global heap object {} in the collection at {addr} holds {} bytes; the element \
|
||
says {} × {base_size}",
|
||
vl.object_index,
|
||
data.len(),
|
||
vl.length
|
||
)));
|
||
}
|
||
Ok(Some(data))
|
||
}
|
||
|
||
/// One element (the first [`element_size`](Self::element_size) bytes of
|
||
/// `elem`) of a variable-length sequence whose base type is `base_size`
|
||
/// bytes: its `length × base_size` bytes, or `None` for a null element
|
||
/// (heap address 0).
|
||
pub fn element(
|
||
&mut self,
|
||
elem: &[u8],
|
||
base_size: usize,
|
||
) -> Result<Option<&'a [u8]>, FormatError> {
|
||
let vl = parse_vl_references(elem, 1, self.offset_size)?;
|
||
self.resolve(&vl[0], base_size)
|
||
}
|
||
|
||
/// One variable-length string element: its bytes up to the first NUL,
|
||
/// or `None` for a null element (h5dump prints it as `NULL`, h5py
|
||
/// returns it as empty).
|
||
pub fn string_element(&mut self, elem: &[u8]) -> Result<Option<&'a [u8]>, FormatError> {
|
||
Ok(self.element(elem, 1)?.map(cut_at_nul))
|
||
}
|
||
|
||
/// The strings of the variable-length string elements in `raw`, as
|
||
/// bytes. A string ends at its first NUL, as libhdf5 returns it (it
|
||
/// converts each to a C string); a null element is empty.
|
||
pub fn string_bytes(&mut self, raw: &[u8]) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
self.elements(raw)?
|
||
.iter()
|
||
.map(|vl| Ok(self.resolve(vl, 1)?.map(cut_at_nul).unwrap_or(&[]).to_vec()))
|
||
.collect()
|
||
}
|
||
|
||
/// The strings of the variable-length string elements in `raw`, decoded
|
||
/// as UTF-8 with invalid sequences replaced by U+FFFD (see
|
||
/// [`string_bytes`](Self::string_bytes) for the exact bytes).
|
||
pub fn strings(&mut self, raw: &[u8]) -> Result<Vec<String>, FormatError> {
|
||
Ok(self
|
||
.string_bytes(raw)?
|
||
.into_iter()
|
||
.map(|b| match String::from_utf8(b) {
|
||
Ok(s) => s,
|
||
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// The sequences of the variable-length sequence elements in `raw`, each
|
||
/// as its `length × base_size` bytes in the base type's encoding.
|
||
pub fn sequences(&mut self, raw: &[u8], base_size: usize) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
if base_size == 0 {
|
||
return Err(FormatError::VlDataError(
|
||
"variable-length sequence of a zero-size base type".into(),
|
||
));
|
||
}
|
||
self.elements(raw)?
|
||
.iter()
|
||
.map(|vl| Ok(self.resolve(vl, base_size)?.unwrap_or(&[]).to_vec()))
|
||
.collect()
|
||
}
|
||
}
|
||
|
||
/// A string's bytes up to its first NUL.
|
||
fn cut_at_nul(s: &[u8]) -> &[u8] {
|
||
&s[..s.iter().position(|&b| b == 0).unwrap_or(s.len())]
|
||
}
|
||
|
||
/// Resolve VL strings from raw data by looking up each element in the global heap.
|
||
///
|
||
/// Reads the first `num_elements` elements of `raw`. Strings end at their
|
||
/// first NUL and invalid UTF-8 is replaced, as in [`VlResolver::strings`].
|
||
pub fn read_vl_strings(
|
||
file_data: &[u8],
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<String>, FormatError> {
|
||
let raw = first_elements(raw_data, num_elements, offset_size)?;
|
||
VlResolver::new(file_data, offset_size, length_size).strings(raw)
|
||
}
|
||
|
||
/// The first `num_elements` elements of `raw`, or an error if it is shorter.
|
||
fn first_elements(raw: &[u8], num_elements: u64, offset_size: u8) -> Result<&[u8], FormatError> {
|
||
let total = usize::try_from(num_elements)
|
||
.ok()
|
||
.and_then(|n| n.checked_mul(element_size(offset_size)))
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: raw.len(),
|
||
})?;
|
||
raw.get(..total).ok_or(FormatError::UnexpectedEof {
|
||
expected: total,
|
||
available: raw.len(),
|
||
})
|
||
}
|
||
|
||
/// Resolve VL sequences from raw data, returning each element's bytes.
|
||
///
|
||
/// Each element is the sequence's full encoding — element count × base type
|
||
/// size bytes, in the base type's byte order — so a sequence of `i32` yields
|
||
/// four bytes per value. Decode it with the base type (e.g.
|
||
/// [`crate::data_read::read_as_i64`]). This does not know the base type, so
|
||
/// it returns each heap object whole; [`VlResolver::sequences`] also checks
|
||
/// the object's size against the element's length.
|
||
pub fn read_vl_bytes(
|
||
file_data: &[u8],
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
let refs = parse_vl_references(raw_data, num_elements, offset_size)?;
|
||
let mut resolver = VlResolver::new(file_data, offset_size, length_size);
|
||
let mut result = Vec::with_capacity(refs.len());
|
||
|
||
for vl in &refs {
|
||
// A heap address of 0 is a null element, as in VlResolver.
|
||
if vl.collection_address == 0 {
|
||
result.push(Vec::new());
|
||
continue;
|
||
}
|
||
// The heap object holds the whole sequence. `vl.length` counts
|
||
// elements, not bytes, so it is only the byte length when the base
|
||
// type is one byte wide.
|
||
let obj = resolver.object(vl)?;
|
||
result.push(obj.to_vec());
|
||
}
|
||
|
||
Ok(result)
|
||
}
|
||
|
||
impl<'a> VlResolver<'a> {
|
||
/// The heap object `vl` points to, whatever its size; its collection is
|
||
/// parsed on first use.
|
||
fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> {
|
||
let addr = vl.collection_address;
|
||
// libhdf5 writes a null element with address 0, never the undefined
|
||
// address, and fails to read one ("addr undefined") even when its
|
||
// length is 0; we returned an empty value.
|
||
if is_undefined_address(addr, self.offset_size) {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"variable-length element (length {}) has the undefined global heap address",
|
||
vl.length
|
||
)));
|
||
}
|
||
if !self.cache.contains_key(&addr) {
|
||
let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: self.file_data.len(),
|
||
})?;
|
||
let index =
|
||
GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?;
|
||
// parse_index checked that the collection lies in the file.
|
||
let end = offset + index.collection_size as usize;
|
||
self.check_overlap(offset, end)?;
|
||
let coll = CachedCollection::new(index);
|
||
if self.cached_bytes.saturating_add(coll.cost()) > self.budget {
|
||
self.cache.clear();
|
||
self.cached_bytes = 0;
|
||
}
|
||
self.cached_bytes += coll.cost();
|
||
self.cache.insert(addr, coll);
|
||
}
|
||
let (start, size) = self.cache[&addr].get(vl.object_index).ok_or(
|
||
FormatError::GlobalHeapObjectNotFound {
|
||
collection_address: addr,
|
||
index: vl.object_index as u16,
|
||
},
|
||
)?;
|
||
Ok(&self.file_data[start..start + size])
|
||
}
|
||
|
||
/// Record the collection at `start..end`, refusing one that overlaps a
|
||
/// collection already read. libhdf5 allocates each collection its own
|
||
/// block; overlapping ones only come from a crafted file, where they let
|
||
/// every byte be parsed again as the objects of each collection.
|
||
fn check_overlap(&mut self, start: usize, end: usize) -> Result<(), FormatError> {
|
||
if let Some(&known) = self.extents.get(&start) {
|
||
return if known == end {
|
||
Ok(())
|
||
} else {
|
||
Err(FormatError::VlDataError(format!(
|
||
"global heap collection at {start} changed size"
|
||
)))
|
||
};
|
||
}
|
||
let before = self.extents.range(..start).next_back();
|
||
let after = self.extents.range(start..).next();
|
||
let clash = match (before, after) {
|
||
(Some((&s, &e)), _) if e > start => Some(s),
|
||
(_, Some((&s, _))) if s < end => Some(s),
|
||
_ => None,
|
||
};
|
||
if let Some(other) = clash {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"global heap collection at {start} overlaps the one at {other}"
|
||
)));
|
||
}
|
||
self.extents.insert(start, end);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Build a global heap collection at given offset in a file buffer.
|
||
fn build_gcol_at(
|
||
file_data: &mut Vec<u8>,
|
||
offset: usize,
|
||
objects: &[(u16, &[u8])], // (index, data)
|
||
) {
|
||
let length_size = 8usize;
|
||
|
||
// Ensure file_data is large enough
|
||
let header_size = 8 + length_size;
|
||
let mut obj_total = 0usize;
|
||
for (_, data) in objects {
|
||
let padded = (data.len() + 7) & !7;
|
||
obj_total += 8 + length_size + padded;
|
||
}
|
||
obj_total += 2; // free space marker
|
||
let collection_size = header_size + obj_total;
|
||
let needed = offset + collection_size;
|
||
if file_data.len() < needed {
|
||
file_data.resize(needed, 0);
|
||
}
|
||
|
||
let mut pos = offset;
|
||
// Signature
|
||
file_data[pos..pos + 4].copy_from_slice(b"GCOL");
|
||
file_data[pos + 4] = 1; // version
|
||
// reserved(3) already 0
|
||
pos += 8;
|
||
file_data[pos..pos + 8].copy_from_slice(&(collection_size as u64).to_le_bytes());
|
||
pos += 8;
|
||
|
||
for (index, data) in objects {
|
||
file_data[pos..pos + 2].copy_from_slice(&index.to_le_bytes());
|
||
file_data[pos + 2..pos + 4].copy_from_slice(&1u16.to_le_bytes()); // ref_count
|
||
// reserved(4) already 0
|
||
pos += 8;
|
||
file_data[pos..pos + 8].copy_from_slice(&(data.len() as u64).to_le_bytes());
|
||
pos += 8;
|
||
file_data[pos..pos + data.len()].copy_from_slice(data);
|
||
let padded = (data.len() + 7) & !7;
|
||
pos += padded;
|
||
}
|
||
// free space marker
|
||
file_data[pos..pos + 2].copy_from_slice(&0u16.to_le_bytes());
|
||
}
|
||
|
||
/// Build VL reference raw data for given strings at a collection address.
|
||
fn build_vl_refs(
|
||
strings: &[&str],
|
||
collection_address: u64,
|
||
start_index: u16,
|
||
offset_size: u8,
|
||
) -> Vec<u8> {
|
||
let mut raw = Vec::new();
|
||
for (i, s) in strings.iter().enumerate() {
|
||
raw.extend_from_slice(&(s.len() as u32).to_le_bytes());
|
||
match offset_size {
|
||
4 => raw.extend_from_slice(&(collection_address as u32).to_le_bytes()),
|
||
8 => raw.extend_from_slice(&collection_address.to_le_bytes()),
|
||
_ => panic!("unsupported"),
|
||
}
|
||
raw.extend_from_slice(&(start_index as u32 + i as u32).to_le_bytes());
|
||
}
|
||
raw
|
||
}
|
||
|
||
#[test]
|
||
fn parse_vl_references_two_elements() {
|
||
let raw = build_vl_refs(&["hello", "world"], 0x1000, 1, 8);
|
||
let refs = parse_vl_references(&raw, 2, 8).unwrap();
|
||
assert_eq!(refs.len(), 2);
|
||
assert_eq!(refs[0].length, 5);
|
||
assert_eq!(refs[0].collection_address, 0x1000);
|
||
assert_eq!(refs[0].object_index, 1);
|
||
assert_eq!(refs[1].length, 5);
|
||
assert_eq!(refs[1].object_index, 2);
|
||
}
|
||
|
||
#[test]
|
||
fn read_vl_strings_from_heap() {
|
||
let gcol_offset = 256usize;
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, gcol_offset, &[(1, b"Alice"), (2, b"Bob")]);
|
||
|
||
let raw = build_vl_refs(&["Alice", "Bob"], gcol_offset as u64, 1, 8);
|
||
let strings = read_vl_strings(&file_data, &raw, 2, 8, 8).unwrap();
|
||
assert_eq!(strings, vec!["Alice", "Bob"]);
|
||
}
|
||
|
||
#[test]
|
||
fn an_undefined_heap_address_is_an_error_even_at_length_0() {
|
||
// libhdf5 fails the read ("addr undefined"); h5py and libhdf5 write
|
||
// a null element with address 0. We returned "".
|
||
let mut file_data = vec![0u8; 256];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"x")]);
|
||
for (os, undef) in [(8u8, u64::MAX), (4, 0xFFFF_FFFF)] {
|
||
for length in [0, 1] {
|
||
let mut raw = element(1, 64, 1, os);
|
||
raw.extend(element(length, undef, 1, os));
|
||
let mut r = VlResolver::new(&file_data, os, 8);
|
||
let e = r.string_bytes(&raw).unwrap_err().to_string();
|
||
assert!(e.contains("undefined"), "{e}");
|
||
assert!(r.sequences(&raw, 1).is_err());
|
||
assert!(r.string_element(&raw[raw.len() / 2..]).is_err());
|
||
let n = 2;
|
||
assert!(read_vl_strings(&file_data, &raw, n, os, 8).is_err());
|
||
assert!(read_vl_bytes(&file_data, &raw, n, os, 8).is_err());
|
||
// The defined element alone still reads.
|
||
assert_eq!(r.strings(&raw[..raw.len() / 2]).unwrap(), ["x"]);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn null_vl_element_zero_address() {
|
||
let mut raw = Vec::new();
|
||
raw.extend_from_slice(&0u32.to_le_bytes());
|
||
raw.extend_from_slice(&0u64.to_le_bytes());
|
||
raw.extend_from_slice(&0u32.to_le_bytes());
|
||
|
||
let file_data = vec![0u8; 16];
|
||
let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap();
|
||
assert_eq!(strings, vec![""]);
|
||
}
|
||
|
||
#[test]
|
||
fn read_vl_bytes_from_heap() {
|
||
let gcol_offset = 128usize;
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(
|
||
&mut file_data,
|
||
gcol_offset,
|
||
&[(1, &[0xDE, 0xAD]), (2, &[0xBE, 0xEF, 0xCA])],
|
||
);
|
||
|
||
let _raw = build_vl_refs(&["ab", "abc"], gcol_offset as u64, 1, 8);
|
||
// Fix lengths to match actual byte lengths
|
||
let mut raw_fixed = Vec::new();
|
||
raw_fixed.extend_from_slice(&2u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&(gcol_offset as u64).to_le_bytes());
|
||
raw_fixed.extend_from_slice(&1u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&3u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&(gcol_offset as u64).to_le_bytes());
|
||
raw_fixed.extend_from_slice(&2u32.to_le_bytes());
|
||
|
||
let bytes = read_vl_bytes(&file_data, &raw_fixed, 2, 8, 8).unwrap();
|
||
assert_eq!(bytes, vec![vec![0xDE, 0xAD], vec![0xBE, 0xEF, 0xCA]]);
|
||
}
|
||
|
||
fn element(length: u32, addr: u64, index: u32, offset_size: u8) -> Vec<u8> {
|
||
let mut raw = length.to_le_bytes().to_vec();
|
||
raw.extend_from_slice(&addr.to_le_bytes()[..offset_size as usize]);
|
||
raw.extend_from_slice(&index.to_le_bytes());
|
||
raw
|
||
}
|
||
|
||
#[test]
|
||
fn strings_end_at_the_first_nul() {
|
||
// libhdf5 hands each VL string over as a C string, so h5py sees
|
||
// "a\0b" as "a"; we used to return the NUL and what followed.
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"a\0b"), (2, b"cd")]);
|
||
let mut raw = element(3, 64, 1, 8);
|
||
raw.extend(element(2, 64, 2, 8));
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
assert_eq!(
|
||
r.string_bytes(&raw).unwrap(),
|
||
vec![b"a".to_vec(), b"cd".to_vec()]
|
||
);
|
||
assert_eq!(
|
||
read_vl_strings(&file_data, &raw, 2, 8, 8).unwrap(),
|
||
["a", "cd"]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn a_heap_object_of_the_wrong_size_is_an_error() {
|
||
// libhdf5: "Expected global heap object size does not match". We
|
||
// used to return the object cut to the element's length.
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"cdefgh"), (2, &[1, 0, 0, 0])]);
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
assert!(r.string_bytes(&element(3, 64, 1, 8)).is_err());
|
||
assert!(r.string_bytes(&element(9, 64, 1, 8)).is_err());
|
||
assert!(read_vl_strings(&file_data, &element(3, 64, 1, 8), 1, 8, 8).is_err());
|
||
// A sequence of one i32 is 4 bytes; of two, 8.
|
||
assert_eq!(
|
||
r.sequences(&element(1, 64, 2, 8), 4).unwrap(),
|
||
vec![vec![1, 0, 0, 0]]
|
||
);
|
||
assert!(r.sequences(&element(2, 64, 2, 8), 4).is_err());
|
||
assert!(r.sequences(&element(1, 64, 2, 8), 0).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn address_zero_is_null_whatever_the_length() {
|
||
// libhdf5 treats a heap address of 0 as a null element.
|
||
let file_data = vec![0u8; 64];
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
assert_eq!(
|
||
r.string_bytes(&element(5, 0, 1, 8)).unwrap(),
|
||
vec![Vec::<u8>::new()]
|
||
);
|
||
assert_eq!(
|
||
r.sequences(&element(5, 0, 1, 8), 4).unwrap(),
|
||
vec![Vec::<u8>::new()]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn four_byte_offsets_use_twelve_byte_elements() {
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"one"), (2, b""), (3, b"three")]);
|
||
let mut raw = element(3, 64, 1, 4);
|
||
raw.extend(element(0, 64, 2, 4));
|
||
raw.extend(element(5, 64, 3, 4));
|
||
assert_eq!(raw.len(), 36);
|
||
let mut r = VlResolver::new(&file_data, 4, 8);
|
||
assert_eq!(r.element_size(), 12);
|
||
assert_eq!(r.strings(&raw).unwrap(), ["one", "", "three"]);
|
||
// Not a whole number of elements.
|
||
assert!(r.strings(&raw[..30]).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn the_cache_stays_within_its_budget_and_rereads_what_it_dropped() {
|
||
// Twenty collections of three objects each; a budget that holds
|
||
// about two of them. Reading every element twice must still return
|
||
// the right strings after the cache is dropped.
|
||
let mut file_data = vec![0u8; 64];
|
||
let mut raw = Vec::new();
|
||
for c in 0..20u64 {
|
||
let at = file_data.len();
|
||
let names: Vec<String> = (0..3).map(|i| format!("c{c}o{i}")).collect();
|
||
let objs: Vec<(u16, &[u8])> = names
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(i, n)| (i as u16 + 1, n.as_bytes()))
|
||
.collect();
|
||
build_gcol_at(&mut file_data, at, &objs);
|
||
for (i, n) in names.iter().enumerate() {
|
||
raw.extend(element(n.len() as u32, at as u64, i as u32 + 1, 8));
|
||
}
|
||
}
|
||
raw.extend(raw.clone());
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
let one = CachedCollection {
|
||
objects: vec![(0, 0, 0); 3],
|
||
}
|
||
.cost();
|
||
r.budget = 2 * one + 1;
|
||
let want: Vec<String> = (0..2)
|
||
.flat_map(|_| (0..20).flat_map(|c| (0..3).map(move |i| format!("c{c}o{i}"))))
|
||
.collect();
|
||
for (k, chunk) in raw.chunks(16).enumerate() {
|
||
assert_eq!(r.strings(chunk).unwrap(), [want[k].clone()]);
|
||
assert!(r.cached_bytes <= r.budget);
|
||
assert!(r.cache.len() <= 2);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn element_size_is_checked_against_the_offset_size() {
|
||
assert!(check_element_size(16, 8).is_ok());
|
||
assert!(check_element_size(12, 4).is_ok());
|
||
assert!(check_element_size(16, 4).is_err());
|
||
assert!(check_element_size(524_304, 8).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn parse_vl_references_truncated_error() {
|
||
let raw = vec![0u8; 10]; // too short for 1 element with offset_size=8
|
||
let err = parse_vl_references(&raw, 1, 8).unwrap_err();
|
||
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
|
||
}
|
||
}
|