osobhandClaude Opus 5.5
41b7837d0a
VlResolver kept an owned copy of every object of every heap collection it parsed, for the whole read. Collections nested inside each other's object data, 32 bytes apart with each element pointing at a different one, made retained memory O(elements x file size): 1.58 GB for a 744 KB file (read_vl_strings did the same before VlResolver). Chaining every collection's objects into one shared run of tiny objects made parse time O(elements x objects) as well. libhdf5 refuses these files. - The cache records where each object lies (GlobalHeapCollection:: parse_index, new) instead of copying it, and is dropped past a 32 MiB budget. - A collection overlapping one already read is an error: libhdf5 gives every collection its own block, so only a crafted file has them. - parse and parse_index refuse a collection that runs past the end of the file and an object that runs past the end of its collection. tests/vl_heap_bounds.rs measures peak heap use with a counting allocator: 129 MB and 350 MB live before on its two crafted files (64 KB and 176 KB), 97 KB and 0.9 MB now. Conformance unchanged at 575 of 697. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
724 lines
27 KiB
Rust
724 lines
27 KiB
Rust
//! Variable-length data reading (VL strings & VL sequences).
|
||
//!
|
||
//! VL data elements in HDF5 store their values in the global heap.
|
||
//! The raw data for each element contains a global heap ID:
|
||
//! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`.
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec};
|
||
#[cfg(feature = "std")]
|
||
use std::collections::BTreeMap;
|
||
|
||
use crate::error::FormatError;
|
||
use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex};
|
||
|
||
/// A parsed variable-length element reference (global heap ID).
|
||
#[derive(Debug, Clone)]
|
||
pub struct VlElement {
|
||
/// Length of the VL data.
|
||
pub length: u32,
|
||
/// Address of the global heap collection containing the data.
|
||
pub collection_address: u64,
|
||
/// Index of the object within the collection.
|
||
pub object_index: u32,
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
match offset.checked_add(needed) {
|
||
Some(end) if end <= data.len() => Ok(()),
|
||
_ => Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
}),
|
||
}
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, offset_size: u8) -> Result<u64, FormatError> {
|
||
let s = offset_size as usize;
|
||
ensure_len(data, pos, s)?;
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match offset_size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(offset_size)),
|
||
})
|
||
}
|
||
|
||
/// Parse VL global heap references from raw attribute/dataset data.
|
||
pub fn parse_vl_references(
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
) -> Result<Vec<VlElement>, FormatError> {
|
||
let elem_size = 4 + offset_size as usize + 4; // length + address + index
|
||
let total =
|
||
(num_elements as usize)
|
||
.checked_mul(elem_size)
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: raw_data.len(),
|
||
})?;
|
||
if raw_data.len() < total {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: total,
|
||
available: raw_data.len(),
|
||
});
|
||
}
|
||
|
||
let mut elements = Vec::with_capacity(num_elements as usize);
|
||
let mut pos = 0;
|
||
|
||
for _ in 0..num_elements {
|
||
let length = u32::from_le_bytes([
|
||
raw_data[pos],
|
||
raw_data[pos + 1],
|
||
raw_data[pos + 2],
|
||
raw_data[pos + 3],
|
||
]);
|
||
pos += 4;
|
||
|
||
let collection_address = read_offset(raw_data, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
|
||
let object_index = u32::from_le_bytes([
|
||
raw_data[pos],
|
||
raw_data[pos + 1],
|
||
raw_data[pos + 2],
|
||
raw_data[pos + 3],
|
||
]);
|
||
pos += 4;
|
||
|
||
elements.push(VlElement {
|
||
length,
|
||
collection_address,
|
||
object_index,
|
||
});
|
||
}
|
||
|
||
Ok(elements)
|
||
}
|
||
|
||
/// Check if an address represents an undefined/null address.
|
||
fn is_undefined_address(addr: u64, offset_size: u8) -> bool {
|
||
match offset_size {
|
||
2 => addr == 0xFFFF,
|
||
4 => addr == 0xFFFF_FFFF,
|
||
8 => addr == 0xFFFF_FFFF_FFFF_FFFF,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
/// The size of one variable-length element in a file with `offset_size`-byte
|
||
/// addresses: a sequence length (4), a global heap collection address and an
|
||
/// object index (4). libhdf5 computes it this way rather than trusting the
|
||
/// datatype message (`H5T_set_loc`).
|
||
pub fn element_size(offset_size: u8) -> usize {
|
||
4 + offset_size as usize + 4
|
||
}
|
||
|
||
/// Refuse a variable-length datatype whose stored element size is not the
|
||
/// one this file's offset size implies. Its elements would be laid out with
|
||
/// a stride libhdf5 does not use, so every value after the first would be
|
||
/// read from the wrong place.
|
||
pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), FormatError> {
|
||
let expected = element_size(offset_size);
|
||
if stored_size as usize != expected {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"variable-length datatype stores {stored_size}-byte elements; a file with \
|
||
{offset_size}-byte offsets uses {expected}"
|
||
)));
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// A collection's objects, located in the file data but not copied:
|
||
/// `(index, offset, size)` of the first object with each index, sorted by
|
||
/// index.
|
||
struct CachedCollection {
|
||
objects: Vec<(u16, usize, usize)>,
|
||
}
|
||
|
||
impl CachedCollection {
|
||
fn new(index: GlobalHeapIndex) -> Self {
|
||
let mut objects: Vec<(u16, usize, usize)> = index
|
||
.objects
|
||
.iter()
|
||
.map(|o| (o.index, o.offset, o.size))
|
||
.collect();
|
||
// Stable, so the first object with a repeated index is kept.
|
||
objects.sort_by_key(|o| o.0);
|
||
objects.dedup_by_key(|o| o.0);
|
||
Self { objects }
|
||
}
|
||
|
||
/// What this entry costs to keep, in bytes (roughly).
|
||
fn cost(&self) -> usize {
|
||
64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>()
|
||
}
|
||
|
||
fn get(&self, index: u32) -> Option<(usize, usize)> {
|
||
let index = u16::try_from(index).ok()?;
|
||
let i = self.objects.binary_search_by_key(&index, |o| o.0).ok()?;
|
||
Some((self.objects[i].1, self.objects[i].2))
|
||
}
|
||
}
|
||
|
||
/// How many bytes of collection indexes a [`VlResolver`] keeps before it
|
||
/// drops them and starts again. Values are never copied into the cache, so
|
||
/// this bounds what a read retains however many collections it visits.
|
||
const CACHE_BUDGET: usize = 32 << 20;
|
||
|
||
/// Resolves variable-length elements against a file's global heap, parsing
|
||
/// each heap collection once however many elements point into it.
|
||
///
|
||
/// Values follow libhdf5: an element whose heap address is 0 is null (an
|
||
/// empty string or sequence), and an element whose heap object is not
|
||
/// exactly `length × base size` bytes is an error ("Expected global heap
|
||
/// object size does not match"), not a truncated or padded value.
|
||
///
|
||
/// Memory stays bounded on hostile files: the cache holds where each
|
||
/// object lies, not a copy of it, up to a fixed budget; and collections
|
||
/// that overlap one another are refused (libhdf5 never writes them), so a
|
||
/// file cannot make the resolver parse the same bytes as the objects of
|
||
/// many collections.
|
||
pub struct VlResolver<'a> {
|
||
file_data: &'a [u8],
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
cache: BTreeMap<u64, CachedCollection>,
|
||
cached_bytes: usize,
|
||
budget: usize,
|
||
/// Start → end of every collection parsed so far (kept when the cache
|
||
/// is dropped, to check overlaps).
|
||
extents: BTreeMap<usize, usize>,
|
||
}
|
||
|
||
impl<'a> VlResolver<'a> {
|
||
/// A resolver over `file_data` (the file from its superblock on), with
|
||
/// the superblock's offset and length sizes.
|
||
pub fn new(file_data: &'a [u8], offset_size: u8, length_size: u8) -> Self {
|
||
Self {
|
||
file_data,
|
||
offset_size,
|
||
length_size,
|
||
cache: BTreeMap::new(),
|
||
cached_bytes: 0,
|
||
budget: CACHE_BUDGET,
|
||
extents: BTreeMap::new(),
|
||
}
|
||
}
|
||
|
||
/// The size of one element in this file (see [`element_size`]).
|
||
pub fn element_size(&self) -> usize {
|
||
element_size(self.offset_size)
|
||
}
|
||
|
||
/// Split `raw` into elements; its length must be a whole number of them.
|
||
fn elements(&self, raw: &[u8]) -> Result<Vec<VlElement>, FormatError> {
|
||
let size = self.element_size();
|
||
if !raw.len().is_multiple_of(size) {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"{} bytes is not a whole number of {size}-byte variable-length elements",
|
||
raw.len()
|
||
)));
|
||
}
|
||
parse_vl_references(raw, (raw.len() / size) as u64, self.offset_size)
|
||
}
|
||
|
||
/// The bytes of one element: `length × base_size` bytes from the heap,
|
||
/// or `None` for a null element.
|
||
fn resolve(
|
||
&mut self,
|
||
vl: &VlElement,
|
||
base_size: usize,
|
||
) -> Result<Option<&'a [u8]>, FormatError> {
|
||
let addr = vl.collection_address;
|
||
if addr == 0 {
|
||
return Ok(None);
|
||
}
|
||
if vl.length == 0 && is_undefined_address(addr, self.offset_size) {
|
||
return Ok(Some(&[]));
|
||
}
|
||
let data = self.object(vl)?;
|
||
let expected = (vl.length as usize)
|
||
.checked_mul(base_size)
|
||
.ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?;
|
||
if data.len() != expected {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"global heap object {} in the collection at {addr} holds {} bytes; the element \
|
||
says {} × {base_size}",
|
||
vl.object_index,
|
||
data.len(),
|
||
vl.length
|
||
)));
|
||
}
|
||
Ok(Some(data))
|
||
}
|
||
|
||
/// One element (the first [`element_size`](Self::element_size) bytes of
|
||
/// `elem`) of a variable-length sequence whose base type is `base_size`
|
||
/// bytes: its `length × base_size` bytes, or `None` for a null element
|
||
/// (heap address 0).
|
||
pub fn element(
|
||
&mut self,
|
||
elem: &[u8],
|
||
base_size: usize,
|
||
) -> Result<Option<&'a [u8]>, FormatError> {
|
||
let vl = parse_vl_references(elem, 1, self.offset_size)?;
|
||
self.resolve(&vl[0], base_size)
|
||
}
|
||
|
||
/// One variable-length string element: its bytes up to the first NUL,
|
||
/// or `None` for a null element (h5dump prints it as `NULL`, h5py
|
||
/// returns it as empty).
|
||
pub fn string_element(&mut self, elem: &[u8]) -> Result<Option<&'a [u8]>, FormatError> {
|
||
Ok(self.element(elem, 1)?.map(cut_at_nul))
|
||
}
|
||
|
||
/// The strings of the variable-length string elements in `raw`, as
|
||
/// bytes. A string ends at its first NUL, as libhdf5 returns it (it
|
||
/// converts each to a C string); a null element is empty.
|
||
pub fn string_bytes(&mut self, raw: &[u8]) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
self.elements(raw)?
|
||
.iter()
|
||
.map(|vl| Ok(self.resolve(vl, 1)?.map(cut_at_nul).unwrap_or(&[]).to_vec()))
|
||
.collect()
|
||
}
|
||
|
||
/// The strings of the variable-length string elements in `raw`, decoded
|
||
/// as UTF-8 with invalid sequences replaced by U+FFFD (see
|
||
/// [`string_bytes`](Self::string_bytes) for the exact bytes).
|
||
pub fn strings(&mut self, raw: &[u8]) -> Result<Vec<String>, FormatError> {
|
||
Ok(self
|
||
.string_bytes(raw)?
|
||
.into_iter()
|
||
.map(|b| match String::from_utf8(b) {
|
||
Ok(s) => s,
|
||
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
||
})
|
||
.collect())
|
||
}
|
||
|
||
/// The sequences of the variable-length sequence elements in `raw`, each
|
||
/// as its `length × base_size` bytes in the base type's encoding.
|
||
pub fn sequences(&mut self, raw: &[u8], base_size: usize) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
if base_size == 0 {
|
||
return Err(FormatError::VlDataError(
|
||
"variable-length sequence of a zero-size base type".into(),
|
||
));
|
||
}
|
||
self.elements(raw)?
|
||
.iter()
|
||
.map(|vl| Ok(self.resolve(vl, base_size)?.unwrap_or(&[]).to_vec()))
|
||
.collect()
|
||
}
|
||
}
|
||
|
||
/// A string's bytes up to its first NUL.
|
||
fn cut_at_nul(s: &[u8]) -> &[u8] {
|
||
&s[..s.iter().position(|&b| b == 0).unwrap_or(s.len())]
|
||
}
|
||
|
||
/// Resolve VL strings from raw data by looking up each element in the global heap.
|
||
///
|
||
/// Reads the first `num_elements` elements of `raw`. Strings end at their
|
||
/// first NUL and invalid UTF-8 is replaced, as in [`VlResolver::strings`].
|
||
pub fn read_vl_strings(
|
||
file_data: &[u8],
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<String>, FormatError> {
|
||
let raw = first_elements(raw_data, num_elements, offset_size)?;
|
||
VlResolver::new(file_data, offset_size, length_size).strings(raw)
|
||
}
|
||
|
||
/// The first `num_elements` elements of `raw`, or an error if it is shorter.
|
||
fn first_elements(raw: &[u8], num_elements: u64, offset_size: u8) -> Result<&[u8], FormatError> {
|
||
let total = usize::try_from(num_elements)
|
||
.ok()
|
||
.and_then(|n| n.checked_mul(element_size(offset_size)))
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: raw.len(),
|
||
})?;
|
||
raw.get(..total).ok_or(FormatError::UnexpectedEof {
|
||
expected: total,
|
||
available: raw.len(),
|
||
})
|
||
}
|
||
|
||
/// Resolve VL sequences from raw data, returning each element's bytes.
|
||
///
|
||
/// Each element is the sequence's full encoding — element count × base type
|
||
/// size bytes, in the base type's byte order — so a sequence of `i32` yields
|
||
/// four bytes per value. Decode it with the base type (e.g.
|
||
/// [`crate::data_read::read_as_i64`]). This does not know the base type, so
|
||
/// it returns each heap object whole; [`VlResolver::sequences`] also checks
|
||
/// the object's size against the element's length.
|
||
pub fn read_vl_bytes(
|
||
file_data: &[u8],
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
let refs = parse_vl_references(raw_data, num_elements, offset_size)?;
|
||
let mut resolver = VlResolver::new(file_data, offset_size, length_size);
|
||
let mut result = Vec::with_capacity(refs.len());
|
||
|
||
for vl in &refs {
|
||
if vl.length == 0
|
||
&& (is_undefined_address(vl.collection_address, offset_size)
|
||
|| vl.collection_address == 0)
|
||
{
|
||
result.push(Vec::new());
|
||
continue;
|
||
}
|
||
// The heap object holds the whole sequence. `vl.length` counts
|
||
// elements, not bytes, so it is only the byte length when the base
|
||
// type is one byte wide.
|
||
let obj = resolver.object(vl)?;
|
||
result.push(obj.to_vec());
|
||
}
|
||
|
||
Ok(result)
|
||
}
|
||
|
||
impl<'a> VlResolver<'a> {
|
||
/// The heap object `vl` points to, whatever its size; its collection is
|
||
/// parsed on first use.
|
||
fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> {
|
||
let addr = vl.collection_address;
|
||
if !self.cache.contains_key(&addr) {
|
||
let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: self.file_data.len(),
|
||
})?;
|
||
let index =
|
||
GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?;
|
||
// parse_index checked that the collection lies in the file.
|
||
let end = offset + index.collection_size as usize;
|
||
self.check_overlap(offset, end)?;
|
||
let coll = CachedCollection::new(index);
|
||
if self.cached_bytes.saturating_add(coll.cost()) > self.budget {
|
||
self.cache.clear();
|
||
self.cached_bytes = 0;
|
||
}
|
||
self.cached_bytes += coll.cost();
|
||
self.cache.insert(addr, coll);
|
||
}
|
||
let (start, size) = self.cache[&addr].get(vl.object_index).ok_or(
|
||
FormatError::GlobalHeapObjectNotFound {
|
||
collection_address: addr,
|
||
index: vl.object_index as u16,
|
||
},
|
||
)?;
|
||
Ok(&self.file_data[start..start + size])
|
||
}
|
||
|
||
/// Record the collection at `start..end`, refusing one that overlaps a
|
||
/// collection already read. libhdf5 allocates each collection its own
|
||
/// block; overlapping ones only come from a crafted file, where they let
|
||
/// every byte be parsed again as the objects of each collection.
|
||
fn check_overlap(&mut self, start: usize, end: usize) -> Result<(), FormatError> {
|
||
if let Some(&known) = self.extents.get(&start) {
|
||
return if known == end {
|
||
Ok(())
|
||
} else {
|
||
Err(FormatError::VlDataError(format!(
|
||
"global heap collection at {start} changed size"
|
||
)))
|
||
};
|
||
}
|
||
let before = self.extents.range(..start).next_back();
|
||
let after = self.extents.range(start..).next();
|
||
let clash = match (before, after) {
|
||
(Some((&s, &e)), _) if e > start => Some(s),
|
||
(_, Some((&s, _))) if s < end => Some(s),
|
||
_ => None,
|
||
};
|
||
if let Some(other) = clash {
|
||
return Err(FormatError::VlDataError(format!(
|
||
"global heap collection at {start} overlaps the one at {other}"
|
||
)));
|
||
}
|
||
self.extents.insert(start, end);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Build a global heap collection at given offset in a file buffer.
|
||
fn build_gcol_at(
|
||
file_data: &mut Vec<u8>,
|
||
offset: usize,
|
||
objects: &[(u16, &[u8])], // (index, data)
|
||
) {
|
||
let length_size = 8usize;
|
||
|
||
// Ensure file_data is large enough
|
||
let header_size = 8 + length_size;
|
||
let mut obj_total = 0usize;
|
||
for (_, data) in objects {
|
||
let padded = (data.len() + 7) & !7;
|
||
obj_total += 8 + length_size + padded;
|
||
}
|
||
obj_total += 2; // free space marker
|
||
let collection_size = header_size + obj_total;
|
||
let needed = offset + collection_size;
|
||
if file_data.len() < needed {
|
||
file_data.resize(needed, 0);
|
||
}
|
||
|
||
let mut pos = offset;
|
||
// Signature
|
||
file_data[pos..pos + 4].copy_from_slice(b"GCOL");
|
||
file_data[pos + 4] = 1; // version
|
||
// reserved(3) already 0
|
||
pos += 8;
|
||
file_data[pos..pos + 8].copy_from_slice(&(collection_size as u64).to_le_bytes());
|
||
pos += 8;
|
||
|
||
for (index, data) in objects {
|
||
file_data[pos..pos + 2].copy_from_slice(&index.to_le_bytes());
|
||
file_data[pos + 2..pos + 4].copy_from_slice(&1u16.to_le_bytes()); // ref_count
|
||
// reserved(4) already 0
|
||
pos += 8;
|
||
file_data[pos..pos + 8].copy_from_slice(&(data.len() as u64).to_le_bytes());
|
||
pos += 8;
|
||
file_data[pos..pos + data.len()].copy_from_slice(data);
|
||
let padded = (data.len() + 7) & !7;
|
||
pos += padded;
|
||
}
|
||
// free space marker
|
||
file_data[pos..pos + 2].copy_from_slice(&0u16.to_le_bytes());
|
||
}
|
||
|
||
/// Build VL reference raw data for given strings at a collection address.
|
||
fn build_vl_refs(
|
||
strings: &[&str],
|
||
collection_address: u64,
|
||
start_index: u16,
|
||
offset_size: u8,
|
||
) -> Vec<u8> {
|
||
let mut raw = Vec::new();
|
||
for (i, s) in strings.iter().enumerate() {
|
||
raw.extend_from_slice(&(s.len() as u32).to_le_bytes());
|
||
match offset_size {
|
||
4 => raw.extend_from_slice(&(collection_address as u32).to_le_bytes()),
|
||
8 => raw.extend_from_slice(&collection_address.to_le_bytes()),
|
||
_ => panic!("unsupported"),
|
||
}
|
||
raw.extend_from_slice(&(start_index as u32 + i as u32).to_le_bytes());
|
||
}
|
||
raw
|
||
}
|
||
|
||
#[test]
|
||
fn parse_vl_references_two_elements() {
|
||
let raw = build_vl_refs(&["hello", "world"], 0x1000, 1, 8);
|
||
let refs = parse_vl_references(&raw, 2, 8).unwrap();
|
||
assert_eq!(refs.len(), 2);
|
||
assert_eq!(refs[0].length, 5);
|
||
assert_eq!(refs[0].collection_address, 0x1000);
|
||
assert_eq!(refs[0].object_index, 1);
|
||
assert_eq!(refs[1].length, 5);
|
||
assert_eq!(refs[1].object_index, 2);
|
||
}
|
||
|
||
#[test]
|
||
fn read_vl_strings_from_heap() {
|
||
let gcol_offset = 256usize;
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, gcol_offset, &[(1, b"Alice"), (2, b"Bob")]);
|
||
|
||
let raw = build_vl_refs(&["Alice", "Bob"], gcol_offset as u64, 1, 8);
|
||
let strings = read_vl_strings(&file_data, &raw, 2, 8, 8).unwrap();
|
||
assert_eq!(strings, vec!["Alice", "Bob"]);
|
||
}
|
||
|
||
#[test]
|
||
fn null_vl_element_empty_string() {
|
||
// length=0, address=undefined
|
||
let mut raw = Vec::new();
|
||
raw.extend_from_slice(&0u32.to_le_bytes()); // length=0
|
||
raw.extend_from_slice(&u64::MAX.to_le_bytes()); // undefined address
|
||
raw.extend_from_slice(&0u32.to_le_bytes()); // index
|
||
|
||
let file_data = vec![0u8; 16];
|
||
let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap();
|
||
assert_eq!(strings, vec![""]);
|
||
}
|
||
|
||
#[test]
|
||
fn null_vl_element_zero_address() {
|
||
let mut raw = Vec::new();
|
||
raw.extend_from_slice(&0u32.to_le_bytes());
|
||
raw.extend_from_slice(&0u64.to_le_bytes());
|
||
raw.extend_from_slice(&0u32.to_le_bytes());
|
||
|
||
let file_data = vec![0u8; 16];
|
||
let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap();
|
||
assert_eq!(strings, vec![""]);
|
||
}
|
||
|
||
#[test]
|
||
fn read_vl_bytes_from_heap() {
|
||
let gcol_offset = 128usize;
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(
|
||
&mut file_data,
|
||
gcol_offset,
|
||
&[(1, &[0xDE, 0xAD]), (2, &[0xBE, 0xEF, 0xCA])],
|
||
);
|
||
|
||
let _raw = build_vl_refs(&["ab", "abc"], gcol_offset as u64, 1, 8);
|
||
// Fix lengths to match actual byte lengths
|
||
let mut raw_fixed = Vec::new();
|
||
raw_fixed.extend_from_slice(&2u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&(gcol_offset as u64).to_le_bytes());
|
||
raw_fixed.extend_from_slice(&1u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&3u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&(gcol_offset as u64).to_le_bytes());
|
||
raw_fixed.extend_from_slice(&2u32.to_le_bytes());
|
||
|
||
let bytes = read_vl_bytes(&file_data, &raw_fixed, 2, 8, 8).unwrap();
|
||
assert_eq!(bytes, vec![vec![0xDE, 0xAD], vec![0xBE, 0xEF, 0xCA]]);
|
||
}
|
||
|
||
fn element(length: u32, addr: u64, index: u32, offset_size: u8) -> Vec<u8> {
|
||
let mut raw = length.to_le_bytes().to_vec();
|
||
raw.extend_from_slice(&addr.to_le_bytes()[..offset_size as usize]);
|
||
raw.extend_from_slice(&index.to_le_bytes());
|
||
raw
|
||
}
|
||
|
||
#[test]
|
||
fn strings_end_at_the_first_nul() {
|
||
// libhdf5 hands each VL string over as a C string, so h5py sees
|
||
// "a\0b" as "a"; we used to return the NUL and what followed.
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"a\0b"), (2, b"cd")]);
|
||
let mut raw = element(3, 64, 1, 8);
|
||
raw.extend(element(2, 64, 2, 8));
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
assert_eq!(
|
||
r.string_bytes(&raw).unwrap(),
|
||
vec![b"a".to_vec(), b"cd".to_vec()]
|
||
);
|
||
assert_eq!(
|
||
read_vl_strings(&file_data, &raw, 2, 8, 8).unwrap(),
|
||
["a", "cd"]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn a_heap_object_of_the_wrong_size_is_an_error() {
|
||
// libhdf5: "Expected global heap object size does not match". We
|
||
// used to return the object cut to the element's length.
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"cdefgh"), (2, &[1, 0, 0, 0])]);
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
assert!(r.string_bytes(&element(3, 64, 1, 8)).is_err());
|
||
assert!(r.string_bytes(&element(9, 64, 1, 8)).is_err());
|
||
assert!(read_vl_strings(&file_data, &element(3, 64, 1, 8), 1, 8, 8).is_err());
|
||
// A sequence of one i32 is 4 bytes; of two, 8.
|
||
assert_eq!(
|
||
r.sequences(&element(1, 64, 2, 8), 4).unwrap(),
|
||
vec![vec![1, 0, 0, 0]]
|
||
);
|
||
assert!(r.sequences(&element(2, 64, 2, 8), 4).is_err());
|
||
assert!(r.sequences(&element(1, 64, 2, 8), 0).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn address_zero_is_null_whatever_the_length() {
|
||
// libhdf5 treats a heap address of 0 as a null element.
|
||
let file_data = vec![0u8; 64];
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
assert_eq!(
|
||
r.string_bytes(&element(5, 0, 1, 8)).unwrap(),
|
||
vec![Vec::<u8>::new()]
|
||
);
|
||
assert_eq!(
|
||
r.sequences(&element(5, 0, 1, 8), 4).unwrap(),
|
||
vec![Vec::<u8>::new()]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn four_byte_offsets_use_twelve_byte_elements() {
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, 64, &[(1, b"one"), (2, b""), (3, b"three")]);
|
||
let mut raw = element(3, 64, 1, 4);
|
||
raw.extend(element(0, 64, 2, 4));
|
||
raw.extend(element(5, 64, 3, 4));
|
||
assert_eq!(raw.len(), 36);
|
||
let mut r = VlResolver::new(&file_data, 4, 8);
|
||
assert_eq!(r.element_size(), 12);
|
||
assert_eq!(r.strings(&raw).unwrap(), ["one", "", "three"]);
|
||
// Not a whole number of elements.
|
||
assert!(r.strings(&raw[..30]).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn the_cache_stays_within_its_budget_and_rereads_what_it_dropped() {
|
||
// Twenty collections of three objects each; a budget that holds
|
||
// about two of them. Reading every element twice must still return
|
||
// the right strings after the cache is dropped.
|
||
let mut file_data = vec![0u8; 64];
|
||
let mut raw = Vec::new();
|
||
for c in 0..20u64 {
|
||
let at = file_data.len();
|
||
let names: Vec<String> = (0..3).map(|i| format!("c{c}o{i}")).collect();
|
||
let objs: Vec<(u16, &[u8])> = names
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(i, n)| (i as u16 + 1, n.as_bytes()))
|
||
.collect();
|
||
build_gcol_at(&mut file_data, at, &objs);
|
||
for (i, n) in names.iter().enumerate() {
|
||
raw.extend(element(n.len() as u32, at as u64, i as u32 + 1, 8));
|
||
}
|
||
}
|
||
raw.extend(raw.clone());
|
||
let mut r = VlResolver::new(&file_data, 8, 8);
|
||
let one = CachedCollection {
|
||
objects: vec![(0, 0, 0); 3],
|
||
}
|
||
.cost();
|
||
r.budget = 2 * one + 1;
|
||
let want: Vec<String> = (0..2)
|
||
.flat_map(|_| (0..20).flat_map(|c| (0..3).map(move |i| format!("c{c}o{i}"))))
|
||
.collect();
|
||
for (k, chunk) in raw.chunks(16).enumerate() {
|
||
assert_eq!(r.strings(chunk).unwrap(), [want[k].clone()]);
|
||
assert!(r.cached_bytes <= r.budget);
|
||
assert!(r.cache.len() <= 2);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn element_size_is_checked_against_the_offset_size() {
|
||
assert!(check_element_size(16, 8).is_ok());
|
||
assert!(check_element_size(12, 4).is_ok());
|
||
assert!(check_element_size(16, 4).is_err());
|
||
assert!(check_element_size(524_304, 8).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn parse_vl_references_truncated_error() {
|
||
let raw = vec![0u8; 10]; // too short for 1 element with offset_size=8
|
||
let err = parse_vl_references(&raw, 1, 8).unwrap_err();
|
||
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
|
||
}
|
||
}
|