osobhandClaude Opus 5.5
2f252df084
read_vl_bytes cut each element to the reference's length field, which counts sequence elements, not bytes: a VL int32 [1, 2, 3] came back as 3 bytes. Return the whole global-heap object, which is element count x base size bytes. No in-tree caller depended on the old behaviour. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
343 lines
12 KiB
Rust
343 lines
12 KiB
Rust
//! Variable-length data reading (VL strings & VL sequences).
|
||
//!
|
||
//! VL data elements in HDF5 store their values in the global heap.
|
||
//! The raw data for each element contains a global heap ID:
|
||
//! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`.
|
||
|
||
#[cfg(not(feature = "std"))]
|
||
use alloc::{string::String, vec::Vec};
|
||
|
||
use crate::error::FormatError;
|
||
use crate::global_heap::GlobalHeapCollection;
|
||
|
||
/// A parsed variable-length element reference (global heap ID).
|
||
#[derive(Debug, Clone)]
|
||
pub struct VlElement {
|
||
/// Length of the VL data.
|
||
pub length: u32,
|
||
/// Address of the global heap collection containing the data.
|
||
pub collection_address: u64,
|
||
/// Index of the object within the collection.
|
||
pub object_index: u32,
|
||
}
|
||
|
||
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
|
||
match offset.checked_add(needed) {
|
||
Some(end) if end <= data.len() => Ok(()),
|
||
_ => Err(FormatError::UnexpectedEof {
|
||
expected: offset.saturating_add(needed),
|
||
available: data.len(),
|
||
}),
|
||
}
|
||
}
|
||
|
||
fn read_offset(data: &[u8], pos: usize, offset_size: u8) -> Result<u64, FormatError> {
|
||
let s = offset_size as usize;
|
||
ensure_len(data, pos, s)?;
|
||
let slice = &data[pos..pos + s];
|
||
Ok(match offset_size {
|
||
2 => u16::from_le_bytes([slice[0], slice[1]]) as u64,
|
||
4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64,
|
||
8 => u64::from_le_bytes([
|
||
slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7],
|
||
]),
|
||
_ => return Err(FormatError::InvalidOffsetSize(offset_size)),
|
||
})
|
||
}
|
||
|
||
/// Parse VL global heap references from raw attribute/dataset data.
|
||
pub fn parse_vl_references(
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
) -> Result<Vec<VlElement>, FormatError> {
|
||
let elem_size = 4 + offset_size as usize + 4; // length + address + index
|
||
let total =
|
||
(num_elements as usize)
|
||
.checked_mul(elem_size)
|
||
.ok_or(FormatError::UnexpectedEof {
|
||
expected: usize::MAX,
|
||
available: raw_data.len(),
|
||
})?;
|
||
if raw_data.len() < total {
|
||
return Err(FormatError::UnexpectedEof {
|
||
expected: total,
|
||
available: raw_data.len(),
|
||
});
|
||
}
|
||
|
||
let mut elements = Vec::with_capacity(num_elements as usize);
|
||
let mut pos = 0;
|
||
|
||
for _ in 0..num_elements {
|
||
let length = u32::from_le_bytes([
|
||
raw_data[pos],
|
||
raw_data[pos + 1],
|
||
raw_data[pos + 2],
|
||
raw_data[pos + 3],
|
||
]);
|
||
pos += 4;
|
||
|
||
let collection_address = read_offset(raw_data, pos, offset_size)?;
|
||
pos += offset_size as usize;
|
||
|
||
let object_index = u32::from_le_bytes([
|
||
raw_data[pos],
|
||
raw_data[pos + 1],
|
||
raw_data[pos + 2],
|
||
raw_data[pos + 3],
|
||
]);
|
||
pos += 4;
|
||
|
||
elements.push(VlElement {
|
||
length,
|
||
collection_address,
|
||
object_index,
|
||
});
|
||
}
|
||
|
||
Ok(elements)
|
||
}
|
||
|
||
/// Check if an address represents an undefined/null address.
|
||
fn is_undefined_address(addr: u64, offset_size: u8) -> bool {
|
||
match offset_size {
|
||
2 => addr == 0xFFFF,
|
||
4 => addr == 0xFFFF_FFFF,
|
||
8 => addr == 0xFFFF_FFFF_FFFF_FFFF,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
/// Resolve VL strings from raw data by looking up each element in the global heap.
|
||
pub fn read_vl_strings(
|
||
file_data: &[u8],
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<String>, FormatError> {
|
||
let refs = parse_vl_references(raw_data, num_elements, offset_size)?;
|
||
let mut result = Vec::with_capacity(refs.len());
|
||
|
||
for vl in &refs {
|
||
if vl.length == 0 && is_undefined_address(vl.collection_address, offset_size) {
|
||
result.push(String::new());
|
||
continue;
|
||
}
|
||
if vl.length == 0 && vl.collection_address == 0 {
|
||
result.push(String::new());
|
||
continue;
|
||
}
|
||
|
||
let coll =
|
||
GlobalHeapCollection::parse(file_data, vl.collection_address as usize, length_size)?;
|
||
let obj = coll.get_object(vl.object_index as u16).ok_or(
|
||
FormatError::GlobalHeapObjectNotFound {
|
||
collection_address: vl.collection_address,
|
||
index: vl.object_index as u16,
|
||
},
|
||
)?;
|
||
|
||
// The object data is the raw string bytes
|
||
let len = (vl.length as usize).min(obj.data.len());
|
||
let s = String::from_utf8_lossy(&obj.data[..len]).into_owned();
|
||
result.push(s);
|
||
}
|
||
|
||
Ok(result)
|
||
}
|
||
|
||
/// Resolve VL sequences from raw data, returning each element's bytes.
|
||
///
|
||
/// Each element is the sequence's full encoding — element count × base type
|
||
/// size bytes, in the base type's byte order — so a sequence of `i32` yields
|
||
/// four bytes per value. Decode it with the base type (e.g.
|
||
/// [`crate::data_read::read_as_i64`]).
|
||
pub fn read_vl_bytes(
|
||
file_data: &[u8],
|
||
raw_data: &[u8],
|
||
num_elements: u64,
|
||
offset_size: u8,
|
||
length_size: u8,
|
||
) -> Result<Vec<Vec<u8>>, FormatError> {
|
||
let refs = parse_vl_references(raw_data, num_elements, offset_size)?;
|
||
let mut result = Vec::with_capacity(refs.len());
|
||
|
||
for vl in &refs {
|
||
if vl.length == 0
|
||
&& (is_undefined_address(vl.collection_address, offset_size)
|
||
|| vl.collection_address == 0)
|
||
{
|
||
result.push(Vec::new());
|
||
continue;
|
||
}
|
||
|
||
let coll =
|
||
GlobalHeapCollection::parse(file_data, vl.collection_address as usize, length_size)?;
|
||
let obj = coll.get_object(vl.object_index as u16).ok_or(
|
||
FormatError::GlobalHeapObjectNotFound {
|
||
collection_address: vl.collection_address,
|
||
index: vl.object_index as u16,
|
||
},
|
||
)?;
|
||
|
||
// The heap object holds the whole sequence. `vl.length` counts
|
||
// elements, not bytes, so it is only the byte length when the base
|
||
// type is one byte wide.
|
||
result.push(obj.data.clone());
|
||
}
|
||
|
||
Ok(result)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
/// Build a global heap collection at given offset in a file buffer.
|
||
fn build_gcol_at(
|
||
file_data: &mut Vec<u8>,
|
||
offset: usize,
|
||
objects: &[(u16, &[u8])], // (index, data)
|
||
) {
|
||
let length_size = 8usize;
|
||
|
||
// Ensure file_data is large enough
|
||
let header_size = 8 + length_size;
|
||
let mut obj_total = 0usize;
|
||
for (_, data) in objects {
|
||
let padded = (data.len() + 7) & !7;
|
||
obj_total += 8 + length_size + padded;
|
||
}
|
||
obj_total += 2; // free space marker
|
||
let collection_size = header_size + obj_total;
|
||
let needed = offset + collection_size;
|
||
if file_data.len() < needed {
|
||
file_data.resize(needed, 0);
|
||
}
|
||
|
||
let mut pos = offset;
|
||
// Signature
|
||
file_data[pos..pos + 4].copy_from_slice(b"GCOL");
|
||
file_data[pos + 4] = 1; // version
|
||
// reserved(3) already 0
|
||
pos += 8;
|
||
file_data[pos..pos + 8].copy_from_slice(&(collection_size as u64).to_le_bytes());
|
||
pos += 8;
|
||
|
||
for (index, data) in objects {
|
||
file_data[pos..pos + 2].copy_from_slice(&index.to_le_bytes());
|
||
file_data[pos + 2..pos + 4].copy_from_slice(&1u16.to_le_bytes()); // ref_count
|
||
// reserved(4) already 0
|
||
pos += 8;
|
||
file_data[pos..pos + 8].copy_from_slice(&(data.len() as u64).to_le_bytes());
|
||
pos += 8;
|
||
file_data[pos..pos + data.len()].copy_from_slice(data);
|
||
let padded = (data.len() + 7) & !7;
|
||
pos += padded;
|
||
}
|
||
// free space marker
|
||
file_data[pos..pos + 2].copy_from_slice(&0u16.to_le_bytes());
|
||
}
|
||
|
||
/// Build VL reference raw data for given strings at a collection address.
|
||
fn build_vl_refs(
|
||
strings: &[&str],
|
||
collection_address: u64,
|
||
start_index: u16,
|
||
offset_size: u8,
|
||
) -> Vec<u8> {
|
||
let mut raw = Vec::new();
|
||
for (i, s) in strings.iter().enumerate() {
|
||
raw.extend_from_slice(&(s.len() as u32).to_le_bytes());
|
||
match offset_size {
|
||
4 => raw.extend_from_slice(&(collection_address as u32).to_le_bytes()),
|
||
8 => raw.extend_from_slice(&collection_address.to_le_bytes()),
|
||
_ => panic!("unsupported"),
|
||
}
|
||
raw.extend_from_slice(&(start_index as u32 + i as u32).to_le_bytes());
|
||
}
|
||
raw
|
||
}
|
||
|
||
#[test]
|
||
fn parse_vl_references_two_elements() {
|
||
let raw = build_vl_refs(&["hello", "world"], 0x1000, 1, 8);
|
||
let refs = parse_vl_references(&raw, 2, 8).unwrap();
|
||
assert_eq!(refs.len(), 2);
|
||
assert_eq!(refs[0].length, 5);
|
||
assert_eq!(refs[0].collection_address, 0x1000);
|
||
assert_eq!(refs[0].object_index, 1);
|
||
assert_eq!(refs[1].length, 5);
|
||
assert_eq!(refs[1].object_index, 2);
|
||
}
|
||
|
||
#[test]
|
||
fn read_vl_strings_from_heap() {
|
||
let gcol_offset = 256usize;
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(&mut file_data, gcol_offset, &[(1, b"Alice"), (2, b"Bob")]);
|
||
|
||
let raw = build_vl_refs(&["Alice", "Bob"], gcol_offset as u64, 1, 8);
|
||
let strings = read_vl_strings(&file_data, &raw, 2, 8, 8).unwrap();
|
||
assert_eq!(strings, vec!["Alice", "Bob"]);
|
||
}
|
||
|
||
#[test]
|
||
fn null_vl_element_empty_string() {
|
||
// length=0, address=undefined
|
||
let mut raw = Vec::new();
|
||
raw.extend_from_slice(&0u32.to_le_bytes()); // length=0
|
||
raw.extend_from_slice(&u64::MAX.to_le_bytes()); // undefined address
|
||
raw.extend_from_slice(&0u32.to_le_bytes()); // index
|
||
|
||
let file_data = vec![0u8; 16];
|
||
let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap();
|
||
assert_eq!(strings, vec![""]);
|
||
}
|
||
|
||
#[test]
|
||
fn null_vl_element_zero_address() {
|
||
let mut raw = Vec::new();
|
||
raw.extend_from_slice(&0u32.to_le_bytes());
|
||
raw.extend_from_slice(&0u64.to_le_bytes());
|
||
raw.extend_from_slice(&0u32.to_le_bytes());
|
||
|
||
let file_data = vec![0u8; 16];
|
||
let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap();
|
||
assert_eq!(strings, vec![""]);
|
||
}
|
||
|
||
#[test]
|
||
fn read_vl_bytes_from_heap() {
|
||
let gcol_offset = 128usize;
|
||
let mut file_data = vec![0u8; 512];
|
||
build_gcol_at(
|
||
&mut file_data,
|
||
gcol_offset,
|
||
&[(1, &[0xDE, 0xAD]), (2, &[0xBE, 0xEF, 0xCA])],
|
||
);
|
||
|
||
let _raw = build_vl_refs(&["ab", "abc"], gcol_offset as u64, 1, 8);
|
||
// Fix lengths to match actual byte lengths
|
||
let mut raw_fixed = Vec::new();
|
||
raw_fixed.extend_from_slice(&2u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&(gcol_offset as u64).to_le_bytes());
|
||
raw_fixed.extend_from_slice(&1u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&3u32.to_le_bytes());
|
||
raw_fixed.extend_from_slice(&(gcol_offset as u64).to_le_bytes());
|
||
raw_fixed.extend_from_slice(&2u32.to_le_bytes());
|
||
|
||
let bytes = read_vl_bytes(&file_data, &raw_fixed, 2, 8, 8).unwrap();
|
||
assert_eq!(bytes, vec![vec![0xDE, 0xAD], vec![0xBE, 0xEF, 0xCA]]);
|
||
}
|
||
|
||
#[test]
|
||
fn parse_vl_references_truncated_error() {
|
||
let raw = vec![0u8; 10]; // too short for 1 element with offset_size=8
|
||
let err = parse_vl_references(&raw, 1, 8).unwrap_err();
|
||
assert!(matches!(err, FormatError::UnexpectedEof { .. }));
|
||
}
|
||
}
|