fix(format): return whole VL sequences from read_vl_bytes
read_vl_bytes cut each element to the reference's length field, which counts sequence elements, not bytes: a VL int32 [1, 2, 3] came back as 3 bytes. Return the whole global-heap object, which is element count x base size bytes. No in-tree caller depended on the old behaviour. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -148,7 +148,12 @@ pub fn read_vl_strings(
|
|||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve VL byte sequences from raw data.
|
/// Resolve VL sequences from raw data, returning each element's bytes.
|
||||||
|
///
|
||||||
|
/// Each element is the sequence's full encoding — element count × base type
|
||||||
|
/// size bytes, in the base type's byte order — so a sequence of `i32` yields
|
||||||
|
/// four bytes per value. Decode it with the base type (e.g.
|
||||||
|
/// [`crate::data_read::read_as_i64`]).
|
||||||
pub fn read_vl_bytes(
|
pub fn read_vl_bytes(
|
||||||
file_data: &[u8],
|
file_data: &[u8],
|
||||||
raw_data: &[u8],
|
raw_data: &[u8],
|
||||||
@@ -177,8 +182,10 @@ pub fn read_vl_bytes(
|
|||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let len = (vl.length as usize).min(obj.data.len());
|
// The heap object holds the whole sequence. `vl.length` counts
|
||||||
result.push(obj.data[..len].to_vec());
|
// elements, not bytes, so it is only the byte length when the base
|
||||||
|
// type is one byte wide.
|
||||||
|
result.push(obj.data.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
|
|||||||
@@ -311,3 +311,64 @@ with h5py.File("{path}", "r") as f:
|
|||||||
assert_eq!(bools.read_u64().unwrap(), vec![1, 0, 1]);
|
assert_eq!(bools.read_u64().unwrap(), vec![1, 0, 1]);
|
||||||
assert_eq!(bools.read_i32().unwrap(), vec![1, 0, 1]);
|
assert_eq!(bools.read_i32().unwrap(), vec![1, 0, 1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vl_sequences_of_wide_base_types_read_whole() {
|
||||||
|
// read_vl_bytes took the sequence's element count as its byte length, so
|
||||||
|
// [1, 2, 3] as VL int32 came back as 3 bytes instead of 12.
|
||||||
|
use clawhdf5::Selection;
|
||||||
|
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
|
||||||
|
use clawhdf5_format::vl_data::read_vl_bytes;
|
||||||
|
|
||||||
|
skip_if_no_python!();
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("vlen.h5");
|
||||||
|
let script = format!(
|
||||||
|
r#"{PRELUDE}
|
||||||
|
data = {{
|
||||||
|
"i4": (h5py.vlen_dtype("<i4"), [[1, 2, 3], [], [-5], list(range(40))]),
|
||||||
|
"f8": (h5py.vlen_dtype("<f8"), [[1.5, -2.0], [0.25]]),
|
||||||
|
"u1": (h5py.vlen_dtype("u1"), [[1, 2, 255], []]),
|
||||||
|
}}
|
||||||
|
with h5py.File("{path}", "w") as f:
|
||||||
|
for name, (dt, seqs) in data.items():
|
||||||
|
ds = f.create_dataset(name, (len(seqs),), dtype=dt)
|
||||||
|
for i, s in enumerate(seqs):
|
||||||
|
ds[i] = s
|
||||||
|
with h5py.File("{path}", "r") as f:
|
||||||
|
for name in data:
|
||||||
|
for i, s in enumerate(f[name][()]):
|
||||||
|
emit(f"{{name}}:{{i}}", np.frombuffer(np.asarray(s).tobytes(), "u1"))
|
||||||
|
"#,
|
||||||
|
path = path.display()
|
||||||
|
);
|
||||||
|
let expected = run_python(&script);
|
||||||
|
|
||||||
|
let file = File::open(&path).unwrap();
|
||||||
|
let sb = file.superblock();
|
||||||
|
for (name, count) in [("i4", 4), ("f8", 2), ("u1", 2)] {
|
||||||
|
let raw = file
|
||||||
|
.dataset(name)
|
||||||
|
.unwrap()
|
||||||
|
.read_selection(&Selection::All)
|
||||||
|
.unwrap();
|
||||||
|
let got =
|
||||||
|
read_vl_bytes(file.as_bytes(), &raw, count, sb.offset_size, sb.length_size).unwrap();
|
||||||
|
let want: Vec<Vec<u8>> = (0..count)
|
||||||
|
.map(|i| parse(expected.get(&format!("{name}:{i}")).unwrap()))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(got, want, "{name}");
|
||||||
|
if name == "i4" {
|
||||||
|
let i32_le = Datatype::FixedPoint {
|
||||||
|
size: 4,
|
||||||
|
byte_order: DatatypeByteOrder::LittleEndian,
|
||||||
|
signed: true,
|
||||||
|
bit_offset: 0,
|
||||||
|
bit_precision: 32,
|
||||||
|
};
|
||||||
|
let values = clawhdf5_format::data_read::read_as_i64(&got[0], &i32_le).unwrap();
|
||||||
|
assert_eq!(values, vec![1, 2, 3]);
|
||||||
|
assert_eq!(got[3].len(), 40 * 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user