osobhandClaude Opus 5.5
5a202f3791
libhdf5 fails to read a VL element whose global heap address is
undefined (all 0xff), even at length 0 ("addr undefined"); we returned
"" (or an empty sequence) in every reader. Checked with h5py first:
libhdf5 writes a null element with address 0, which still reads as
empty, and h5py writes "" as a zero-size heap object at a real address,
so no file they write relies on the old behaviour. read_vl_bytes now
treats address 0 as null whatever the length, as VlResolver does.
Tests, each failing before: vl_data unit test (8- and 4-byte offsets,
lengths 0 and 1); clawhdf5 vl_data_interop
a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks
where h5py writes ""); h5rs dump --json and check --data on the patched
`undef` dataset; clawhdf5-wasm vl_strings.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
495 lines
19 KiB
Rust
495 lines
19 KiB
Rust
//! Variable-length data (VL strings and VL sequences) read through the
|
|
//! facade, checked against h5py/libhdf5.
|
|
//!
|
|
//! h5py writes each file — once with the default 8-byte offsets and once
|
|
//! with 4-byte offsets and lengths (`sizeof_addr = 4`) — and prints what
|
|
//! libhdf5 reads back; `File`, `MmapFile` and `LazyFile` must return the same
|
|
//! values. Skipped when python3 with h5py is unavailable, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
// `Selection::slice(&[0..1])` is one range per dimension, not a Vec of a range.
|
|
#![allow(clippy::single_range_in_vec_init)]
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::Path;
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{AttrValue, File, LazyFile, MmapFile, Selection};
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py, numpy"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
/// Run `script` and return its stdout as `key -> value`, one
|
|
/// `key<TAB>value` line per key.
|
|
fn run_python(script: &str) -> HashMap<String, String> {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python");
|
|
assert!(
|
|
output.status.success(),
|
|
"python failed:\n{}",
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
String::from_utf8_lossy(&output.stdout)
|
|
.lines()
|
|
.filter_map(|line| {
|
|
let (k, v) = line.split_once('\t')?;
|
|
Some((k.to_string(), v.to_string()))
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// `hex,hex,...` -> the strings' bytes.
|
|
fn parse_strings(v: &str) -> Vec<Vec<u8>> {
|
|
v.split(',')
|
|
.map(|h| {
|
|
(0..h.len())
|
|
.step_by(2)
|
|
.map(|i| u8::from_str_radix(&h[i..i + 2], 16).unwrap())
|
|
.collect()
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// `1 2 3|| -5` -> sequences.
|
|
fn parse_seqs(v: &str) -> Vec<Vec<f64>> {
|
|
v.split('|')
|
|
.map(|s| s.split_whitespace().map(|x| x.parse().unwrap()).collect())
|
|
.collect()
|
|
}
|
|
|
|
fn utf8(bytes: &[Vec<u8>]) -> Vec<String> {
|
|
bytes
|
|
.iter()
|
|
.map(|b| String::from_utf8(b.clone()).unwrap())
|
|
.collect()
|
|
}
|
|
|
|
/// Writes `vl8.h5` (8-byte offsets) and `vl4.h5` (4-byte offsets and
|
|
/// lengths) into `dir` and prints h5py's reading of both.
|
|
const SCRIPT: &str = r#"
|
|
import sys, h5py, numpy as np
|
|
d = sys.argv[1]
|
|
S = h5py.string_dtype('utf-8'); A = h5py.string_dtype('ascii')
|
|
def make(path, sizes):
|
|
if sizes:
|
|
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(*sizes)
|
|
f = h5py.File(h5py.h5f.create(path.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl))
|
|
else:
|
|
f = h5py.File(path, 'w')
|
|
f.create_dataset('scalar_utf8', data='héllo', dtype=S)
|
|
f.create_dataset('scalar_ascii', data=b'hello', dtype=A)
|
|
f.create_dataset('d1', data=np.array(['a', '', 'ccc', 'δδ'], dtype=object), dtype=S)
|
|
f.create_dataset('d2', data=np.array([['x', 'yy', 'zzz'], ['', 'w', 'vv']], dtype=object), dtype=S)
|
|
f.create_dataset('chunked', data=np.array(['s%d' % i * (i % 5) for i in range(100)], dtype=object),
|
|
dtype=S, chunks=(7,), compression='gzip')
|
|
f.create_dataset('chunked2d', data=np.array([['r%dc%d' % (r, c) for c in range(9)] for r in range(11)], dtype=object),
|
|
dtype=S, chunks=(4, 4), compression='gzip', shuffle=True)
|
|
f.create_dataset('unwritten', shape=(5,), dtype=S, chunks=(2,))
|
|
p = f.create_dataset('partial', shape=(6,), dtype=S, chunks=(2,)); p[0] = 'first'; p[5] = 'last'
|
|
f.create_dataset('contig_empty', shape=(3,), dtype=S)
|
|
dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE); dcpl.set_layout(h5py.h5d.COMPACT)
|
|
f.create_dataset('compact', data=np.array(['c1', '', 'c3'], dtype=object), dtype=S, dcpl=dcpl)
|
|
assert f['compact'].id.get_create_plist().get_layout() == h5py.h5d.COMPACT
|
|
f.attrs['vlattr'] = 'attr-value'
|
|
f.attrs.create('vlattr_arr', np.array(['p', 'qq', ''], dtype=object), dtype=S)
|
|
ct = np.dtype([('id', '<i4'), ('name', S), ('v', '<f8')])
|
|
arr = np.zeros(3, dtype=ct); arr['id'] = [1, 2, 3]; arr['name'] = ['one', '', 'three']; arr['v'] = [.5, 1.5, 2.5]
|
|
f.create_dataset('compound', data=arr)
|
|
f.attrs.create('compound_attr', arr)
|
|
v = f.create_dataset('vlen_i4', shape=(3,), dtype=h5py.vlen_dtype(np.dtype('<i4')))
|
|
v[0] = [1, 2, 3]; v[1] = []; v[2] = [-5]
|
|
v = f.create_dataset('vlen_f8', shape=(2, 2), dtype=h5py.vlen_dtype(np.dtype('<f8')), chunks=(1, 2), compression='gzip')
|
|
v[0, 0] = [1.5]; v[0, 1] = [2.5, 3.5]; v[1, 1] = [9.0]
|
|
v = f.create_dataset('vlen_u2_be', shape=(2,), dtype=h5py.vlen_dtype(np.dtype('>u2')))
|
|
v[0] = [1, 65535]; v[1] = [300]
|
|
f.attrs.create('vlen_attr', np.array([np.array([1, 2], dtype='<i8'), np.array([3], dtype='<i8')], dtype=object),
|
|
dtype=h5py.vlen_dtype(np.dtype('<i8')))
|
|
f.close()
|
|
|
|
def hexes(a):
|
|
return ','.join(bytes(x).hex() for x in np.asarray(a, dtype=object).ravel())
|
|
def seqs(a):
|
|
return '|'.join(' '.join(repr(float(x)) for x in s) for s in np.asarray(a, dtype=object).ravel())
|
|
|
|
for tag, sizes in (('8', None), ('4', (4, 4))):
|
|
path = '%s/vl%s.h5' % (d, tag)
|
|
make(path, sizes)
|
|
with h5py.File(path, 'r') as f:
|
|
for name in ('compact', 'scalar_utf8', 'scalar_ascii', 'd1', 'd2', 'chunked', 'chunked2d', 'unwritten',
|
|
'partial', 'contig_empty'):
|
|
v = f[name][()]
|
|
print('%s:%s\t%s' % (tag, name, hexes([v] if np.ndim(v) == 0 else v)))
|
|
print('%s:d2[1,1:3]\t%s' % (tag, hexes(f['d2'][1, 1:3])))
|
|
print('%s:chunked[5:60:3]\t%s' % (tag, hexes(f['chunked'][5:60:3])))
|
|
print('%s:chunked2d[2:9:2,3:8]\t%s' % (tag, hexes(f['chunked2d'][2:9:2, 3:8])))
|
|
print('%s:compound.name\t%s' % (tag, hexes(f['compound']['name'])))
|
|
print('%s:compound_attr.name\t%s' % (tag, hexes(f.attrs['compound_attr']['name'])))
|
|
print('%s:vlattr\t%s' % (tag, hexes([f.attrs['vlattr'].encode()])))
|
|
print('%s:vlattr_arr\t%s' % (tag, hexes([s.encode() for s in f.attrs['vlattr_arr']])))
|
|
for name in ('vlen_i4', 'vlen_f8'):
|
|
print('%s:%s\t%s' % (tag, name, seqs(f[name][()])))
|
|
print('%s:vlen_f8[1,:]\t%s' % (tag, seqs(f['vlen_f8'][1, :])))
|
|
print('%s:vlen_attr\t%s' % (tag, seqs(f.attrs['vlen_attr'])))
|
|
"#;
|
|
|
|
fn make_files(dir: &Path) -> HashMap<String, String> {
|
|
let script = format!(
|
|
"import sys; sys.argv = ['x', {:?}]\n{SCRIPT}",
|
|
dir.display().to_string()
|
|
);
|
|
run_python(&script)
|
|
}
|
|
|
|
const STRING_DATASETS: [&str; 10] = [
|
|
"compact",
|
|
"scalar_utf8",
|
|
"scalar_ascii",
|
|
"d1",
|
|
"d2",
|
|
"chunked",
|
|
"chunked2d",
|
|
"unwritten",
|
|
"partial",
|
|
"contig_empty",
|
|
];
|
|
|
|
#[test]
|
|
fn vl_string_datasets_read_like_h5py() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let expected = make_files(dir.path());
|
|
for tag in ["8", "4"] {
|
|
let path = dir.path().join(format!("vl{tag}.h5"));
|
|
let file = File::open(&path).unwrap();
|
|
let mmap = MmapFile::open(&path).unwrap();
|
|
let lazy = LazyFile::open_mmap(&path).unwrap();
|
|
for name in STRING_DATASETS {
|
|
let want = parse_strings(&expected[&format!("{tag}:{name}")]);
|
|
let ctx = format!("vl{tag}.h5 {name}");
|
|
let ds = file.dataset(name).unwrap();
|
|
assert_eq!(ds.read_string_bytes().unwrap(), want, "{ctx}");
|
|
assert_eq!(ds.read_string().unwrap(), utf8(&want), "{ctx}");
|
|
let m = mmap.dataset(name).unwrap();
|
|
assert_eq!(m.read_string_bytes().unwrap(), want, "{ctx} (mmap)");
|
|
assert_eq!(m.read_string().unwrap(), utf8(&want), "{ctx} (mmap)");
|
|
let l = lazy.dataset(name).unwrap();
|
|
assert_eq!(l.read_string_bytes().unwrap(), want, "{ctx} (lazy)");
|
|
assert_eq!(l.read_string().unwrap(), utf8(&want), "{ctx} (lazy)");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vl_string_selections_read_like_h5py() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let expected = make_files(dir.path());
|
|
let hyperslab = |start: &[u64], stride: &[u64], count: &[u64]| Selection::Hyperslab {
|
|
start: start.to_vec(),
|
|
stride: stride.to_vec(),
|
|
count: count.to_vec(),
|
|
block: vec![1; start.len()],
|
|
};
|
|
let cases = [
|
|
("d2", "d2[1,1:3]", Selection::slice(&[1..2, 1..3])),
|
|
("chunked", "chunked[5:60:3]", hyperslab(&[5], &[3], &[19])),
|
|
(
|
|
"chunked2d",
|
|
"chunked2d[2:9:2,3:8]",
|
|
hyperslab(&[2, 3], &[2, 1], &[4, 5]),
|
|
),
|
|
];
|
|
for tag in ["8", "4"] {
|
|
let file = File::open(dir.path().join(format!("vl{tag}.h5"))).unwrap();
|
|
for (name, key, sel) in &cases {
|
|
let want = utf8(&parse_strings(&expected[&format!("{tag}:{key}")]));
|
|
let got = file
|
|
.dataset(name)
|
|
.unwrap()
|
|
.read_string_selection(sel)
|
|
.unwrap();
|
|
assert_eq!(got, want, "vl{tag}.h5 {key}");
|
|
}
|
|
// A selection of VL integers is not strings.
|
|
assert!(
|
|
file.dataset("vlen_i4")
|
|
.unwrap()
|
|
.read_string_selection(&Selection::slice(&[0..1]))
|
|
.is_err()
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vl_values_in_compounds_and_attributes_read_like_h5py() {
|
|
// With 4-byte offsets these failed with GlobalHeapObjectNotFound or came
|
|
// back as `AttrValue::Raw`: the VL type claimed 16-byte elements and the
|
|
// global heap was read without the padding libhdf5 puts after its
|
|
// headers.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let expected = make_files(dir.path());
|
|
for tag in ["8", "4"] {
|
|
let file = File::open(dir.path().join(format!("vl{tag}.h5"))).unwrap();
|
|
let want = |key: &str| utf8(&parse_strings(&expected[&format!("{tag}:{key}")]));
|
|
|
|
let attrs = file.root().attrs().unwrap();
|
|
match &attrs["vlattr"] {
|
|
AttrValue::String(s) => assert_eq!(*s, want("vlattr")[0], "vl{tag}.h5"),
|
|
other => panic!("vl{tag}.h5 vlattr: {other:?}"),
|
|
}
|
|
match &attrs["vlattr_arr"] {
|
|
AttrValue::StringArray(s) => assert_eq!(*s, want("vlattr_arr"), "vl{tag}.h5"),
|
|
other => panic!("vl{tag}.h5 vlattr_arr: {other:?}"),
|
|
}
|
|
|
|
// Compound with a VL string member: dataset and attribute.
|
|
let ds = file.dataset("compound").unwrap();
|
|
let dt = ds.raw_datatype().unwrap();
|
|
let raw = ds.read_selection(&Selection::All).unwrap();
|
|
let fields = clawhdf5_format::data_read::read_compound_fields(&raw, &dt).unwrap();
|
|
let name = fields.iter().find(|f| f.name == "name").unwrap();
|
|
assert_eq!(
|
|
file.decode_strings(&name.datatype, &name.raw_data).unwrap(),
|
|
want("compound.name"),
|
|
"vl{tag}.h5 compound"
|
|
);
|
|
let id = fields.iter().find(|f| f.name == "id").unwrap();
|
|
assert_eq!(
|
|
clawhdf5_format::data_read::read_as_i64(&id.raw_data, &id.datatype).unwrap(),
|
|
vec![1, 2, 3]
|
|
);
|
|
let v = fields.iter().find(|f| f.name == "v").unwrap();
|
|
assert_eq!(
|
|
clawhdf5_format::data_read::read_as_f64(&v.raw_data, &v.datatype).unwrap(),
|
|
vec![0.5, 1.5, 2.5]
|
|
);
|
|
|
|
let AttrValue::Raw { datatype, data, .. } = &attrs["compound_attr"] else {
|
|
panic!("compound attribute is Raw");
|
|
};
|
|
let fields = clawhdf5_format::data_read::read_compound_fields(data, datatype).unwrap();
|
|
let name = fields.iter().find(|f| f.name == "name").unwrap();
|
|
assert_eq!(
|
|
file.decode_strings(&name.datatype, &name.raw_data).unwrap(),
|
|
want("compound_attr.name"),
|
|
"vl{tag}.h5 compound attribute"
|
|
);
|
|
|
|
// A VL sequence attribute.
|
|
let AttrValue::Raw { datatype, data, .. } = &attrs["vlen_attr"] else {
|
|
panic!("vlen attribute is Raw");
|
|
};
|
|
let got: Vec<Vec<i64>> = file.decode_vlen(datatype, data).unwrap();
|
|
let want_seqs = parse_seqs(&expected[&format!("{tag}:vlen_attr")]);
|
|
assert_eq!(
|
|
got,
|
|
want_seqs
|
|
.iter()
|
|
.map(|s| s.iter().map(|&x| x as i64).collect::<Vec<_>>())
|
|
.collect::<Vec<_>>(),
|
|
"vl{tag}.h5 vlen_attr"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vl_sequence_datasets_read_like_h5py() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let expected = make_files(dir.path());
|
|
for tag in ["8", "4"] {
|
|
let path = dir.path().join(format!("vl{tag}.h5"));
|
|
let file = File::open(&path).unwrap();
|
|
let seqs = |key: &str| parse_seqs(&expected[&format!("{tag}:{key}")]);
|
|
|
|
let i4 = file.dataset("vlen_i4").unwrap();
|
|
let want: Vec<Vec<i32>> = seqs("vlen_i4")
|
|
.iter()
|
|
.map(|s| s.iter().map(|&x| x as i32).collect())
|
|
.collect();
|
|
assert_eq!(i4.read_vlen::<i32>().unwrap(), want, "vl{tag}.h5 vlen_i4");
|
|
let as_f64: Vec<Vec<f64>> = i4.read_vlen().unwrap();
|
|
assert_eq!(as_f64, seqs("vlen_i4"));
|
|
|
|
let f8 = file.dataset("vlen_f8").unwrap();
|
|
assert_eq!(
|
|
f8.read_vlen::<f64>().unwrap(),
|
|
seqs("vlen_f8"),
|
|
"vl{tag}.h5"
|
|
);
|
|
assert_eq!(
|
|
f8.read_vlen_selection::<f64>(&Selection::slice(&[1..2, 0..2]))
|
|
.unwrap(),
|
|
seqs("vlen_f8[1,:]"),
|
|
"vl{tag}.h5 vlen_f8[1,:]"
|
|
);
|
|
|
|
// h5py returns big-endian VL elements byte-swapped (an h5py bug, see
|
|
// CONFORMANCE.md); the values written are [1, 65535] and [300].
|
|
assert_eq!(
|
|
file.dataset("vlen_u2_be")
|
|
.unwrap()
|
|
.read_vlen::<u64>()
|
|
.unwrap(),
|
|
vec![vec![1, 65535], vec![300]]
|
|
);
|
|
|
|
let mmap = MmapFile::open(&path).unwrap();
|
|
assert_eq!(
|
|
mmap.dataset("vlen_f8").unwrap().read_vlen::<f64>().unwrap(),
|
|
seqs("vlen_f8")
|
|
);
|
|
let lazy = LazyFile::open_mmap(&path).unwrap();
|
|
assert_eq!(
|
|
lazy.dataset("vlen_f8").unwrap().read_vlen::<f64>().unwrap(),
|
|
seqs("vlen_f8")
|
|
);
|
|
|
|
// Wrong kind of data is an error, not a value.
|
|
assert!(i4.read_string().is_err());
|
|
assert!(i4.read_string_bytes().is_err());
|
|
assert!(file.dataset("d1").unwrap().read_vlen::<f64>().is_err());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vl_strings_end_at_nul_and_mis_sized_elements_fail_like_h5py() {
|
|
// h5py cannot write a VL string with a NUL in it, so the file is patched:
|
|
// one string gets an embedded NUL, and two elements get a length that
|
|
// disagrees with their heap object. libhdf5 returns the string up to the
|
|
// NUL and refuses the others ("Expected global heap object size does
|
|
// not match"); we used to return the NUL and a truncated string.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("patched.h5");
|
|
let script = format!(
|
|
r#"
|
|
import struct, h5py, numpy as np
|
|
path = {path:?}
|
|
with h5py.File(path, 'w') as f:
|
|
f.create_dataset('d', data=np.array(['aXb', 'cdefgh', 'ij', 'ok'], dtype=object),
|
|
dtype=h5py.string_dtype())
|
|
s = f.create_dataset('seq', shape=(2,), dtype=h5py.vlen_dtype(np.dtype('<i4')))
|
|
s[0] = np.array([1, 2, 3], dtype='<i4'); s[1] = np.array([4], dtype='<i4')
|
|
off = f['d'].id.get_offset(); soff = f['seq'].id.get_offset()
|
|
b = bytearray(open(path, 'rb').read())
|
|
i = b.index(b'aXb'); b[i + 1] = 0
|
|
struct.pack_into('<I', b, off + 16, 3) # 'cdefgh': length 6 -> 3
|
|
struct.pack_into('<I', b, off + 32, 9) # 'ij': length 2 -> 9
|
|
struct.pack_into('<I', b, soff, 2) # [1, 2, 3]: length 3 -> 2
|
|
open(path, 'wb').write(bytes(b))
|
|
with h5py.File(path, 'r') as f:
|
|
for i in range(4):
|
|
try:
|
|
print('d%d\t%s' % (i, f['d'][i].hex()))
|
|
except OSError as e:
|
|
print('d%d\terror' % i)
|
|
for i in range(2):
|
|
try:
|
|
print('seq%d\t%s' % (i, ' '.join(str(x) for x in f['seq'][i])))
|
|
except OSError as e:
|
|
print('seq%d\terror' % i)
|
|
"#,
|
|
path = path.display().to_string()
|
|
);
|
|
let expected = run_python(&script);
|
|
assert_eq!(expected["d0"], "61", "h5py cuts 'a\\0b' at the NUL");
|
|
assert_eq!(expected["d1"], "error");
|
|
assert_eq!(expected["d2"], "error");
|
|
assert_eq!(expected["d3"], "6f6b");
|
|
assert_eq!(expected["seq0"], "error");
|
|
assert_eq!(expected["seq1"], "4");
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let d = file.dataset("d").unwrap();
|
|
let one = |i: u64| d.read_string_selection(&Selection::slice(&[i..i + 1]));
|
|
assert_eq!(one(0).unwrap(), vec!["a"]);
|
|
assert!(one(1).is_err());
|
|
assert!(one(2).is_err());
|
|
assert_eq!(one(3).unwrap(), vec!["ok"]);
|
|
assert!(d.read_string().is_err());
|
|
let seq = file.dataset("seq").unwrap();
|
|
let one = |i: u64| seq.read_vlen_selection::<i32>(&Selection::slice(&[i..i + 1]));
|
|
assert!(one(0).is_err());
|
|
assert_eq!(one(1).unwrap(), vec![vec![4]]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_vl_element_at_the_undefined_heap_address_fails_like_h5py() {
|
|
// libhdf5 writes a null element with heap address 0 (h5py reads it as
|
|
// b''), and an empty string as a real zero-size heap object; neither
|
|
// uses the undefined address. An element of length 0 at the undefined
|
|
// address fails in libhdf5 ("addr undefined"); we returned "".
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("undef.h5");
|
|
let script = format!(
|
|
r#"
|
|
import struct, h5py, numpy as np
|
|
path = {path:?}
|
|
with h5py.File(path, 'w') as f:
|
|
f.create_dataset('d', data=np.array(['x', '', 'yz', ''], dtype=object), dtype=h5py.string_dtype())
|
|
off = f['d'].id.get_offset()
|
|
b = bytearray(open(path, 'rb').read())
|
|
# h5py's '' (element 3): length 0 at a real heap address, not 0 or all 0xff.
|
|
length, addr, _ = struct.unpack_from('<IQI', b, off + 48)
|
|
print('empty\t%d %d' % (length, addr not in (0, 2**64 - 1)))
|
|
struct.pack_into('<IQI', b, off + 16, 0, 2**64 - 1, 1)
|
|
open(path, 'wb').write(bytes(b))
|
|
with h5py.File(path, 'r') as f:
|
|
for i in range(4):
|
|
try:
|
|
print('d%d\t%s' % (i, f['d'][i].hex()))
|
|
except OSError as e:
|
|
print('d%d\terror %s' % (i, 'addr undefined' in str(e)))
|
|
"#,
|
|
path = path.display().to_string()
|
|
);
|
|
let expected = run_python(&script);
|
|
assert_eq!(expected["empty"], "0 1", "h5py writes '' at a real address");
|
|
assert_eq!(expected["d0"], "78");
|
|
assert_eq!(expected["d1"], "error True");
|
|
assert_eq!(expected["d2"], "797a");
|
|
assert_eq!(expected["d3"], "");
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let d = file.dataset("d").unwrap();
|
|
let one = |i: u64| d.read_string_selection(&Selection::slice(&[i..i + 1]));
|
|
assert_eq!(one(0).unwrap(), vec!["x"]);
|
|
let e = one(1).unwrap_err().to_string();
|
|
assert!(e.contains("undefined global heap address"), "{e}");
|
|
assert_eq!(one(2).unwrap(), vec!["yz"]);
|
|
assert_eq!(one(3).unwrap(), vec![""]);
|
|
assert!(d.read_string().is_err());
|
|
assert!(d.read_string_bytes().is_err());
|
|
}
|