fix(tools,wasm): resolve VL data through the library's VlResolver
h5rs (dump, ls, diff, check --data) kept its own lenient VL decoder: a heap object longer than its element was cut to the element's length (libhdf5 and h5py refuse it), a null string printed "" where h5dump prints NULL, the stored element size was trusted, and every heap collection was kept as an owned copy for the whole run. It now resolves each element with VlResolver::element / string_element (new: one element in place, borrowing from the file), and refuses a VL type whose stored element size is not 4 + offset size + 4, as File does. H5::heap_object and its cache are gone. h5diff compares a null VL string equal to an empty one; so does h5rs diff. clawhdf5-wasm already resolved VL strings with read_vl_strings; it now uses VlResolver and checks the stored element size before reading, as File::read_string does. Tests (h5py writes the files, patched for "a\0b", a null element and mis-sized heap objects, with 8- and 4-byte offsets): - h5rs_interop dump_prints_vl_data_like_h5dump: byte-identical to h5dump; - dump_json_vl_values_match_h5py: h5py's values, errors where h5py fails; - check_data_flags_mis_sized_vl_heap_objects; - clawhdf5-wasm tests/vl_strings.rs: wasm, File and h5py agree. All four fail before. check --data over the 150 cve_hdf5 CVE and fuzzer files now passes 15 (h5dump rejects 8 of them), was 16 and 9: the stored-size check flags cve-2024-32608. h5rs-check-ok-files.sh --data: 0 of 422 flagged; h5rs-fuzz.sh: clean on 180 files. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,121 @@
|
||||
"""Write the variable-length data files the h5rs VL tests run on.
|
||||
|
||||
usage: gen_vl_files.py OUTDIR
|
||||
|
||||
For 8-byte (`vl8`) and 4-byte (`vl4`) offsets, writes OUTDIR/vl8.h5 and
|
||||
OUTDIR/vl4.h5, which libhdf5 reads in full, and OUTDIR/bad8.h5 and
|
||||
OUTDIR/bad4.h5, whose `bad` and `badseq` elements 0 have a length that
|
||||
disagrees with their global heap object (libhdf5: "Expected global heap
|
||||
object size does not match"). h5py cannot write a VL string with a NUL in
|
||||
it or a null element in a contiguous dataset, so those are patched in.
|
||||
|
||||
Prints one JSON object: for each file, each dataset's values as h5py reads
|
||||
them one element at a time (strings as text, sequences as lists, a compound
|
||||
as a list of its fields), with null for an element h5py cannot read; the
|
||||
root attribute `va`; and the addresses of the `bad` elements' collections.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
out = sys.argv[1]
|
||||
S = h5py.string_dtype("utf-8")
|
||||
I4 = h5py.vlen_dtype(np.dtype("<i4"))
|
||||
|
||||
|
||||
def create(path, sizes):
|
||||
if sizes is None:
|
||||
return h5py.File(path, "w")
|
||||
fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
|
||||
fcpl.set_sizes(*sizes)
|
||||
return h5py.File(h5py.h5f.create(path.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl))
|
||||
|
||||
|
||||
def element(length, addr, index, os_):
|
||||
return struct.pack("<I", length) + addr.to_bytes(os_, "little") + struct.pack("<I", index)
|
||||
|
||||
|
||||
def good(path, sizes):
|
||||
os_ = 8 if sizes is None else sizes[0]
|
||||
with create(path, sizes) as f:
|
||||
f.create_dataset(
|
||||
"d", data=np.array(["aXb", "", "ok", "zz", "hello"], dtype=object), dtype=S
|
||||
)
|
||||
u = f.create_dataset("u", shape=(4,), dtype=S, chunks=(1,))
|
||||
u[1] = "w"
|
||||
s = f.create_dataset("seq", shape=(3,), dtype=I4)
|
||||
s[0] = [1, 2, 3]
|
||||
s[1] = []
|
||||
s[2] = [-5]
|
||||
s = f.create_dataset("sequ", shape=(3,), dtype=I4, chunks=(1,))
|
||||
s[0] = [7, 8]
|
||||
ct = np.dtype([("id", "<i4"), ("name", S)])
|
||||
arr = np.zeros(3, dtype=ct)
|
||||
arr["id"] = [1, 2, 3]
|
||||
arr["name"] = ["one", "", "three"]
|
||||
f.create_dataset("cmp", data=arr)
|
||||
f.attrs.create("va", np.array(["p", "", "q"], dtype=object), dtype=S)
|
||||
off = f["d"].id.get_offset()
|
||||
b = bytearray(open(path, "rb").read())
|
||||
i = b.index(b"aXb")
|
||||
b[i + 1] = 0 # "a\0b"
|
||||
es = 8 + os_
|
||||
b[off + 2 * es : off + 3 * es] = element(2, 0, 1, os_) # "ok" -> null
|
||||
open(path, "wb").write(bytes(b))
|
||||
|
||||
|
||||
def bad(path, sizes):
|
||||
os_ = 8 if sizes is None else sizes[0]
|
||||
with create(path, sizes) as f:
|
||||
f.create_dataset("bad", data=np.array(["cdefgh", "ok"], dtype=object), dtype=S)
|
||||
s = f.create_dataset("badseq", shape=(2,), dtype=I4)
|
||||
s[0] = [1, 2, 3]
|
||||
s[1] = [4]
|
||||
off, soff = f["bad"].id.get_offset(), f["badseq"].id.get_offset()
|
||||
b = bytearray(open(path, "rb").read())
|
||||
gcol = int.from_bytes(b[off + 4 : off + 4 + os_], "little")
|
||||
struct.pack_into("<I", b, off, 3) # "cdefgh": length 6 -> 3
|
||||
struct.pack_into("<I", b, soff, 2) # [1, 2, 3]: length 3 -> 2
|
||||
open(path, "wb").write(bytes(b))
|
||||
return gcol
|
||||
|
||||
|
||||
def value(v):
|
||||
if isinstance(v, bytes):
|
||||
return v.decode()
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
if isinstance(v, np.void):
|
||||
return [value(x) for x in v]
|
||||
if isinstance(v, np.ndarray):
|
||||
return [value(x) for x in v]
|
||||
return v.item() if hasattr(v, "item") else v
|
||||
|
||||
|
||||
def read(ds):
|
||||
got = []
|
||||
for i in range(ds.shape[0]):
|
||||
try:
|
||||
got.append(value(ds[i]))
|
||||
except OSError:
|
||||
got.append(None)
|
||||
return got
|
||||
|
||||
|
||||
result = {}
|
||||
for tag, sizes in (("8", None), ("4", (4, 4))):
|
||||
g, x = os.path.join(out, f"vl{tag}.h5"), os.path.join(out, f"bad{tag}.h5")
|
||||
good(g, sizes)
|
||||
gcol = bad(x, sizes)
|
||||
with h5py.File(g, "r") as f:
|
||||
result[f"vl{tag}"] = {n: read(f[n]) for n in ("d", "u", "seq", "sequ", "cmp")}
|
||||
result[f"vl{tag}"]["va"] = [value(s) for s in f.attrs["va"]]
|
||||
with h5py.File(x, "r") as f:
|
||||
result[f"bad{tag}"] = {n: read(f[n]) for n in ("bad", "badseq")}
|
||||
result[f"bad{tag}"]["gcol"] = gcol
|
||||
json.dump(result, sys.stdout)
|
||||
@@ -773,3 +773,125 @@ fn every_subcommand_rejects_a_non_hdf5_file_cleanly() {
|
||||
assert_eq!(code(&h5rs(&["ls"])), 2);
|
||||
assert_eq!(code(&h5rs(&["--help"])), 0);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// variable-length data
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Runs `tests/gen_vl_files.py`: VL strings (with an embedded NUL, empty
|
||||
/// and null elements), VL sequences, a VL compound member and a VL
|
||||
/// attribute, with 8- and 4-byte offsets, plus files whose heap objects
|
||||
/// disagree with their elements' lengths.
|
||||
fn generate_vl() -> Option<Files> {
|
||||
if missing(python_available(), "python3 with h5py") {
|
||||
return None;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_vl_files.py");
|
||||
let out = Command::new(python())
|
||||
.arg(&script)
|
||||
.arg(dir.path())
|
||||
.output()
|
||||
.expect("run gen_vl_files.py");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"gen_vl_files.py failed:\n{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let values = serde_json::from_slice(&out.stdout).expect("gen_vl_files.py output");
|
||||
Some(Files { dir, values })
|
||||
}
|
||||
|
||||
/// `dump` resolves VL elements through the library's `VlResolver`, as
|
||||
/// libhdf5 does: "a\0b" prints as "a", a null string as NULL (it printed
|
||||
/// ""), and with 4-byte offsets too; the output is h5dump's byte for byte.
|
||||
#[test]
|
||||
fn dump_prints_vl_data_like_h5dump() {
|
||||
let Some(f) = generate_vl() else { return };
|
||||
for name in ["vl8.h5", "vl4.h5"] {
|
||||
let p = f.p(name);
|
||||
let ours = stdout(&h5rs(&["dump", &p]));
|
||||
assert!(
|
||||
ours.contains(r#"(0): "a", "", NULL, "zz", "hello""#),
|
||||
"{name}:\n{ours}"
|
||||
);
|
||||
assert!(ours.contains(r#"(0): NULL, "w", NULL, NULL"#), "{name}");
|
||||
assert!(ours.contains("(0): (1, 2, 3), (), (-5)"), "{name}");
|
||||
if missing(tool_available("h5dump"), "h5dump") {
|
||||
continue;
|
||||
}
|
||||
let reference = run("h5dump", &[&p]);
|
||||
assert!(reference.status.success(), "{name}: {reference:?}");
|
||||
assert_eq!(ours, stdout(&reference).replacen(&p, name, 1), "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `dump --json` gives the values h5py reads, element by element; and an
|
||||
/// element whose heap object is not its length × base size is an error, as
|
||||
/// in h5py, not a truncated value (it printed "cde" and (1, 2)).
|
||||
#[test]
|
||||
fn dump_json_vl_values_match_h5py() {
|
||||
let Some(f) = generate_vl() else { return };
|
||||
for tag in ["8", "4"] {
|
||||
let (good, bad) = (format!("vl{tag}"), format!("bad{tag}"));
|
||||
let o = h5rs(&["dump", "--json", &f.p(&format!("{good}.h5"))]);
|
||||
assert!(o.status.success(), "{good}: {o:?}");
|
||||
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
||||
let want = &f.values[&good];
|
||||
for d in doc["datasets"].as_object().unwrap().values() {
|
||||
let path = d["alias"][0].as_str().unwrap();
|
||||
assert_eq!(d["value"], want[&path[1..]], "{good}: {path}");
|
||||
}
|
||||
let attrs = &doc["groups"][doc["root"].as_str().unwrap()]["attributes"];
|
||||
assert_eq!(attrs[0]["name"], "va");
|
||||
assert_eq!(attrs[0]["value"], want["va"], "{good}: va");
|
||||
|
||||
let o = h5rs(&["dump", "--json", &f.p(&format!("{bad}.h5"))]);
|
||||
let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
|
||||
let want = &f.values[&bad];
|
||||
for d in doc["datasets"].as_object().unwrap().values() {
|
||||
let path = d["alias"][0].as_str().unwrap();
|
||||
let got = d["value"].as_array().unwrap();
|
||||
let want = want[&path[1..]].as_array().unwrap();
|
||||
assert_eq!(got.len(), want.len(), "{bad}: {path}");
|
||||
for (g, w) in got.iter().zip(want) {
|
||||
if w.is_null() {
|
||||
// h5py cannot read it: neither can we.
|
||||
let e = g["error"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("{bad}: {path}: {g}"));
|
||||
assert!(e.contains("holds"), "{bad}: {path}: {e}");
|
||||
} else {
|
||||
assert_eq!(g, w, "{bad}: {path}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `check --data` holds VL elements to libhdf5's rule: a heap object whose
|
||||
/// size is not exactly the element's length × base size is a problem (it
|
||||
/// only caught objects shorter than the element).
|
||||
#[test]
|
||||
fn check_data_flags_mis_sized_vl_heap_objects() {
|
||||
let Some(f) = generate_vl() else { return };
|
||||
for tag in ["8", "4"] {
|
||||
let o = h5rs(&["check", "--data", &f.p(&format!("vl{tag}.h5"))]);
|
||||
let s = stdout(&o);
|
||||
assert_eq!(code(&o), 0, "vl{tag}: {s}");
|
||||
assert!(
|
||||
s.contains("global heap collections read: 1"),
|
||||
"vl{tag}: {s}"
|
||||
);
|
||||
|
||||
let o = h5rs(&["check", "--data", &f.p(&format!("bad{tag}.h5"))]);
|
||||
let s = stdout(&o);
|
||||
assert_eq!(code(&o), 1, "bad{tag}: {s}");
|
||||
let at = f.values[format!("bad{tag}")]["gcol"].as_u64().unwrap();
|
||||
for (path, what) in [("/bad", "6 bytes"), ("/badseq", "12 bytes")] {
|
||||
let want = format!("problem: {at:#x} {path}: variable-length data: global heap object");
|
||||
assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}");
|
||||
assert!(s.contains(what), "bad{tag}: {s}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user