Files
clawhdf5/crates/clawhdf5/tests/header_validation_interop.rs
T
osobhandClaude Opus 5.5 993214723e test: a v2 header message running into the checksum is refused, as in libhdf5
The review read libhdf5's H5O__chunk_deserialize as accepting a v2
message that runs up to 4 bytes into the chunk's checksum, since it
bounds message bodies by the whole chunk buffer. It does not accept it:
the message loop stops at the checksum, and the checksum read that
follows starts past it and overruns the chunk ("ran off end of input
buffer while decoding"). h5py refuses such files whether the message
runs 1, 4 or 5 bytes in, and so does clawhdf5, with its own error text.
No code change; the test pins the agreement and a comment records why.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-26 01:32:00 -05:00

511 lines
18 KiB
Rust

//! Corrupt files that libhdf5 refuses must be refused here too, not read.
//!
//! h5py writes a valid file, the script corrupts a copy the way a damaged or
//! malicious file would be, and records whether h5py (libhdf5) still opens
//! and reads the object. clawhdf5 must agree: read the valid file, refuse
//! each corrupt one. Skipped when python3 with h5py is unavailable, unless
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
use std::path::Path;
use std::process::Command;
use clawhdf5::File;
fn python() -> String {
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}
fn interop_required() -> bool {
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}
fn python_available() -> bool {
Command::new(python())
.args(["-c", "import h5py, numpy"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
macro_rules! skip_if_no_python {
() => {
if !python_available() {
assert!(
!interop_required(),
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
);
eprintln!("SKIP: python3 with h5py not available");
return;
}
};
}
/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
/// `d` the output directory) and then, for every `NAME.h5` it wrote,
/// prints `NAME ok` when h5py opens and reads dataset `d` and `NAME ERROR`
/// otherwise. Returns those lines, sorted.
fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
let script = format!(
r#"
import h5py, numpy as np, struct, os, glob
d = "{dir}"
{body}
for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
name = os.path.basename(path)[:-3]
try:
with h5py.File(path, "r") as f:
f["d"][()]
print(name, "ok")
except Exception:
print(name, "ERROR")
"#,
dir = dir.display()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
let mut lines: Vec<String> = String::from_utf8_lossy(&out.stdout)
.lines()
.map(str::to_owned)
.collect();
lines.sort();
lines
}
/// Whether clawhdf5 opens and reads dataset `d` of `path` (as raw bytes of
/// whatever type it has).
fn clawhdf5_reads(path: &Path) -> Result<(), String> {
let file = File::open(path).map_err(|e| format!("open: {e}"))?;
let ds = file.dataset("d").map_err(|e| format!("dataset: {e}"))?;
ds.dtype().map_err(|e| format!("dtype: {e}"))?;
ds.shape().map_err(|e| format!("shape: {e}"))?;
file.read_multi(&["d"])
.map(|_| ())
.map_err(|e| format!("read: {e}"))
}
/// h5py's verdict for each file must be `expected`, and clawhdf5 must read
/// exactly the files h5py reads.
fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
assert_eq!(verdicts, expected, "h5py's view changed");
for line in verdicts {
let (name, verdict) = line.split_once(' ').unwrap();
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")));
match verdict {
"ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
_ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
}
}
}
#[test]
fn chunk_dimensions_libhdf5_refuses_are_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
// A chunked int32 dataset (chunk 37, element size 4 after it in the
// layout message), with libver earliest (layout v3) and latest (v4).
// The corrupt copies set the chunk dimension to 0 (which read as all
// fill values), to 0x80000000 (an 8 GiB chunk) and to 38, which the
// chunk index's offsets 37 and 74 are not multiples of (libhdf5: "bad
// coordinate offset"; the chunks were read at the wrong place). A v4
// layout indexes chunks by position, not offset, but both libraries
// refuse the changed chunk grid there too.
let verdicts = h5py_verdicts(
dir.path(),
r#"
for libver in ("earliest", "latest"):
good = os.path.join(d, f"{libver}_good.h5")
with h5py.File(good, "w", libver=libver) as f:
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,))
data = bytearray(open(good, "rb").read())
if libver == "earliest":
at = data.find(struct.pack("<II", 37, 4))
width = 4
else:
# v4: flags, ndims, bytes per dim, then the dims
at = data.find(bytes([4, 2, 0, 2]))
width = data[at + 4]
at += 5
assert at > 0
for name, value in (("zero", 0), ("huge", 0x80000000), ("offgrid", 38)):
bad = bytearray(data)
if value >= 1 << (8 * width):
continue
bad[at:at + width] = value.to_bytes(width, "little")
open(os.path.join(d, f"{libver}_{name}.h5"), "wb").write(bad)
"#,
);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&[
"earliest_good ok",
"earliest_huge ERROR",
"earliest_offgrid ERROR",
"earliest_zero ERROR",
"latest_good ok",
"latest_offgrid ERROR",
"latest_zero ERROR",
],
);
}
/// Python: `fix_ohdr(buf, off)` recomputes the Jenkins lookup3 checksum of
/// the version-2 object header chunk 0 at `off`, so a field can be changed
/// in a `libver="latest"` file without the checksum failing first.
const FIX_OHDR_PY: &str = r#"
def _rot(x, k):
return ((x << k) | (x >> (32 - k))) & 0xFFFFFFFF
def lookup3(data):
M = 0xFFFFFFFF
n = len(data); a = b = c = (0xDEADBEEF + n) & M; i = 0
w = lambda j: int.from_bytes(data[j:j + 4], "little")
while n > 12:
a = (a + w(i)) & M; b = (b + w(i + 4)) & M; c = (c + w(i + 8)) & M
a = (a - c) & M; a ^= _rot(c, 4); c = (c + b) & M
b = (b - a) & M; b ^= _rot(a, 6); a = (a + c) & M
c = (c - b) & M; c ^= _rot(b, 8); b = (b + a) & M
a = (a - c) & M; a ^= _rot(c, 16); c = (c + b) & M
b = (b - a) & M; b ^= _rot(a, 19); a = (a + c) & M
c = (c - b) & M; c ^= _rot(b, 4); b = (b + a) & M
n -= 12; i += 12
if n == 0:
return c
t = bytes(data[i:]) + bytes(12)
w = lambda j: int.from_bytes(t[j:j + 4], "little")
a = (a + w(0)) & M; b = (b + w(4)) & M; c = (c + w(8)) & M
c ^= b; c = (c - _rot(b, 14)) & M
a ^= c; a = (a - _rot(c, 11)) & M
b ^= a; b = (b - _rot(a, 25)) & M
c ^= b; c = (c - _rot(b, 16)) & M
a ^= c; a = (a - _rot(c, 4)) & M
b ^= a; b = (b - _rot(a, 14)) & M
c ^= b; c = (c - _rot(b, 24)) & M
return c
def fix_ohdr(buf, off):
assert buf[off:off + 4] == b"OHDR"
flags = buf[off + 5]; p = off + 6
if flags & 0x20: p += 16
if flags & 0x10: p += 4
width = 1 << (flags & 3)
end = p + width + int.from_bytes(buf[p:p + width], "little")
buf[end:end + 4] = lookup3(bytes(buf[off:end])).to_bytes(4, "little")
"#;
/// A chunked layout records the element size as its last dimension; libhdf5
/// refuses a dataset whose datatype has another size ("stored datatype size
/// in chunk layout does not match datatype description"). This read the
/// chunks laid out with the wrong element size.
#[test]
fn chunk_layout_element_size_must_match_the_datatype() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let body = format!(
"{FIX_OHDR_PY}{}",
r#"
for libver in ("earliest", "latest"):
good = os.path.join(d, f"{libver}_good.h5")
with h5py.File(good, "w", libver=libver) as f:
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,))
data = bytearray(open(good, "rb").read())
if libver == "earliest":
at = data.find(struct.pack("<II", 37, 4)) + 4
width = 4
else:
# v4: version, class, flags, ndims, bytes per dim, then the dims
at = data.find(bytes([4, 2, 0, 2, 1, 37, 4])) + 6
width = 1
assert at > 6
for size in (2, 8):
bad = bytearray(data)
bad[at:at + width] = size.to_bytes(width, "little")
if libver == "latest":
fix_ohdr(bad, bad.rfind(b"OHDR", 0, at))
open(os.path.join(d, f"{libver}_size{size}.h5"), "wb").write(bad)
"#
);
let verdicts = h5py_verdicts(dir.path(), &body);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&[
"earliest_good ok",
"earliest_size2 ERROR",
"earliest_size8 ERROR",
"latest_good ok",
"latest_size2 ERROR",
"latest_size8 ERROR",
],
);
for name in ["earliest_size2", "latest_size8"] {
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5"))).unwrap_err();
assert!(
err.contains("stored datatype size in chunk layout"),
"{name}: {err}"
);
}
}
/// A v2 object header message whose size runs past the messages into the
/// chunk's checksum is refused by libhdf5 whether it runs 1 byte or more
/// into the checksum (its loop stops at the checksum, and then the checksum
/// read and the size check fail), so it is refused here too.
#[test]
fn v2_header_message_running_into_the_checksum_is_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
let body = format!(
"{FIX_OHDR_PY}{}",
r#"
good = os.path.join(d, "good.h5")
with h5py.File(good, "w", libver="latest") as f:
f.create_dataset("d", data=np.arange(4, dtype="<i4"))
raw = bytearray(open(good, "rb").read())
off = raw.rfind(b"OHDR")
flags = raw[off + 5]; p = off + 6
if flags & 0x20: p += 16
if flags & 0x10: p += 4
width = 1 << (flags & 3)
end = p + width + int.from_bytes(raw[p:p + width], "little"); p += width
hdr = 6 if flags & 0x04 else 4
last = p
while p + hdr <= end:
last = p
p += hdr + int.from_bytes(raw[p + 1:p + 3], "little")
assert p == end
size = int.from_bytes(raw[last + 1:last + 3], "little")
for k in (1, 4, 5):
bad = bytearray(raw)
bad[last + 1:last + 3] = (size + k).to_bytes(2, "little")
fix_ohdr(bad, off)
open(os.path.join(d, f"into_checksum_{k}.h5"), "wb").write(bad)
"#
);
let verdicts = h5py_verdicts(dir.path(), &body);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&[
"good ok",
"into_checksum_1 ERROR",
"into_checksum_4 ERROR",
"into_checksum_5 ERROR",
],
);
}
#[test]
fn truncated_files_are_refused_and_nothing_past_the_end_of_file_is_read() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
// The superblock records where the file's data ends. A copy missing its
// last bytes is truncated: libhdf5 refuses to open it (this read what
// was left). Bytes appended after the end are not part of the file, and
// a superblock moved by prepending a user block (its recorded base
// address now wrong) has its end of file moved with it; both still read.
let verdicts = h5py_verdicts(
dir.path(),
r#"
for libver in ("earliest", "latest"):
good = os.path.join(d, f"{libver}_good.h5")
with h5py.File(good, "w", libver=libver) as f:
f.create_dataset("d", data=np.arange(100, dtype="<i4"))
f.attrs["note"] = "x" * 64
data = open(good, "rb").read()
open(os.path.join(d, f"{libver}_truncated.h5"), "wb").write(data[:-8])
open(os.path.join(d, f"{libver}_appended.h5"), "wb").write(data + b"\0" * 64)
open(os.path.join(d, f"{libver}_moved.h5"), "wb").write(b"\0" * 512 + data)
"#,
);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&[
"earliest_appended ok",
"earliest_good ok",
"earliest_moved ok",
"earliest_truncated ERROR",
"latest_appended ok",
"latest_good ok",
"latest_moved ok",
"latest_truncated ERROR",
],
);
for name in ["earliest_truncated", "latest_truncated"] {
let err = File::open(dir.path().join(format!("{name}.h5")))
.err()
.unwrap_or_else(|| panic!("{name} opened"));
assert!(err.to_string().contains("truncated file"), "{name}: {err}");
}
}
#[test]
fn header_and_datatype_damage_libhdf5_refuses_is_refused() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
// Each case damages one field of a valid file h5py wrote (libver
// earliest, so version-1 object headers); all of these read before.
let verdicts = h5py_verdicts(
dir.path(),
r#"
def write(name, make):
path = os.path.join(d, name + ".h5")
with h5py.File(path, "w", libver="earliest") as f:
make(f)
return bytearray(open(path, "rb").read())
def save(name, data):
open(os.path.join(d, name + ".h5"), "wb").write(data)
# A layout message (type 8, 24 bytes, v1 header) flagged shareable.
data = write("layout_good", lambda f: f.create_dataset("d", data=np.arange(4, dtype="<i4")))
at = data.find(bytes([8, 0, 24, 0]))
assert at > 0
bad = bytearray(data); bad[at + 4] |= 0x40
save("layout_shareable", bad)
# A message size that is not a multiple of 8 in a v1 header.
bad = bytearray(data); bad[at + 2] = 23
save("layout_unaligned", bad)
# A compound whose second field repeats the first's name, or overlaps it.
dt = np.dtype([("aa", "<i4"), ("bb", "<i4")])
data = write("compound_good", lambda f: f.create_dataset("d", data=np.zeros(2, dt)))
at = data.find(b"bb\0")
bad = bytearray(data); bad[at:at + 2] = b"aa"
save("compound_duplicate", bad)
bad = bytearray(data); bad[at + 8:at + 12] = struct.pack("<I", 2)
save("compound_overlap", bad)
# An enum member with an empty name.
et = h5py.enum_dtype({"RED": 0, "GREEN": 1}, basetype="i4")
data = write("enum_good", lambda f: f.create_dataset("d", data=np.zeros(2, "<i4"), dtype=et))
at = data.find(b"RED\0")
bad = bytearray(data); bad[at:at + 3] = b"\0\0\0"
save("enum_empty_name", bad)
# A float whose exponent overlaps its mantissa (exponent at bit 20).
data = write("float_good", lambda f: f.create_dataset("d", data=np.zeros(3, "<f4")))
at = data.find(bytes([0, 0, 32, 0, 23, 8, 0, 23]))
bad = bytearray(data); bad[at + 4] = 20
save("float_overlap", bad)
"#,
);
assert_agrees_with_h5py(
dir.path(),
&verdicts,
&[
"compound_duplicate ERROR",
"compound_good ok",
"compound_overlap ERROR",
"enum_empty_name ERROR",
"enum_good ok",
"float_good ok",
"float_overlap ERROR",
"layout_good ok",
"layout_shareable ERROR",
"layout_unaligned ERROR",
],
);
}
/// Runs a Python script (with `h5py`, `numpy as np` and `struct` imported,
/// `d` the output directory) and fails the test if it fails.
fn run_python(dir: &Path, body: &str) {
let script = format!(
"import h5py, numpy as np, struct, os\nd = \"{}\"\n{body}",
dir.display()
);
let out = Command::new(python())
.args(["-c", &script])
.output()
.expect("failed to run python");
assert!(
out.status.success(),
"python failed:\n{}",
String::from_utf8_lossy(&out.stderr)
);
}
/// libhdf5 limits a chunk to under 4 GiB only when a version-1 B-tree
/// indexes it; HDF5 2.0 writes larger chunks with layout version 5 (libver
/// v200), and h5py reads them. These were refused as "chunk size must be <
/// 4GB". Ignored by default: h5py writes a 4 GiB chunk and both libraries
/// hold it in memory (about 9 GiB in all).
#[test]
#[ignore = "writes and reads a 4 GiB chunk (about 9 GiB of memory)"]
fn chunks_of_4_gib_and_more_read_with_layout_v5() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
with h5py.File(os.path.join(d, "big.h5"), "w", libver=("v200", "v200")) as f:
ds = f.create_dataset("d", shape=(10,), maxshape=(None,), chunks=(2**29 + 1,),
dtype="<f8", compression="gzip", compression_opts=1)
ds[:] = np.arange(10.0)
with h5py.File(os.path.join(d, "big.h5"), "r") as f:
assert list(f["d"][:]) == list(np.arange(10.0))
"#,
);
let file = File::open(dir.path().join("big.h5")).unwrap();
let values = file.dataset("d").unwrap().read_f64().unwrap();
assert_eq!(values, (0..10).map(f64::from).collect::<Vec<_>>());
}
/// A variable-length compound member takes 4 + offset size + 4 bytes, which
/// is 12 in a file with 4-byte offsets, and libhdf5 checks for overlapping
/// members with that stored size. A member right after one was refused as
/// "member overlaps with previous member" (the check took the 16 bytes of
/// an 8-byte-offset file), and with it every attribute of the object.
#[test]
fn variable_length_compound_members_in_files_with_4_byte_offsets() {
skip_if_no_python!();
let dir = tempfile::tempdir().unwrap();
run_python(
dir.path(),
r#"
from h5py import h5f, h5p
dt = np.dtype([("s", h5py.string_dtype()), ("i", "<i4")])
a = np.array([("x", 1), ("yy", 2)], dtype=dt)
for libver in ("earliest", "latest"):
fcpl = h5p.create(h5p.FILE_CREATE)
fcpl.set_sizes(4, 4)
fapl = h5p.create(h5p.FILE_ACCESS)
low = h5f.LIBVER_EARLIEST if libver == "earliest" else h5f.LIBVER_LATEST
fapl.set_libver_bounds(low, h5f.LIBVER_LATEST)
path = os.path.join(d, f"{libver}.h5")
with h5py.File(h5f.create(path.encode(), h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)) as f:
f.create_dataset("c", data=a)
f.attrs["c"] = a
f.attrs["note"] = "kept"
with h5py.File(path, "r") as f:
assert f["c"][1]["i"] == 2
assert f.attrs["note"] == "kept"
"#,
);
for libver in ["earliest", "latest"] {
let file = File::open(dir.path().join(format!("{libver}.h5"))).unwrap();
let dtype = file.dataset("c").unwrap().dtype();
assert!(
matches!(&dtype, Ok(clawhdf5::DType::Compound(fields))
if fields.iter().map(|f| f.0.as_str()).eq(["s", "i"])),
"{libver}: {dtype:?}"
);
// (Variable-length values in files with 4-byte offsets are not
// decoded yet; see docs/known-issues.md. The attributes must at
// least be listed without errors.)
let (attrs, errors) = file.root().attrs_with_errors().unwrap();
assert!(errors.is_empty(), "{libver}: {errors:?}");
assert!(
attrs.contains_key("c") && attrs.contains_key("note"),
"{libver}: {attrs:?}"
);
}
}