osobhandClaude Opus 5.5
742ed4dfb8
libhdf5 does not walk the chunk B-tree to read a dataset: it looks each chunk up (H5B_find with H5D__btree_cmp3 and H5D__btree_found), asking for the element-size coordinate as 0. collect_chunk_info_checked now parses the tree with its keys and returns each stored chunk only when that lookup, replayed over the scaled keys, finds it. A key with a non-zero element-size coordinate is therefore found in a 1-D dataset (cmp3 compares only the first coordinate there, and found compares with <=) and missed in a dataset of rank 2 or more, which reads fill values. The previous commit refused every such key, which refused 1-D files libhdf5 reads correctly; before that, the rank-2 case read the chunk's data where h5py reads fill values (cve-2025-44905 /Shuffle_float_data_le, now identical to h5py, so it leaves the conformance report's list of libhdf5 bugs). Test: chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them compares 1-D and 2-D files against h5py's values. It fails on the previous commit (the 1-D file is refused) and with the refusal removed (the 2-D file reads 0..23 where h5py reads fill values). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
719 lines
27 KiB
Rust
719 lines
27 KiB
Rust
//! Corrupt files that libhdf5 refuses must be refused here too, not read.
|
|
//!
|
|
//! h5py writes a valid file, the script corrupts a copy the way a damaged or
|
|
//! malicious file would be, and records whether h5py (libhdf5) still opens
|
|
//! and reads the object. clawhdf5 must agree: read the valid file, refuse
|
|
//! each corrupt one. Skipped when python3 with h5py is unavailable, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::path::Path;
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::File;
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py, numpy"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
/// Runs `body` (Python, with `h5py`, `numpy as np`, `struct` imported and
|
|
/// `d` the output directory) and then, for every `NAME.h5` it wrote,
|
|
/// prints `NAME ok` when h5py opens and reads dataset `d` (or the dataset
|
|
/// the body names in `DSET`) and `NAME ERROR` otherwise. Returns those
|
|
/// lines, sorted.
|
|
fn h5py_verdicts(dir: &Path, body: &str) -> Vec<String> {
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, struct, os, glob
|
|
d = "{dir}"
|
|
{body}
|
|
for path in sorted(glob.glob(os.path.join(d, "*.h5"))):
|
|
name = os.path.basename(path)[:-3]
|
|
try:
|
|
with h5py.File(path, "r") as f:
|
|
f[globals().get("DSET", "d")][()]
|
|
print(name, "ok")
|
|
except Exception:
|
|
print(name, "ERROR")
|
|
"#,
|
|
dir = dir.display()
|
|
);
|
|
let out = Command::new(python())
|
|
.args(["-c", &script])
|
|
.output()
|
|
.expect("failed to run python");
|
|
assert!(
|
|
out.status.success(),
|
|
"python failed:\n{}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
let mut lines: Vec<String> = String::from_utf8_lossy(&out.stdout)
|
|
.lines()
|
|
.map(str::to_owned)
|
|
.collect();
|
|
lines.sort();
|
|
lines
|
|
}
|
|
|
|
/// Whether clawhdf5 opens and reads dataset `dset` of `path` (as raw bytes
|
|
/// of whatever type it has).
|
|
fn clawhdf5_reads(path: &Path, dset: &str) -> Result<(), String> {
|
|
let file = File::open(path).map_err(|e| format!("open: {e}"))?;
|
|
let ds = file.dataset(dset).map_err(|e| format!("dataset: {e}"))?;
|
|
ds.dtype().map_err(|e| format!("dtype: {e}"))?;
|
|
ds.shape().map_err(|e| format!("shape: {e}"))?;
|
|
file.read_multi(&[dset])
|
|
.map(|_| ())
|
|
.map_err(|e| format!("read: {e}"))
|
|
}
|
|
|
|
/// h5py's verdict for each file must be `expected`, and clawhdf5 must read
|
|
/// exactly the files h5py reads.
|
|
fn assert_agrees_with_h5py(dir: &Path, verdicts: &[String], expected: &[&str]) {
|
|
assert_agrees_with_h5py_on(dir, verdicts, expected, "d");
|
|
}
|
|
|
|
/// [`assert_agrees_with_h5py`] reading dataset `dset`.
|
|
fn assert_agrees_with_h5py_on(dir: &Path, verdicts: &[String], expected: &[&str], dset: &str) {
|
|
assert_eq!(verdicts, expected, "h5py's view changed");
|
|
for line in verdicts {
|
|
let (name, verdict) = line.split_once(' ').unwrap();
|
|
let ours = clawhdf5_reads(&dir.join(format!("{name}.h5")), dset);
|
|
match verdict {
|
|
"ok" => assert!(ours.is_ok(), "{name}: h5py reads it, we fail: {ours:?}"),
|
|
_ => assert!(ours.is_err(), "{name}: h5py refuses it, we read it"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn chunk_dimensions_libhdf5_refuses_are_refused() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
// A chunked int32 dataset (chunk 37, element size 4 after it in the
|
|
// layout message), with libver earliest (layout v3) and latest (v4).
|
|
// The corrupt copies set the chunk dimension to 0 (which read as all
|
|
// fill values), to 0x80000000 (an 8 GiB chunk) and to 38, which the
|
|
// chunk index's offsets 37 and 74 are not multiples of (libhdf5: "bad
|
|
// coordinate offset"; the chunks were read at the wrong place). A v4
|
|
// layout indexes chunks by position, not offset, but both libraries
|
|
// refuse the changed chunk grid there too.
|
|
let verdicts = h5py_verdicts(
|
|
dir.path(),
|
|
r#"
|
|
for libver in ("earliest", "latest"):
|
|
good = os.path.join(d, f"{libver}_good.h5")
|
|
with h5py.File(good, "w", libver=libver) as f:
|
|
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,))
|
|
data = bytearray(open(good, "rb").read())
|
|
if libver == "earliest":
|
|
at = data.find(struct.pack("<II", 37, 4))
|
|
width = 4
|
|
else:
|
|
# v4: flags, ndims, bytes per dim, then the dims
|
|
at = data.find(bytes([4, 2, 0, 2]))
|
|
width = data[at + 4]
|
|
at += 5
|
|
assert at > 0
|
|
for name, value in (("zero", 0), ("huge", 0x80000000), ("offgrid", 38)):
|
|
bad = bytearray(data)
|
|
if value >= 1 << (8 * width):
|
|
continue
|
|
bad[at:at + width] = value.to_bytes(width, "little")
|
|
open(os.path.join(d, f"{libver}_{name}.h5"), "wb").write(bad)
|
|
"#,
|
|
);
|
|
assert_agrees_with_h5py(
|
|
dir.path(),
|
|
&verdicts,
|
|
&[
|
|
"earliest_good ok",
|
|
"earliest_huge ERROR",
|
|
"earliest_offgrid ERROR",
|
|
"earliest_zero ERROR",
|
|
"latest_good ok",
|
|
"latest_offgrid ERROR",
|
|
"latest_zero ERROR",
|
|
],
|
|
);
|
|
}
|
|
|
|
/// Python: `fix_ohdr(buf, off)` recomputes the Jenkins lookup3 checksum of
|
|
/// the version-2 object header chunk 0 at `off`, so a field can be changed
|
|
/// in a `libver="latest"` file without the checksum failing first.
|
|
const FIX_OHDR_PY: &str = r#"
|
|
def _rot(x, k):
|
|
return ((x << k) | (x >> (32 - k))) & 0xFFFFFFFF
|
|
def lookup3(data):
|
|
M = 0xFFFFFFFF
|
|
n = len(data); a = b = c = (0xDEADBEEF + n) & M; i = 0
|
|
w = lambda j: int.from_bytes(data[j:j + 4], "little")
|
|
while n > 12:
|
|
a = (a + w(i)) & M; b = (b + w(i + 4)) & M; c = (c + w(i + 8)) & M
|
|
a = (a - c) & M; a ^= _rot(c, 4); c = (c + b) & M
|
|
b = (b - a) & M; b ^= _rot(a, 6); a = (a + c) & M
|
|
c = (c - b) & M; c ^= _rot(b, 8); b = (b + a) & M
|
|
a = (a - c) & M; a ^= _rot(c, 16); c = (c + b) & M
|
|
b = (b - a) & M; b ^= _rot(a, 19); a = (a + c) & M
|
|
c = (c - b) & M; c ^= _rot(b, 4); b = (b + a) & M
|
|
n -= 12; i += 12
|
|
if n == 0:
|
|
return c
|
|
t = bytes(data[i:]) + bytes(12)
|
|
w = lambda j: int.from_bytes(t[j:j + 4], "little")
|
|
a = (a + w(0)) & M; b = (b + w(4)) & M; c = (c + w(8)) & M
|
|
c ^= b; c = (c - _rot(b, 14)) & M
|
|
a ^= c; a = (a - _rot(c, 11)) & M
|
|
b ^= a; b = (b - _rot(a, 25)) & M
|
|
c ^= b; c = (c - _rot(b, 16)) & M
|
|
a ^= c; a = (a - _rot(c, 4)) & M
|
|
b ^= a; b = (b - _rot(a, 14)) & M
|
|
c ^= b; c = (c - _rot(b, 24)) & M
|
|
return c
|
|
def fix_ohdr(buf, off):
|
|
assert buf[off:off + 4] == b"OHDR"
|
|
flags = buf[off + 5]; p = off + 6
|
|
if flags & 0x20: p += 16
|
|
if flags & 0x10: p += 4
|
|
width = 1 << (flags & 3)
|
|
end = p + width + int.from_bytes(buf[p:p + width], "little")
|
|
buf[end:end + 4] = lookup3(bytes(buf[off:end])).to_bytes(4, "little")
|
|
"#;
|
|
|
|
/// A chunked layout records the element size as its last dimension; libhdf5
|
|
/// refuses a dataset whose datatype has another size ("stored datatype size
|
|
/// in chunk layout does not match datatype description"). This read the
|
|
/// chunks laid out with the wrong element size.
|
|
#[test]
|
|
fn chunk_layout_element_size_must_match_the_datatype() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let body = format!(
|
|
"{FIX_OHDR_PY}{}",
|
|
r#"
|
|
for libver in ("earliest", "latest"):
|
|
good = os.path.join(d, f"{libver}_good.h5")
|
|
with h5py.File(good, "w", libver=libver) as f:
|
|
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,))
|
|
data = bytearray(open(good, "rb").read())
|
|
if libver == "earliest":
|
|
at = data.find(struct.pack("<II", 37, 4)) + 4
|
|
width = 4
|
|
else:
|
|
# v4: version, class, flags, ndims, bytes per dim, then the dims
|
|
at = data.find(bytes([4, 2, 0, 2, 1, 37, 4])) + 6
|
|
width = 1
|
|
assert at > 6
|
|
for size in (2, 8):
|
|
bad = bytearray(data)
|
|
bad[at:at + width] = size.to_bytes(width, "little")
|
|
if libver == "latest":
|
|
fix_ohdr(bad, bad.rfind(b"OHDR", 0, at))
|
|
open(os.path.join(d, f"{libver}_size{size}.h5"), "wb").write(bad)
|
|
"#
|
|
);
|
|
let verdicts = h5py_verdicts(dir.path(), &body);
|
|
assert_agrees_with_h5py(
|
|
dir.path(),
|
|
&verdicts,
|
|
&[
|
|
"earliest_good ok",
|
|
"earliest_size2 ERROR",
|
|
"earliest_size8 ERROR",
|
|
"latest_good ok",
|
|
"latest_size2 ERROR",
|
|
"latest_size8 ERROR",
|
|
],
|
|
);
|
|
for name in ["earliest_size2", "latest_size8"] {
|
|
let err = clawhdf5_reads(&dir.path().join(format!("{name}.h5")), "d").unwrap_err();
|
|
assert!(
|
|
err.contains("stored datatype size in chunk layout"),
|
|
"{name}: {err}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A v2 object header message whose size runs past the messages into the
|
|
/// chunk's checksum is refused by libhdf5 whether it runs 1 byte or more
|
|
/// into the checksum (its loop stops at the checksum, and then the checksum
|
|
/// read and the size check fail), so it is refused here too.
|
|
#[test]
|
|
fn v2_header_message_running_into_the_checksum_is_refused() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let body = format!(
|
|
"{FIX_OHDR_PY}{}",
|
|
r#"
|
|
good = os.path.join(d, "good.h5")
|
|
with h5py.File(good, "w", libver="latest") as f:
|
|
f.create_dataset("d", data=np.arange(4, dtype="<i4"))
|
|
raw = bytearray(open(good, "rb").read())
|
|
off = raw.rfind(b"OHDR")
|
|
flags = raw[off + 5]; p = off + 6
|
|
if flags & 0x20: p += 16
|
|
if flags & 0x10: p += 4
|
|
width = 1 << (flags & 3)
|
|
end = p + width + int.from_bytes(raw[p:p + width], "little"); p += width
|
|
hdr = 6 if flags & 0x04 else 4
|
|
last = p
|
|
while p + hdr <= end:
|
|
last = p
|
|
p += hdr + int.from_bytes(raw[p + 1:p + 3], "little")
|
|
assert p == end
|
|
size = int.from_bytes(raw[last + 1:last + 3], "little")
|
|
for k in (1, 4, 5):
|
|
bad = bytearray(raw)
|
|
bad[last + 1:last + 3] = (size + k).to_bytes(2, "little")
|
|
fix_ohdr(bad, off)
|
|
open(os.path.join(d, f"into_checksum_{k}.h5"), "wb").write(bad)
|
|
"#
|
|
);
|
|
let verdicts = h5py_verdicts(dir.path(), &body);
|
|
assert_agrees_with_h5py(
|
|
dir.path(),
|
|
&verdicts,
|
|
&[
|
|
"good ok",
|
|
"into_checksum_1 ERROR",
|
|
"into_checksum_4 ERROR",
|
|
"into_checksum_5 ERROR",
|
|
],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn truncated_files_are_refused_and_nothing_past_the_end_of_file_is_read() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
// The superblock records where the file's data ends. A copy missing its
|
|
// last bytes is truncated: libhdf5 refuses to open it (this read what
|
|
// was left). Bytes appended after the end are not part of the file, and
|
|
// a superblock moved by prepending a user block (its recorded base
|
|
// address now wrong) has its end of file moved with it; both still read.
|
|
let verdicts = h5py_verdicts(
|
|
dir.path(),
|
|
r#"
|
|
for libver in ("earliest", "latest"):
|
|
good = os.path.join(d, f"{libver}_good.h5")
|
|
with h5py.File(good, "w", libver=libver) as f:
|
|
f.create_dataset("d", data=np.arange(100, dtype="<i4"))
|
|
f.attrs["note"] = "x" * 64
|
|
data = open(good, "rb").read()
|
|
open(os.path.join(d, f"{libver}_truncated.h5"), "wb").write(data[:-8])
|
|
open(os.path.join(d, f"{libver}_appended.h5"), "wb").write(data + b"\0" * 64)
|
|
open(os.path.join(d, f"{libver}_moved.h5"), "wb").write(b"\0" * 512 + data)
|
|
"#,
|
|
);
|
|
assert_agrees_with_h5py(
|
|
dir.path(),
|
|
&verdicts,
|
|
&[
|
|
"earliest_appended ok",
|
|
"earliest_good ok",
|
|
"earliest_moved ok",
|
|
"earliest_truncated ERROR",
|
|
"latest_appended ok",
|
|
"latest_good ok",
|
|
"latest_moved ok",
|
|
"latest_truncated ERROR",
|
|
],
|
|
);
|
|
for name in ["earliest_truncated", "latest_truncated"] {
|
|
let err = File::open(dir.path().join(format!("{name}.h5")))
|
|
.err()
|
|
.unwrap_or_else(|| panic!("{name} opened"));
|
|
assert!(err.to_string().contains("truncated file"), "{name}: {err}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn header_and_datatype_damage_libhdf5_refuses_is_refused() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
// Each case damages one field of a valid file h5py wrote (libver
|
|
// earliest, so version-1 object headers); all of these read before.
|
|
let verdicts = h5py_verdicts(
|
|
dir.path(),
|
|
r#"
|
|
def write(name, make):
|
|
path = os.path.join(d, name + ".h5")
|
|
with h5py.File(path, "w", libver="earliest") as f:
|
|
make(f)
|
|
return bytearray(open(path, "rb").read())
|
|
|
|
def save(name, data):
|
|
open(os.path.join(d, name + ".h5"), "wb").write(data)
|
|
|
|
# A layout message (type 8, 24 bytes, v1 header) flagged shareable.
|
|
data = write("layout_good", lambda f: f.create_dataset("d", data=np.arange(4, dtype="<i4")))
|
|
at = data.find(bytes([8, 0, 24, 0]))
|
|
assert at > 0
|
|
bad = bytearray(data); bad[at + 4] |= 0x40
|
|
save("layout_shareable", bad)
|
|
# A message size that is not a multiple of 8 in a v1 header.
|
|
bad = bytearray(data); bad[at + 2] = 23
|
|
save("layout_unaligned", bad)
|
|
|
|
# A compound whose second field repeats the first's name, or overlaps it.
|
|
dt = np.dtype([("aa", "<i4"), ("bb", "<i4")])
|
|
data = write("compound_good", lambda f: f.create_dataset("d", data=np.zeros(2, dt)))
|
|
at = data.find(b"bb\0")
|
|
bad = bytearray(data); bad[at:at + 2] = b"aa"
|
|
save("compound_duplicate", bad)
|
|
bad = bytearray(data); bad[at + 8:at + 12] = struct.pack("<I", 2)
|
|
save("compound_overlap", bad)
|
|
|
|
# An enum member with an empty name.
|
|
et = h5py.enum_dtype({"RED": 0, "GREEN": 1}, basetype="i4")
|
|
data = write("enum_good", lambda f: f.create_dataset("d", data=np.zeros(2, "<i4"), dtype=et))
|
|
at = data.find(b"RED\0")
|
|
bad = bytearray(data); bad[at:at + 3] = b"\0\0\0"
|
|
save("enum_empty_name", bad)
|
|
|
|
# A float whose exponent overlaps its mantissa (exponent at bit 20).
|
|
data = write("float_good", lambda f: f.create_dataset("d", data=np.zeros(3, "<f4")))
|
|
at = data.find(bytes([0, 0, 32, 0, 23, 8, 0, 23]))
|
|
bad = bytearray(data); bad[at + 4] = 20
|
|
save("float_overlap", bad)
|
|
"#,
|
|
);
|
|
assert_agrees_with_h5py(
|
|
dir.path(),
|
|
&verdicts,
|
|
&[
|
|
"compound_duplicate ERROR",
|
|
"compound_good ok",
|
|
"compound_overlap ERROR",
|
|
"enum_empty_name ERROR",
|
|
"enum_good ok",
|
|
"float_good ok",
|
|
"float_overlap ERROR",
|
|
"layout_good ok",
|
|
"layout_shareable ERROR",
|
|
"layout_unaligned ERROR",
|
|
],
|
|
);
|
|
}
|
|
|
|
/// Runs a Python script (with `h5py`, `numpy as np` and `struct` imported,
|
|
/// `d` the output directory) and fails the test if it fails.
|
|
fn run_python(dir: &Path, body: &str) {
|
|
let script = format!(
|
|
"import h5py, numpy as np, struct, os\nd = \"{}\"\n{body}",
|
|
dir.display()
|
|
);
|
|
let out = Command::new(python())
|
|
.args(["-c", &script])
|
|
.output()
|
|
.expect("failed to run python");
|
|
assert!(
|
|
out.status.success(),
|
|
"python failed:\n{}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
}
|
|
|
|
/// libhdf5 limits a chunk to under 4 GiB only when a version-1 B-tree
|
|
/// indexes it; HDF5 2.0 writes larger chunks with layout version 5 (libver
|
|
/// v200), and h5py reads them. These were refused as "chunk size must be <
|
|
/// 4GB". Ignored by default: h5py writes a 4 GiB chunk and both libraries
|
|
/// hold it in memory (about 9 GiB in all).
|
|
#[test]
|
|
#[ignore = "writes and reads a 4 GiB chunk (about 9 GiB of memory)"]
|
|
fn chunks_of_4_gib_and_more_read_with_layout_v5() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
run_python(
|
|
dir.path(),
|
|
r#"
|
|
with h5py.File(os.path.join(d, "big.h5"), "w", libver=("v200", "v200")) as f:
|
|
ds = f.create_dataset("d", shape=(10,), maxshape=(None,), chunks=(2**29 + 1,),
|
|
dtype="<f8", compression="gzip", compression_opts=1)
|
|
ds[:] = np.arange(10.0)
|
|
with h5py.File(os.path.join(d, "big.h5"), "r") as f:
|
|
assert list(f["d"][:]) == list(np.arange(10.0))
|
|
"#,
|
|
);
|
|
let file = File::open(dir.path().join("big.h5")).unwrap();
|
|
let values = file.dataset("d").unwrap().read_f64().unwrap();
|
|
assert_eq!(values, (0..10).map(f64::from).collect::<Vec<_>>());
|
|
}
|
|
|
|
/// A variable-length compound member takes 4 + offset size + 4 bytes, which
|
|
/// is 12 in a file with 4-byte offsets, and libhdf5 checks for overlapping
|
|
/// members with that stored size. A member right after one was refused as
|
|
/// "member overlaps with previous member" (the check took the 16 bytes of
|
|
/// an 8-byte-offset file), and with it every attribute of the object.
|
|
#[test]
|
|
fn variable_length_compound_members_in_files_with_4_byte_offsets() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
run_python(
|
|
dir.path(),
|
|
r#"
|
|
from h5py import h5f, h5p
|
|
dt = np.dtype([("s", h5py.string_dtype()), ("i", "<i4")])
|
|
a = np.array([("x", 1), ("yy", 2)], dtype=dt)
|
|
for libver in ("earliest", "latest"):
|
|
fcpl = h5p.create(h5p.FILE_CREATE)
|
|
fcpl.set_sizes(4, 4)
|
|
fapl = h5p.create(h5p.FILE_ACCESS)
|
|
low = h5f.LIBVER_EARLIEST if libver == "earliest" else h5f.LIBVER_LATEST
|
|
fapl.set_libver_bounds(low, h5f.LIBVER_LATEST)
|
|
path = os.path.join(d, f"{libver}.h5")
|
|
with h5py.File(h5f.create(path.encode(), h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)) as f:
|
|
f.create_dataset("c", data=a)
|
|
f.attrs["c"] = a
|
|
f.attrs["note"] = "kept"
|
|
with h5py.File(path, "r") as f:
|
|
assert f["c"][1]["i"] == 2
|
|
assert f.attrs["note"] == "kept"
|
|
"#,
|
|
);
|
|
for libver in ["earliest", "latest"] {
|
|
let file = File::open(dir.path().join(format!("{libver}.h5"))).unwrap();
|
|
let dtype = file.dataset("c").unwrap().dtype();
|
|
assert!(
|
|
matches!(&dtype, Ok(clawhdf5::DType::Compound(fields))
|
|
if fields.iter().map(|f| f.0.as_str()).eq(["s", "i"])),
|
|
"{libver}: {dtype:?}"
|
|
);
|
|
// (Variable-length values in files with 4-byte offsets are not
|
|
// decoded yet; see docs/known-issues.md. The attributes must at
|
|
// least be listed without errors.)
|
|
let (attrs, errors) = file.root().attrs_with_errors().unwrap();
|
|
assert!(errors.is_empty(), "{libver}: {errors:?}");
|
|
assert!(
|
|
attrs.contains_key("c") && attrs.contains_key("note"),
|
|
"{libver}: {attrs:?}"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// cve-2024-32624 `/Dset_OBJREF`: a dataspace whose element count times the
|
|
/// element size overflows 64 bits. libhdf5 refuses to open the dataset
|
|
/// ("size of dataset's storage overflowed"); `File::dataset` used to open it
|
|
/// and report its shape, and only reading failed. The same for storage that
|
|
/// runs past the end of the file ("invalid dataset size, likely file
|
|
/// corruption").
|
|
#[test]
|
|
fn dataset_storage_libhdf5_refuses_at_open_is_refused_at_open() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
// A contiguous int64 dataset of 3 elements, libver earliest (a version 1
|
|
// dataspace holding the size and the maximum size). "overflow" makes
|
|
// both 2^62 + 2 (x 8 bytes overflows); "pasteof" makes both 1000 (the
|
|
// storage runs past the end of the file).
|
|
let verdicts = h5py_verdicts(
|
|
dir.path(),
|
|
r#"
|
|
good = os.path.join(d, "good.h5")
|
|
with h5py.File(good, "w", libver="earliest") as f:
|
|
f.create_dataset("d", data=np.array([7, 8, 9], dtype="<i8"))
|
|
data = bytearray(open(good, "rb").read())
|
|
three = struct.pack("<QQ", 3, 3)
|
|
at = data.find(three)
|
|
assert at > 0 and data.find(three, at + 1) < 0
|
|
for name, n in (("overflow", (1 << 62) + 2), ("pasteof", 1000)):
|
|
bad = bytearray(data)
|
|
bad[at:at + 16] = struct.pack("<QQ", n, n)
|
|
open(os.path.join(d, f"{name}.h5"), "wb").write(bad)
|
|
"#,
|
|
);
|
|
assert_agrees_with_h5py(
|
|
dir.path(),
|
|
&verdicts,
|
|
&["good ok", "overflow ERROR", "pasteof ERROR"],
|
|
);
|
|
for name in ["overflow", "pasteof"] {
|
|
let path = dir.path().join(format!("{name}.h5"));
|
|
let file = File::open(&path).unwrap();
|
|
let err = file.dataset("d").unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("invalid dataset storage"),
|
|
"{name}: {err}"
|
|
);
|
|
assert!(file.root().dataset("d").is_err(), "{name}");
|
|
let mm = clawhdf5::MmapFile::open(&path).unwrap();
|
|
assert!(mm.dataset("d").is_err(), "{name}: MmapFile");
|
|
}
|
|
}
|
|
|
|
/// libhdf5 decodes the superblock extension's messages when it opens a file
|
|
/// and refuses the file when one does not decode: cve-2020-10810 (a File
|
|
/// Space Info message too short for what it announces), cve-2020-10812 (a
|
|
/// metadata cache image past the end of the file). We did not look at those
|
|
/// messages and opened such files.
|
|
#[test]
|
|
fn superblock_extension_messages_libhdf5_refuses_are_refused() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let mdc = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/h5clear_mdc_image.h5");
|
|
let body = format!(
|
|
r#"
|
|
{FIX_OHDR_PY}
|
|
def ext_addr(buf):
|
|
assert buf[8] in (2, 3)
|
|
return int.from_bytes(buf[20:28], "little")
|
|
def save(name, buf):
|
|
open(os.path.join(d, name + ".h5"), "wb").write(buf)
|
|
|
|
# A paged file: its superblock extension holds a File Space Info message
|
|
# (version 1, strategy page, not persisting, threshold 1, page size 4096).
|
|
DSET = "DSET"
|
|
good = os.path.join(d, "paged_good.h5")
|
|
with h5py.File(good, "w", libver="latest", fs_strategy="page", fs_page_size=4096) as f:
|
|
f.create_dataset("DSET", data=np.arange(10, dtype="<i4"))
|
|
data = bytearray(open(good, "rb").read())
|
|
ext = ext_addr(data)
|
|
at = data.find(struct.pack("<QQ", 1, 4096), ext)
|
|
assert at > ext
|
|
# Page size 256 (under libhdf5's minimum of 512).
|
|
bad = bytearray(data); bad[at + 8:at + 16] = struct.pack("<Q", 256); fix_ohdr(bad, ext)
|
|
save("paged_small_page", bad)
|
|
# Persisting free space, without the manager addresses that then follow.
|
|
bad = bytearray(data); bad[at - 1] = 1; fix_ohdr(bad, ext)
|
|
save("paged_persist_short", bad)
|
|
|
|
# h5clear_mdc_image.h5 (from libhdf5's tests): a metadata cache image, the
|
|
# root group's header only in the image. The Metadata Cache Image message
|
|
# (type 0x18) records the image's address and length.
|
|
data = bytearray(open(r"{mdc}", "rb").read())
|
|
save("mdc_good", data)
|
|
ext = ext_addr(data)
|
|
at = data.find(bytes([0x18, 17, 0]), ext)
|
|
assert at > ext
|
|
length = at + 5 + 8
|
|
bad = bytearray(data); bad[length:length + 8] = struct.pack("<Q", 1 << 28); fix_ohdr(bad, ext)
|
|
save("mdc_past_eof", bad)
|
|
"#,
|
|
mdc = mdc.display()
|
|
);
|
|
let verdicts = h5py_verdicts(dir.path(), &body);
|
|
assert_agrees_with_h5py_on(
|
|
dir.path(),
|
|
&verdicts,
|
|
&[
|
|
"mdc_good ok",
|
|
"mdc_past_eof ERROR",
|
|
"paged_good ok",
|
|
"paged_persist_short ERROR",
|
|
"paged_small_page ERROR",
|
|
],
|
|
"DSET",
|
|
);
|
|
}
|
|
|
|
/// An unfiltered chunk the index records at less than the chunk's size
|
|
/// (`cve-2025-44904`): HDF5 2.0 fills the rest of the chunk with whatever
|
|
/// its buffer held, and later libhdf5 releases refuse it ("incorrect chunk
|
|
/// size returned from index for unfiltered chunk"); we used to read the
|
|
/// rest as zeros, and now refuse it.
|
|
#[test]
|
|
fn unfiltered_chunk_of_the_wrong_size_is_refused() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
run_python(
|
|
dir.path(),
|
|
r#"
|
|
good = os.path.join(d, "good.h5")
|
|
with h5py.File(good, "w", libver="earliest") as f:
|
|
f.create_dataset("d", data=np.arange(100, dtype="<i4"), chunks=(37,), fillvalue=-1)
|
|
data = bytearray(open(good, "rb").read())
|
|
tree = data.find(b"TREE")
|
|
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
|
|
tree = data.find(b"TREE", tree + 1)
|
|
assert tree > 0 and data[tree + 5] == 0
|
|
key = tree + 8 + 16 # the first chunk's key: size, filter mask, offsets
|
|
second = key + 24 + 8 # a key (4 + 4 + 2 x 8 bytes), then a child address
|
|
assert struct.unpack_from("<IIQQ", data, second) == (148, 0, 37, 0)
|
|
bad = bytearray(data); struct.pack_into("<I", bad, second, 100)
|
|
open(os.path.join(d, "short_chunk.h5"), "wb").write(bad)
|
|
"#,
|
|
);
|
|
assert_eq!(
|
|
clawhdf5_reads(&dir.path().join("good.h5"), "d"),
|
|
Ok(()),
|
|
"good"
|
|
);
|
|
let err = clawhdf5_reads(&dir.path().join("short_chunk.h5"), "d").unwrap_err();
|
|
assert!(err.starts_with("read:"), "short_chunk: {err}");
|
|
}
|
|
|
|
/// A v1 B-tree chunk key whose element-size coordinate is not 0. libhdf5
|
|
/// looks each chunk up with that coordinate set to 0 (`H5B_find`,
|
|
/// `H5D__btree_cmp3`, `H5D__btree_found`): in a 1-D dataset it still finds
|
|
/// the chunk, in a 2-D one it does not and reads fill values
|
|
/// (`cve-2025-44905` `/Shuffle_float_data_le`). Both must read exactly what
|
|
/// h5py reads: the 1-D file was refused, and before that the 2-D chunk was
|
|
/// read as if its key were well formed.
|
|
#[test]
|
|
fn chunk_keys_with_an_element_offset_read_as_libhdf5_reads_them() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
run_python(
|
|
dir.path(),
|
|
r#"
|
|
def corrupt(name, shape, chunks, which, element_offset):
|
|
path = os.path.join(d, name + ".h5")
|
|
with h5py.File(path, "w", libver="earliest") as f:
|
|
f.create_dataset("d", data=np.arange(np.prod(shape), dtype="<i4").reshape(shape),
|
|
chunks=chunks, fillvalue=-1)
|
|
data = bytearray(open(path, "rb").read())
|
|
tree = data.find(b"TREE")
|
|
while data[tree + 4] != 1: # the chunk index, not the root group's B-tree
|
|
tree = data.find(b"TREE", tree + 1)
|
|
assert tree > 0 and data[tree + 5] == 0
|
|
ndims = len(shape) + 1
|
|
key_size = 8 + 8 * ndims
|
|
key = tree + 8 + 16 + which * (key_size + 8)
|
|
last = key + 8 + 8 * (ndims - 1)
|
|
assert struct.unpack_from("<Q", data, last) == (0,)
|
|
struct.pack_into("<Q", data, last, element_offset)
|
|
open(path, "wb").write(data)
|
|
with h5py.File(path, "r") as f:
|
|
open(path + ".txt", "w").write(" ".join(map(str, f["d"][()].ravel())))
|
|
|
|
corrupt("one_d", (100,), (37,), 1, 4096)
|
|
corrupt("two_d", (4, 6), (2, 3), 1, 4)
|
|
corrupt("two_d_first", (4, 6), (2, 3), 0, 8)
|
|
"#,
|
|
);
|
|
for name in ["one_d", "two_d", "two_d_first"] {
|
|
let path = dir.path().join(format!("{name}.h5"));
|
|
let expected: Vec<i32> = std::fs::read_to_string(dir.path().join(format!("{name}.h5.txt")))
|
|
.unwrap()
|
|
.split_whitespace()
|
|
.map(|v| v.parse().unwrap())
|
|
.collect();
|
|
let file = File::open(&path).unwrap();
|
|
let got = file.dataset("d").unwrap().read_i32();
|
|
assert_eq!(got.as_deref().ok(), Some(&expected[..]), "{name}: {got:?}");
|
|
}
|
|
}
|