osobhandClaude Opus 5.5
159e588550
Our checksum reduced its sums with `% 65535`; libhdf5's H5_checksum_fletcher32 folds them with `(s & 0xffff) + (s >> 16)`, which leaves 0xffff where the modulo leaves 0. On about one chunk in 32768 libhdf5 refused the chunks we wrote and we refused the chunks it wrote. Every release since v2.1.0 is affected. clawhdf5_format::checksum::fletcher32 is a port of H5_checksum_fletcher32 and the filter's only implementation. Verification also accepts the byte-swapped form libhdf5 accepts (1.6.2 and earlier) and the `% 65535` form earlier releases wrote, so their files stay readable. The new interop test compares the checksum with libhdf5's own function (ctypes) on every 1- and 2-byte input and 40 000 random and fold-heavy inputs, and moves fold-case chunks between h5py and FileBuilder/FileEditor in both directions; with the old filters.rs the three file tests fail. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
322 lines
11 KiB
Rust
322 lines
11 KiB
Rust
//! Fletcher-32 against libhdf5.
|
|
//!
|
|
//! libhdf5's `H5_checksum_fletcher32` reduces its sums with the
|
|
//! ones'-complement fold `(s & 0xffff) + (s >> 16)`, which leaves 0xffff
|
|
//! where `% 65535` leaves 0. Our checksum once used `% 65535`, so on about
|
|
//! one chunk in 32768 (a sum that is a non-zero multiple of 65535) libhdf5
|
|
//! rejected the chunks we wrote and we rejected the chunks it wrote.
|
|
//!
|
|
//! - The checksum is compared with libhdf5's own `H5_checksum_fletcher32`,
|
|
//! called through ctypes from the library h5py loads, over every one-byte
|
|
//! and two-byte input and a large corpus of random and fold-heavy inputs.
|
|
//! - Chunks engineered to hit the fold are written by `FileBuilder` and by
|
|
//! `FileEditor` and read by h5py, and written by h5py and read by us.
|
|
//!
|
|
//! Skipped when python3 with h5py is unavailable, unless
|
|
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
|
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{File, FileBuilder, FileEditor};
|
|
use clawhdf5_format::checksum::fletcher32;
|
|
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
fn have_h5py() -> bool {
|
|
let ok = Command::new(python())
|
|
.args(["-c", "import h5py, numpy"])
|
|
.output()
|
|
.is_ok_and(|o| o.status.success());
|
|
if !ok {
|
|
assert!(
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
}
|
|
ok
|
|
}
|
|
|
|
fn run_python(script: &str, args: &[&str]) -> String {
|
|
let out = Command::new(python())
|
|
.arg("-c")
|
|
.arg(script)
|
|
.args(args)
|
|
.output()
|
|
.expect("failed to run python");
|
|
assert!(
|
|
out.status.success(),
|
|
"python failed:\nSTDOUT: {}\nSTDERR: {}",
|
|
String::from_utf8_lossy(&out.stdout),
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
String::from_utf8_lossy(&out.stdout).trim().to_string()
|
|
}
|
|
|
|
fn tmp(name: &str) -> std::path::PathBuf {
|
|
let dir = std::env::temp_dir().join(format!("clawhdf5_fletcher32_{}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
dir.join(name)
|
|
}
|
|
|
|
/// The checksum our code computed before it was fixed: each sum reduced
|
|
/// `% 65535`. Only used to show that the test data hits the disagreement.
|
|
fn fletcher32_mod(data: &[u8]) -> u32 {
|
|
let (mut s1, mut s2) = (0u64, 0u64);
|
|
for w in data.chunks(2) {
|
|
let v = (u64::from(w[0]) << 8) | w.get(1).map_or(0, |&b| u64::from(b));
|
|
s1 = (s1 + v) % 65535;
|
|
s2 = (s2 + s1) % 65535;
|
|
}
|
|
((s2 as u32) << 16) | s1 as u32
|
|
}
|
|
|
|
/// Splitmix64, so the data is the same on every run.
|
|
struct Rng(u64);
|
|
impl Rng {
|
|
fn next(&mut self) -> u64 {
|
|
self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15);
|
|
let mut z = self.0;
|
|
z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
|
|
z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
|
|
z ^ (z >> 31)
|
|
}
|
|
}
|
|
|
|
/// Writes every case of the file `argv[1]` (u32 LE length + bytes) back to
|
|
/// `argv[2]` as libhdf5's checksum of each, u32 LE.
|
|
const LIBHDF5_CHECKSUMS: &str = r#"
|
|
import ctypes, glob, os, struct, sys
|
|
import h5py
|
|
cands = glob.glob(os.path.join(os.path.dirname(h5py.__file__), os.pardir, 'h5py.libs', 'libhdf5-*.so*'))
|
|
cands += glob.glob(os.path.join(os.path.dirname(h5py.__file__), '.dylibs', 'libhdf5*.dylib'))
|
|
if cands:
|
|
lib = ctypes.CDLL(cands[0])
|
|
else:
|
|
# A system h5py links the system libhdf5, already loaded.
|
|
import h5py.h5
|
|
lib = ctypes.CDLL(h5py.h5.__file__)
|
|
f = lib.H5_checksum_fletcher32
|
|
f.restype = ctypes.c_uint32
|
|
f.argtypes = [ctypes.c_char_p, ctypes.c_size_t]
|
|
data = open(sys.argv[1], 'rb').read()
|
|
out = bytearray()
|
|
i = 0
|
|
while i < len(data):
|
|
(n,) = struct.unpack_from('<I', data, i)
|
|
i += 4
|
|
b = data[i:i + n]
|
|
i += n
|
|
out += struct.pack('<I', f(b, n))
|
|
open(sys.argv[2], 'wb').write(out)
|
|
"#;
|
|
|
|
#[test]
|
|
fn checksum_matches_libhdf5() {
|
|
if !have_h5py() {
|
|
return;
|
|
}
|
|
let mut cases: Vec<Vec<u8>> = Vec::new();
|
|
// Every one-byte input (the odd-length path alone) and every one-word
|
|
// input (65535 = 0xffff is the smallest fold).
|
|
cases.extend((0..=255u8).map(|b| vec![b]));
|
|
cases.extend((0..=u16::MAX).map(|w| w.to_be_bytes().to_vec()));
|
|
let mut rng = Rng(0x5eed_f1e7);
|
|
// Words drawn from values that make multiples of 65535 frequent, at
|
|
// lengths around the 360-word block boundaries, odd and even.
|
|
const FOLDY: [u16; 6] = [0, 1, 0xfffe, 0xffff, 0x8000, 0x7fff];
|
|
for _ in 0..40_000 {
|
|
let len = match rng.next() % 4 {
|
|
0 => (rng.next() % 16) as usize,
|
|
1 => 718 + (rng.next() % 6) as usize,
|
|
2 => 1438 + (rng.next() % 6) as usize,
|
|
_ => (rng.next() % 3000) as usize,
|
|
};
|
|
let foldy = rng.next().is_multiple_of(2);
|
|
let mut v = Vec::with_capacity(len + 1);
|
|
while v.len() < len {
|
|
let w = if foldy {
|
|
FOLDY[(rng.next() % 6) as usize]
|
|
} else {
|
|
rng.next() as u16
|
|
};
|
|
v.extend_from_slice(&w.to_be_bytes());
|
|
}
|
|
v.truncate(len);
|
|
cases.push(v);
|
|
}
|
|
// Long runs of 0xff: sums are multiples of 65535 at every block.
|
|
for len in [720, 721, 1440, 1441, 7200, 65536, 65537] {
|
|
cases.push(vec![0xff; len]);
|
|
}
|
|
let mut blob = Vec::new();
|
|
for c in &cases {
|
|
blob.extend_from_slice(&(c.len() as u32).to_le_bytes());
|
|
blob.extend_from_slice(c);
|
|
}
|
|
let input = tmp("cases.bin");
|
|
let output = tmp("sums.bin");
|
|
std::fs::write(&input, &blob).unwrap();
|
|
run_python(
|
|
LIBHDF5_CHECKSUMS,
|
|
&[input.to_str().unwrap(), output.to_str().unwrap()],
|
|
);
|
|
let sums = std::fs::read(&output).unwrap();
|
|
assert_eq!(sums.len(), cases.len() * 4);
|
|
let mut folds = 0;
|
|
for (c, s) in cases.iter().zip(sums.as_chunks::<4>().0) {
|
|
let want = u32::from_le_bytes(*s);
|
|
assert_eq!(
|
|
fletcher32(c),
|
|
want,
|
|
"checksum of {} bytes {:02x?}...",
|
|
c.len(),
|
|
&c[..c.len().min(16)]
|
|
);
|
|
if fletcher32_mod(c) != want {
|
|
folds += 1;
|
|
}
|
|
}
|
|
// The corpus must exercise the case `% 65535` got wrong.
|
|
assert!(folds > 500, "only {folds} fold cases");
|
|
}
|
|
|
|
const CHUNK: usize = 8;
|
|
|
|
/// `n` chunks of `CHUNK` bytes, each one a chunk on which the old
|
|
/// `% 65535` checksum and libhdf5's differ (sum1, sum2 or both a non-zero
|
|
/// multiple of 65535), with an ordinary chunk between them.
|
|
fn fold_chunks(n: usize) -> Vec<u8> {
|
|
let mut rng = Rng(42);
|
|
let mut out = Vec::new();
|
|
let mut found = 0;
|
|
while found < n {
|
|
// Build a chunk whose sum1 is a multiple of 65535 half the time,
|
|
// otherwise search at random for a sum2 fold.
|
|
let mut c: Vec<u8> = (0..CHUNK).map(|_| rng.next() as u8).collect();
|
|
if found % 2 == 0 {
|
|
let words: u64 = c[..CHUNK - 2]
|
|
.chunks(2)
|
|
.map(|w| (u64::from(w[0]) << 8) | u64::from(w[1]))
|
|
.sum();
|
|
let last = ((65535 - words % 65535) % 65535) as u16;
|
|
c[CHUNK - 2..].copy_from_slice(&last.to_be_bytes());
|
|
}
|
|
if fletcher32(&c) != fletcher32_mod(&c) {
|
|
out.extend_from_slice(&c);
|
|
out.extend((0..CHUNK).map(|i| i as u8 + 1));
|
|
found += 1;
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_reads_fold_case_chunks_we_write() {
|
|
if !have_h5py() {
|
|
return;
|
|
}
|
|
let data = fold_chunks(32);
|
|
// FileBuilder.
|
|
let built = tmp("built.h5");
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("d")
|
|
.with_u8_data(&data)
|
|
.with_chunks(&[CHUNK as u64])
|
|
.with_fletcher32();
|
|
b.write(&built).unwrap();
|
|
// FileEditor, into a dataset h5py created.
|
|
let edited = tmp("edited.h5");
|
|
run_python(
|
|
"import sys, h5py, numpy as np\n\
|
|
with h5py.File(sys.argv[1], 'w') as f:\n\
|
|
\x20 f.create_dataset('d', data=np.zeros(int(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)",
|
|
&[edited.to_str().unwrap(), &data.len().to_string()],
|
|
);
|
|
FileEditor::open(&edited)
|
|
.unwrap()
|
|
.write_all("d", &data)
|
|
.unwrap();
|
|
for path in [&built, &edited] {
|
|
let got = run_python(
|
|
"import sys, h5py\n\
|
|
with h5py.File(sys.argv[1], 'r') as f:\n\
|
|
\x20 assert f['d'].fletcher32\n\
|
|
\x20 print(f['d'][:].tobytes().hex())",
|
|
&[path.to_str().unwrap()],
|
|
);
|
|
assert_eq!(got, hex(&data), "{}", path.display());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn we_read_fold_case_chunks_h5py_writes() {
|
|
if !have_h5py() {
|
|
return;
|
|
}
|
|
let data = fold_chunks(32);
|
|
let path = tmp("h5py.h5");
|
|
run_python(
|
|
"import sys, h5py, numpy as np\n\
|
|
with h5py.File(sys.argv[1], 'w') as f:\n\
|
|
\x20 f.create_dataset('d', data=np.frombuffer(bytes.fromhex(sys.argv[2]), 'u1'), chunks=(8,), fletcher32=True)",
|
|
&[path.to_str().unwrap(), &hex(&data)],
|
|
);
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("d").unwrap();
|
|
assert_eq!(
|
|
ds.read_selection(&clawhdf5_format::selection::Selection::All)
|
|
.unwrap(),
|
|
data
|
|
);
|
|
}
|
|
|
|
/// A checksum stored with the bytes of each 16-bit half swapped, as
|
|
/// libhdf5 1.6.2 and earlier wrote it, is accepted as libhdf5 accepts it;
|
|
/// so is the `% 65535` form clawhdf5 v2.7.0 and earlier wrote, so that
|
|
/// their files stay readable.
|
|
#[test]
|
|
fn legacy_checksums_are_accepted() {
|
|
use clawhdf5_format::filter_pipeline::{FILTER_FLETCHER32, FilterDescription, FilterPipeline};
|
|
let payload = [1u8, 2, 3, 4, 5];
|
|
let sum = fletcher32(&payload);
|
|
let swapped = ((sum & 0x00ff_00ff) << 8) | ((sum >> 8) & 0x00ff_00ff);
|
|
assert_ne!(sum, swapped);
|
|
let pipeline = FilterPipeline {
|
|
version: 2,
|
|
filters: vec![FilterDescription {
|
|
filter_id: FILTER_FLETCHER32,
|
|
name: None,
|
|
client_data: vec![],
|
|
flags: 0,
|
|
}],
|
|
};
|
|
for stored in [sum, swapped] {
|
|
let mut chunk = payload.to_vec();
|
|
chunk.extend_from_slice(&stored.to_le_bytes());
|
|
let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1)
|
|
.unwrap();
|
|
assert_eq!(out, payload);
|
|
}
|
|
// Our old checksum of a fold-case chunk.
|
|
let fold = fold_chunks(1);
|
|
let fold = &fold[..CHUNK];
|
|
let old = fletcher32_mod(fold);
|
|
assert_ne!(old, fletcher32(fold));
|
|
let mut chunk = fold.to_vec();
|
|
chunk.extend_from_slice(&old.to_le_bytes());
|
|
let out = clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, CHUNK, 1).unwrap();
|
|
assert_eq!(out, fold);
|
|
let mut chunk = payload.to_vec();
|
|
chunk.extend_from_slice(&(sum ^ 1).to_le_bytes());
|
|
assert!(
|
|
clawhdf5_format::filters::decompress_chunk(&chunk, &pipeline, payload.len(), 1).is_err()
|
|
);
|
|
}
|
|
|
|
fn hex(b: &[u8]) -> String {
|
|
b.iter().map(|x| format!("{x:02x}")).collect()
|
|
}
|