diff --git a/crates/clawhdf5-tools/tests/edit_interop.rs b/crates/clawhdf5-tools/tests/edit_interop.rs index cb345f8..3a37e83 100644 --- a/crates/clawhdf5-tools/tests/edit_interop.rs +++ b/crates/clawhdf5-tools/tests/edit_interop.rs @@ -113,7 +113,8 @@ impl Model { .fold(0u64, |a, (&x, &d)| a * d + x) as usize } - /// Grow to `shape`, new elements `fill`. + /// Change the extent to `shape`: elements inside both keep their + /// values, new ones are `fill`. fn resize(&mut self, shape: &[u64], fill: i32) { let old = self.clone(); *self = Self::new(shape, |_| fill); @@ -125,8 +126,10 @@ impl Model { c[d] = r % old.shape[d]; r /= old.shape[d]; } - let i = self.index(&c); - self.data[i] = old.data[flat as usize]; + if c.iter().zip(shape).all(|(x, s)| x < s) { + let i = self.index(&c); + self.data[i] = old.data[flat as usize]; + } } } @@ -293,9 +296,24 @@ fn append_many_gzip() { } } -/// Random operations — grow, hyperslab writes, point writes, attributes — -/// on a 2-D dataset with one unlimited dimension, checked against a model -/// after every few operations. +/// A random attribute value: a scalar, an int64 array, a short string or +/// one larger than a heap's managed-object limit (a huge heap object once +/// the attributes are in dense storage). +fn random_attr(rng: &mut Rng) -> AttrValue { + match rng.below(8) { + 0..=2 => AttrValue::I64(rng.next() as i64 >> 3), + 3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()), + 5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)), + _ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)), + } +} + +/// Random operations — growth and shrinking along any dimension, hyperslab +/// and point writes, attributes (enough names to move them to dense storage +/// on version-2 object headers, replaced with values of any size) — on a +/// 2-D dataset with one unlimited dimension and one with two (a version-2 +/// B-tree chunk index under `v114`/`latest`), checked against a model (and +/// through h5py, numpy) after every few operations. fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { let dir = tmpdir(); let path = dir.path().join(format!("rand_{tag}.h5")); @@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) { with h5py.File({p:?}, 'w', libver={libver}) as f:\n\ \x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \ dtype=' = Vec::new(); + let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)]; + models[0].write_block(&[1, 2], &[2, 4], &[5; 8]); + models[1].write_block(&[0, 1], &[4, 4], &[8; 16]); + let fills = [-9, 3]; + let names = ["m", "b"]; + let mut attrs: Vec<(String, AttrValue)> = Vec::new(); let mut rng = Rng(seed); let mut ed = FileEditor::open(&path).unwrap(); - for step in 0..120 { - match rng.below(10) { - 0..=1 => { - let rows = m.shape[0] + 1 + rng.below(5); - ed.resize("m", &[rows, 7]).unwrap(); - m.resize(&[rows, 7], -9); + for step in 0..160 { + let d = rng.below(2) as usize; + let name = names[d]; + let m = &mut models[d]; + match rng.below(12) { + 0..=2 => { + // Grow or shrink: dimension 1 of "m" is fixed at 7. + let rows = rng.below(m.shape[0] + 6); + let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) }; + ed.resize(name, &[rows, cols]).unwrap(); + m.resize(&[rows, cols], fills[d]); } - 2..=6 => { + 3..=7 if m.shape.iter().all(|&s| s > 0) => { let r0 = rng.below(m.shape[0]); - let c0 = rng.below(7); + let c0 = rng.below(m.shape[1]); let cnt = [ 1 + rng.below((m.shape[0] - r0).min(6)), - 1 + rng.below(7 - c0), + 1 + rng.below((m.shape[1] - c0).min(6)), ]; let n = cnt[0] * cnt[1]; let vals: Vec = (0..n).map(|_| (rng.next() % 100_000) as i32).collect(); - ed.write_values("m", &block(&[r0, c0], &cnt), &vals) + ed.write_values(name, &block(&[r0, c0], &cnt), &vals) .unwrap(); m.write_block(&[r0, c0], &cnt, &vals); } - 7 => { + 8 if m.shape.iter().all(|&s| s > 0) => { let pts: Vec> = (0..1 + rng.below(4)) - .map(|_| vec![rng.below(m.shape[0]), rng.below(7)]) + .map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])]) .collect(); let vals: Vec = pts.iter().map(|_| rng.next() as i32).collect(); - ed.write_values("m", &Selection::Points(pts.clone()), &vals) + ed.write_values(name, &Selection::Points(pts.clone()), &vals) .unwrap(); for (p, v) in pts.iter().zip(&vals) { let i = m.index(p); m.data[i] = *v; } } - _ => { - let k = rng.below(6); - let name = format!("a{k}"); - let v = rng.next() as i64; - match ed.set_attr("m", &name, &AttrValue::I64(v)) { + 9..=11 => { + let k = rng.below(20); + let aname = format!("a{k}"); + let v = random_attr(&mut rng); + match ed.set_attr("m", &aname, &v) { Ok(()) => { - attrs.retain(|(n, _)| *n != name); - attrs.push((name, v)); + attrs.retain(|(n, _)| *n != aname); + attrs.push((aname, v)); } - Err(e) => panic!("set_attr {name}: {e}"), + // Replacing the only attribute in a heap block with one + // of another size would have libhdf5 free the block. + Err(Error::Unsupported(msg)) if msg.contains("last object") => {} + Err(e) => panic!("set_attr {aname}: {e}"), } } + _ => {} } - if step % 30 == 29 { + if step % 40 == 39 { drop(ed); - verify(&path, "m", &m); + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_tools(&path, h5dump); check_attrs(&path, "m", &attrs); ed = FileEditor::open(&path).unwrap(); } } drop(ed); - verify(&path, "m", &m); + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_attrs(&path, "m", &attrs); py(&format!( "import h5py, numpy as np\n\ with h5py.File({p:?}, 'r+') as f:\n\ - \x20 d = f['m']\n\ - \x20 n = d.shape[0]\n\ - \x20 d.resize((n + 3, 7))\n\ - \x20 d[n:, :] = 42\n\ - \x20 d.attrs['from_h5py'] = 1.5\n", + \x20 for name, cols in (('m', 7), ('b', None)):\n\ + \x20 d = f[name]\n\ + \x20 n, c = d.shape\n\ + \x20 d.resize((n + 3, cols or c + 2))\n\ + \x20 d[n:, :] = 42\n\ + \x20 f['m'].attrs['from_h5py'] = 1.5\n", p = path.to_str().unwrap() )); - let n = m.shape[0]; - m.resize(&[n + 3, 7], -9); - m.write_block(&[n, 0], &[3, 7], &[42; 21]); - verify(&path, "m", &m); + for (d, m) in models.iter_mut().enumerate() { + let (n, c) = (m.shape[0], m.shape[1]); + let c2 = if d == 0 { 7 } else { c + 2 }; + m.resize(&[n + 3, c2], fills[d]); + m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]); + } + for (n, m) in names.iter().zip(&models) { + verify(&path, n, m); + } check_tools(&path, h5dump); + check_attrs(&path, "m", &attrs); } -fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) { +/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of +/// its attributes agrees with libhdf5's object info). +fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) { let f = File::open(path).unwrap(); let got = f.dataset(obj).unwrap().attrs().unwrap(); for (n, v) in attrs { - match got.get(n) { - Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"), - other => panic!("attribute {n}: {other:?}"), - } + let g = got + .get(n) + .unwrap_or_else(|| panic!("attribute {n} missing")); + // Our reader reports a one-element array as a scalar. + let v = match v { + AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]), + v => v, + }; + assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}"); } - let want: Vec = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect(); - py(&format!( + let want: Vec = attrs + .iter() + .map(|(n, v)| { + let pv = match v { + AttrValue::I64(x) => format!("{x}"), + AttrValue::I64Array(a) => format!("{a:?}"), + AttrValue::String(s) => format!("{s:?}"), + other => panic!("{other:?}"), + }; + format!("{n:?}: {pv}") + }) + .collect(); + let script = format!( "import h5py\n\ f = h5py.File({p:?}, 'r')\n\ want = {{{w}}}\n\ - got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\ - assert got == want, (got, want)\n", + a = f[{obj:?}].attrs\n\ + def norm(v):\n\ + \x20 v = v.decode() if isinstance(v, bytes) else v\n\ + \x20 return v.tolist() if hasattr(v, 'tolist') else v\n\ + got = {{k: norm(v) for k, v in a.items() if k in want}}\n\ + assert got == want, sorted(set(want) ^ set(got))\n\ + assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n", p = path.to_str().unwrap(), w = want.join(", ") - )); + ); + let sp = path.with_extension("attrs.py"); + std::fs::write(&sp, script).unwrap(); + let o = Command::new(python()).arg(&sp).output().unwrap(); + assert!(o.status.success(), "attribute check failed:\n{}", text(&o)); } #[test] @@ -411,7 +482,11 @@ fn random_operations_match_a_model() { if !tools_ok() { return; } - let mut seed = 1; + // CLAWHDF5_EDIT_SEED runs the same workloads with other random choices. + let mut seed = std::env::var("CLAWHDF5_EDIT_SEED") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(1); for (i, (lv, dump)) in LIBVERS.iter().enumerate() { // h5dump has no LZF decoder (h5py's own filter). for (j, (extra, lzf)) in [