crates/clawhdf5-tools/tests/edit_interop.rs

@@ -113,7 +113,8 @@ impl Model {
             .fold(0u64, |a, (&x, &d)| a * d + x) as usize
     }
 
-    /// Grow to `shape`, new elements `fill`.
+    /// Change the extent to `shape`: elements inside both keep their
+    /// values, new ones are `fill`.
     fn resize(&mut self, shape: &[u64], fill: i32) {
         let old = self.clone();
         *self = Self::new(shape, |_| fill);
@@ -125,10 +126,12 @@ impl Model {
                 c[d] = r % old.shape[d];
                 r /= old.shape[d];
             }
+            if c.iter().zip(shape).all(|(x, s)| x < s) {
                 let i = self.index(&c);
                 self.data[i] = old.data[flat as usize];
             }
         }
+    }
 
     /// Apply a hyperslab write of `vals` (row-major over the block).
     fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
@@ -293,9 +296,24 @@ fn append_many_gzip() {
     }
 }
 
-/// Random operations — grow, hyperslab writes, point writes, attributes —
+/// A random attribute value: a scalar, an int64 array, a short string or
-/// on a 2-D dataset with one unlimited dimension, checked against a model
+/// one larger than a heap's managed-object limit (a huge heap object once
-/// after every few operations.
+/// the attributes are in dense storage).
+fn random_attr(rng: &mut Rng) -> AttrValue {
+    match rng.below(8) {
+        0..=2 => AttrValue::I64(rng.next() as i64 >> 3),
+        3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()),
+        5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)),
+        _ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)),
+    }
+}
+
+/// Random operations — growth and shrinking along any dimension, hyperslab
+/// and point writes, attributes (enough names to move them to dense storage
+/// on version-2 object headers, replaced with values of any size) — on a
+/// 2-D dataset with one unlimited dimension and one with two (a version-2
+/// B-tree chunk index under `v114`/`latest`), checked against a model (and
+/// through h5py, numpy) after every few operations.
 fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
     let dir = tmpdir();
     let path = dir.path().join(format!("rand_{tag}.h5"));
@@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
          with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
          \x20   f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \
          dtype='<i4', fillvalue=-9{extra})\n\
-         \x20   f['m'][1:3, 2:6] = 5\n",
+         \x20   f['m'][1:3, 2:6] = 5\n\
+         \x20   f.create_dataset('b', shape=(5, 6), maxshape=(None, None), chunks=(2, 4), \
+         dtype='<i4', fillvalue=3{extra})\n\
+         \x20   f['b'][0:4, 1:5] = 8\n",
         p = path.to_str().unwrap()
     ));
-    let mut m = Model::new(&[4, 7], |_| -9);
+    let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)];
-    m.write_block(&[1, 2], &[2, 4], &[5; 8]);
+    models[0].write_block(&[1, 2], &[2, 4], &[5; 8]);
-    let mut attrs: Vec<(String, i64)> = Vec::new();
+    models[1].write_block(&[0, 1], &[4, 4], &[8; 16]);
+    let fills = [-9, 3];
+    let names = ["m", "b"];
+    let mut attrs: Vec<(String, AttrValue)> = Vec::new();
     let mut rng = Rng(seed);
     let mut ed = FileEditor::open(&path).unwrap();
-    for step in 0..120 {
+    for step in 0..160 {
-        match rng.below(10) {
+        let d = rng.below(2) as usize;
-            0..=1 => {
+        let name = names[d];
-                let rows = m.shape[0] + 1 + rng.below(5);
+        let m = &mut models[d];
-                ed.resize("m", &[rows, 7]).unwrap();
+        match rng.below(12) {
-                m.resize(&[rows, 7], -9);
+            0..=2 => {
+                // Grow or shrink: dimension 1 of "m" is fixed at 7.
+                let rows = rng.below(m.shape[0] + 6);
+                let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) };
+                ed.resize(name, &[rows, cols]).unwrap();
+                m.resize(&[rows, cols], fills[d]);
             }
-            2..=6 => {
+            3..=7 if m.shape.iter().all(|&s| s > 0) => {
                 let r0 = rng.below(m.shape[0]);
-                let c0 = rng.below(7);
+                let c0 = rng.below(m.shape[1]);
                 let cnt = [
                     1 + rng.below((m.shape[0] - r0).min(6)),
-                    1 + rng.below(7 - c0),
+                    1 + rng.below((m.shape[1] - c0).min(6)),
                 ];
                 let n = cnt[0] * cnt[1];
                 let vals: Vec<i32> = (0..n).map(|_| (rng.next() % 100_000) as i32).collect();
-                ed.write_values("m", &block(&[r0, c0], &cnt), &vals)
+                ed.write_values(name, &block(&[r0, c0], &cnt), &vals)
                     .unwrap();
                 m.write_block(&[r0, c0], &cnt, &vals);
             }
-            7 => {
+            8 if m.shape.iter().all(|&s| s > 0) => {
                 let pts: Vec<Vec<u64>> = (0..1 + rng.below(4))
-                    .map(|_| vec![rng.below(m.shape[0]), rng.below(7)])
+                    .map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])])
                     .collect();
                 let vals: Vec<i32> = pts.iter().map(|_| rng.next() as i32).collect();
-                ed.write_values("m", &Selection::Points(pts.clone()), &vals)
+                ed.write_values(name, &Selection::Points(pts.clone()), &vals)
                     .unwrap();
                 for (p, v) in pts.iter().zip(&vals) {
                     let i = m.index(p);
                     m.data[i] = *v;
                 }
             }
-            _ => {
+            9..=11 => {
-                let k = rng.below(6);
+                let k = rng.below(20);
-                let name = format!("a{k}");
+                let aname = format!("a{k}");
-                let v = rng.next() as i64;
+                let v = random_attr(&mut rng);
-                match ed.set_attr("m", &name, &AttrValue::I64(v)) {
+                match ed.set_attr("m", &aname, &v) {
                     Ok(()) => {
-                        attrs.retain(|(n, _)| *n != name);
+                        attrs.retain(|(n, _)| *n != aname);
-                        attrs.push((name, v));
+                        attrs.push((aname, v));
                     }
-                    Err(e) => panic!("set_attr {name}: {e}"),
+                    // Replacing the only attribute in a heap block with one
+                    // of another size would have libhdf5 free the block.
+                    Err(Error::Unsupported(msg)) if msg.contains("last object") => {}
+                    Err(e) => panic!("set_attr {aname}: {e}"),
                 }
             }
+            _ => {}
         }
-        if step % 30 == 29 {
+        if step % 40 == 39 {
             drop(ed);
-            verify(&path, "m", &m);
+            for (n, m) in names.iter().zip(&models) {
+                verify(&path, n, m);
+            }
             check_tools(&path, h5dump);
             check_attrs(&path, "m", &attrs);
             ed = FileEditor::open(&path).unwrap();
         }
     }
     drop(ed);
-    verify(&path, "m", &m);
+    for (n, m) in names.iter().zip(&models) {
+        verify(&path, n, m);
+    }
     check_attrs(&path, "m", &attrs);
     py(&format!(
         "import h5py, numpy as np\n\
          with h5py.File({p:?}, 'r+') as f:\n\
-         \x20   d = f['m']\n\
+         \x20   for name, cols in (('m', 7), ('b', None)):\n\
-         \x20   n = d.shape[0]\n\
+         \x20       d = f[name]\n\
-         \x20   d.resize((n + 3, 7))\n\
+         \x20       n, c = d.shape\n\
+         \x20       d.resize((n + 3, cols or c + 2))\n\
          \x20       d[n:, :] = 42\n\
-         \x20   d.attrs['from_h5py'] = 1.5\n",
+         \x20   f['m'].attrs['from_h5py'] = 1.5\n",
         p = path.to_str().unwrap()
     ));
-    let n = m.shape[0];
+    for (d, m) in models.iter_mut().enumerate() {
-    m.resize(&[n + 3, 7], -9);
+        let (n, c) = (m.shape[0], m.shape[1]);
-    m.write_block(&[n, 0], &[3, 7], &[42; 21]);
+        let c2 = if d == 0 { 7 } else { c + 2 };
-    verify(&path, "m", &m);
+        m.resize(&[n + 3, c2], fills[d]);
+        m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]);
+    }
+    for (n, m) in names.iter().zip(&models) {
+        verify(&path, n, m);
+    }
     check_tools(&path, h5dump);
+    check_attrs(&path, "m", &attrs);
 }
 
-fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) {
+/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of
+/// its attributes agrees with libhdf5's object info).
+fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) {
     let f = File::open(path).unwrap();
     let got = f.dataset(obj).unwrap().attrs().unwrap();
     for (n, v) in attrs {
-        match got.get(n) {
+        let g = got
-            Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"),
+            .get(n)
-            other => panic!("attribute {n}: {other:?}"),
+            .unwrap_or_else(|| panic!("attribute {n} missing"));
+        // Our reader reports a one-element array as a scalar.
+        let v = match v {
+            AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]),
+            v => v,
+        };
+        assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}");
     }
-    }
+    let want: Vec<String> = attrs
-    let want: Vec<String> = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect();
+        .iter()
-    py(&format!(
+        .map(|(n, v)| {
+            let pv = match v {
+                AttrValue::I64(x) => format!("{x}"),
+                AttrValue::I64Array(a) => format!("{a:?}"),
+                AttrValue::String(s) => format!("{s:?}"),
+                other => panic!("{other:?}"),
+            };
+            format!("{n:?}: {pv}")
+        })
+        .collect();
+    let script = format!(
         "import h5py\n\
          f = h5py.File({p:?}, 'r')\n\
          want = {{{w}}}\n\
-         got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\
+         a = f[{obj:?}].attrs\n\
-         assert got == want, (got, want)\n",
+         def norm(v):\n\
+         \x20   v = v.decode() if isinstance(v, bytes) else v\n\
+         \x20   return v.tolist() if hasattr(v, 'tolist') else v\n\
+         got = {{k: norm(v) for k, v in a.items() if k in want}}\n\
+         assert got == want, sorted(set(want) ^ set(got))\n\
+         assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n",
         p = path.to_str().unwrap(),
         w = want.join(", ")
-    ));
+    );
+    let sp = path.with_extension("attrs.py");
+    std::fs::write(&sp, script).unwrap();
+    let o = Command::new(python()).arg(&sp).output().unwrap();
+    assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
 }
 
 #[test]
@@ -411,7 +482,11 @@ fn random_operations_match_a_model() {
     if !tools_ok() {
         return;
     }
-    let mut seed = 1;
+    // CLAWHDF5_EDIT_SEED runs the same workloads with other random choices.
+    let mut seed = std::env::var("CLAWHDF5_EDIT_SEED")
+        .ok()
+        .and_then(|s| s.parse::<u64>().ok())
+        .unwrap_or(1);
     for (i, (lv, dump)) in LIBVERS.iter().enumerate() {
         // h5dump has no LZF decoder (h5py's own filter).
         for (j, (extra, lzf)) in [