tests: random editor operations include shrinking, 2-D growth, dense attributes
random_operations_match_a_model (edit_interop) now drives, on every libver (earliest, v114, latest) and filter set (none, gzip + shuffle + fletcher32, LZF): - a dataset with one unlimited dimension and one with two (a version-2 B-tree chunk index under v114/latest), resized to random shapes that shrink and grow any resizable dimension, with block and point writes; - attributes on the first dataset under 20 names with values of random types and sizes (scalars, int64 arrays, short strings, strings above the heap's managed limit), so they move to dense storage on version-2 headers and are replaced by values of other sizes; against a model where shrunk-away elements that come back read as the fill value, compared with our reader and with h5py/numpy every 40 steps, with h5dump and h5rs check; h5py then grows both datasets and adds an attribute. The attribute check also compares h5py's attribute count with libhdf5's object info. CLAWHDF5_EDIT_SEED reruns the workloads with other random choices (seeds 1000-4000 pass). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -113,7 +113,8 @@ impl Model {
|
|||||||
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Grow to `shape`, new elements `fill`.
|
/// Change the extent to `shape`: elements inside both keep their
|
||||||
|
/// values, new ones are `fill`.
|
||||||
fn resize(&mut self, shape: &[u64], fill: i32) {
|
fn resize(&mut self, shape: &[u64], fill: i32) {
|
||||||
let old = self.clone();
|
let old = self.clone();
|
||||||
*self = Self::new(shape, |_| fill);
|
*self = Self::new(shape, |_| fill);
|
||||||
@@ -125,8 +126,10 @@ impl Model {
|
|||||||
c[d] = r % old.shape[d];
|
c[d] = r % old.shape[d];
|
||||||
r /= old.shape[d];
|
r /= old.shape[d];
|
||||||
}
|
}
|
||||||
let i = self.index(&c);
|
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
||||||
self.data[i] = old.data[flat as usize];
|
let i = self.index(&c);
|
||||||
|
self.data[i] = old.data[flat as usize];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -293,9 +296,24 @@ fn append_many_gzip() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Random operations — grow, hyperslab writes, point writes, attributes —
|
/// A random attribute value: a scalar, an int64 array, a short string or
|
||||||
/// on a 2-D dataset with one unlimited dimension, checked against a model
|
/// one larger than a heap's managed-object limit (a huge heap object once
|
||||||
/// after every few operations.
|
/// the attributes are in dense storage).
|
||||||
|
fn random_attr(rng: &mut Rng) -> AttrValue {
|
||||||
|
match rng.below(8) {
|
||||||
|
0..=2 => AttrValue::I64(rng.next() as i64 >> 3),
|
||||||
|
3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()),
|
||||||
|
5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)),
|
||||||
|
_ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Random operations — growth and shrinking along any dimension, hyperslab
|
||||||
|
/// and point writes, attributes (enough names to move them to dense storage
|
||||||
|
/// on version-2 object headers, replaced with values of any size) — on a
|
||||||
|
/// 2-D dataset with one unlimited dimension and one with two (a version-2
|
||||||
|
/// B-tree chunk index under `v114`/`latest`), checked against a model (and
|
||||||
|
/// through h5py, numpy) after every few operations.
|
||||||
fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
|
fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
|
||||||
let dir = tmpdir();
|
let dir = tmpdir();
|
||||||
let path = dir.path().join(format!("rand_{tag}.h5"));
|
let path = dir.path().join(format!("rand_{tag}.h5"));
|
||||||
@@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
|
|||||||
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
||||||
\x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \
|
\x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \
|
||||||
dtype='<i4', fillvalue=-9{extra})\n\
|
dtype='<i4', fillvalue=-9{extra})\n\
|
||||||
\x20 f['m'][1:3, 2:6] = 5\n",
|
\x20 f['m'][1:3, 2:6] = 5\n\
|
||||||
|
\x20 f.create_dataset('b', shape=(5, 6), maxshape=(None, None), chunks=(2, 4), \
|
||||||
|
dtype='<i4', fillvalue=3{extra})\n\
|
||||||
|
\x20 f['b'][0:4, 1:5] = 8\n",
|
||||||
p = path.to_str().unwrap()
|
p = path.to_str().unwrap()
|
||||||
));
|
));
|
||||||
let mut m = Model::new(&[4, 7], |_| -9);
|
let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)];
|
||||||
m.write_block(&[1, 2], &[2, 4], &[5; 8]);
|
models[0].write_block(&[1, 2], &[2, 4], &[5; 8]);
|
||||||
let mut attrs: Vec<(String, i64)> = Vec::new();
|
models[1].write_block(&[0, 1], &[4, 4], &[8; 16]);
|
||||||
|
let fills = [-9, 3];
|
||||||
|
let names = ["m", "b"];
|
||||||
|
let mut attrs: Vec<(String, AttrValue)> = Vec::new();
|
||||||
let mut rng = Rng(seed);
|
let mut rng = Rng(seed);
|
||||||
let mut ed = FileEditor::open(&path).unwrap();
|
let mut ed = FileEditor::open(&path).unwrap();
|
||||||
for step in 0..120 {
|
for step in 0..160 {
|
||||||
match rng.below(10) {
|
let d = rng.below(2) as usize;
|
||||||
0..=1 => {
|
let name = names[d];
|
||||||
let rows = m.shape[0] + 1 + rng.below(5);
|
let m = &mut models[d];
|
||||||
ed.resize("m", &[rows, 7]).unwrap();
|
match rng.below(12) {
|
||||||
m.resize(&[rows, 7], -9);
|
0..=2 => {
|
||||||
|
// Grow or shrink: dimension 1 of "m" is fixed at 7.
|
||||||
|
let rows = rng.below(m.shape[0] + 6);
|
||||||
|
let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) };
|
||||||
|
ed.resize(name, &[rows, cols]).unwrap();
|
||||||
|
m.resize(&[rows, cols], fills[d]);
|
||||||
}
|
}
|
||||||
2..=6 => {
|
3..=7 if m.shape.iter().all(|&s| s > 0) => {
|
||||||
let r0 = rng.below(m.shape[0]);
|
let r0 = rng.below(m.shape[0]);
|
||||||
let c0 = rng.below(7);
|
let c0 = rng.below(m.shape[1]);
|
||||||
let cnt = [
|
let cnt = [
|
||||||
1 + rng.below((m.shape[0] - r0).min(6)),
|
1 + rng.below((m.shape[0] - r0).min(6)),
|
||||||
1 + rng.below(7 - c0),
|
1 + rng.below((m.shape[1] - c0).min(6)),
|
||||||
];
|
];
|
||||||
let n = cnt[0] * cnt[1];
|
let n = cnt[0] * cnt[1];
|
||||||
let vals: Vec<i32> = (0..n).map(|_| (rng.next() % 100_000) as i32).collect();
|
let vals: Vec<i32> = (0..n).map(|_| (rng.next() % 100_000) as i32).collect();
|
||||||
ed.write_values("m", &block(&[r0, c0], &cnt), &vals)
|
ed.write_values(name, &block(&[r0, c0], &cnt), &vals)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
m.write_block(&[r0, c0], &cnt, &vals);
|
m.write_block(&[r0, c0], &cnt, &vals);
|
||||||
}
|
}
|
||||||
7 => {
|
8 if m.shape.iter().all(|&s| s > 0) => {
|
||||||
let pts: Vec<Vec<u64>> = (0..1 + rng.below(4))
|
let pts: Vec<Vec<u64>> = (0..1 + rng.below(4))
|
||||||
.map(|_| vec![rng.below(m.shape[0]), rng.below(7)])
|
.map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])])
|
||||||
.collect();
|
.collect();
|
||||||
let vals: Vec<i32> = pts.iter().map(|_| rng.next() as i32).collect();
|
let vals: Vec<i32> = pts.iter().map(|_| rng.next() as i32).collect();
|
||||||
ed.write_values("m", &Selection::Points(pts.clone()), &vals)
|
ed.write_values(name, &Selection::Points(pts.clone()), &vals)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
for (p, v) in pts.iter().zip(&vals) {
|
for (p, v) in pts.iter().zip(&vals) {
|
||||||
let i = m.index(p);
|
let i = m.index(p);
|
||||||
m.data[i] = *v;
|
m.data[i] = *v;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {
|
9..=11 => {
|
||||||
let k = rng.below(6);
|
let k = rng.below(20);
|
||||||
let name = format!("a{k}");
|
let aname = format!("a{k}");
|
||||||
let v = rng.next() as i64;
|
let v = random_attr(&mut rng);
|
||||||
match ed.set_attr("m", &name, &AttrValue::I64(v)) {
|
match ed.set_attr("m", &aname, &v) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
attrs.retain(|(n, _)| *n != name);
|
attrs.retain(|(n, _)| *n != aname);
|
||||||
attrs.push((name, v));
|
attrs.push((aname, v));
|
||||||
}
|
}
|
||||||
Err(e) => panic!("set_attr {name}: {e}"),
|
// Replacing the only attribute in a heap block with one
|
||||||
|
// of another size would have libhdf5 free the block.
|
||||||
|
Err(Error::Unsupported(msg)) if msg.contains("last object") => {}
|
||||||
|
Err(e) => panic!("set_attr {aname}: {e}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
_ => {}
|
||||||
}
|
}
|
||||||
if step % 30 == 29 {
|
if step % 40 == 39 {
|
||||||
drop(ed);
|
drop(ed);
|
||||||
verify(&path, "m", &m);
|
for (n, m) in names.iter().zip(&models) {
|
||||||
|
verify(&path, n, m);
|
||||||
|
}
|
||||||
check_tools(&path, h5dump);
|
check_tools(&path, h5dump);
|
||||||
check_attrs(&path, "m", &attrs);
|
check_attrs(&path, "m", &attrs);
|
||||||
ed = FileEditor::open(&path).unwrap();
|
ed = FileEditor::open(&path).unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
drop(ed);
|
drop(ed);
|
||||||
verify(&path, "m", &m);
|
for (n, m) in names.iter().zip(&models) {
|
||||||
|
verify(&path, n, m);
|
||||||
|
}
|
||||||
check_attrs(&path, "m", &attrs);
|
check_attrs(&path, "m", &attrs);
|
||||||
py(&format!(
|
py(&format!(
|
||||||
"import h5py, numpy as np\n\
|
"import h5py, numpy as np\n\
|
||||||
with h5py.File({p:?}, 'r+') as f:\n\
|
with h5py.File({p:?}, 'r+') as f:\n\
|
||||||
\x20 d = f['m']\n\
|
\x20 for name, cols in (('m', 7), ('b', None)):\n\
|
||||||
\x20 n = d.shape[0]\n\
|
\x20 d = f[name]\n\
|
||||||
\x20 d.resize((n + 3, 7))\n\
|
\x20 n, c = d.shape\n\
|
||||||
\x20 d[n:, :] = 42\n\
|
\x20 d.resize((n + 3, cols or c + 2))\n\
|
||||||
\x20 d.attrs['from_h5py'] = 1.5\n",
|
\x20 d[n:, :] = 42\n\
|
||||||
|
\x20 f['m'].attrs['from_h5py'] = 1.5\n",
|
||||||
p = path.to_str().unwrap()
|
p = path.to_str().unwrap()
|
||||||
));
|
));
|
||||||
let n = m.shape[0];
|
for (d, m) in models.iter_mut().enumerate() {
|
||||||
m.resize(&[n + 3, 7], -9);
|
let (n, c) = (m.shape[0], m.shape[1]);
|
||||||
m.write_block(&[n, 0], &[3, 7], &[42; 21]);
|
let c2 = if d == 0 { 7 } else { c + 2 };
|
||||||
verify(&path, "m", &m);
|
m.resize(&[n + 3, c2], fills[d]);
|
||||||
|
m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]);
|
||||||
|
}
|
||||||
|
for (n, m) in names.iter().zip(&models) {
|
||||||
|
verify(&path, n, m);
|
||||||
|
}
|
||||||
check_tools(&path, h5dump);
|
check_tools(&path, h5dump);
|
||||||
|
check_attrs(&path, "m", &attrs);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) {
|
/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of
|
||||||
|
/// its attributes agrees with libhdf5's object info).
|
||||||
|
fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) {
|
||||||
let f = File::open(path).unwrap();
|
let f = File::open(path).unwrap();
|
||||||
let got = f.dataset(obj).unwrap().attrs().unwrap();
|
let got = f.dataset(obj).unwrap().attrs().unwrap();
|
||||||
for (n, v) in attrs {
|
for (n, v) in attrs {
|
||||||
match got.get(n) {
|
let g = got
|
||||||
Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"),
|
.get(n)
|
||||||
other => panic!("attribute {n}: {other:?}"),
|
.unwrap_or_else(|| panic!("attribute {n} missing"));
|
||||||
}
|
// Our reader reports a one-element array as a scalar.
|
||||||
|
let v = match v {
|
||||||
|
AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]),
|
||||||
|
v => v,
|
||||||
|
};
|
||||||
|
assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}");
|
||||||
}
|
}
|
||||||
let want: Vec<String> = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect();
|
let want: Vec<String> = attrs
|
||||||
py(&format!(
|
.iter()
|
||||||
|
.map(|(n, v)| {
|
||||||
|
let pv = match v {
|
||||||
|
AttrValue::I64(x) => format!("{x}"),
|
||||||
|
AttrValue::I64Array(a) => format!("{a:?}"),
|
||||||
|
AttrValue::String(s) => format!("{s:?}"),
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
};
|
||||||
|
format!("{n:?}: {pv}")
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let script = format!(
|
||||||
"import h5py\n\
|
"import h5py\n\
|
||||||
f = h5py.File({p:?}, 'r')\n\
|
f = h5py.File({p:?}, 'r')\n\
|
||||||
want = {{{w}}}\n\
|
want = {{{w}}}\n\
|
||||||
got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\
|
a = f[{obj:?}].attrs\n\
|
||||||
assert got == want, (got, want)\n",
|
def norm(v):\n\
|
||||||
|
\x20 v = v.decode() if isinstance(v, bytes) else v\n\
|
||||||
|
\x20 return v.tolist() if hasattr(v, 'tolist') else v\n\
|
||||||
|
got = {{k: norm(v) for k, v in a.items() if k in want}}\n\
|
||||||
|
assert got == want, sorted(set(want) ^ set(got))\n\
|
||||||
|
assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n",
|
||||||
p = path.to_str().unwrap(),
|
p = path.to_str().unwrap(),
|
||||||
w = want.join(", ")
|
w = want.join(", ")
|
||||||
));
|
);
|
||||||
|
let sp = path.with_extension("attrs.py");
|
||||||
|
std::fs::write(&sp, script).unwrap();
|
||||||
|
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||||
|
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -411,7 +482,11 @@ fn random_operations_match_a_model() {
|
|||||||
if !tools_ok() {
|
if !tools_ok() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let mut seed = 1;
|
// CLAWHDF5_EDIT_SEED runs the same workloads with other random choices.
|
||||||
|
let mut seed = std::env::var("CLAWHDF5_EDIT_SEED")
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| s.parse::<u64>().ok())
|
||||||
|
.unwrap_or(1);
|
||||||
for (i, (lv, dump)) in LIBVERS.iter().enumerate() {
|
for (i, (lv, dump)) in LIBVERS.iter().enumerate() {
|
||||||
// h5dump has no LZF decoder (h5py's own filter).
|
// h5dump has no LZF decoder (h5py's own filter).
|
||||||
for (j, (extra, lzf)) in [
|
for (j, (extra, lzf)) in [
|
||||||
|
|||||||
Reference in New Issue
Block a user