tests: resize and write clawhdf5-written datasets on every chunk index
Version-2 B-tree (its writer's own node size and single-leaf layout), Extensible Array and Fixed Array datasets written by FileBuilder, with and without deflate, resized up and down along both dimensions and written at random against a model; h5py, h5dump and h5rs check read the result, and h5py resizes and rewrites every dataset afterwards. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -1450,3 +1450,83 @@ fn last_huge_attribute_replaced() {
|
|||||||
check_tools(&b, true);
|
check_tools(&b, true);
|
||||||
check_attr_values(&b, &want);
|
check_attr_values(&b, &want);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Datasets clawhdf5 writes — a version-2 B-tree index (two unlimited
|
||||||
|
/// dimensions, its own node size and a single leaf sized to its records),
|
||||||
|
/// an Extensible Array, a Fixed Array (fixed maximum shape), deflated and
|
||||||
|
/// not — resized up and down and written at random, against a model; h5py,
|
||||||
|
/// h5dump and `h5rs check` read the result and h5py goes on.
|
||||||
|
#[test]
|
||||||
|
fn resize_clawhdf5_written_datasets() {
|
||||||
|
if !tools_ok() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let dir = tmpdir();
|
||||||
|
let path = dir.path().join("ours_resize.h5");
|
||||||
|
let mut b = clawhdf5::FileBuilder::new();
|
||||||
|
let grid: Vec<i32> = (0..48).collect();
|
||||||
|
let specs: [(&str, [u64; 2], [u64; 2], bool); 4] = [
|
||||||
|
("bt2", [6, 8], [u64::MAX, u64::MAX], false),
|
||||||
|
("bt2_gz", [6, 8], [u64::MAX, u64::MAX], true),
|
||||||
|
("ea", [6, 8], [u64::MAX, 8], true),
|
||||||
|
("fa", [6, 8], [20, 12], false),
|
||||||
|
];
|
||||||
|
for (name, shape, max, gz) in specs {
|
||||||
|
let d = b
|
||||||
|
.create_dataset(name)
|
||||||
|
.with_i32_data(&grid)
|
||||||
|
.with_shape(&shape)
|
||||||
|
.with_maxshape(&max)
|
||||||
|
.with_chunks(&[2, 3]);
|
||||||
|
if gz {
|
||||||
|
d.with_deflate(4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.write(&path).unwrap();
|
||||||
|
assert_eq!(chunk_index(&path, "bt2").0, 5, "version-2 B-tree index");
|
||||||
|
assert_eq!(chunk_index(&path, "ea").0, 4, "Extensible Array index");
|
||||||
|
let mut models: Vec<Model> = specs
|
||||||
|
.iter()
|
||||||
|
.map(|_| Model::new(&[6, 8], |i| i as i32))
|
||||||
|
.collect();
|
||||||
|
let mut rng = Rng(77);
|
||||||
|
let mut ed = FileEditor::open(&path).unwrap();
|
||||||
|
for step in 0..120 {
|
||||||
|
let k = rng.below(4) as usize;
|
||||||
|
let (name, _, max, _) = specs[k];
|
||||||
|
let m = &mut models[k];
|
||||||
|
if step % 4 == 0 {
|
||||||
|
let s: Vec<u64> = (0..2)
|
||||||
|
.map(|d| rng.below((m.shape[d] * 2 + 4).min(max[d]) + 1))
|
||||||
|
.collect();
|
||||||
|
ed.resize(name, &s).unwrap();
|
||||||
|
m.resize(&s, 0);
|
||||||
|
} else if m.shape.iter().all(|&s| s > 0) {
|
||||||
|
let st: Vec<u64> = m.shape.iter().map(|&s| rng.below(s)).collect();
|
||||||
|
let cnt: Vec<u64> = (0..2)
|
||||||
|
.map(|d| 1 + rng.below((m.shape[d] - st[d]).min(5)))
|
||||||
|
.collect();
|
||||||
|
let vals: Vec<i32> = (0..cnt[0] * cnt[1]).map(|_| rng.next() as i32).collect();
|
||||||
|
ed.write_values(name, &block(&st, &cnt), &vals).unwrap();
|
||||||
|
m.write_block(&st, &cnt, &vals);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(ed);
|
||||||
|
for ((name, ..), m) in specs.iter().zip(&models) {
|
||||||
|
verify(&path, name, m);
|
||||||
|
}
|
||||||
|
check_tools(&path, true);
|
||||||
|
py(&format!(
|
||||||
|
"import h5py, numpy as np\n\
|
||||||
|
with h5py.File({p:?}, 'r+') as f:\n\
|
||||||
|
\x20 for n in ['bt2', 'bt2_gz', 'ea', 'fa']:\n\
|
||||||
|
\x20 d = f[n]\n\
|
||||||
|
\x20 d.resize((6, 8))\n\
|
||||||
|
\x20 d[...] = np.arange(48, dtype='<i4').reshape(6, 8) * 2\n",
|
||||||
|
p = path.to_str().unwrap()
|
||||||
|
));
|
||||||
|
for (name, ..) in specs {
|
||||||
|
verify(&path, name, &Model::new(&[6, 8], |i| i as i32 * 2));
|
||||||
|
}
|
||||||
|
check_tools(&path, true);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user