Merge branch 'feat/p3-editor-coverage' into feat/p3-remote-editor
# Conflicts: # CHANGELOG.md # CLAUDE.md # docs/design/range-reads.md
This commit is contained in:
osobh
File diff suppressed because it is too large
Load Diff
@@ -113,7 +113,8 @@ impl Model {
|
||||
.fold(0u64, |a, (&x, &d)| a * d + x) as usize
|
||||
}
|
||||
|
||||
/// Grow to `shape`, new elements `fill`.
|
||||
/// Change the extent to `shape`: elements inside both keep their
|
||||
/// values, new ones are `fill`.
|
||||
fn resize(&mut self, shape: &[u64], fill: i32) {
|
||||
let old = self.clone();
|
||||
*self = Self::new(shape, |_| fill);
|
||||
@@ -125,8 +126,10 @@ impl Model {
|
||||
c[d] = r % old.shape[d];
|
||||
r /= old.shape[d];
|
||||
}
|
||||
let i = self.index(&c);
|
||||
self.data[i] = old.data[flat as usize];
|
||||
if c.iter().zip(shape).all(|(x, s)| x < s) {
|
||||
let i = self.index(&c);
|
||||
self.data[i] = old.data[flat as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -293,9 +296,24 @@ fn append_many_gzip() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Random operations — grow, hyperslab writes, point writes, attributes —
|
||||
/// on a 2-D dataset with one unlimited dimension, checked against a model
|
||||
/// after every few operations.
|
||||
/// A random attribute value: a scalar, an int64 array, a short string or
|
||||
/// one larger than a heap's managed-object limit (a huge heap object once
|
||||
/// the attributes are in dense storage).
|
||||
fn random_attr(rng: &mut Rng) -> AttrValue {
|
||||
match rng.below(8) {
|
||||
0..=2 => AttrValue::I64(rng.next() as i64 >> 3),
|
||||
3..=4 => AttrValue::I64Array((0..1 + rng.below(40)).map(|k| k as i64 * 7).collect()),
|
||||
5..=6 => AttrValue::String("s".repeat(1 + rng.below(200) as usize)),
|
||||
_ => AttrValue::String("h".repeat(5000 + rng.below(100) as usize)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Random operations — growth and shrinking along any dimension, hyperslab
|
||||
/// and point writes, attributes (enough names to move them to dense storage
|
||||
/// on version-2 object headers, replaced with values of any size) — on a
|
||||
/// 2-D dataset with one unlimited dimension and one with two (a version-2
|
||||
/// B-tree chunk index under `v114`/`latest`), checked against a model (and
|
||||
/// through h5py, numpy) after every few operations.
|
||||
fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
|
||||
let dir = tmpdir();
|
||||
let path = dir.path().join(format!("rand_{tag}.h5"));
|
||||
@@ -304,106 +322,159 @@ fn random_ops(libver: &str, h5dump: bool, extra: &str, tag: &str, seed: u64) {
|
||||
with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
|
||||
\x20 f.create_dataset('m', shape=(4, 7), maxshape=(None, 7), chunks=(3, 4), \
|
||||
dtype='<i4', fillvalue=-9{extra})\n\
|
||||
\x20 f['m'][1:3, 2:6] = 5\n",
|
||||
\x20 f['m'][1:3, 2:6] = 5\n\
|
||||
\x20 f.create_dataset('b', shape=(5, 6), maxshape=(None, None), chunks=(2, 4), \
|
||||
dtype='<i4', fillvalue=3{extra})\n\
|
||||
\x20 f['b'][0:4, 1:5] = 8\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let mut m = Model::new(&[4, 7], |_| -9);
|
||||
m.write_block(&[1, 2], &[2, 4], &[5; 8]);
|
||||
let mut attrs: Vec<(String, i64)> = Vec::new();
|
||||
let mut models = [Model::new(&[4, 7], |_| -9), Model::new(&[5, 6], |_| 3)];
|
||||
models[0].write_block(&[1, 2], &[2, 4], &[5; 8]);
|
||||
models[1].write_block(&[0, 1], &[4, 4], &[8; 16]);
|
||||
let fills = [-9, 3];
|
||||
let names = ["m", "b"];
|
||||
let mut attrs: Vec<(String, AttrValue)> = Vec::new();
|
||||
let mut rng = Rng(seed);
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
for step in 0..120 {
|
||||
match rng.below(10) {
|
||||
0..=1 => {
|
||||
let rows = m.shape[0] + 1 + rng.below(5);
|
||||
ed.resize("m", &[rows, 7]).unwrap();
|
||||
m.resize(&[rows, 7], -9);
|
||||
for step in 0..160 {
|
||||
let d = rng.below(2) as usize;
|
||||
let name = names[d];
|
||||
let m = &mut models[d];
|
||||
match rng.below(12) {
|
||||
0..=2 => {
|
||||
// Grow or shrink: dimension 1 of "m" is fixed at 7.
|
||||
let rows = rng.below(m.shape[0] + 6);
|
||||
let cols = if d == 0 { 7 } else { rng.below(m.shape[1] + 5) };
|
||||
ed.resize(name, &[rows, cols]).unwrap();
|
||||
m.resize(&[rows, cols], fills[d]);
|
||||
}
|
||||
2..=6 => {
|
||||
3..=7 if m.shape.iter().all(|&s| s > 0) => {
|
||||
let r0 = rng.below(m.shape[0]);
|
||||
let c0 = rng.below(7);
|
||||
let c0 = rng.below(m.shape[1]);
|
||||
let cnt = [
|
||||
1 + rng.below((m.shape[0] - r0).min(6)),
|
||||
1 + rng.below(7 - c0),
|
||||
1 + rng.below((m.shape[1] - c0).min(6)),
|
||||
];
|
||||
let n = cnt[0] * cnt[1];
|
||||
let vals: Vec<i32> = (0..n).map(|_| (rng.next() % 100_000) as i32).collect();
|
||||
ed.write_values("m", &block(&[r0, c0], &cnt), &vals)
|
||||
ed.write_values(name, &block(&[r0, c0], &cnt), &vals)
|
||||
.unwrap();
|
||||
m.write_block(&[r0, c0], &cnt, &vals);
|
||||
}
|
||||
7 => {
|
||||
8 if m.shape.iter().all(|&s| s > 0) => {
|
||||
let pts: Vec<Vec<u64>> = (0..1 + rng.below(4))
|
||||
.map(|_| vec![rng.below(m.shape[0]), rng.below(7)])
|
||||
.map(|_| vec![rng.below(m.shape[0]), rng.below(m.shape[1])])
|
||||
.collect();
|
||||
let vals: Vec<i32> = pts.iter().map(|_| rng.next() as i32).collect();
|
||||
ed.write_values("m", &Selection::Points(pts.clone()), &vals)
|
||||
ed.write_values(name, &Selection::Points(pts.clone()), &vals)
|
||||
.unwrap();
|
||||
for (p, v) in pts.iter().zip(&vals) {
|
||||
let i = m.index(p);
|
||||
m.data[i] = *v;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let k = rng.below(6);
|
||||
let name = format!("a{k}");
|
||||
let v = rng.next() as i64;
|
||||
match ed.set_attr("m", &name, &AttrValue::I64(v)) {
|
||||
9..=11 => {
|
||||
let k = rng.below(20);
|
||||
let aname = format!("a{k}");
|
||||
let v = random_attr(&mut rng);
|
||||
match ed.set_attr("m", &aname, &v) {
|
||||
Ok(()) => {
|
||||
attrs.retain(|(n, _)| *n != name);
|
||||
attrs.push((name, v));
|
||||
attrs.retain(|(n, _)| *n != aname);
|
||||
attrs.push((aname, v));
|
||||
}
|
||||
Err(e) => panic!("set_attr {name}: {e}"),
|
||||
// Replacing the only attribute in a heap block with one
|
||||
// of another size would have libhdf5 free the block.
|
||||
Err(Error::Unsupported(msg)) if msg.contains("last object") => {}
|
||||
Err(e) => panic!("set_attr {aname}: {e}"),
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if step % 30 == 29 {
|
||||
if step % 40 == 39 {
|
||||
drop(ed);
|
||||
verify(&path, "m", &m);
|
||||
for (n, m) in names.iter().zip(&models) {
|
||||
verify(&path, n, m);
|
||||
}
|
||||
check_tools(&path, h5dump);
|
||||
check_attrs(&path, "m", &attrs);
|
||||
ed = FileEditor::open(&path).unwrap();
|
||||
}
|
||||
}
|
||||
drop(ed);
|
||||
verify(&path, "m", &m);
|
||||
for (n, m) in names.iter().zip(&models) {
|
||||
verify(&path, n, m);
|
||||
}
|
||||
check_attrs(&path, "m", &attrs);
|
||||
py(&format!(
|
||||
"import h5py, numpy as np\n\
|
||||
with h5py.File({p:?}, 'r+') as f:\n\
|
||||
\x20 d = f['m']\n\
|
||||
\x20 n = d.shape[0]\n\
|
||||
\x20 d.resize((n + 3, 7))\n\
|
||||
\x20 d[n:, :] = 42\n\
|
||||
\x20 d.attrs['from_h5py'] = 1.5\n",
|
||||
\x20 for name, cols in (('m', 7), ('b', None)):\n\
|
||||
\x20 d = f[name]\n\
|
||||
\x20 n, c = d.shape\n\
|
||||
\x20 d.resize((n + 3, cols or c + 2))\n\
|
||||
\x20 d[n:, :] = 42\n\
|
||||
\x20 f['m'].attrs['from_h5py'] = 1.5\n",
|
||||
p = path.to_str().unwrap()
|
||||
));
|
||||
let n = m.shape[0];
|
||||
m.resize(&[n + 3, 7], -9);
|
||||
m.write_block(&[n, 0], &[3, 7], &[42; 21]);
|
||||
verify(&path, "m", &m);
|
||||
for (d, m) in models.iter_mut().enumerate() {
|
||||
let (n, c) = (m.shape[0], m.shape[1]);
|
||||
let c2 = if d == 0 { 7 } else { c + 2 };
|
||||
m.resize(&[n + 3, c2], fills[d]);
|
||||
m.write_block(&[n, 0], &[3, c2], &vec![42; (3 * c2) as usize]);
|
||||
}
|
||||
for (n, m) in names.iter().zip(&models) {
|
||||
verify(&path, n, m);
|
||||
}
|
||||
check_tools(&path, h5dump);
|
||||
check_attrs(&path, "m", &attrs);
|
||||
}
|
||||
|
||||
fn check_attrs(path: &Path, obj: &str, attrs: &[(String, i64)]) {
|
||||
/// Our reader and h5py see `attrs` on dataset `obj` (and h5py's count of
|
||||
/// its attributes agrees with libhdf5's object info).
|
||||
fn check_attrs(path: &Path, obj: &str, attrs: &[(String, AttrValue)]) {
|
||||
let f = File::open(path).unwrap();
|
||||
let got = f.dataset(obj).unwrap().attrs().unwrap();
|
||||
for (n, v) in attrs {
|
||||
match got.get(n) {
|
||||
Some(AttrValue::I64(g)) => assert_eq!(g, v, "attribute {n}"),
|
||||
other => panic!("attribute {n}: {other:?}"),
|
||||
}
|
||||
let g = got
|
||||
.get(n)
|
||||
.unwrap_or_else(|| panic!("attribute {n} missing"));
|
||||
// Our reader reports a one-element array as a scalar.
|
||||
let v = match v {
|
||||
AttrValue::I64Array(a) if a.len() == 1 => &AttrValue::I64(a[0]),
|
||||
v => v,
|
||||
};
|
||||
assert_eq!(format!("{g:?}"), format!("{v:?}"), "attribute {n}");
|
||||
}
|
||||
let want: Vec<String> = attrs.iter().map(|(n, v)| format!("{n:?}: {v}")).collect();
|
||||
py(&format!(
|
||||
let want: Vec<String> = attrs
|
||||
.iter()
|
||||
.map(|(n, v)| {
|
||||
let pv = match v {
|
||||
AttrValue::I64(x) => format!("{x}"),
|
||||
AttrValue::I64Array(a) => format!("{a:?}"),
|
||||
AttrValue::String(s) => format!("{s:?}"),
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
format!("{n:?}: {pv}")
|
||||
})
|
||||
.collect();
|
||||
let script = format!(
|
||||
"import h5py\n\
|
||||
f = h5py.File({p:?}, 'r')\n\
|
||||
want = {{{w}}}\n\
|
||||
got = {{k: int(v) for k, v in f[{obj:?}].attrs.items() if k in want}}\n\
|
||||
assert got == want, (got, want)\n",
|
||||
a = f[{obj:?}].attrs\n\
|
||||
def norm(v):\n\
|
||||
\x20 v = v.decode() if isinstance(v, bytes) else v\n\
|
||||
\x20 return v.tolist() if hasattr(v, 'tolist') else v\n\
|
||||
got = {{k: norm(v) for k, v in a.items() if k in want}}\n\
|
||||
assert got == want, sorted(set(want) ^ set(got))\n\
|
||||
assert len(a) == h5py.h5o.get_info(f[{obj:?}].id).num_attrs == len(list(a))\n",
|
||||
p = path.to_str().unwrap(),
|
||||
w = want.join(", ")
|
||||
));
|
||||
);
|
||||
let sp = path.with_extension("attrs.py");
|
||||
std::fs::write(&sp, script).unwrap();
|
||||
let o = Command::new(python()).arg(&sp).output().unwrap();
|
||||
assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -411,7 +482,11 @@ fn random_operations_match_a_model() {
|
||||
if !tools_ok() {
|
||||
return;
|
||||
}
|
||||
let mut seed = 1;
|
||||
// CLAWHDF5_EDIT_SEED runs the same workloads with other random choices.
|
||||
let mut seed = std::env::var("CLAWHDF5_EDIT_SEED")
|
||||
.ok()
|
||||
.and_then(|s| s.parse::<u64>().ok())
|
||||
.unwrap_or(1);
|
||||
for (i, (lv, dump)) in LIBVERS.iter().enumerate() {
|
||||
// h5dump has no LZF decoder (h5py's own filter).
|
||||
for (j, (extra, lzf)) in [
|
||||
@@ -752,19 +827,20 @@ fn overwrite_every_layout() {
|
||||
}
|
||||
check_tools(&path, *dump);
|
||||
}
|
||||
// A version-2 B-tree index can take new chunks only from libhdf5 for
|
||||
// now: growing works, writing the new chunks is refused and changes
|
||||
// nothing.
|
||||
// A version-2 B-tree index takes new chunks too.
|
||||
if *lv != "'earliest'" {
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
ed.resize("bt2", &[8, 6]).unwrap();
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
unsupported(ed.write_values("bt2", &block(&[6, 0], &[2, 6]), &[5; 12]));
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit changed the file"
|
||||
);
|
||||
models[12].resize(&[8, 6], 0);
|
||||
ed.resize("bt2", &[8, 7]).unwrap();
|
||||
models[12].resize(&[8, 7], 0);
|
||||
let vals: Vec<i32> = (0..23).collect();
|
||||
ed.write_values("bt2", &block(&[6, 0], &[2, 7]), &vals[..14])
|
||||
.unwrap();
|
||||
models[12].write_block(&[6, 0], &[2, 7], &vals[..14]);
|
||||
ed.write_values("bt2", &block(&[0, 6], &[6, 1]), &vals[14..20])
|
||||
.unwrap();
|
||||
models[12].write_block(&[0, 6], &[6, 1], &vals[14..20]);
|
||||
drop(ed);
|
||||
verify(&path, "bt2", &models[12]);
|
||||
}
|
||||
// libhdf5 goes on modifying what we wrote.
|
||||
py(&format!(
|
||||
@@ -835,19 +911,16 @@ fn attributes_in_place() {
|
||||
.unwrap();
|
||||
want.push(("d", "units".into(), AttrValue::String("km".into())));
|
||||
if *lv != "'earliest'" {
|
||||
// Up to the compact limit (8) and no further; attributes in
|
||||
// dense storage and tracked creation order are refused, and a
|
||||
// refused edit writes nothing.
|
||||
// Up to the compact limit (8), then into dense storage; objects
|
||||
// already in dense storage and ones tracking creation order.
|
||||
ed.set_attr("g", "eighth", &AttrValue::I64(8)).unwrap();
|
||||
want.push(("g", "eighth".into(), AttrValue::I64(8)));
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
unsupported(ed.set_attr("g", "ninth", &AttrValue::I64(9)));
|
||||
unsupported(ed.set_attr("dense", "k0", &AttrValue::I64(1)));
|
||||
unsupported(ed.set_attr("tracked", "b", &AttrValue::I64(1)));
|
||||
assert!(
|
||||
std::fs::read(&path).unwrap() == before,
|
||||
"a refused edit changed the file"
|
||||
);
|
||||
ed.set_attr("g", "ninth", &AttrValue::I64(9)).unwrap();
|
||||
want.push(("g", "ninth".into(), AttrValue::I64(9)));
|
||||
ed.set_attr("dense", "k0", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("dense", "k0".into(), AttrValue::I64(1)));
|
||||
ed.set_attr("tracked", "b", &AttrValue::I64(1)).unwrap();
|
||||
want.push(("tracked", "b".into(), AttrValue::I64(1)));
|
||||
}
|
||||
drop(ed);
|
||||
check_tools(&path, *dump);
|
||||
@@ -1027,7 +1100,7 @@ fn refused_edits_change_nothing() {
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
let mut ed = FileEditor::open(&path).unwrap();
|
||||
unsupported(ed.write_all("s", &[0u8; 32]));
|
||||
unsupported(ed.resize("x", &[4]));
|
||||
unsupported(ed.resize("c", &[4]));
|
||||
unsupported(
|
||||
ed.resize("c", &[6, 1])
|
||||
.map_err(|_| Error::Unsupported(String::new())),
|
||||
@@ -1124,9 +1197,10 @@ fn out_of_order_chunk_creation_matches_libhdf5() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Not a check: prints how much space an append workload leaks (the editor
|
||||
/// never reuses space), against libhdf5 doing the same appends and against
|
||||
/// `h5repack` of each. Run with `--ignored --nocapture`.
|
||||
/// Not a check: prints how much space an append workload leaks (one
|
||||
/// editor for the whole workload, which reuses the space it frees but not
|
||||
/// space it cannot fit a grown chunk into), against libhdf5 doing the same
|
||||
/// appends and against `h5repack` of each. Run with `--ignored --nocapture`.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn measure_append_waste() {
|
||||
|
||||
Reference in New Issue
Block a user