format: chunk grids with a zero extent have no chunks (no division by zero)
A Fixed or Extensible Array index whose maximum extent (the current one when no maximum is recorded) is 0 along a dimension has a zero stride for every dimension before it; ChunkGrid::offsets divided by it. The unfixed editor made such files by resizing a clawhdf5-written dataset to a zero extent: `h5rs check` panicked and the next resize raised an internal error (12 of the reviewer's random-edit seeds 10..39). Such an index has no slot for any chunk of the dataset; offsets now returns None. Tests: chunk_grid::zero_extent_has_no_chunks; edit_interop's zero_extent_resizes_without_a_recorded_maximum on a file the unfixed editor left (fixture) and on a 2.7.0-written file taken through zero extents with `h5rs check --data` and h5dump at every step; test_edit.py random edits on seeds 10..39 of a clawhdf5-written file. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -2,6 +2,22 @@
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
### Correctness: zero extents in Fixed/Extensible Array chunk indexes (2026-09-27)
|
||||||
|
- **A chunked dataset whose maximum (or, with none recorded, current)
|
||||||
|
extent is 0 along a dimension made the reader divide by zero** (fixed
|
||||||
|
2026-09-27): `h5rs check` panicked ("attempt to divide by zero",
|
||||||
|
`chunk_grid.rs`) and the next `FileEditor::resize` failed with an
|
||||||
|
internal error. The unfixed editor produced such files by resizing a
|
||||||
|
clawhdf5-written dataset to a zero extent (12 of 30 extra random-edit
|
||||||
|
seeds on clawhdf5-written files). Such an index has no slot for any
|
||||||
|
chunk of the dataset, and `ChunkGrid::offsets` now says so instead of
|
||||||
|
dividing by the zero stride. Tests: `chunk_grid`'s
|
||||||
|
`zero_extent_has_no_chunks`, `edit_interop.rs`'s
|
||||||
|
`zero_extent_resizes_without_a_recorded_maximum` (a file the unfixed
|
||||||
|
editor left checks clean and resizes on; a 2.7.0-written file through
|
||||||
|
zero extents checks clean at each step), and `test_edit.py`'s random
|
||||||
|
edits on seeds 10 to 39 of a clawhdf5-written file.
|
||||||
|
|
||||||
### Correctness: resizing chunked datasets with no recorded maximum (2026-09-27)
|
### Correctness: resizing chunked datasets with no recorded maximum (2026-09-27)
|
||||||
- **`FileEditor::resize` scrambled the values of a chunked dataset whose
|
- **`FileEditor::resize` scrambled the values of a chunked dataset whose
|
||||||
dataspace records no maximum dimensions when it shrank it** (fixed
|
dataspace records no maximum dimensions when it shrank it** (fixed
|
||||||
|
|||||||
@@ -135,6 +135,12 @@ impl ChunkGrid {
|
|||||||
let mut rem = index;
|
let mut rem = index;
|
||||||
for p in 0..rank {
|
for p in 0..rank {
|
||||||
let d = self.order[p];
|
let d = self.order[p];
|
||||||
|
// A zero stride: a later dimension has no chunks (its maximum,
|
||||||
|
// or with none recorded its current extent, is 0), so no slot of
|
||||||
|
// the index is a chunk of the dataset.
|
||||||
|
if self.down[p] == 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
let scaled = rem / self.down[p];
|
let scaled = rem / self.down[p];
|
||||||
rem %= self.down[p];
|
rem %= self.down[p];
|
||||||
if scaled >= self.cur_chunks[d] {
|
if scaled >= self.cur_chunks[d] {
|
||||||
@@ -193,6 +199,28 @@ mod tests {
|
|||||||
assert_eq!(g.offsets(11), Some(vec![2, 3]));
|
assert_eq!(g.offsets(11), Some(vec![2, 3]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zero_extent_has_no_chunks() {
|
||||||
|
// No maximum recorded and a zero current dimension: every stride
|
||||||
|
// before it is 0 (this divided by zero).
|
||||||
|
let g = ChunkGrid::fixed_array(&[1, 0], None, &[6, 6]).unwrap();
|
||||||
|
for i in 0..16 {
|
||||||
|
assert_eq!(g.offsets(i), None);
|
||||||
|
}
|
||||||
|
let g = ChunkGrid::fixed_array(&[0, 0, 3], Some(&[4, 0, 3]), &[2, 2, 3]).unwrap();
|
||||||
|
for i in 0..16 {
|
||||||
|
assert_eq!(g.offsets(i), None);
|
||||||
|
}
|
||||||
|
let g = ChunkGrid::extensible_array(&[0, 5], Some(&[u64::MAX, 0]), &[2, 2]).unwrap();
|
||||||
|
for i in 0..16 {
|
||||||
|
assert_eq!(g.offsets(i), None);
|
||||||
|
}
|
||||||
|
// A zero last dimension leaves the other strides alone.
|
||||||
|
let g = ChunkGrid::fixed_array(&[4, 0], Some(&[4, 6]), &[2, 3]).unwrap();
|
||||||
|
assert_eq!(g.offsets(0), None);
|
||||||
|
assert_eq!(g.linear_index(&[1, 1]), 3);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn rejects_two_unlimited_dims_after_the_first() {
|
fn rejects_two_unlimited_dims_after_the_first() {
|
||||||
assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err());
|
assert!(ChunkGrid::fixed_array(&[4, 6], Some(&[u64::MAX, u64::MAX]), &[2, 3]).is_err());
|
||||||
|
|||||||
@@ -445,6 +445,13 @@ def test_random_edits_match_h5py(h5py, tmp_path, source, seed):
|
|||||||
h5dump_reads(ours_path, base)
|
h5dump_reads(ours_path, base)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("seed", range(10, 40))
|
||||||
|
def test_random_edits_on_clawhdf5_files(h5py, tmp_path, seed):
|
||||||
|
"""More random sequences on a clawhdf5-written file, whose resizes to
|
||||||
|
zero extents once left files `h5rs check` could not read."""
|
||||||
|
test_random_edits_match_h5py(h5py, tmp_path, "clawhdf5", seed)
|
||||||
|
|
||||||
|
|
||||||
def resized_model(before, shape, fill):
|
def resized_model(before, shape, fill):
|
||||||
"""`before` resized to `shape` as HDF5 resizes: elements inside both
|
"""`before` resized to `shape` as HDF5 resizes: elements inside both
|
||||||
extents keep their values, the others read as the fill value."""
|
extents keep their values, the others read as the fill value."""
|
||||||
|
|||||||
@@ -1509,3 +1509,68 @@ fn unencodable_filters_are_unsupported() {
|
|||||||
"a refused edit changed the file"
|
"a refused edit changed the file"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn fixture(dir: &Path, name: &str) -> std::path::PathBuf {
|
||||||
|
let path = dir.join(name);
|
||||||
|
std::fs::copy(
|
||||||
|
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("../clawhdf5/tests/fixtures")
|
||||||
|
.join(name),
|
||||||
|
&path,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
path
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Zero extents on chunked datasets with no recorded maximum. The unfixed
|
||||||
|
/// editor left `chunk_zero_extent_no_maxshape.h5` (a 2.7.0-written file
|
||||||
|
/// resized to 1x0): a Fixed Array whose maximum, taken from the current
|
||||||
|
/// dimensions, has no chunks along one dimension, so every stride before it
|
||||||
|
/// is 0 — `h5rs check` panicked dividing by it and the next resize failed
|
||||||
|
/// with an internal error. Such a file must check clean and resize on; a
|
||||||
|
/// 2.7.0-written file taken through zero extents by the fixed editor must
|
||||||
|
/// check clean at every step and read the fill value where it grew.
|
||||||
|
#[test]
|
||||||
|
fn zero_extent_resizes_without_a_recorded_maximum() {
|
||||||
|
if !tools_ok() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let dir = tmpdir();
|
||||||
|
let path = fixture(dir.path(), "chunk_zero_extent_no_maxshape.h5");
|
||||||
|
check_tools(&path, true);
|
||||||
|
let mut ed = FileEditor::open(&path).unwrap();
|
||||||
|
ed.resize("d", &[0, 0]).unwrap();
|
||||||
|
ed.resize("z", &[0, 0]).unwrap();
|
||||||
|
// Their maximum is now what the index was laid out by (1 x 0).
|
||||||
|
ed.resize("d", &[1, 0]).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
ed.resize("d", &[1, 1]),
|
||||||
|
Err(Error::InvalidArgument(_))
|
||||||
|
));
|
||||||
|
drop(ed);
|
||||||
|
check_tools(&path, true);
|
||||||
|
|
||||||
|
let path = fixture(dir.path(), "chunked_no_maxshape_v2_7_0.h5");
|
||||||
|
for shape in [[15, 15], [3, 2], [1, 1], [1, 0], [0, 0], [0, 20], [20, 20]] {
|
||||||
|
let mut ed = FileEditor::open(&path).unwrap();
|
||||||
|
ed.resize("d", &shape).unwrap();
|
||||||
|
ed.resize("z", &shape).unwrap();
|
||||||
|
drop(ed);
|
||||||
|
check_tools(&path, true);
|
||||||
|
}
|
||||||
|
let f = File::open(&path).unwrap();
|
||||||
|
for name in ["d", "z"] {
|
||||||
|
let d = f.dataset(name).unwrap();
|
||||||
|
assert_eq!(d.shape().unwrap(), [20, 20]);
|
||||||
|
assert!(d.read_f32().unwrap().iter().all(|&v| v == 0.0), "{name}");
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
py(&format!(
|
||||||
|
"import h5py\n\
|
||||||
|
with h5py.File({:?}) as f:\n\
|
||||||
|
\x20 print(int(abs(f['d'][()]).sum() + abs(f['z'][()]).sum()), f['d'].maxshape)",
|
||||||
|
path.to_str().unwrap()
|
||||||
|
)),
|
||||||
|
"0 (20, 20)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
Binary file not shown.
Reference in New Issue
Block a user