fix(format): lay Fixed/Extensible Array chunk indexes out by max dims
Both indexes place each chunk at a linear index computed from the dataset's maximum dimensions (libhdf5's max_down_chunks), and the Extensible Array first swizzles its unlimited dimension to the slowest position. We linearised by the current dimensions, so any dataset whose shape was smaller than its maxshape, or whose unlimited dimension was not the first, read back scrambled without an error: h5py libver="latest" files with maxshape (10, None) or (20, 10), and the libhdf5 test files h5fc_ext*.h5 and test_ld.h5. The linearisation now lives in chunk_grid (shared with the writers), and slots beyond the current extent are ignored as the library does. read_fixed_array_chunks / read_extensible_array_chunks take the dataspace's max dimensions. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,256 @@
|
||||
//! Fixed Array / Extensible Array chunk-index interop with libhdf5 (via h5py).
|
||||
//!
|
||||
//! Both indexes place each chunk at a linear index computed from the
|
||||
//! dataset's *maximum* dimensions, and the Extensible Array additionally
|
||||
//! moves its unlimited dimension to the slowest-varying position. Getting
|
||||
//! either wrong reads (or writes) every chunk after the first row in the
|
||||
//! wrong place, silently, so these tests compare every value.
|
||||
//!
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn run_python(script: &str) -> String {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
if !output.status.success() {
|
||||
panic!(
|
||||
"Python script failed:\nSTDOUT: {}\nSTDERR: {}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
||||
}
|
||||
|
||||
/// Row-major `arange` of `shape`, cropped to `crop` (the current extent).
|
||||
fn arange_cropped(full: &[usize], crop: &[usize]) -> Vec<i32> {
|
||||
let n: usize = crop.iter().product();
|
||||
let mut out = Vec::with_capacity(n);
|
||||
for flat in 0..n {
|
||||
let mut rem = flat;
|
||||
let mut src = 0usize;
|
||||
let mut stride = 1usize;
|
||||
let mut coords = vec![0usize; crop.len()];
|
||||
for d in (0..crop.len()).rev() {
|
||||
coords[d] = rem % crop[d];
|
||||
rem /= crop[d];
|
||||
}
|
||||
for d in (0..full.len()).rev() {
|
||||
src += coords[d] * stride;
|
||||
stride *= full[d];
|
||||
}
|
||||
out.push(src as i32);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One `i4` dataset, filled with `arange` over `full` and then resized to
|
||||
/// `shape` (equal to `full` unless the case shrinks it).
|
||||
struct Case {
|
||||
name: &'static str,
|
||||
full: Vec<usize>,
|
||||
shape: Vec<usize>,
|
||||
chunks: Vec<usize>,
|
||||
maxshape: &'static str,
|
||||
extra: &'static str,
|
||||
index: &'static str,
|
||||
}
|
||||
|
||||
fn py_tuple(v: &[usize]) -> String {
|
||||
let parts: Vec<String> = v.iter().map(|x| x.to_string()).collect();
|
||||
format!("({},)", parts.join(","))
|
||||
}
|
||||
|
||||
/// Have h5py (`libver="latest"`, so Fixed/Extensible Array indexes) write
|
||||
/// every case to one file, then read each back and compare every value.
|
||||
fn check_h5py_written(cases: &[Case]) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("h5py_chunk_index.h5");
|
||||
let path_str = path.display().to_string();
|
||||
|
||||
let mut script =
|
||||
format!("import h5py, numpy as np\nf = h5py.File(r'{path_str}', 'w', libver='latest')\n");
|
||||
for c in cases {
|
||||
script += &format!(
|
||||
"d = f.create_dataset('{name}', data=np.arange({n}, dtype='i4').reshape({full}), \
|
||||
chunks={chunks}, maxshape={maxshape}{extra})\n\
|
||||
d.resize({shape})\n",
|
||||
name = c.name,
|
||||
n = c.full.iter().product::<usize>(),
|
||||
full = py_tuple(&c.full),
|
||||
chunks = py_tuple(&c.chunks),
|
||||
maxshape = c.maxshape,
|
||||
extra = c.extra,
|
||||
shape = py_tuple(&c.shape),
|
||||
);
|
||||
}
|
||||
script += "f.close()\n";
|
||||
run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for c in cases {
|
||||
let ds = file.dataset(c.name).unwrap();
|
||||
let shape: Vec<usize> = ds.shape().unwrap().iter().map(|&d| d as usize).collect();
|
||||
assert_eq!(shape, c.shape, "{}: shape", c.name);
|
||||
let got = ds.read_i32().unwrap();
|
||||
let want = arange_cropped(&c.full, &c.shape);
|
||||
let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count();
|
||||
assert_eq!(
|
||||
got,
|
||||
want,
|
||||
"{}: {bad} of {} values differ (index {})",
|
||||
c.name,
|
||||
want.len(),
|
||||
c.index
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// h5py-written Extensible Array whose unlimited dimension is not the first,
|
||||
/// with the current shape smaller than the finite maximum: the library
|
||||
/// swizzles the unlimited dimension to the slowest position and strides the
|
||||
/// rest by their maximum chunk counts.
|
||||
#[test]
|
||||
fn h5py_extensible_array_partial_extent_reads_correctly() {
|
||||
skip_if_no_python!();
|
||||
check_h5py_written(&[
|
||||
// The `ea_fa_partial.h5` repro from the conformance sweep.
|
||||
Case {
|
||||
name: "ea_10_none",
|
||||
full: vec![4, 6],
|
||||
shape: vec![4, 6],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(10, None)",
|
||||
extra: "",
|
||||
index: "EA, unlimited dim 1",
|
||||
},
|
||||
Case {
|
||||
name: "ea_none_10",
|
||||
full: vec![4, 6],
|
||||
shape: vec![4, 6],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(None, 10)",
|
||||
extra: "",
|
||||
index: "EA, unlimited dim 0",
|
||||
},
|
||||
Case {
|
||||
name: "ea_3d_mid",
|
||||
full: vec![3, 4, 5],
|
||||
shape: vec![3, 4, 5],
|
||||
chunks: vec![2, 3, 2],
|
||||
maxshape: "(5, None, 7)",
|
||||
extra: "",
|
||||
index: "EA, unlimited dim 1 of 3",
|
||||
},
|
||||
Case {
|
||||
name: "ea_3d_last_gzip",
|
||||
full: vec![3, 4, 5],
|
||||
shape: vec![3, 4, 5],
|
||||
chunks: vec![2, 3, 2],
|
||||
maxshape: "(5, 9, None)",
|
||||
extra: ", compression='gzip'",
|
||||
index: "EA, unlimited dim 2 of 3, filtered",
|
||||
},
|
||||
// Many chunks: crosses data blocks, super blocks and paging.
|
||||
Case {
|
||||
name: "ea_many",
|
||||
full: vec![3, 1500],
|
||||
shape: vec![3, 1500],
|
||||
chunks: vec![1, 1],
|
||||
maxshape: "(4, None)",
|
||||
extra: "",
|
||||
index: "EA, 4500 slots",
|
||||
},
|
||||
// Shrunk after writing: chunks beyond the extent must be ignored.
|
||||
Case {
|
||||
name: "ea_shrunk",
|
||||
full: vec![8, 9],
|
||||
shape: vec![3, 4],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(10, None)",
|
||||
extra: "",
|
||||
index: "EA, shrunk",
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
/// h5py-written Fixed Array with the current shape smaller than a finite
|
||||
/// maxshape: the index has one slot per chunk of the *maximum* extent.
|
||||
#[test]
|
||||
fn h5py_fixed_array_partial_extent_reads_correctly() {
|
||||
skip_if_no_python!();
|
||||
check_h5py_written(&[
|
||||
Case {
|
||||
name: "fa_20_10",
|
||||
full: vec![4, 6],
|
||||
shape: vec![4, 6],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(20, 10)",
|
||||
extra: "",
|
||||
index: "FA",
|
||||
},
|
||||
Case {
|
||||
name: "fa_3d_gzip",
|
||||
full: vec![3, 4, 5],
|
||||
shape: vec![3, 4, 5],
|
||||
chunks: vec![2, 3, 2],
|
||||
maxshape: "(6, 8, 10)",
|
||||
extra: ", compression='gzip'",
|
||||
index: "FA, filtered",
|
||||
},
|
||||
// Paged (> 1024 slots) with most of them beyond the extent.
|
||||
Case {
|
||||
name: "fa_paged",
|
||||
full: vec![30, 50],
|
||||
shape: vec![30, 50],
|
||||
chunks: vec![1, 1],
|
||||
maxshape: "(40, 60)",
|
||||
extra: "",
|
||||
index: "FA, 2400 slots, paged",
|
||||
},
|
||||
Case {
|
||||
name: "fa_shrunk",
|
||||
full: vec![8, 9],
|
||||
shape: vec![5, 2],
|
||||
chunks: vec![2, 3],
|
||||
maxshape: "(20, 10)",
|
||||
extra: "",
|
||||
index: "FA, shrunk",
|
||||
},
|
||||
]);
|
||||
}
|
||||
Reference in New Issue
Block a user