fix(format): keep maxshape == shape datasets contiguous
Any maxshape forced chunked storage, even one equal to the shape, which cannot grow. h5py and the library store such a dataset contiguously; we now do too unless chunks (or a filter) are requested. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -1124,7 +1124,12 @@ impl FileWriter {
|
|||||||
let is_chunked: Vec<bool> = all_ds
|
let is_chunked: Vec<bool> = all_ds
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(i, d)| !is_vds[i] && (d.chunk_options.is_chunked() || d.maxshape.is_some()))
|
.map(|(i, d)| {
|
||||||
|
// Only a dataset that can grow needs chunks; a maxshape equal
|
||||||
|
// to the shape is as fixed as no maxshape at all.
|
||||||
|
let resizable = d.maxshape.as_ref().is_some_and(|m| *m != d.ds.dimensions);
|
||||||
|
!is_vds[i] && (d.chunk_options.is_chunked() || resizable)
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
// Determine which datasets use compact storage
|
// Determine which datasets use compact storage
|
||||||
let is_compact: Vec<bool> = all_ds
|
let is_compact: Vec<bool> = all_ds
|
||||||
|
|||||||
@@ -547,3 +547,47 @@ fn btree_v2_index_past_one_leaf_is_refused() {
|
|||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
assert!(b.write(dir.path().join("too_many.h5")).is_err());
|
assert!(b.write(dir.path().join("too_many.h5")).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A maxshape equal to the shape cannot grow, so it needs no chunks: the
|
||||||
|
/// dataset stays contiguous (as h5py makes it) unless chunks are requested.
|
||||||
|
#[test]
|
||||||
|
fn maxshape_equal_to_shape_stays_contiguous() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("ms_eq.h5");
|
||||||
|
let data: Vec<i32> = (0..40).collect();
|
||||||
|
let mut b = FileBuilder::new();
|
||||||
|
b.create_dataset("plain")
|
||||||
|
.with_i32_data(&data)
|
||||||
|
.with_shape(&[40])
|
||||||
|
.with_maxshape(&[40]);
|
||||||
|
b.create_dataset("chunked")
|
||||||
|
.with_i32_data(&data)
|
||||||
|
.with_shape(&[40])
|
||||||
|
.with_maxshape(&[40])
|
||||||
|
.with_chunks(&[8]);
|
||||||
|
b.write(&path).unwrap();
|
||||||
|
|
||||||
|
let file = File::open(&path).unwrap();
|
||||||
|
let plain = file.dataset("plain").unwrap();
|
||||||
|
assert_eq!(plain.read_i32().unwrap(), data);
|
||||||
|
assert_eq!(plain.max_dimensions().unwrap(), Some(vec![40]));
|
||||||
|
assert!(
|
||||||
|
plain.read_raw_ref().unwrap().is_some(),
|
||||||
|
"maxshape == shape should be contiguous"
|
||||||
|
);
|
||||||
|
let chunked = file.dataset("chunked").unwrap();
|
||||||
|
assert_eq!(chunked.read_i32().unwrap(), data);
|
||||||
|
assert!(chunked.read_raw_ref().unwrap().is_none());
|
||||||
|
|
||||||
|
skip_if_no_python!();
|
||||||
|
let out = run_python(&format!(
|
||||||
|
"import h5py, numpy as np\n\
|
||||||
|
f = h5py.File(r'{}', 'r')\n\
|
||||||
|
for n in ('plain', 'chunked'):\n\
|
||||||
|
\x20 d = f[n]\n\
|
||||||
|
\x20 assert np.array_equal(d[()], np.arange(40, dtype='i4')), n\n\
|
||||||
|
\x20 print(n, d.chunks, d.maxshape)\n",
|
||||||
|
path.display()
|
||||||
|
));
|
||||||
|
assert_eq!(out, "plain None (40,)\nchunked (8,) (40,)");
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user