fix(format): keep maxshape == shape datasets contiguous

Any maxshape forced chunked storage, even one equal to the shape, which
cannot grow. h5py and the library store such a dataset contiguously; we
now do too unless chunks (or a filter) are requested.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-25 21:15:55 -05:00
co-authored by Claude Opus 5.5
parent 1dba7b465a
commit f5505fb03d
2 changed files with 50 additions and 1 deletions
+6 -1
View File
@@ -1124,7 +1124,12 @@ impl FileWriter {
let is_chunked: Vec<bool> = all_ds let is_chunked: Vec<bool> = all_ds
.iter() .iter()
.enumerate() .enumerate()
.map(|(i, d)| !is_vds[i] && (d.chunk_options.is_chunked() || d.maxshape.is_some())) .map(|(i, d)| {
// Only a dataset that can grow needs chunks; a maxshape equal
// to the shape is as fixed as no maxshape at all.
let resizable = d.maxshape.as_ref().is_some_and(|m| *m != d.ds.dimensions);
!is_vds[i] && (d.chunk_options.is_chunked() || resizable)
})
.collect(); .collect();
// Determine which datasets use compact storage // Determine which datasets use compact storage
let is_compact: Vec<bool> = all_ds let is_compact: Vec<bool> = all_ds
@@ -547,3 +547,47 @@ fn btree_v2_index_past_one_leaf_is_refused() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();
assert!(b.write(dir.path().join("too_many.h5")).is_err()); assert!(b.write(dir.path().join("too_many.h5")).is_err());
} }
/// A maxshape equal to the shape cannot grow, so it needs no chunks: the
/// dataset stays contiguous (as h5py makes it) unless chunks are requested.
#[test]
fn maxshape_equal_to_shape_stays_contiguous() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("ms_eq.h5");
let data: Vec<i32> = (0..40).collect();
let mut b = FileBuilder::new();
b.create_dataset("plain")
.with_i32_data(&data)
.with_shape(&[40])
.with_maxshape(&[40]);
b.create_dataset("chunked")
.with_i32_data(&data)
.with_shape(&[40])
.with_maxshape(&[40])
.with_chunks(&[8]);
b.write(&path).unwrap();
let file = File::open(&path).unwrap();
let plain = file.dataset("plain").unwrap();
assert_eq!(plain.read_i32().unwrap(), data);
assert_eq!(plain.max_dimensions().unwrap(), Some(vec![40]));
assert!(
plain.read_raw_ref().unwrap().is_some(),
"maxshape == shape should be contiguous"
);
let chunked = file.dataset("chunked").unwrap();
assert_eq!(chunked.read_i32().unwrap(), data);
assert!(chunked.read_raw_ref().unwrap().is_none());
skip_if_no_python!();
let out = run_python(&format!(
"import h5py, numpy as np\n\
f = h5py.File(r'{}', 'r')\n\
for n in ('plain', 'chunked'):\n\
\x20 d = f[n]\n\
\x20 assert np.array_equal(d[()], np.arange(40, dtype='i4')), n\n\
\x20 print(n, d.chunks, d.maxshape)\n",
path.display()
));
assert_eq!(out, "plain None (40,)\nchunked (8,) (40,)");
}