fix(format): write chunk indexes over the max extent, swizzled for EA

The writer indexed chunks by their position in the current shape, the
same mistake the reader had. With a finite maxshape larger than the shape
the Fixed Array was sized for the shape, so libhdf5 looked up chunks past
its end ("addr overflow"); with the unlimited dimension anywhere but first,
e.g. maxshape (20, None), libhdf5 swizzles that dimension to the slowest
position and read our Extensible Array scrambled. Two unlimited dimensions
produced a file libhdf5 refused to open ("already found unlimited
dimension").

Chunks are now placed with the shared chunk_grid linearisation: Fixed
Array slots cover every chunk of the maximum extent (unwritten ones
undefined), Extensible Array indexes are swizzled, Single Chunk is only
used when the maximum extent is one chunk, and a maxshape that is smaller
than the shape, has more than one unlimited dimension, or would need an
absurd Fixed Array is an error instead of a bad file.
build_chunked_data_from_precompressed now returns a Result.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5/tests/chunk_index_interop.rs

@@ -373,6 +373,73 @@ fn check_we_write(cases: &[WriteCase]) {
             "h5dump failed: {stderr}"
         );
     }
+
+    // Let libhdf5 grow every resizable dataset by two chunks per dimension
+    // (capped at the maxshape) and rewrite it, which updates our index in
+    // place and inserts new chunks into it. Then both readers must agree.
+    let script = format!(
+        r#"
+import h5py, numpy as np
+grown = {{}}
+with h5py.File(r'{path_str}', 'r+') as f:
+    for name in f:
+        d = f[name]
+        if d.chunks is None:
+            continue
+        new = tuple(s + 2 * c if m is None else min(m, s + 2 * c)
+                    for s, m, c in zip(d.shape, d.maxshape, d.chunks))
+        if new == d.shape:
+            continue
+        old = d[()]
+        full = np.full(new, -7, 'i4')
+        full[tuple(slice(0, s) for s in old.shape)] = old
+        d.resize(new)
+        d[...] = full
+        grown[name] = (list(old.shape), list(new))
+with h5py.File(r'{path_str}', 'r') as f:
+    for name, (old, new) in grown.items():
+        want = np.full(new, -7, 'i4')
+        want[tuple(slice(0, s) for s in old)] = np.arange(int(np.prod(old)), dtype='i4').reshape(old)
+        assert np.array_equal(f[name][()], want), name
+for name, (old, new) in grown.items():
+    print(name, ','.join(map(str, old)), ','.join(map(str, new)))
+"#
+    );
+    let out = run_python(&script);
+    let growable = cases
+        .iter()
+        .filter(|c| c.maxshape.as_ref().is_some_and(|m| *m != c.shape))
+        .count();
+    assert_eq!(out.lines().count(), growable, "libhdf5 grew: {out}");
+    let dims = |s: &str| -> Vec<usize> { s.split(',').map(|x| x.parse().unwrap()).collect() };
+    let file = File::open(&path).unwrap();
+    for line in out.lines() {
+        let mut parts = line.split(' ');
+        let (name, old, new) = (
+            parts.next().unwrap(),
+            dims(parts.next().unwrap()),
+            dims(parts.next().unwrap()),
+        );
+        let got = file.dataset(name).unwrap().read_i32().unwrap();
+        let n: usize = new.iter().product();
+        let mut want = vec![-7i32; n];
+        for (flat, w) in want.iter_mut().enumerate() {
+            let mut rem = flat;
+            let mut coords = vec![0usize; new.len()];
+            for d in (0..new.len()).rev() {
+                coords[d] = rem % new[d];
+                rem /= new[d];
+            }
+            if coords.iter().zip(&old).all(|(c, o)| c < o) {
+                *w = coords.iter().zip(&old).fold(0, |acc, (c, o)| acc * o + c) as i32;
+            }
+        }
+        let bad = got.iter().zip(&want).filter(|(a, b)| a != b).count();
+        assert!(
+            got.len() == n && bad == 0,
+            "{name}: after libhdf5 grew it, our reader got {bad} of {n} values wrong"
+        );
+    }
 }
 
 /// A Fixed Array with more than 1024 elements must be paged, or libhdf5
@@ -411,3 +478,53 @@ fn we_write_extensible_array_past_index_block() {
     cases.push(wcase("ea_140000", &[140_000], &[1], Some(unl)));
     check_we_write(&cases);
 }
+
+/// A maxshape larger than the shape: the index must be laid out over the
+/// chunks of the maximum extent (libhdf5 read our Fixed Array past its end:
+/// "addr overflow"), and an Extensible Array whose unlimited dimension is not
+/// the first must swizzle it to the slowest position (libhdf5 read our
+/// `(20, None)` dataset scrambled).
+#[test]
+fn we_write_maxshape_larger_than_shape() {
+    const U: u64 = u64::MAX;
+    let mut cases = vec![
+        // Fixed Array over the maximum extent.
+        wcase("fa2d_finite_max", &[20, 30], &[5, 5], Some(&[40, 60])),
+        wcase("fa1d_finite_max", &[40], &[4], Some(&[100])),
+        wcase("fa3d_edges", &[6, 7, 8], &[4, 3, 5], Some(&[10, 9, 20])),
+        wcase("fa_paged_max", &[30, 50], &[1, 1], Some(&[40, 60])),
+        wcase("fa_one_chunk_now", &[5], &[5], Some(&[50])),
+        // Extensible Array, unlimited dimension first (no swizzle) ...
+        wcase("ea2d_unl_fin", &[20, 30], &[5, 5], Some(&[U, 30])),
+        wcase("ea2d_unl_fin_max", &[20, 30], &[5, 5], Some(&[U, 60])),
+        // ... and not first (swizzled).
+        wcase("ea2d_fin_unl", &[20, 30], &[5, 5], Some(&[20, U])),
+        wcase("ea2d_fin_max_unl", &[20, 30], &[5, 5], Some(&[40, U])),
+        wcase("ea3d_mid", &[6, 7, 8], &[4, 3, 5], Some(&[10, U, 20])),
+        // Past the index block and into super blocks, swizzled.
+        wcase("ea2d_many", &[3, 2000], &[1, 1], Some(&[4, U])),
+    ];
+    let mut filtered = wcase(
+        "ea3d_last_deflate",
+        &[6, 7, 8],
+        &[4, 3, 5],
+        Some(&[6, 8, U]),
+    );
+    filtered.deflate = true;
+    cases.push(filtered);
+    check_we_write(&cases);
+}
+
+/// More than one unlimited dimension needs a B-tree v2 chunk index; the
+/// writer must not produce a file libhdf5 cannot open.
+#[test]
+fn two_unlimited_dims_are_refused() {
+    let mut b = FileBuilder::new();
+    b.create_dataset("d")
+        .with_i32_data(&(0..600).collect::<Vec<i32>>())
+        .with_shape(&[20, 30])
+        .with_chunks(&[5, 5])
+        .with_maxshape(&[u64::MAX, u64::MAX]);
+    let dir = tempfile::tempdir().unwrap();
+    assert!(b.write(dir.path().join("unl_unl.h5")).is_err());
+}