//! `clawhdf5::FileEditor` beyond overwrites and one-dimensional appends,
//! against libhdf5: chunks added to version-2 B-tree chunk indexes (two or
//! more unlimited dimensions), datasets shrunk, attributes in dense storage
//! (and moved there), and space reused within an editing session. Files are
//! written by h5py (`earliest`, `v110`, HDF5 2.0's `latest`) and by
//! clawhdf5; after each edit h5py must read exactly what a model says,
//! h5dump must read the file, `h5rs check --data` must find nothing, and our
//! reader must agree; h5py then opens the file `r+` and goes on. Where the
//! same operations can be done by libhdf5, the structures it builds (B-tree
//! shapes, heap and free-space bookkeeping) are compared with the editor's.
//!
//! Needs python3 with h5py and numpy (`CLAWHDF5_PYTHON`) and h5dump; skips
//! when they are missing, unless `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::path::Path;
use std::process::{Command, Output};

use clawhdf5::{Error, File, FileEditor, Selection};

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn available(cmd: &str, args: &[&str]) -> bool {
    Command::new(cmd)
        .args(args)
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

/// Whether the tools are here; false (skip) or a panic when interop is
/// required.
fn tools_ok() -> bool {
    let ok =
        available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
    if !ok {
        assert!(
            !interop_required(),
            "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
        );
        eprintln!("SKIP: h5py/numpy or h5dump not available");
    }
    ok
}

fn py(script: &str) -> String {
    let o = Command::new(python())
        .args(["-c", script])
        .output()
        .expect("run python");
    assert!(
        o.status.success(),
        "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
        String::from_utf8_lossy(&o.stdout),
        String::from_utf8_lossy(&o.stderr)
    );
    String::from_utf8_lossy(&o.stdout).trim().to_string()
}

fn text(o: &Output) -> String {
    format!(
        "{}{}",
        String::from_utf8_lossy(&o.stdout),
        String::from_utf8_lossy(&o.stderr)
    )
}

/// The file is valid for libhdf5's tools and for `h5rs check --data`.
/// HDF5 2.0 `latest` files are beyond h5dump 1.14.
fn check_tools(path: &Path, h5dump: bool) {
    let p = path.to_str().unwrap();
    let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
        .args(["check", "--data", "-q", p])
        .output()
        .unwrap();
    assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
    if h5dump {
        let o = Command::new("h5dump")
            .args(["-o", "/dev/null", p])
            .output()
            .unwrap();
        assert!(
            o.status.success() && o.stderr.is_empty(),
            "h5dump {p}:\n{}",
            text(&o)
        );
    }
}

/// A dataset's expected contents.
#[derive(Clone, Debug)]
struct Model {
    shape: Vec<u64>,
    /// Row-major values.
    data: Vec<i32>,
}

impl Model {
    fn new(shape: &[u64], f: impl Fn(u64) -> i32) -> Self {
        let n: u64 = shape.iter().product();
        Self {
            shape: shape.to_vec(),
            data: (0..n).map(f).collect(),
        }
    }

    fn index(&self, c: &[u64]) -> usize {
        c.iter()
            .zip(&self.shape)
            .fold(0u64, |a, (&x, &d)| a * d + x) as usize
    }

    /// Change the extent to `shape`: elements inside both keep their
    /// values, new ones are `fill`.
    fn resize(&mut self, shape: &[u64], fill: i32) {
        let old = self.clone();
        *self = Self::new(shape, |_| fill);
        let n: u64 = old.shape.iter().product();
        for flat in 0..n {
            let mut c = vec![0u64; old.shape.len()];
            let mut r = flat;
            for d in (0..c.len()).rev() {
                c[d] = r % old.shape[d];
                r /= old.shape[d];
            }
            if c.iter().zip(shape).all(|(x, s)| x < s) {
                let i = self.index(&c);
                self.data[i] = old.data[flat as usize];
            }
        }
    }

    /// Apply a hyperslab write of `vals` (row-major over the block).
    fn write_block(&mut self, start: &[u64], count: &[u64], vals: &[i32]) {
        let n: u64 = count.iter().product();
        for flat in 0..n {
            let mut c = vec![0u64; count.len()];
            let mut r = flat;
            for d in (0..c.len()).rev() {
                c[d] = start[d] + r % count[d];
                r /= count[d];
            }
            let i = self.index(&c);
            self.data[i] = vals[flat as usize];
        }
    }
}

fn block(start: &[u64], count: &[u64]) -> Selection {
    Selection::Hyperslab {
        start: start.to_vec(),
        stride: vec![1; start.len()],
        count: count.to_vec(),
        block: vec![1; start.len()],
    }
}

/// h5py and our reader both read `m` from dataset `name`.
fn verify(path: &Path, name: &str, m: &Model) {
    let f = File::open(path).unwrap();
    let ds = f.dataset(name).unwrap();
    assert_eq!(ds.shape().unwrap(), m.shape, "our shape of {name}");
    let got = ds.read_i32().unwrap();
    if let Some(i) = (0..got.len()).find(|&i| got[i] != m.data[i]) {
        panic!(
            "our values of {name} in {}: element {i} (shape {:?}) is {}, expected {}",
            path.display(),
            m.shape,
            got[i],
            m.data[i]
        );
    }
    let exp = path.with_extension("expect");
    let bytes: Vec<u8> = m.data.iter().flat_map(|v| v.to_le_bytes()).collect();
    std::fs::write(&exp, bytes).unwrap();
    let shape: Vec<String> = m.shape.iter().map(|d| d.to_string()).collect();
    py(&format!(
        "import h5py, numpy as np\n\
         f = h5py.File({p:?}, 'r')\n\
         a = f[{name:?}][()]\n\
         e = np.fromfile({e:?}, dtype='<i4').reshape(({s},))\n\
         assert a.shape == e.shape, (a.shape, e.shape)\n\
         bad = np.argwhere(a != e)\n\
         assert len(bad) == 0, ('mismatch at', bad[:5], a[tuple(bad[0])], e[tuple(bad[0])])\n",
        p = path.to_str().unwrap(),
        e = exp.to_str().unwrap(),
        s = shape.join(",")
    ));
}

/// A deterministic pseudo-random sequence (SplitMix64).
struct Rng(u64);
impl Rng {
    fn next(&mut self) -> u64 {
        self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
        let mut z = self.0;
        z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
        z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
        z ^ (z >> 31)
    }
    fn below(&mut self, n: u64) -> u64 {
        self.next() % n
    }
}

fn tmpdir() -> tempfile::TempDir {
    let base = std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"));
    tempfile::TempDir::new_in(base).unwrap()
}

fn unsupported<T: std::fmt::Debug>(r: Result<T, Error>) {
    match r {
        Err(Error::Unsupported(_)) => {}
        other => panic!("expected Error::Unsupported, got {other:?}"),
    }
}

fn le(b: &[u8]) -> u64 {
    b.iter()
        .take(8)
        .enumerate()
        .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i)))
}

fn enc_size(v: u64) -> usize {
    ((63 - v.max(1).leading_zeros()) / 8 + 1) as usize
}

/// The shape of the version-2 B-tree whose header is at `hdr` in `b`
/// (8-byte addresses and lengths): its depth, record count, and every
/// node's (depth, records) breadth-first.
fn bt2_shape_at(b: &[u8], hdr: usize) -> (u16, u64, Vec<(u16, u64)>) {
    let h = &b[hdr..];
    assert_eq!(&h[0..4], b"BTHD");
    let node_size = u64::from(u32::from_le_bytes(h[6..10].try_into().unwrap()));
    let rs = u64::from(u16::from_le_bytes(h[10..12].try_into().unwrap()));
    let depth = u16::from_le_bytes(h[12..14].try_into().unwrap());
    let root = le(&h[16..24]);
    let root_n = le(&h[24..26]);
    let total = le(&h[26..34]);
    // Node geometry, as H5B2__hdr_init computes it.
    let leaf_max = (node_size - 10) / rs;
    let nrec_w = enc_size(leaf_max);
    let mut cum = vec![(leaf_max, 0usize)];
    for d in 1..=u64::from(depth) {
        let (below, below_w) = cum[d as usize - 1];
        let ptr = 8 + nrec_w as u64 + if d > 1 { below_w as u64 } else { 0 };
        let max = (node_size - 10 - ptr) / (rs + ptr);
        let c = (max + 1) * below + max;
        cum.push((c, enc_size(c)));
    }
    let mut out = Vec::new();
    let mut level = vec![(root, root_n)];
    let mut dep = depth;
    loop {
        let mut next = Vec::new();
        for &(addr, n) in &level {
            out.push((dep, n));
            if dep == 0 {
                assert_eq!(&b[addr as usize..addr as usize + 4], b"BTLF");
                continue;
            }
            let node = &b[addr as usize..];
            assert_eq!(&node[0..4], b"BTIN");
            let mut q = 6 + (n * rs) as usize;
            let all_w = if dep > 1 { cum[dep as usize - 1].1 } else { 0 };
            for _ in 0..=n {
                let a = le(&node[q..q + 8]);
                let c = le(&node[q + 8..q + 8 + nrec_w]);
                q += 8 + nrec_w + all_w;
                next.push((a, c));
            }
        }
        if dep == 0 {
            break;
        }
        dep -= 1;
        level = next;
    }
    (depth, total, out)
}

/// The shape of the file's only chunk B-tree (record type 10 or 11).
fn chunk_bt2_shape(path: &Path) -> (u16, u64, Vec<(u16, u64)>) {
    let b = std::fs::read(path).unwrap();
    let hdrs: Vec<usize> = b
        .windows(6)
        .enumerate()
        .filter(|(_, w)| &w[0..5] == b"BTHD\0" && (w[5] == 10 || w[5] == 11))
        .map(|(i, _)| i)
        .collect();
    assert_eq!(hdrs.len(), 1, "one chunk B-tree in {}", path.display());
    bt2_shape_at(&b, hdrs[0])
}

/// Grow a dataset with two unlimited dimensions in both directions many
/// times, writing the new parts, with libhdf5 (its chunk cache off, so each
/// chunk enters the index as it is written, in the same order as the
/// editor's) and with the editor: the same values, and the same B-tree —
/// depth, record count, and every node's record count — through leaf and
/// internal splits, redistributions and a depth increase.
fn bt2_growth(libver: &str, h5dump: bool, extra: &str, tag: &str) {
    let dir = tmpdir();
    let a = dir.path().join(format!("bt2_{tag}_h5py.h5"));
    let b = dir.path().join(format!("bt2_{tag}_edit.h5"));
    // Extents: both dimensions grow; new columns fall between existing
    // chunks in the index's key order (row first).
    let steps: Vec<(u64, u64)> = (1..=12u64).map(|k| (7 * k, 6 * k + k % 3)).collect();
    let val = |r: u64, c: u64| (r * 1000 + c) as i32;
    let create = |p: &Path, write: bool| {
        py(&format!(
            "import h5py, numpy as np\n\
             with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
             \x20   d = f.create_dataset('x', shape=(0, 0), maxshape=(None, None), chunks=(1, 1), \
             dtype='<i4'{extra})\n\
             \x20   if {write}:\n\
             \x20       r0, c0 = 0, 0\n\
             \x20       for r, c in {steps:?}:\n\
             \x20           d.resize((r, c))\n\
             \x20           g = np.add.outer(np.arange(r) * 1000, np.arange(c)).astype('<i4')\n\
             \x20           if c > c0 and r0 > 0: d[0:r0, c0:c] = g[0:r0, c0:c]\n\
             \x20           if r > r0: d[r0:r, :] = g[r0:r, :]\n\
             \x20           r0, c0 = r, c\n",
            p = p.to_str().unwrap(),
            write = if write { "True" } else { "False" },
        ))
    };
    create(&a, true);
    create(&b, false);
    let mut m = Model::new(&[0, 0], |_| 0);
    let mut ed = FileEditor::open(&b).unwrap();
    let (mut r0, mut c0) = (0u64, 0u64);
    for &(r, c) in &steps {
        ed.resize("x", &[r, c]).unwrap();
        m.resize(&[r, c], 0);
        if c > c0 && r0 > 0 {
            let vals: Vec<i32> = (0..r0)
                .flat_map(|i| (c0..c).map(move |j| val(i, j)))
                .collect();
            ed.write_values("x", &block(&[0, c0], &[r0, c - c0]), &vals)
                .unwrap();
            m.write_block(&[0, c0], &[r0, c - c0], &vals);
        }
        if r > r0 {
            let vals: Vec<i32> = (r0..r)
                .flat_map(|i| (0..c).map(move |j| val(i, j)))
                .collect();
            ed.write_values("x", &block(&[r0, 0], &[r - r0, c]), &vals)
                .unwrap();
            m.write_block(&[r0, 0], &[r - r0, c], &vals);
        }
        (r0, c0) = (r, c);
    }
    drop(ed);
    verify(&b, "x", &m);
    verify(&a, "x", &m);
    check_tools(&b, h5dump);
    let (sa, sb) = (chunk_bt2_shape(&a), chunk_bt2_shape(&b));
    assert!(sa.0 >= 1, "the test should build an internal level: {sa:?}");
    assert_eq!(sb, sa, "B-tree shape differs from libhdf5's");
    // libhdf5 goes on growing what the editor built.
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   d = f['x']\n\
         \x20   r, c = d.shape\n\
         \x20   d.resize((r + 3, c + 2))\n\
         \x20   d[:, c:] = -1\n\
         \x20   d[r:, :] = -2\n",
        p = b.to_str().unwrap()
    ));
    let (r, c) = (m.shape[0], m.shape[1]);
    m.resize(&[r + 3, c + 2], 0);
    m.write_block(&[0, c], &[r + 3, 2], &vec![-1; (2 * (r + 3)) as usize]);
    m.write_block(&[r, 0], &[3, c + 2], &vec![-2; (3 * (c + 2)) as usize]);
    verify(&b, "x", &m);
    check_tools(&b, h5dump);
}

/// Dataset `name`'s layout: its chunk index type (0 for a version-1
/// B-tree), index address and chunk rank (element-size dimension
/// included).
fn chunk_index(path: &Path, name: &str) -> (u8, Option<u64>, usize) {
    use clawhdf5_format::data_layout::DataLayout;
    use clawhdf5_format::message_type::MessageType;
    use clawhdf5_format::object_header::ObjectHeader;
    let f = File::open(path).unwrap();
    let sb = f.superblock();
    let (os, ls) = (sb.offset_size, sb.length_size);
    let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, name).unwrap();
    let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
    let m = oh
        .messages
        .iter()
        .find(|m| m.msg_type == MessageType::DataLayout)
        .unwrap();
    match DataLayout::parse(&m.data, os, ls).unwrap() {
        DataLayout::Chunked {
            chunk_dimensions,
            btree_address,
            chunk_index_type,
            ..
        } => (
            chunk_index_type.unwrap_or(0),
            btree_address,
            chunk_dimensions.len(),
        ),
        other => panic!("{other:?}"),
    }
}

/// A version-1 chunk B-tree's shape: every node's (level, entries),
/// breadth-first from the root.
fn bt1_shape(b: &[u8], root: u64, ndims: usize) -> Vec<(u8, u64)> {
    let ks = 8 + 8 * ndims;
    let mut out = Vec::new();
    let mut level = vec![root];
    while !level.is_empty() {
        let mut next = Vec::new();
        for a in level {
            let n = &b[a as usize..];
            assert_eq!(&n[0..5], b"TREE\x01", "node at {a}");
            let lv = n[5];
            let e = le(&n[6..8]);
            out.push((lv, e));
            if lv > 0 {
                for i in 0..e as usize {
                    next.push(le(
                        &n[24 + (i + 1) * ks + i * 8..24 + (i + 1) * ks + i * 8 + 8]
                    ));
                }
            }
        }
        level = next;
    }
    out
}

/// The shape of dataset `name`'s chunk index, for comparing with the one
/// libhdf5 builds: B-tree node shapes, Extensible Array statistics, or
/// nothing to compare (Fixed Array, implicit).
fn index_shape(path: &Path, name: &str) -> String {
    let b = std::fs::read(path).unwrap();
    match chunk_index(path, name) {
        (_, None, _) => "none".into(),
        (0, Some(a), nd) => format!("bt1 {:?}", bt1_shape(&b, a, nd)),
        (5, Some(a), _) => format!("bt2 {:?}", bt2_shape_at(&b, a as usize)),
        (4, Some(a), _) => {
            let s: Vec<u64> = (0..6)
                .map(|k| le(&b[a as usize + 12 + 8 * k..a as usize + 20 + 8 * k]))
                .collect();
            format!("ea {s:?}")
        }
        (t, Some(_), _) => format!("type {t}"),
    }
}

/// One step of a resize workload.
#[derive(Debug, Clone)]
enum Op {
    Resize(Vec<u64>),
    /// A block write; values from `vals`.
    Write(Vec<u64>, Vec<u64>, u64),
}

fn op_vals(seed: u64, n: u64) -> Vec<i32> {
    (0..n)
        .map(|i| ((i * 31 + seed * 7919) % 100_000) as i32)
        .collect()
}

/// Random resizes (shrinking and growing any dimension within `max`)
/// and block writes.
fn random_resize_ops(rng: &mut Rng, start: &[u64], max: &[u64], n: usize) -> Vec<Op> {
    let mut shape = start.to_vec();
    let mut ops = Vec::new();
    for k in 0..n {
        if k % 3 == 0 {
            shape = (0..shape.len())
                .map(|d| {
                    let cap = max[d].min(shape[d] * 2 + 6);
                    rng.below(cap + 1)
                })
                .collect();
            ops.push(Op::Resize(shape.clone()));
        } else if shape.iter().all(|&s| s > 0) {
            let st: Vec<u64> = shape.iter().map(|&s| rng.below(s)).collect();
            let cnt: Vec<u64> = (0..shape.len())
                .map(|d| 1 + rng.below((shape[d] - st[d]).min(6)))
                .collect();
            ops.push(Op::Write(st, cnt, rng.next() % 1000));
        }
    }
    ops
}

/// The python statements applying `ops` to dataset `d`.
fn py_ops(ops: &[Op]) -> String {
    let mut s = String::new();
    for op in ops {
        match op {
            Op::Resize(shape) => {
                s += &format!("\x20   d.resize({shape:?})\n")
                    .replace('[', "(")
                    .replace(']', ",)");
            }
            Op::Write(st, cnt, seed) => {
                let n: u64 = cnt.iter().product();
                let sl: Vec<String> = st
                    .iter()
                    .zip(cnt)
                    .map(|(a, c)| format!("{a}:{}", a + c))
                    .collect();
                s += &format!(
                    "\x20   d[{}] = ((np.arange({n}, dtype=np.int64) * 31 + {seed} * 7919) % 100000)\
                     .astype('<i4').reshape({cnt:?})\n",
                    sl.join(", ")
                )
                .replace("reshape([", "reshape((")
                .replace("])\n", ",))\n");
            }
        }
    }
    s
}

/// The same resize workload — shrinking and growing along every
/// dimension, with writes — done by libhdf5 (chunk cache off) and by the
/// editor on dataset `x` created by `create` (python, `f` open): the same
/// values as a model where elements that come back after a shrink read as
/// the fill value, the same chunk index shape, a file h5py/h5dump/`h5rs
/// check` accept, and h5py can go on.
fn shrink_workload(tag: &str, libver: &str, create: &str, fill: i32, h5dump: bool, seed: u64) {
    let dir = tmpdir();
    let a = dir.path().join(format!("shrink_{tag}_h5py.h5"));
    let b = dir.path().join(format!("shrink_{tag}_edit.h5"));
    let mk = |p: &Path| {
        py(&format!(
            "import h5py, numpy as np\n\
             from h5py import h5p, h5d, h5s, h5t\n\
             with h5py.File({p:?}, 'w', libver={libver}, rdcc_nbytes=0) as f:\n\
             {create}",
            p = p.to_str().unwrap()
        ))
    };
    mk(&a);
    mk(&b);
    let (start, max) = py(&format!(
        "import h5py\n\
         d = h5py.File({p:?}, 'r')['x']\n\
         print(list(d.shape), [m if m is not None else 10**9 for m in d.maxshape])\n",
        p = b.to_str().unwrap()
    ))
    .split_once("] [")
    .map(|(x, y)| {
        let p = |s: &str| -> Vec<u64> {
            s.trim_matches(|c| c == '[' || c == ']')
                .split(',')
                .filter(|t| !t.trim().is_empty())
                .map(|t| t.trim().parse().unwrap())
                .collect()
        };
        (p(x), p(y))
    })
    .unwrap();
    let initial = py(&format!(
        "import h5py\n\
         d = h5py.File({p:?}, 'r')['x']\n\
         print(' '.join(str(v) for v in d[()].ravel()))\n",
        p = b.to_str().unwrap()
    ));
    let mut m = Model {
        shape: start.clone(),
        data: initial
            .split_whitespace()
            .map(|v| v.parse().unwrap())
            .collect(),
    };
    let mut rng = Rng(seed);
    let ops = random_resize_ops(&mut rng, &start, &max, 60);
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+', rdcc_nbytes=0) as f:\n\
         \x20   d = f['x']\n{o}",
        p = a.to_str().unwrap(),
        o = py_ops(&ops)
    ));
    let mut ed = FileEditor::open(&b).unwrap();
    for (k, op) in ops.iter().enumerate() {
        match op {
            Op::Resize(s) => {
                ed.resize("x", s)
                    .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
                m.resize(s, fill);
            }
            Op::Write(st, cnt, seed) => {
                let vals = op_vals(*seed, cnt.iter().product());
                ed.write_values("x", &block(st, cnt), &vals)
                    .unwrap_or_else(|e| panic!("{tag} op {k} {op:?}: {e}"));
                m.write_block(st, cnt, &vals);
            }
        }
    }
    drop(ed);
    if tag == "implicit" {
        // An implicit index cannot drop chunks: libhdf5 leaves the data of
        // chunks wholly outside a shrunk extent in place and does not
        // refill those that come back unless they lie past the extent it
        // grows from, so they can show old values. The editor must match
        // libhdf5, not the model.
        let got = py(&format!(
            "import h5py\n\
             d = h5py.File({p:?}, 'r')['x']\n\
             print(' '.join(str(v) for v in d[()].ravel()))\n",
            p = a.to_str().unwrap()
        ));
        m.data = got.split_whitespace().map(|v| v.parse().unwrap()).collect();
    }
    verify(&a, "x", &m);
    verify(&b, "x", &m);
    check_tools(&b, h5dump);
    assert_eq!(
        index_shape(&b, "x"),
        index_shape(&a, "x"),
        "{tag}: chunk index differs from libhdf5's"
    );
    // libhdf5 goes on: grow back to the start shape and write everything.
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   d = f['x']\n\
         \x20   d.resize({s:?})\n\
         \x20   d[...] = 3\n",
        p = b.to_str().unwrap(),
        s = start
    ));
    let n: u64 = start.iter().product();
    verify(
        &b,
        "x",
        &Model {
            shape: start.clone(),
            data: vec![3; n as usize],
        },
    );
    check_tools(&b, h5dump);
}

/// Shrinking (h5py's `Dataset.resize` to a smaller shape) along every
/// dimension, mixed with growth and writes, on every chunk index: the
/// version-1 B-tree (`earliest`), Extensible Array (one unlimited
/// dimension), version-2 B-tree (two), Fixed Array (fixed maximum shape)
/// and implicit (early allocation), unfiltered and deflated.
#[test]
fn shrink_matches_libhdf5() {
    if !tools_ok() {
        return;
    }
    let kinds: &[(&str, &str, i32)] = &[
        (
            "ea",
            "\x20   d = f.create_dataset('x', data=np.arange(40, dtype='<i4').reshape(8, 5), \
             maxshape=(None, 5), chunks=(3, 2), fillvalue=-1{z})\n",
            -1,
        ),
        (
            "bt2",
            "\x20   d = f.create_dataset('x', data=np.arange(48, dtype='<i4').reshape(8, 6), \
             maxshape=(None, None), chunks=(3, 2){z})\n",
            0,
        ),
        (
            "fa",
            "\x20   d = f.create_dataset('x', data=np.arange(70, dtype='<i4').reshape(10, 7), \
             maxshape=(12, 9), chunks=(3, 4), fillvalue=5{z})\n",
            5,
        ),
        (
            "1d",
            "\x20   d = f.create_dataset('x', data=np.arange(30, dtype='<i4'), \
             maxshape=(None,), chunks=(4,){z})\n",
            0,
        ),
    ];
    let mut seed = 100;
    for (li, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
        .iter()
        .enumerate()
    {
        for (kind, create, fill) in kinds {
            for (zi, z) in ["", ", compression='gzip', shuffle=True"]
                .iter()
                .enumerate()
            {
                seed += 1;
                let tag = format!("{kind}_{li}_{zi}");
                shrink_workload(&tag, lv, &create.replace("{z}", z), *fill, *dump, seed);
            }
        }
    }
    // Implicit index: early allocation, no filters, fixed maximum shape.
    shrink_workload(
        "implicit",
        "'latest'",
        "\x20   dcpl = h5p.create(h5p.DATASET_CREATE)\n\
         \x20   dcpl.set_chunk((3, 4))\n\
         \x20   dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
         \x20   dcpl.set_fill_value(np.array([7], dtype='<i4'))\n\
         \x20   h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 12)), dcpl=dcpl)\n\
         \x20   f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
        7,
        false,
        99,
    );
}

/// An implicit chunk index (early allocation, no filters, fixed maximum
/// shape) on a dataset smaller than its maximum: libhdf5 places each chunk
/// by its position in the maximum chunk grid. The reader once used the
/// current grid, and returned other chunks' values for every row after the
/// first.
#[test]
fn implicit_index_below_its_maximum_reads_like_libhdf5() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let path = dir.path().join("implicit_max.h5");
    py(&format!(
        "import h5py, numpy as np\n\
         from h5py import h5p, h5d, h5s, h5t\n\
         with h5py.File({p:?}, 'w', libver='latest') as f:\n\
         \x20   dcpl = h5p.create(h5p.DATASET_CREATE)\n\
         \x20   dcpl.set_chunk((3, 4))\n\
         \x20   dcpl.set_alloc_time(h5d.ALLOC_TIME_EARLY)\n\
         \x20   h5d.create(f.id, b'x', h5t.STD_I32LE, h5s.create_simple((8, 9), (14, 21)), dcpl=dcpl)\n\
         \x20   f['x'][...] = np.arange(72, dtype='<i4').reshape(8, 9)\n",
        p = path.to_str().unwrap()
    ));
    assert_eq!(chunk_index(&path, "x").0, 2, "implicit index");
    verify(&path, "x", &Model::new(&[8, 9], |i| i as i32));
    // The editor places new values by the same grid.
    let mut ed = FileEditor::open(&path).unwrap();
    ed.write_values("x", &block(&[4, 5], &[2, 2]), &[-1, -2, -3, -4])
        .unwrap();
    drop(ed);
    let mut m = Model::new(&[8, 9], |i| i as i32);
    m.write_block(&[4, 5], &[2, 2], &[-1, -2, -3, -4]);
    verify(&path, "x", &m);
    check_tools(&path, false);
}

#[test]
fn btree2_chunk_index_growth_matches_libhdf5() {
    if !tools_ok() {
        return;
    }
    for (i, (lv, dump)) in [("'v110'", true), ("'latest'", false)].iter().enumerate() {
        bt2_growth(lv, *dump, "", &format!("plain{i}"));
        bt2_growth(lv, *dump, ", compression='gzip'", &format!("gzip{i}"));
    }
}

/// Random writes into a dataset with two unlimited dimensions (chunks
/// created in any order) and growth, with a small node size so the tree
/// gets deep, against a model; then h5py continues.
#[test]
fn btree2_random_chunk_order() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let path = dir.path().join("bt2_random.h5");
    // A 512-byte node holds 20 unfiltered 2-D records (depth 2 at a few
    // hundred chunks).
    py(&format!(
        "import h5py, numpy as np\n\
         from h5py import h5p, h5d, h5s, h5t\n\
         with h5py.File({p:?}, 'w', libver='latest') as f:\n\
         \x20   f.create_dataset('x', shape=(40, 40), maxshape=(None, None), chunks=(2, 2), \
         dtype='<i4', fillvalue=-7)\n",
        p = path.to_str().unwrap()
    ));
    let mut m = Model::new(&[40, 40], |_| -7);
    let mut rng = Rng(42);
    let mut ed = FileEditor::open(&path).unwrap();
    for step in 0..400 {
        if step % 50 == 49 {
            let (r, c) = (m.shape[0] + rng.below(5), m.shape[1] + rng.below(5));
            ed.resize("x", &[r, c]).unwrap();
            m.resize(&[r, c], -7);
        }
        let r0 = rng.below(m.shape[0]);
        let c0 = rng.below(m.shape[1]);
        let cnt = [
            1 + rng.below((m.shape[0] - r0).min(3)),
            1 + rng.below((m.shape[1] - c0).min(3)),
        ];
        let vals: Vec<i32> = (0..cnt[0] * cnt[1])
            .map(|_| (rng.next() % 1_000_000) as i32)
            .collect();
        ed.write_values("x", &block(&[r0, c0], &cnt), &vals)
            .unwrap();
        m.write_block(&[r0, c0], &cnt, &vals);
        if step % 100 == 99 {
            drop(ed);
            verify(&path, "x", &m);
            check_tools(&path, false);
            ed = FileEditor::open(&path).unwrap();
        }
    }
    drop(ed);
    verify(&path, "x", &m);
    let shape = chunk_bt2_shape(&path);
    assert!(shape.0 >= 1, "{shape:?}");
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   f['x'][...] = 5\n",
        p = path.to_str().unwrap()
    ));
    let n = m.data.len();
    m.data = vec![5; n];
    verify(&path, "x", &m);
    check_tools(&path, false);
}

// ---- dense attributes ----

/// An attribute value both sides can write with the same encoded size: a
/// 1-D int64 array or a fixed-length string.
#[derive(Clone, Debug, PartialEq)]
enum AV {
    Ints(Vec<i64>),
    Str(String),
}

impl AV {
    fn value(&self) -> clawhdf5::AttrValue {
        match self {
            AV::Ints(v) => clawhdf5::AttrValue::I64Array(v.clone()),
            AV::Str(s) => clawhdf5::AttrValue::String(s.clone()),
        }
    }

    fn py(&self) -> String {
        match self {
            AV::Ints(v) => format!("np.array({v:?}, dtype='<i8')"),
            AV::Str(s) => format!("np.bytes_({s:?})"),
        }
    }

    fn py_expect(&self) -> String {
        match self {
            AV::Ints(v) => format!("{v:?}"),
            AV::Str(s) => format!("{s:?}"),
        }
    }

    fn make(i: u64, salt: u64) -> Self {
        match i % 11 {
            5 => AV::Str("h".repeat(5000 + (salt % 50) as usize)),
            1 | 4 | 8 => AV::Str(
                (0..10 + (i * 13 + salt) % 300)
                    .map(|k| (b'a' + ((k + salt) % 26) as u8) as char)
                    .collect(),
            ),
            _ => AV::Ints(
                (0..1 + (i + salt) % 9)
                    .map(|k| (k * 31 + salt) as i64)
                    .collect(),
            ),
        }
    }
}

/// An object's dense attribute storage as libhdf5 would compare it: the
/// heap's statistics and root shape, its free sections (heap offsets and
/// sizes), and the shapes of the name and creation-order index B-trees.
fn dense_info(path: &Path, obj: &str) -> String {
    use clawhdf5_format::message_type::MessageType;
    use clawhdf5_format::object_header::ObjectHeader;
    let f = File::open(path).unwrap();
    let sb = f.superblock();
    let (os, ls) = (sb.offset_size, sb.length_size);
    assert_eq!((os, ls), (8, 8));
    let a = clawhdf5_format::group_v2::resolve_path_any(f.as_bytes(), sb, obj).unwrap();
    let oh = ObjectHeader::parse(f.as_bytes(), a as usize, os, ls).unwrap();
    let Some(m) = oh
        .messages
        .iter()
        .find(|m| m.msg_type == MessageType::AttributeInfo)
    else {
        return "no attribute info".into();
    };
    let d = &m.data;
    let mut p = 2 + if d[1] & 1 != 0 { 2 } else { 0 };
    let heap = le(&d[p..p + 8]);
    p += 8;
    let name = le(&d[p..p + 8]);
    let order = (d[1] & 2 != 0).then(|| le(&d[p + 8..p + 16]));
    if heap == u64::MAX {
        return "compact".into();
    }
    let b = std::fs::read(path).unwrap();
    let h = &b[heap as usize..];
    assert_eq!(&h[0..4], b"FRHP");
    // next huge id, huge bt2, free, fs, man size, alloc, iter, nobjs, huge
    // size, huge objs (8-byte fields from offset 14).
    let stat = |k: usize| le(&h[14 + 8 * k..22 + 8 * k]);
    let fs = stat(3);
    // Root rows: after 12 8-byte fields, width, start, max direct, max
    // index, start rows, root address.
    let rows_at = 14 + 96 + 2 + 16 + 2 + 2 + 8;
    let mut out = format!(
        "heap next_huge={} free={} man={} alloc={} iter={} nobjs={} huge={}/{} rows={}",
        stat(0),
        stat(2),
        stat(4),
        stat(5),
        stat(6),
        stat(7),
        stat(8),
        stat(9),
        le(&h[rows_at..rows_at + 2])
    );
    if fs != u64::MAX {
        let s = &b[fs as usize..];
        assert_eq!(&s[0..4], b"FSHD");
        let tot = le(&s[6..14]);
        let n = le(&s[14..22]);
        // counts (4 x 8), 4 x u16, max section size, section info address.
        let at = 6 + 32 + 8 + 8;
        let sect_addr = le(&s[at..at + 8]);
        let sect_size = le(&s[at + 8..at + 16]);
        let ss = &b[sect_addr as usize..(sect_addr + sect_size) as usize];
        // Sections after the prefix (signature, version, header address);
        // trailing zero padding and the checksum are left out.
        let body = &ss[13..ss.len() - 4];
        let used = body.len() - body.iter().rev().take_while(|&&x| x == 0).count();
        out += &format!(" fs tot={tot} n={n} sections={}", hex(&body[..used]));
    } else {
        out += " no-fs";
    }
    out += &format!(" names={:?}", bt2_shape_at(&b, name as usize));
    if let Some(o) = order {
        out += &format!(" order={:?}", bt2_shape_at(&b, o as usize));
    }
    out
}

fn hex(b: &[u8]) -> String {
    b.iter().map(|x| format!("{x:02x}")).collect()
}

/// h5py and our reader both see `want` on each object (other attributes
/// may exist; these must have these values), and h5py's attribute count
/// agrees with libhdf5's object info.
fn check_attr_values(path: &Path, want: &[(String, String, AV)]) {
    let f = File::open(path).unwrap();
    for (o, n, v) in want {
        let attrs = if o == "d" {
            f.dataset(o).unwrap().attrs().unwrap()
        } else {
            f.group(o).unwrap().attrs().unwrap()
        };
        let got = attrs
            .get(n.as_str())
            .unwrap_or_else(|| panic!("{o}/{n} missing ({} attributes)", attrs.len()));
        match (got, v) {
            (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
            (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
            (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
            other => panic!("{o}/{n}: {other:?}"),
        }
    }
    let exp: Vec<String> = want
        .iter()
        .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
        .collect();
    let script = format!(
        "import h5py, numpy as np\n\
         f = h5py.File({p:?}, 'r')\n\
         want = [{w}]\n\
         for o, n, v in want:\n\
         \x20   a = f[o].attrs[n]\n\
         \x20   a = a.decode() if isinstance(a, bytes) else a\n\
         \x20   a = a.tolist() if hasattr(a, 'tolist') else a\n\
         \x20   a = [a] if isinstance(a, int) else a\n\
         \x20   assert a == v, (o, n, a if len(str(a)) < 200 else len(a), v if len(str(v)) < 200 else len(v))\n\
         for o in set(x[0] for x in want):\n\
         \x20   assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n\
         \x20   assert len(list(f[o].attrs)) == len(f[o].attrs)\n",
        p = path.to_str().unwrap(),
        w = exp.join(", ")
    );
    let sp = path.with_extension("check.py");
    std::fs::write(&sp, script).unwrap();
    let o = Command::new(python()).arg(&sp).output().unwrap();
    assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
}

/// Run python statements (inside `with h5py.File(path, 'r+') as f:`).
fn py_r_plus(path: &Path, lines: &[String]) {
    let script = format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n{l}",
        p = path.to_str().unwrap(),
        l = lines.concat()
    );
    let sp = path.with_extension("ops.py");
    std::fs::write(&sp, script).unwrap();
    let o = Command::new(python()).arg(&sp).output().unwrap();
    assert!(o.status.success(), "h5py workload failed:\n{}", text(&o));
}

type AttrOp = (String, String, AV);

fn set_want(want: &mut Vec<AttrOp>, o: &str, n: &str, v: AV) {
    want.retain(|(wo, wn, _)| !(wo == o && wn == n));
    want.push((o.into(), n.into(), v));
}

/// The same attribute workload by libhdf5 and by the editor on objects
/// with compact attributes that move to dense storage (a plain group, a
/// group tracking and indexing creation order, a dataset): 40 new
/// attributes each (some larger than the heap's 4 KiB managed limit), then
/// same-size rewrites — the heaps, their free space and both index B-trees
/// must come out as libhdf5 makes them; then replacements of another size,
/// then h5py adds, deletes and rewrites attributes.
fn dense_workload(libver: &str, h5dump: bool, tag: &str) {
    let dir = tmpdir();
    let a = dir.path().join(format!("dense_{tag}_h5py.h5"));
    let b = dir.path().join(format!("dense_{tag}_edit.h5"));
    for p in [&a, &b] {
        py(&format!(
            "import h5py, numpy as np\n\
             with h5py.File({p:?}, 'w', libver={libver}) as f:\n\
             \x20   objs = [f.create_group('g'), f.create_group('t', track_order=True), \
             f.create_dataset('d', data=np.arange(4, dtype='<i4'))]\n\
             \x20   for o in objs:\n\
             \x20       for i in range(3): o.attrs.create(f'c{{i}}', np.array([i, i], dtype='<i8'))\n",
            p = p.to_str().unwrap()
        ));
    }
    let objs = ["g", "t", "d"];
    let mut want: Vec<AttrOp> = Vec::new();
    for o in objs {
        for i in 0..3i64 {
            want.push((o.into(), format!("c{i}"), AV::Ints(vec![i, i])));
        }
    }
    // Phase 1: new attributes.
    let mut ops: Vec<AttrOp> = Vec::new();
    for i in 0..40u64 {
        for (k, o) in objs.iter().enumerate() {
            ops.push((o.to_string(), format!("n{i}"), AV::make(i, k as u64)));
        }
    }
    let lines: Vec<String> = ops
        .iter()
        .map(|(o, n, v)| format!("\x20   f[{o:?}].attrs.create({n:?}, {})\n", v.py()))
        .collect();
    py_r_plus(&a, &lines);
    let mut ed = FileEditor::open(&b).unwrap();
    for (o, n, v) in &ops {
        ed.set_attr(o, n, &v.value())
            .unwrap_or_else(|e| panic!("{tag}: set {o}/{n}: {e}"));
        set_want(&mut want, o, n, v.clone());
    }
    drop(ed);
    check_tools(&b, h5dump);
    check_attr_values(&b, &want);
    check_attr_values(&a, &want);
    for o in objs {
        assert_eq!(
            dense_info(&b, o),
            dense_info(&a, o),
            "{tag}: dense storage of {o} differs from libhdf5's after insertions"
        );
    }
    // Phase 2: same-size rewrites (H5Awrite in place).
    let ops2: Vec<AttrOp> = ops
        .iter()
        .step_by(4)
        .map(|(o, n, v)| {
            let nv = match v {
                AV::Ints(x) => AV::Ints(x.iter().map(|y| y * 3 + 1).collect()),
                AV::Str(s) => AV::Str(s.chars().rev().collect()),
            };
            (o.clone(), n.clone(), nv)
        })
        .collect();
    let lines: Vec<String> = ops2
        .iter()
        .map(|(o, n, v)| format!("\x20   f[{o:?}].attrs.modify({n:?}, {})\n", v.py()))
        .collect();
    py_r_plus(&a, &lines);
    let mut ed = FileEditor::open(&b).unwrap();
    for (o, n, v) in &ops2 {
        ed.set_attr(o, n, &v.value()).unwrap();
        set_want(&mut want, o, n, v.clone());
    }
    drop(ed);
    check_tools(&b, h5dump);
    check_attr_values(&b, &want);
    for o in objs {
        assert_eq!(
            dense_info(&b, o),
            dense_info(&a, o),
            "{tag}: dense storage of {o} differs from libhdf5's after rewrites"
        );
    }
    // Phase 3: replacements of another size (values only: h5py replaces
    // through a temporary attribute and a rename).
    let mut ed = FileEditor::open(&b).unwrap();
    for (k, (o, n, v)) in ops.iter().enumerate().filter(|(k, _)| k % 5 == 2) {
        let nv = match v {
            AV::Ints(x) => AV::Ints((0..x.len() as i64 + 3).collect()),
            AV::Str(s) => AV::Str(format!("{s}-{k}")),
        };
        let before = std::fs::read(&b).unwrap();
        match ed.set_attr(o, n, &nv.value()) {
            Ok(()) => set_want(&mut want, o, n, nv),
            Err(Error::Unsupported(msg)) => {
                assert!(msg.contains("last object"), "{tag}: {o}/{n}: {msg}");
                assert!(std::fs::read(&b).unwrap() == before, "refused edit wrote");
            }
            Err(e) => panic!("{tag}: replace {o}/{n}: {e}"),
        }
    }
    drop(ed);
    check_tools(&b, h5dump);
    check_attr_values(&b, &want);
    // libhdf5 goes on: adds, deletes and rewrites.
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   for o in ['g', 't', 'd']:\n\
         \x20       for i in range(5): f[o].attrs[f'late{{i}}'] = np.arange(i + 1)\n\
         \x20       del f[o].attrs['n3']\n\
         \x20       del f[o].attrs['c1']\n\
         \x20       f[o].attrs['n7'] = 'rewritten by h5py'\n",
        p = b.to_str().unwrap()
    ));
    want.retain(|(_, n, _)| n != "n3" && n != "c1" && n != "n7");
    check_tools(&b, h5dump);
    check_attr_values(&b, &want);
}

#[test]
fn dense_attributes_match_libhdf5() {
    if !tools_ok() {
        return;
    }
    for (i, (lv, dump)) in [("'earliest'", true), ("'v110'", true), ("'latest'", false)]
        .iter()
        .enumerate()
    {
        dense_workload(lv, *dump, &format!("{i}"));
    }
}

/// Dense attributes in files clawhdf5 writes (its own heap and index
/// layout, no free-space manager), with and without tracked creation
/// order: attributes added (moving a dataset's attributes to dense storage
/// too), rewritten and replaced, then h5py goes on.
#[test]
fn dense_attributes_on_clawhdf5_files() {
    if !tools_ok() {
        return;
    }
    for track in [false, true] {
        let dir = tmpdir();
        let path = dir.path().join(format!("ours_dense_{track}.h5"));
        let mut b = clawhdf5::FileBuilder::new();
        b.track_order(track);
        for i in 0..12i64 {
            b.set_attr(&format!("r{i}"), clawhdf5::AttrValue::I64Array(vec![i; 3]));
        }
        b.create_dataset("d")
            .with_i32_data(&[1, 2, 3])
            .set_attr("c0", clawhdf5::AttrValue::I64Array(vec![5, 5]));
        b.write(&path).unwrap();
        let mut want: Vec<AttrOp> = (0..12i64)
            .map(|i| ("/".to_string(), format!("r{i}"), AV::Ints(vec![i; 3])))
            .collect();
        want.push(("d".into(), "c0".into(), AV::Ints(vec![5, 5])));
        let mut ed = FileEditor::open(&path).unwrap();
        for i in 0..30u64 {
            for (k, o) in ["/", "d"].iter().enumerate() {
                // Small attributes: a larger one needs a heap block bigger than
                // the next (see dense_attribute_refusals_change_nothing).
                let v = match AV::make(i, k as u64 + 7) {
                    AV::Str(s) if s.len() > 400 => AV::Str(s[..400].to_string()),
                    v => v,
                };
                ed.set_attr(o, &format!("n{i}"), &v.value())
                    .unwrap_or_else(|e| panic!("{track} {o}/n{i}: {e}"));
                set_want(&mut want, o, &format!("n{i}"), v);
            }
        }
        // Rewrites in place, then replacements of another size.
        for i in (0..12i64).step_by(3) {
            let v = AV::Ints(vec![-i; 3]);
            ed.set_attr("/", &format!("r{i}"), &v.value()).unwrap();
            set_want(&mut want, "/", &format!("r{i}"), v);
        }
        for i in (1..30u64).step_by(7) {
            let v = AV::Str(format!("replaced {i}"));
            match ed.set_attr("d", &format!("n{i}"), &v.value()) {
                Ok(()) => set_want(&mut want, "d", &format!("n{i}"), v),
                Err(Error::Unsupported(msg)) => assert!(msg.contains("last object"), "{msg}"),
                Err(e) => panic!("{e}"),
            }
        }
        drop(ed);
        check_tools(&path, true);
        let want_py: Vec<AttrOp> = want
            .iter()
            .map(|(o, n, v)| {
                let o = if o == "/" { "/".to_string() } else { o.clone() };
                (o, n.clone(), v.clone())
            })
            .collect();
        check_root_and_attrs(&path, &want_py);
        py(&format!(
            "import h5py, numpy as np\n\
             with h5py.File({p:?}, 'r+') as f:\n\
             \x20   for o in ['/', 'd']:\n\
             \x20       f[o].attrs['from_h5py'] = np.arange(3)\n\
             \x20       del f[o].attrs['n2']\n",
            p = path.to_str().unwrap()
        ));
        want.retain(|(_, n, _)| n != "n2");
        check_tools(&path, true);
        check_root_and_attrs(&path, &want);
    }
}

/// `check_attr_values`, with the root group as "/".
fn check_root_and_attrs(path: &Path, want: &[AttrOp]) {
    let f = File::open(path).unwrap();
    for (o, n, v) in want {
        let attrs = match o.as_str() {
            "/" => f.root().attrs().unwrap(),
            "d" => f.dataset(o).unwrap().attrs().unwrap(),
            _ => f.group(o).unwrap().attrs().unwrap(),
        };
        let got = attrs
            .get(n.as_str())
            .unwrap_or_else(|| panic!("{o}/{n} missing"));
        match (got, v) {
            (clawhdf5::AttrValue::I64Array(g), AV::Ints(w)) => assert_eq!(g, w, "{o}/{n}"),
            (clawhdf5::AttrValue::I64(g), AV::Ints(w)) => assert_eq!(&vec![*g], w, "{o}/{n}"),
            (clawhdf5::AttrValue::String(g), AV::Str(w)) => assert_eq!(g, w, "{o}/{n}"),
            other => panic!("{o}/{n}: {other:?}"),
        }
    }
    let exp: Vec<String> = want
        .iter()
        .map(|(o, n, v)| format!("({o:?}, {n:?}, {})", v.py_expect()))
        .collect();
    let script = format!(
        "import h5py, numpy as np\n\
         f = h5py.File({p:?}, 'r')\n\
         want = [{w}]\n\
         for o, n, v in want:\n\
         \x20   a = f[o].attrs[n]\n\
         \x20   a = a.decode() if isinstance(a, bytes) else a\n\
         \x20   a = a.tolist() if hasattr(a, 'tolist') else a\n\
         \x20   a = [a] if isinstance(a, int) else a\n\
         \x20   assert a == v, (o, n)\n\
         for o in set(x[0] for x in want):\n\
         \x20   assert len(f[o].attrs) == h5py.h5o.get_info(f[o].id).num_attrs\n",
        p = path.to_str().unwrap(),
        w = exp.join(", ")
    );
    let sp = path.with_extension("check.py");
    std::fs::write(&sp, script).unwrap();
    let o = Command::new(python()).arg(&sp).output().unwrap();
    assert!(o.status.success(), "attribute check failed:\n{}", text(&o));
}

/// What the editor refuses in dense storage — an object larger than the
/// next heap block (libhdf5 would skip blocks and record their space as
/// free, which this editor does not do) — is `Error::Unsupported`, and the
/// file is left byte for byte as it was.
#[test]
fn dense_attribute_refusals_change_nothing() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let path = dir.path().join("dense_refuse.h5");
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'w', libver='v110') as f:\n\
         \x20   g = f.create_group('g')\n\
         \x20   for i in range(12): g.attrs[f'k{{i}}'] = i\n",
        p = path.to_str().unwrap()
    ));
    let before = std::fs::read(&path).unwrap();
    let mut ed = FileEditor::open(&path).unwrap();
    unsupported(ed.set_attr("g", "big", &clawhdf5::AttrValue::String("x".repeat(2000))));
    drop(ed);
    assert!(
        std::fs::read(&path).unwrap() == before,
        "a refused edit wrote"
    );
    // What libhdf5 does with it instead works on the untouched file.
    py(&format!(
        "import h5py\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   f['g'].attrs['big'] = 'x' * 2000\n\
         \x20   assert len(f['g'].attrs) == 13\n",
        p = path.to_str().unwrap()
    ));
    check_tools(&path, true);
}

/// Space one edit frees is reused by later edits of the same editor: the
/// chunks a shrink removes are where the chunks of the following growth
/// go, so the file does not grow; with a new editor per edit (nothing to
/// reuse) it does. h5py, h5dump and `h5rs check` read the result, and
/// h5py goes on.
#[test]
fn freed_space_is_reused_within_a_session() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let mut sizes = Vec::new();
    for session in [true, false] {
        let path = dir.path().join(format!("reuse_{session}.h5"));
        py(&format!(
            "import h5py, numpy as np\n\
             with h5py.File({p:?}, 'w', libver='v110') as f:\n\
             \x20   f.create_dataset('x', data=np.zeros(2000, dtype='<i4'), maxshape=(None,), \
             chunks=(100,), compression='gzip')\n",
            p = path.to_str().unwrap()
        ));
        let mut rng = Rng(5);
        let vals: Vec<i32> = (0..2000).map(|_| rng.next() as i32).collect();
        let mut ed = FileEditor::open(&path).unwrap();
        ed.write_values("x", &Selection::All, &vals).unwrap();
        drop(ed);
        let len0 = std::fs::metadata(&path).unwrap().len();
        let mut ed = FileEditor::open(&path).unwrap();
        ed.resize("x", &[1000]).unwrap();
        if session {
            assert!(ed.reusable_bytes() > 0);
        } else {
            ed = {
                drop(ed);
                FileEditor::open(&path).unwrap()
            };
        }
        ed.resize("x", &[2000]).unwrap();
        ed.write_values("x", &block(&[1000], &[1000]), &vals[1000..])
            .unwrap();
        if session {
            assert_eq!(ed.reusable_bytes(), 0, "every freed chunk is reused");
        }
        drop(ed);
        let len1 = std::fs::metadata(&path).unwrap().len();
        sizes.push((len0, len1));
        let m = Model {
            shape: vec![2000],
            data: vals.clone(),
        };
        verify(&path, "x", &m);
        check_tools(&path, true);
        py(&format!(
            "import h5py, numpy as np\n\
             with h5py.File({p:?}, 'r+') as f:\n\
             \x20   f['x'].resize((2100,))\n\
             \x20   f['x'][2000:] = 9\n",
            p = path.to_str().unwrap()
        ));
        let mut m = m;
        m.resize(&[2100], 0);
        m.write_block(&[2000], &[100], &[9; 100]);
        verify(&path, "x", &m);
        check_tools(&path, true);
    }
    let (reuse, fresh) = (sizes[0], sizes[1]);
    assert_eq!(
        reuse.1, reuse.0,
        "a session reusing freed chunks does not grow the file"
    );
    assert!(fresh.1 > fresh.0, "without reuse the file grows");
}

/// Replacing the last huge attribute (larger than the heap's managed
/// limit) of an object with a small one deletes the heap's huge-object
/// B-tree, as libhdf5 does when it closes the heap (`H5HF__huge_term`). A
/// heap left with an empty huge-object B-tree made read-only libhdf5 fail
/// to list the attributes ("no write intent on file").
#[test]
fn last_huge_attribute_replaced() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let a = dir.path().join("huge_h5py.h5");
    let b = dir.path().join("huge_edit.h5");
    for p in [&a, &b] {
        py(&format!(
            "import h5py, numpy as np\n\
             with h5py.File({p:?}, 'w', libver='v110') as f:\n\
             \x20   g = f.create_group('g')\n\
             \x20   for i in range(10): g.attrs.create(f'k{{i}}', np.array([i], dtype='<i8'))\n\
             \x20   g.attrs.create('big', np.bytes_('h' * 6000))\n",
            p = p.to_str().unwrap()
        ));
    }
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   f['g'].attrs.create('big', np.array([1, 2], dtype='<i8'))\n",
        p = a.to_str().unwrap()
    ));
    let mut ed = FileEditor::open(&b).unwrap();
    ed.set_attr("g", "big", &clawhdf5::AttrValue::I64Array(vec![1, 2]))
        .unwrap();
    drop(ed);
    check_tools(&b, true);
    let mut want: Vec<AttrOp> = (0..10i64)
        .map(|i| ("g".to_string(), format!("k{i}"), AV::Ints(vec![i])))
        .collect();
    want.push(("g".into(), "big".into(), AV::Ints(vec![1, 2])));
    check_attr_values(&b, &want);
    let info = |p: &Path| {
        let s = dense_info(p, "g");
        s[..s.find(" fs ").or(s.find(" no-fs")).unwrap()].to_string()
    };
    assert_eq!(info(&b), info(&a), "heap after the replacement");
    // A huge attribute again starts the huge-object B-tree over.
    let mut ed = FileEditor::open(&b).unwrap();
    ed.set_attr("g", "big2", &clawhdf5::AttrValue::String("x".repeat(7000)))
        .unwrap();
    drop(ed);
    want.push(("g".into(), "big2".into(), AV::Str("x".repeat(7000))));
    check_tools(&b, true);
    check_attr_values(&b, &want);
}

/// Datasets clawhdf5 writes — a version-2 B-tree index (two unlimited
/// dimensions, its own node size and a single leaf sized to its records),
/// an Extensible Array, a Fixed Array (fixed maximum shape), deflated and
/// not — resized up and down and written at random, against a model; h5py,
/// h5dump and `h5rs check` read the result and h5py goes on.
#[test]
fn resize_clawhdf5_written_datasets() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let path = dir.path().join("ours_resize.h5");
    let mut b = clawhdf5::FileBuilder::new();
    let grid: Vec<i32> = (0..48).collect();
    let specs: [(&str, [u64; 2], [u64; 2], bool); 4] = [
        ("bt2", [6, 8], [u64::MAX, u64::MAX], false),
        ("bt2_gz", [6, 8], [u64::MAX, u64::MAX], true),
        ("ea", [6, 8], [u64::MAX, 8], true),
        ("fa", [6, 8], [20, 12], false),
    ];
    for (name, shape, max, gz) in specs {
        let d = b
            .create_dataset(name)
            .with_i32_data(&grid)
            .with_shape(&shape)
            .with_maxshape(&max)
            .with_chunks(&[2, 3]);
        if gz {
            d.with_deflate(4);
        }
    }
    b.write(&path).unwrap();
    assert_eq!(chunk_index(&path, "bt2").0, 5, "version-2 B-tree index");
    assert_eq!(chunk_index(&path, "ea").0, 4, "Extensible Array index");
    let mut models: Vec<Model> = specs
        .iter()
        .map(|_| Model::new(&[6, 8], |i| i as i32))
        .collect();
    let mut rng = Rng(77);
    let mut ed = FileEditor::open(&path).unwrap();
    for step in 0..120 {
        let k = rng.below(4) as usize;
        let (name, _, max, _) = specs[k];
        let m = &mut models[k];
        if step % 4 == 0 {
            let s: Vec<u64> = (0..2)
                .map(|d| rng.below((m.shape[d] * 2 + 4).min(max[d]) + 1))
                .collect();
            ed.resize(name, &s).unwrap();
            m.resize(&s, 0);
        } else if m.shape.iter().all(|&s| s > 0) {
            let st: Vec<u64> = m.shape.iter().map(|&s| rng.below(s)).collect();
            let cnt: Vec<u64> = (0..2)
                .map(|d| 1 + rng.below((m.shape[d] - st[d]).min(5)))
                .collect();
            let vals: Vec<i32> = (0..cnt[0] * cnt[1]).map(|_| rng.next() as i32).collect();
            ed.write_values(name, &block(&st, &cnt), &vals).unwrap();
            m.write_block(&st, &cnt, &vals);
        }
    }
    drop(ed);
    for ((name, ..), m) in specs.iter().zip(&models) {
        verify(&path, name, m);
    }
    check_tools(&path, true);
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   for n in ['bt2', 'bt2_gz', 'ea', 'fa']:\n\
         \x20       d = f[n]\n\
         \x20       d.resize((6, 8))\n\
         \x20       d[...] = np.arange(48, dtype='<i4').reshape(6, 8) * 2\n",
        p = path.to_str().unwrap()
    ));
    for (name, ..) in specs {
        verify(&path, name, &Model::new(&[6, 8], |i| i as i32 * 2));
    }
    check_tools(&path, true);
}