//! Files written with `FileBuilder::libver_bounds(LibVer::V18, LibVer::V18)`
//! must be readable by HDF5 1.8. Every writer feature is written under that
//! bound and read back by HDF5 1.8.23's h5dump (values dumped in binary and
//! compared), by h5py (libhdf5 2.x), by h5dump 1.14, by clawhdf5 and by
//! `h5rs check --data`; then `FileEditor` grows, appends to and annotates
//! the file (splitting version-1 B-tree nodes) and every reader checks it
//! again. The version-1 B-trees we write are compared node by node with
//! the ones libhdf5 writes for the same data under h5py's
//! `libver=('v108', 'latest')`.
//!
//! HDF5 1.8 is found through `CLAWHDF5_H5DUMP18` (the path to its h5dump)
//! or at `~/.cache/hdf5-1.8.23/bin/h5dump`, where
//! `scripts/build-hdf5-1.8.sh` builds it; without it the 1.8 checks are
//! skipped (CI has no HDF5 1.8), even with `CLAWHDF5_REQUIRE_INTEROP=1`.
//! h5py/numpy (`CLAWHDF5_PYTHON`) and h5dump are needed otherwise; they
//! skip when missing unless `CLAWHDF5_REQUIRE_INTEROP=1`.

use std::path::{Path, PathBuf};
use std::process::{Command, Output};

use clawhdf5::{AttrValue, File, FileBuilder, FileEditor, LibVer, Selection};
use clawhdf5_format::datatype::{CharacterSet, Datatype, StringPadding};
use clawhdf5_format::file_writer::{CompoundTypeBuilder, EnumTypeBuilder};
use clawhdf5_format::type_builders::make_i32_type;

fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn available(cmd: &str, args: &[&str]) -> bool {
    Command::new(cmd)
        .args(args)
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

fn tools_ok() -> bool {
    let ok =
        available(&python(), &["-c", "import h5py, numpy"]) && available("h5dump", &["--version"]);
    if !ok {
        assert!(
            !interop_required(),
            "CLAWHDF5_REQUIRE_INTEROP=1 but h5py/numpy or h5dump is not available"
        );
        eprintln!("SKIP: h5py/numpy or h5dump not available");
    }
    ok
}

/// HDF5 1.8's h5dump, when there is one.
fn h5dump18() -> Option<PathBuf> {
    let p = match std::env::var_os("CLAWHDF5_H5DUMP18") {
        Some(p) => PathBuf::from(p),
        None => PathBuf::from(std::env::var_os("HOME")?).join(".cache/hdf5-1.8.23/bin/h5dump"),
    };
    let o = Command::new(&p).arg("--version").output().ok()?;
    let v = String::from_utf8_lossy(&o.stdout).to_string();
    if !v.contains("1.8.") {
        eprintln!("SKIP 1.8 checks: {} is not HDF5 1.8 ({v})", p.display());
        return None;
    }
    Some(p)
}

fn py(script: &str) -> String {
    let o = Command::new(python())
        .args(["-c", script])
        .output()
        .expect("run python");
    assert!(
        o.status.success(),
        "python failed:\n{script}\nSTDOUT: {}\nSTDERR: {}",
        String::from_utf8_lossy(&o.stdout),
        String::from_utf8_lossy(&o.stderr)
    );
    String::from_utf8_lossy(&o.stdout).trim().to_string()
}

fn text(o: &Output) -> String {
    format!(
        "{}{}",
        String::from_utf8_lossy(&o.stdout),
        String::from_utf8_lossy(&o.stderr)
    )
}

fn tmpdir() -> tempfile::TempDir {
    tempfile::TempDir::new_in(env!("CARGO_TARGET_TMPDIR")).unwrap()
}

/// The hyperslab of `count` elements from `start`.
fn block(start: &[u64], count: &[u64]) -> Selection {
    Selection::Hyperslab {
        start: start.to_vec(),
        stride: vec![1; start.len()],
        count: count.to_vec(),
        block: vec![1; start.len()],
    }
}

fn le<T: Copy, const N: usize>(v: &[T], f: impl Fn(T) -> [u8; N]) -> Vec<u8> {
    v.iter().flat_map(|&x| f(x)).collect()
}

/// The datasets of the test file and the bytes each holds (little-endian,
/// row-major, as h5py's `tobytes()` and h5dump's `-b LE` give them).
struct Expect {
    datasets: Vec<(String, Vec<u8>)>,
}

impl Expect {
    fn set(&mut self, name: &str, bytes: Vec<u8>) {
        match self.datasets.iter_mut().find(|(n, _)| n == name) {
            Some(e) => e.1 = bytes,
            None => self.datasets.push((name.to_string(), bytes)),
        }
    }
}

const MANY: usize = 100_000;

/// Write every feature under the 1.8 bound.
fn write_file(path: &Path) -> Expect {
    let mut e = Expect {
        datasets: Vec::new(),
    };
    let mut b = FileBuilder::new();
    b.libver_bounds(LibVer::V18, LibVer::V18);

    // Contiguous, with dense attributes (more than 8).
    let v: Vec<f64> = (0..1000).map(|i| i as f64 * 0.5).collect();
    let d = b.create_dataset("contig").with_f64_data(&v);
    for i in 0..12 {
        d.set_attr(&format!("a{i:02}"), AttrValue::I64(i));
    }
    e.set("/contig", le(&v, f64::to_le_bytes));
    b.create_dataset("empty").with_f64_data(&[]);
    e.set("/empty", vec![]);
    b.create_dataset("scalar")
        .with_f64_data(&[2.5])
        .with_shape(&[]);
    e.set("/scalar", 2.5f64.to_le_bytes().to_vec());
    let v: Vec<i32> = (0..16).map(|i| i * 3 - 7).collect();
    b.create_dataset("compact").with_i32_data(&v).compact();
    e.set("/compact", le(&v, i32::to_le_bytes));
    let v: Vec<f32> = (0..20).map(|i| i as f32 / 3.0).collect();
    b.create_dataset("f32").with_f32_data(&v);
    e.set("/f32", le(&v, f32::to_le_bytes));
    let v: Vec<f32> = vec![0.5, -2.0, 1024.0, 0.0];
    b.create_dataset("f16").with_f16_data(&v);
    e.set(
        "/f16",
        le(&v, |x| {
            clawhdf5_format::float16::f32_to_f16_bits(x).to_le_bytes()
        }),
    );

    // Chunked with every built-in filter HDF5 1.8 has, and edge chunks.
    let v: Vec<f32> = (0..60 * 70).map(|i| (i % 97) as f32 * 1.25).collect();
    b.create_dataset("chunked")
        .with_f32_data(&v)
        .with_shape(&[60, 70])
        .with_chunks(&[16, 16])
        .with_deflate(6)
        .with_shuffle()
        .with_fletcher32();
    e.set("/chunked", le(&v, f32::to_le_bytes));
    // What the 1.10 indexes would be: Extensible Array, version-2 B-tree,
    // Fixed Array, single chunk. All become version-1 B-trees.
    let v: Vec<i32> = (0..25).collect();
    b.create_dataset("resizable")
        .with_i32_data(&v)
        .with_maxshape(&[u64::MAX])
        .with_chunks(&[4]);
    e.set("/resizable", le(&v, i32::to_le_bytes));
    let v: Vec<i64> = (0..30).map(|i| i * 1_000_000_007).collect();
    b.create_dataset("resizable2")
        .with_i64_data(&v)
        .with_shape(&[5, 6])
        .with_maxshape(&[u64::MAX, u64::MAX])
        .with_chunks(&[2, 4]);
    e.set("/resizable2", le(&v, i64::to_le_bytes));
    let v: Vec<i32> = (0..10).map(|i| -i).collect();
    b.create_dataset("fixedmax")
        .with_i32_data(&v)
        .with_maxshape(&[100])
        .with_chunks(&[3])
        .with_deflate(1);
    e.set("/fixedmax", le(&v, i32::to_le_bytes));
    let v: Vec<i32> = (0..8).map(|i| i * i).collect();
    b.create_dataset("single")
        .with_i32_data(&v)
        .with_chunks(&[8]);
    e.set("/single", le(&v, i32::to_le_bytes));
    // Enough chunks for a three-level tree, and a 2-D two-level one.
    let v: Vec<i32> = (0..MANY as i32).map(|i| i ^ 0x5a5a).collect();
    b.create_dataset("many")
        .with_i32_data(&v)
        .with_maxshape(&[u64::MAX])
        .with_chunks(&[1]);
    e.set("/many", le(&v, i32::to_le_bytes));
    let v: Vec<f64> = (0..100 * 100).map(|i| i as f64).collect();
    b.create_dataset("grid")
        .with_f64_data(&v)
        .with_shape(&[100, 100])
        .with_chunks(&[1, 1]);
    e.set("/grid", le(&v, f64::to_le_bytes));
    b.create_dataset("empty_chunked")
        .with_f64_data(&[])
        .with_maxshape(&[u64::MAX])
        .with_chunks(&[10]);
    e.set("/empty_chunked", vec![]);
    let v: Vec<i32> = (0..12).collect();
    b.create_dataset("filled")
        .with_i32_data(&v)
        .with_maxshape(&[u64::MAX])
        .with_chunks(&[5])
        .with_fill_value(&(-9i32).to_le_bytes());
    e.set("/filled", le(&v, i32::to_le_bytes));

    // Datatypes.
    let raw = b"abcdehello\0\0\0\0\0".to_vec();
    b.create_dataset("strings").with_compound_data(
        Datatype::String {
            size: 5,
            padding: StringPadding::NullPad,
            charset: CharacterSet::Ascii,
        },
        raw.clone(),
        3,
    );
    e.set("/strings", raw);
    let ct = CompoundTypeBuilder::new()
        .i32_field("a")
        .f64_field("b")
        .build();
    let mut raw = Vec::new();
    for i in 0..5i32 {
        raw.extend_from_slice(&i.to_le_bytes());
        raw.extend_from_slice(&(f64::from(i) * 1.5).to_le_bytes());
    }
    b.create_dataset("compound")
        .with_compound_data(ct, raw.clone(), 5);
    e.set("/compound", raw);
    let et = EnumTypeBuilder::i32_based()
        .value("RED", 0)
        .value("GREEN", 1)
        .value("BLUE", 7)
        .build();
    let v = [0, 7, 1, 1, 0];
    b.create_dataset("enum").with_enum_i32_data(et, &v);
    e.set("/enum", le(&v, i32::to_le_bytes));
    let v: Vec<i32> = (0..24).collect();
    b.create_dataset("array").with_array_data(
        make_i32_type(),
        &[2, 3],
        le(&v, i32::to_le_bytes),
        4,
    );
    e.set("/array", le(&v, i32::to_le_bytes));
    let v: Vec<i64> = vec![-1, 0, i64::MAX];
    b.create_dataset("i64").with_i64_data(&v);
    e.set("/i64", le(&v, i64::to_le_bytes));

    // Groups: compact with links of every kind, dense (links and
    // attributes), and tracking creation order.
    let mut g = b.create_group("g_compact");
    g.create_dataset("x").with_i32_data(&[1, 2, 3]);
    e.set("/g_compact/x", le(&[1i32, 2, 3], i32::to_le_bytes));
    g.add_soft_link("soft", "/contig");
    g.add_external_link("ext", "other.h5", "/data");
    g.set_attr("title", AttrValue::String("compact group".into()));
    b.add_group(g.finish());
    b.add_hard_link("hard", "/g_compact/x");
    let mut g = b.create_group("g_dense");
    for i in 0..20 {
        let v = [i, i + 1];
        g.create_dataset(&format!("d{i:02}")).with_i32_data(&v);
        e.set(&format!("/g_dense/d{i:02}"), le(&v, i32::to_le_bytes));
    }
    for i in 0..12 {
        g.set_attr(&format!("attr{i:02}"), AttrValue::F64(f64::from(i) / 4.0));
    }
    b.add_group(g.finish());
    let mut g = b.create_group("g_order");
    g.track_order(true);
    for i in 0..10 {
        let n = format!("z{}", 9 - i);
        g.create_dataset(&n).with_i32_data(&[i]);
        e.set(&format!("/g_order/{n}"), i.to_le_bytes().to_vec());
    }
    for i in 0..10 {
        g.set_attr(&format!("b{}", 9 - i), AttrValue::I64(i));
    }
    b.add_group(g.finish());
    b.create_dataset("deep/er/path").with_i32_data(&[42]);
    e.set("/deep/er/path", 42i32.to_le_bytes().to_vec());

    b.set_attr("version", AttrValue::F64(1.8));
    b.set_attr("ints", AttrValue::I64Array(vec![1, -2, 3]));
    b.set_attr("text", AttrValue::String("readable by 1.8".into()));
    b.set_attr(
        "texts",
        AttrValue::StringArray(vec!["a".into(), "bb".into(), "ccc".into()]),
    );
    b.set_attr("big", AttrValue::U64(u64::MAX));
    b.write(path).unwrap();
    e
}

/// Structural facts of a file: superblock and layout message versions.
fn check_versions(path: &Path) {
    use clawhdf5_format::message_type::MessageType;
    use clawhdf5_format::object_header::ObjectHeader;
    let bytes = std::fs::read(path).unwrap();
    assert_eq!(bytes[8], 2, "superblock version");
    let f = File::open(path).unwrap();
    let sb = f.superblock().clone();
    for name in [
        "contig",
        "compact",
        "chunked",
        "resizable",
        "resizable2",
        "many",
    ] {
        let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, name).unwrap();
        let oh = ObjectHeader::parse(&bytes, addr as usize, 8, 8).unwrap();
        let layout = oh
            .messages
            .iter()
            .find(|m| m.msg_type == MessageType::DataLayout)
            .unwrap();
        assert_eq!(layout.data[0], 3, "layout version of {name}");
    }
}

/// clawhdf5 reads every dataset's bytes back.
fn check_ours(path: &Path, e: &Expect) {
    let f = File::open(path).unwrap();
    for (name, want) in &e.datasets {
        let got = f
            .dataset(name)
            .unwrap()
            .read_selection(&Selection::All)
            .unwrap();
        assert!(got == *want, "our read of {name} differs");
    }
    let attrs = f.dataset("contig").unwrap().attrs().unwrap();
    assert!((12..=13).contains(&attrs.len()), "{attrs:?}");
}

/// h5py (libhdf5 2.x) reads every dataset's bytes back.
fn check_h5py(path: &Path, e: &Expect, dir: &Path) {
    let mut script = format!(
        "import h5py, numpy as np\nf = h5py.File({p:?}, 'r')\n",
        p = path.to_str().unwrap()
    );
    for (i, (name, want)) in e.datasets.iter().enumerate() {
        let exp = dir.join(format!("expect{i}.bin"));
        std::fs::write(&exp, want).unwrap();
        script.push_str(&format!(
            "a = np.ascontiguousarray(f[{name:?}][()]).tobytes()\n\
             assert a == open({x:?}, 'rb').read(), {name:?}\n",
            x = exp.to_str().unwrap()
        ));
    }
    script.push_str(
        "assert f.attrs['text'] in (b'readable by 1.8', 'readable by 1.8')\n\
         assert list(f.attrs['ints']) == [1, -2, 3]\n\
         assert len(f['g_dense'].attrs) == 12 and len(f['g_dense']) == 20\n\
         assert list(f['g_order']) == ['z9', 'z8', 'z7', 'z6', 'z5', 'z4', 'z3', 'z2', 'z1', 'z0']\n\
         assert f['g_compact/soft'].shape == (1000,)\n\
         assert f['resizable'].maxshape == (None,)\n\
         assert f['filled'].fillvalue == -9\n\
         print('ok')\n",
    );
    assert_eq!(py(&script), "ok");
}

/// h5dump (1.14) and `h5rs check --data` accept the file.
fn check_tools(path: &Path) {
    let p = path.to_str().unwrap();
    let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
        .args(["check", "--data", "-q", p])
        .output()
        .unwrap();
    assert!(o.status.success(), "h5rs check --data {p}:\n{}", text(&o));
    let o = Command::new("h5dump")
        .args(["-o", "/dev/null", p])
        .output()
        .unwrap();
    assert!(
        o.status.success() && o.stderr.is_empty(),
        "h5dump {p}:\n{}",
        text(&o)
    );
}

/// HDF5 1.8's h5dump reads the whole file exactly as h5dump 1.14 does, and
/// dumps each numeric dataset's values as the bytes we wrote.
fn check_18(h5dump: &Path, path: &Path, e: &Expect, dir: &Path) {
    let p = path.to_str().unwrap();
    let o = Command::new(h5dump).arg(p).output().unwrap();
    assert!(
        o.status.success() && o.stderr.is_empty(),
        "h5dump 1.8 {p}:\n{}",
        text(&o)
    );
    let out18 = String::from_utf8_lossy(&o.stdout).to_string();
    assert!(out18.contains("EXTERNAL_LINK \"ext\""), "{out18}");
    let o = Command::new("h5dump").arg(p).output().unwrap();
    assert!(o.status.success(), "h5dump {p}:\n{}", text(&o));
    // HDF5 1.8 has no name for IEEE half floats.
    let out = String::from_utf8_lossy(&o.stdout).replace(
        "H5T_IEEE_F16LE",
        "16-bit little-endian floating-point 16-bit precision",
    );
    if let Some((n, (a, b))) = out18
        .lines()
        .zip(out.lines())
        .enumerate()
        .find(|(_, (a, b))| a != b)
    {
        panic!("h5dump 1.8 and h5dump differ at line {}:\n{a}\n{b}", n + 1);
    }
    assert_eq!(out18.lines().count(), out.lines().count());
    // `-b` writes nothing for compounds, enums, arrays, strings and half
    // floats (HDF5 1.8
    // has no native half float), for h5py's files too: those are covered
    // by the text above.
    let skip = ["/f16", "/strings", "/compound", "/enum", "/array"];
    for (i, (name, want)) in e.datasets.iter().enumerate() {
        if want.is_empty() || skip.contains(&name.as_str()) {
            continue;
        }
        let bin = dir.join(format!("dump18_{i}.bin"));
        let o = Command::new(h5dump)
            .args(["-d", name, "-b", "LE", "-o"])
            .arg(&bin)
            .arg(p)
            .output()
            .unwrap();
        assert!(o.status.success(), "h5dump 1.8 -d {name}:\n{}", text(&o));
        let got = std::fs::read(&bin).unwrap();
        assert!(got == *want, "HDF5 1.8 read of {name} differs");
    }
}

fn check_all(path: &Path, e: &Expect, dir: &Path, h5dump: Option<&Path>) {
    check_ours(path, e);
    check_h5py(path, e, dir);
    check_tools(path);
    if let Some(h) = h5dump {
        check_18(h, path, e, dir);
    }
}

#[test]
fn every_feature_reads_in_hdf5_1_8() {
    if !tools_ok() {
        return;
    }
    let h18 = h5dump18();
    let dir = tmpdir();
    let path = dir.path().join("v18.h5");
    let mut e = write_file(&path);
    check_versions(&path);
    check_all(&path, &e, dir.path(), h18.as_deref());

    // FileEditor: grow the unlimited datasets (splitting B-tree nodes),
    // overwrite values in filtered and unfiltered chunks, set attributes
    // (compact and dense).
    let mut ed = FileEditor::open(&path).unwrap();
    let mut res: Vec<i32> = (0..25).collect();
    for round in 0..300usize {
        let n = res.len() as u64;
        let add = 1 + (round % 9) as u64;
        ed.resize("resizable", &[n + add]).unwrap();
        let vals: Vec<i32> = (0..add).map(|k| (n + k) as i32 * 7 - 3).collect();
        ed.write_values("resizable", &block(&[n], &[add]), &vals)
            .unwrap();
        res.extend(&vals);
    }
    e.set("/resizable", le(&res, i32::to_le_bytes));
    let mut many: Vec<i32> = (0..MANY as i32).map(|i| i ^ 0x5a5a).collect();
    ed.resize("many", &[MANY as u64 + 500]).unwrap();
    let vals: Vec<i32> = (0..500).collect();
    ed.write_values("many", &block(&[MANY as u64], &[500]), &vals)
        .unwrap();
    many.extend(&vals);
    many[12_345] = -1;
    ed.write_values("many", &block(&[12_345], &[1]), &[-1i32])
        .unwrap();
    e.set("/many", le(&many, i32::to_le_bytes));
    let mut chunked: Vec<f32> = (0..60 * 70).map(|i| (i % 97) as f32 * 1.25).collect();
    let row: Vec<f32> = (0..70).map(|i| -(i as f32)).collect();
    ed.write_values("chunked", &block(&[31, 0], &[1, 70]), &row)
        .unwrap();
    chunked[31 * 70..32 * 70].copy_from_slice(&row);
    e.set("/chunked", le(&chunked, f32::to_le_bytes));
    ed.resize("resizable2", &[9, 6]).unwrap();
    let mut r2: Vec<i64> = (0..30).map(|i| i * 1_000_000_007).collect();
    r2.extend(std::iter::repeat_n(0, 24));
    e.set("/resizable2", le(&r2, i64::to_le_bytes));
    ed.set_attr("contig", "added", &AttrValue::F64(3.25))
        .unwrap();
    ed.set_attr("g_dense", "attr03", &AttrValue::F64(-1.0))
        .unwrap();
    ed.set_attr("/", "text", &AttrValue::String("edited".into()))
        .unwrap();
    drop(ed);

    check_ours(&path, &e);
    let f = File::open(&path).unwrap();
    assert!(matches!(
        f.dataset("contig").unwrap().attr("added").unwrap(),
        Some(AttrValue::F64(v)) if v == 3.25
    ));
    drop(f);
    // Everything but the root's "text" attribute is as check_h5py expects.
    py(&format!(
        "import h5py\nf = h5py.File({p:?}, 'r')\n\
         assert f.attrs['text'] in (b'edited', 'edited')\n\
         assert f['contig'].attrs['added'] == 3.25\n\
         assert f['g_dense'].attrs['attr03'] == -1.0\n",
        p = path.to_str().unwrap()
    ));
    let o = Command::new(env!("CARGO_BIN_EXE_h5rs"))
        .args(["check", "--data", "-q", path.to_str().unwrap()])
        .output()
        .unwrap();
    assert!(o.status.success(), "h5rs check after edits:\n{}", text(&o));
    if let Some(h) = h18.as_deref() {
        check_18(h, &path, &e, dir.path());
    }

    // h5py (libhdf5 2.x) goes on appending to the 1.8-format file, and 1.8
    // still reads it.
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'r+') as f:\n\
         \x20   d = f['resizable']\n\
         \x20   n = d.shape[0]\n\
         \x20   d.resize((n + 100,))\n\
         \x20   d[n:] = np.arange(100, dtype='<i4') + 5000\n",
        p = path.to_str().unwrap()
    ));
    res.extend((0..100).map(|k| 5000 + k));
    e.set("/resizable", le(&res, i32::to_le_bytes));
    check_ours(&path, &e);
    if let Some(h) = h18.as_deref() {
        check_18(h, &path, &e, dir.path());
    }
}

// ---- The trees against libhdf5's ----

/// One node of a version-1 chunk B-tree: level, and per key its stored
/// size and offsets; children below.
#[derive(Debug, PartialEq)]
struct TreeNode {
    level: u8,
    keys: Vec<(u32, Vec<u64>)>,
    children: Vec<TreeNode>,
}

fn read_tree(bytes: &[u8], addr: u64, ndims: usize, sizes: bool) -> TreeNode {
    let a = addr as usize;
    assert_eq!(&bytes[a..a + 4], b"TREE");
    let level = bytes[a + 5];
    let n = u16::from_le_bytes([bytes[a + 6], bytes[a + 7]]) as usize;
    let mut p = a + 24;
    let mut keys = Vec::new();
    let mut kids = Vec::new();
    for i in 0..=n {
        let size = u32::from_le_bytes(bytes[p..p + 4].try_into().unwrap());
        let offs = (0..ndims)
            .map(|d| u64::from_le_bytes(bytes[p + 8 + 8 * d..p + 16 + 8 * d].try_into().unwrap()))
            .collect();
        keys.push((if sizes { size } else { 0 }, offs));
        p += 8 + 8 * ndims;
        if i < n {
            kids.push(u64::from_le_bytes(bytes[p..p + 8].try_into().unwrap()));
            p += 8;
        }
    }
    let children = if level > 0 {
        kids.iter()
            .map(|&c| read_tree(bytes, c, ndims, sizes))
            .collect()
    } else {
        Vec::new()
    };
    TreeNode {
        level,
        keys,
        children,
    }
}

/// Where two trees first differ (libhdf5's first).
fn first_difference(a: &TreeNode, b: &TreeNode, at: &str) -> Option<String> {
    if a.level != b.level || a.keys.len() != b.keys.len() {
        return Some(format!(
            "{at}: level {} with {} keys vs level {} with {} keys",
            a.level,
            a.keys.len(),
            b.level,
            b.keys.len()
        ));
    }
    if let Some(i) = (0..a.keys.len()).find(|&i| a.keys[i] != b.keys[i]) {
        return Some(format!("{at} key {i}: {:?} vs {:?}", a.keys[i], b.keys[i]));
    }
    a.children
        .iter()
        .zip(&b.children)
        .enumerate()
        .find_map(|(i, (x, y))| first_difference(x, y, &format!("{at}/{i}")))
}

/// The chunk B-tree of dataset `name`: (tree, ndims) from its layout.
fn tree_of(path: &Path, name: &str, sizes: bool) -> TreeNode {
    use clawhdf5_format::message_type::MessageType;
    use clawhdf5_format::object_header::ObjectHeader;
    let bytes = std::fs::read(path).unwrap();
    let f = File::open(path).unwrap();
    let sb = f.superblock().clone();
    let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, name).unwrap();
    let oh = ObjectHeader::parse(&bytes, addr as usize, 8, 8).unwrap();
    let l = &oh
        .messages
        .iter()
        .find(|m| m.msg_type == MessageType::DataLayout)
        .unwrap()
        .data;
    assert_eq!((l[0], l[1]), (3, 2), "{name}: layout v3, chunked");
    let ndims = l[2] as usize;
    let root = u64::from_le_bytes(l[3..11].try_into().unwrap());
    read_tree(&bytes, root, ndims, sizes)
}

/// Our version-1 chunk B-trees are libhdf5's, node for node (levels, child
/// counts, every key's offsets, and its chunk size where the chunks are
/// the same bytes), for 1-D, 2-D and 3-D datasets with two- and three-level
/// trees, filtered or not.
///
/// libhdf5 inserts each chunk into the tree when it leaves its chunk cache.
/// A whole-dataset write with no cache (`rdcc_nbytes=0`, or chunks larger
/// than the cache) inserts them in row-major order, as we build the tree;
/// with the default cache small chunks of a 1-D dataset still arrive in
/// order, but those of a multi-dimensional one arrive in the order the
/// cache's hash evicts them, which gives the same keys in differently
/// filled nodes. Both trees index the same chunks; we do not model the
/// cache.
#[test]
fn chunk_btrees_match_libhdf5() {
    if !tools_ok() {
        return;
    }
    let dir = tmpdir();
    let theirs = dir.path().join("libhdf5.h5");
    py(&format!(
        "import h5py, numpy as np\n\
         with h5py.File({p:?}, 'w', libver=('v108', 'latest'), rdcc_nbytes=0) as f:\n\
         \x20   f.create_dataset('d1000', data=np.arange(10000.0), chunks=(10,))\n\
         \x20   f.create_dataset('d999', data=np.arange(9990.0), chunks=(10,))\n\
         \x20   f.create_dataset('big', data=np.arange(100000, dtype='<i4'), chunks=(1,), maxshape=(None,))\n\
         \x20   f.create_dataset('grid', data=np.arange(10000.0).reshape(100, 100), chunks=(1, 1))\n\
         \x20   f.create_dataset('cube', data=np.arange(27000, dtype='<i2').reshape(30, 30, 30), chunks=(2, 3, 5))\n\
         \x20   f.create_dataset('gz', data=np.arange(20000, dtype='<i8') % 13, chunks=(7,), compression='gzip')\n",
        p = theirs.to_str().unwrap()
    ));
    let ours = dir.path().join("ours.h5");
    let mut b = FileBuilder::new();
    b.libver_bounds(LibVer::V18, LibVer::V18);
    let v: Vec<f64> = (0..10000).map(f64::from).collect();
    b.create_dataset("d1000")
        .with_f64_data(&v)
        .with_chunks(&[10]);
    b.create_dataset("d999")
        .with_f64_data(&v[..9990])
        .with_chunks(&[10]);
    let v: Vec<i32> = (0..100_000).collect();
    b.create_dataset("big")
        .with_i32_data(&v)
        .with_chunks(&[1])
        .with_maxshape(&[u64::MAX]);
    let v: Vec<f64> = (0..10000).map(f64::from).collect();
    b.create_dataset("grid")
        .with_f64_data(&v)
        .with_shape(&[100, 100])
        .with_chunks(&[1, 1]);
    let raw: Vec<u8> = (0..27000i16).flat_map(|x| x.to_le_bytes()).collect();
    b.create_dataset("cube")
        .with_compound_data(
            clawhdf5_format::datatype::Datatype::FixedPoint {
                size: 2,
                byte_order: clawhdf5_format::datatype::DatatypeByteOrder::LittleEndian,
                signed: true,
                bit_offset: 0,
                bit_precision: 16,
            },
            raw,
            27000,
        )
        .with_shape(&[30, 30, 30])
        .with_chunks(&[2, 3, 5]);
    let v: Vec<i64> = (0..20000).map(|i| i % 13).collect();
    b.create_dataset("gz")
        .with_i64_data(&v)
        .with_chunks(&[7])
        .with_deflate(4);
    b.write(&ours).unwrap();
    for (name, sizes) in [
        ("d1000", true),
        ("d999", true),
        ("big", true),
        ("grid", true),
        ("cube", true),
        // Compressed sizes differ between zlib-rs and zlib.
        ("gz", false),
    ] {
        let a = tree_of(&theirs, name, sizes);
        let b = tree_of(&ours, name, sizes);
        if let Some(d) = first_difference(&a, &b, "root") {
            panic!("{name}: our chunk B-tree differs from libhdf5's: {d}");
        }
    }
}