clawhdf5/crates/clawhdf5-netcdf4/tests/interop_tests.rs

//! NetCDF-4 interop tests: Python creates NetCDF-4 files, clawhdf5-netcdf4 reads them.
//!
//! Tests are skipped if python3 or netCDF4/xarray Python packages are not available.

mod common;

use std::process::Command;

use clawhdf5_netcdf4::{AttrValue, NcType, NetCDF4File};

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/// The Python interpreter to drive interop checks with.
///
/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which
/// on a PEP 668 "externally managed" system is the only place it can be
/// installed. Without it the suite silently skips, and a silent skip here is
/// how a datatype bug once reached a release.
fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency
/// is a test failure instead of a silent skip.
fn interop_required() -> bool {
    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
}

fn netcdf4_python_available() -> bool {
    Command::new(python())
        .args(["-c", "import netCDF4; print(netCDF4.__version__)"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

fn xarray_available() -> bool {
    Command::new(python())
        .args(["-c", "import xarray; print(xarray.__version__)"])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

macro_rules! skip_if_no_netcdf4 {
    () => {
        if !netcdf4_python_available() {
            assert!(
                !interop_required(),
                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with netCDF4 is not available"
            );
            eprintln!("SKIP: python3 with netCDF4 not available");
            return;
        }
    };
}

macro_rules! skip_if_no_xarray {
    () => {
        if !xarray_available() {
            assert!(
                !interop_required(),
                "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with xarray is not available"
            );
            eprintln!("SKIP: python3 with xarray not available");
            return;
        }
    };
}

fn run_python(script: &str) {
    let output = Command::new(python())
        .args(["-c", script])
        .output()
        .expect("failed to run python3");
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        let stdout = String::from_utf8_lossy(&output.stdout);
        panic!("Python script failed:\nSTDOUT: {stdout}\nSTDERR: {stderr}");
    }
}

// ===========================================================================
// 1. Python netCDF4 creates file with dims, vars, CF attrs -> read with clawhdf5
// ===========================================================================

#[test]
fn netcdf4_python_creates_cf_file_clawhdf5_reads() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("cf_test.nc");
    let path_str = path.display().to_string();

    let script = format!(
        r#"
import netCDF4 as nc
import numpy as np

ds = nc.Dataset("{path_str}", "w", format="NETCDF4")
ds.Conventions = "CF-1.8"
ds.title = "Test CF Dataset"

# Dimensions
lat_dim = ds.createDimension("lat", 3)
lon_dim = ds.createDimension("lon", 4)
time_dim = ds.createDimension("time", None)  # unlimited

# Coordinate variables
lat = ds.createVariable("lat", "f4", ("lat",))
lat.units = "degrees_north"
lat.standard_name = "latitude"
lat[:] = [10.0, 20.0, 30.0]

lon = ds.createVariable("lon", "f4", ("lon",))
lon.units = "degrees_east"
lon.standard_name = "longitude"
lon[:] = [-120.0, -110.0, -100.0, -90.0]

time = ds.createVariable("time", "f8", ("time",))
time.units = "hours since 2000-01-01"
time.calendar = "standard"
time[:] = [0.0, 6.0, 12.0]

# Data variable
temp = ds.createVariable("temperature", "f4", ("time", "lat", "lon"),
                         fill_value=-9999.0)
temp.units = "K"
temp.long_name = "Air Temperature"
temp.standard_name = "air_temperature"
data = np.arange(36, dtype=np.float32).reshape(3, 3, 4) + 270.0
temp[:] = data

ds.close()
"#
    );
    run_python(&script);

    let file = NetCDF4File::open(&path).unwrap();

    // Check dimensions
    let dims = file.dimensions().unwrap();
    let dim_names: Vec<&str> = dims.iter().map(|d| d.name.as_str()).collect();
    assert!(dim_names.contains(&"lat"));
    assert!(dim_names.contains(&"lon"));
    assert!(dim_names.contains(&"time"));

    let lat_dim = dims.iter().find(|d| d.name == "lat").unwrap();
    assert_eq!(lat_dim.size, 3);

    let lon_dim = dims.iter().find(|d| d.name == "lon").unwrap();
    assert_eq!(lon_dim.size, 4);

    let time_dim = dims.iter().find(|d| d.name == "time").unwrap();
    assert_eq!(time_dim.size, 3);

    // Check variables
    let variables = file.variables().unwrap();
    let var_names: Vec<String> = variables.iter().map(|v| v.name().to_string()).collect();
    assert!(var_names.contains(&"lat".to_string()));
    assert!(var_names.contains(&"lon".to_string()));
    assert!(var_names.contains(&"time".to_string()));
    assert!(var_names.contains(&"temperature".to_string()));

    // Read lat values
    let lat_var = file.variable("lat").unwrap();
    let lat_vals = lat_var.read_raw_f32().unwrap();
    assert_eq!(lat_vals, vec![10.0f32, 20.0, 30.0]);

    // Check CF attributes on temperature
    let mut temp_var = file.variable("temperature").unwrap();
    let cf = temp_var.cf_attributes().unwrap();
    assert_eq!(cf.units.as_deref(), Some("K"));
    assert_eq!(cf.long_name.as_deref(), Some("Air Temperature"));
    assert_eq!(cf.standard_name.as_deref(), Some("air_temperature"));

    // Read temperature data
    let temp_vals = temp_var.read_raw_f32().unwrap();
    assert_eq!(temp_vals.len(), 36); // 3 * 3 * 4
    assert!((temp_vals[0] - 270.0).abs() < 0.01);
    assert!((temp_vals[35] - 305.0).abs() < 0.01);

    // Check global attributes
    let global_attrs = file.global_attrs().unwrap();
    assert!(matches!(global_attrs.get("Conventions"), Some(AttrValue::String(s)) if s == "CF-1.8"));
    assert!(
        matches!(global_attrs.get("title"), Some(AttrValue::String(s)) if s == "Test CF Dataset")
    );
}

// ===========================================================================
// 2. Python xarray creates file -> clawhdf5 reads dimensions and variables
// ===========================================================================

#[test]
fn xarray_creates_file_clawhdf5_reads() {
    skip_if_no_xarray!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("xarray_test.nc");
    let path_str = path.display().to_string();

    let script = format!(
        r#"
import xarray as xr
import numpy as np
import pandas as pd

# Create xarray Dataset
times = pd.date_range("2020-01-01", periods=5, freq="D")
lats = [10.0, 20.0, 30.0]
lons = [-120.0, -110.0]

temp = np.random.RandomState(42).randn(5, 3, 2).astype(np.float64) * 10 + 280
precip = np.random.RandomState(123).rand(5, 3, 2).astype(np.float64) * 50

ds = xr.Dataset(
    {{
        "temperature": (["time", "lat", "lon"], temp, {{"units": "K", "long_name": "Temperature"}}),
        "precipitation": (["time", "lat", "lon"], precip, {{"units": "mm/day", "long_name": "Precipitation"}}),
    }},
    coords={{
        "time": times,
        "lat": lats,
        "lon": lons,
    }},
    attrs={{"Conventions": "CF-1.8", "source": "xarray test"}},
)
ds.to_netcdf("{path_str}", engine="netcdf4")
"#
    );
    run_python(&script);

    let file = NetCDF4File::open(&path).unwrap();

    // Check dimensions
    let dims = file.dimensions().unwrap();
    let dim_names: Vec<&str> = dims.iter().map(|d| d.name.as_str()).collect();
    assert!(dim_names.contains(&"lat"));
    assert!(dim_names.contains(&"lon"));
    assert!(dim_names.contains(&"time"));

    let lat_dim = dims.iter().find(|d| d.name == "lat").unwrap();
    assert_eq!(lat_dim.size, 3);

    let lon_dim = dims.iter().find(|d| d.name == "lon").unwrap();
    assert_eq!(lon_dim.size, 2);

    // Check variables exist
    let variables = file.variables().unwrap();
    let var_names: Vec<String> = variables.iter().map(|v| v.name().to_string()).collect();
    assert!(var_names.contains(&"temperature".to_string()));
    assert!(var_names.contains(&"precipitation".to_string()));

    // Read temperature variable
    let mut temp_var = file.variable("temperature").unwrap();
    let shape = temp_var.shape().unwrap();
    assert_eq!(shape, vec![5, 3, 2]); // time=5, lat=3, lon=2
    let temp_vals = temp_var.read_raw_f64().unwrap();
    assert_eq!(temp_vals.len(), 30); // 5*3*2

    // Read precipitation variable
    let precip_var = file.variable("precipitation").unwrap();
    let precip_vals = precip_var.read_raw_f64().unwrap();
    assert_eq!(precip_vals.len(), 30);

    // Check CF attributes
    let cf = temp_var.cf_attributes().unwrap();
    assert_eq!(cf.units.as_deref(), Some("K"));
    assert_eq!(cf.long_name.as_deref(), Some("Temperature"));

    // Check global attributes
    let global_attrs = file.global_attrs().unwrap();
    assert!(matches!(global_attrs.get("Conventions"), Some(AttrValue::String(s)) if s == "CF-1.8"));
}

// ===========================================================================
// 3. Python netCDF4 creates file with groups -> clawhdf5 reads groups
// ===========================================================================

#[test]
fn netcdf4_python_creates_grouped_file_clawhdf5_reads() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("grouped_test.nc");
    let path_str = path.display().to_string();

    let script = format!(
        r#"
import netCDF4 as nc
import numpy as np

ds = nc.Dataset("{path_str}", "w", format="NETCDF4")
ds.title = "Grouped NetCDF4 file"

# Root-level dimension and variable
ds.createDimension("x", 5)
x_var = ds.createVariable("x", "f8", ("x",))
x_var[:] = [1.0, 2.0, 3.0, 4.0, 5.0]

# Group: surface
surface = ds.createGroup("surface")
surface.description = "Surface observations"
surface.createDimension("station", 3)
temp = surface.createVariable("temperature", "f4", ("station",))
temp.units = "K"
temp[:] = [288.0, 290.0, 285.0]

# Group: upper_air
upper = ds.createGroup("upper_air")
upper.description = "Upper air soundings"
upper.createDimension("level", 4)
press = upper.createVariable("pressure", "f4", ("level",))
press.units = "hPa"
press[:] = [1000.0, 850.0, 500.0, 200.0]

ds.close()
"#
    );
    run_python(&script);

    let file = NetCDF4File::open(&path).unwrap();

    // Check root variable
    let x_var = file.variable("x").unwrap();
    let x_vals = x_var.read_raw_f64().unwrap();
    assert_eq!(x_vals, vec![1.0, 2.0, 3.0, 4.0, 5.0]);

    // Check group names
    let group_names = file.group_names().unwrap();
    assert!(group_names.contains(&"surface".to_string()));
    assert!(group_names.contains(&"upper_air".to_string()));

    // Check surface group
    let surface = file.group("surface").unwrap();
    let surface_attrs = surface.attrs().unwrap();
    assert!(matches!(
        surface_attrs.get("description"),
        Some(AttrValue::String(s)) if s == "Surface observations"
    ));

    let surf_vars = surface.variables().unwrap();
    let surf_var_names: Vec<String> = surf_vars.iter().map(|v| v.name().to_string()).collect();
    assert!(surf_var_names.contains(&"temperature".to_string()));

    let temp_var = surface.variable("temperature").unwrap();
    let temp_vals = temp_var.read_raw_f32().unwrap();
    assert_eq!(temp_vals, vec![288.0f32, 290.0, 285.0]);

    // Check upper_air group
    let upper = file.group("upper_air").unwrap();
    let press_var = upper.variable("pressure").unwrap();
    let press_vals = press_var.read_raw_f32().unwrap();
    assert_eq!(press_vals, vec![1000.0f32, 850.0, 500.0, 200.0]);
}

#[test]
fn netcdf4_python_string_variable_clawhdf5_reads() {
    // NC_STRING variables are HDF5 variable-length strings, which
    // `read_string` refused ("expected String, got VariableLength") until
    // 2026-09-26.
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("strings.nc");
    let path_str = path.display().to_string();
    let script = format!(
        r#"
import netCDF4 as nc
import numpy as np
ds = nc.Dataset("{path_str}", "w", format="NETCDF4")
ds.createDimension("station", 4)
v = ds.createVariable("name", str, ("station",))
v[:] = np.array(["Oslo", "", "São Paulo", "x"], dtype=object)
ds.close()
"#
    );
    run_python(&script);

    let file = NetCDF4File::open(&path).unwrap();
    let names = file.variable("name").unwrap().read_string().unwrap();
    assert_eq!(names, vec!["Oslo", "", "São Paulo", "x"]);
}

// ===========================================================================
// Unlimited dimensions: the length netCDF-C reports
// ===========================================================================

/// An unlimited dimension's length is the largest extent of the variables
/// using it, in any group (netCDF-C's `nc4_find_dim_len`), not its dimension
/// scale's extent (which netCDF-C leaves at 0): variables of different
/// lengths, one in a subgroup, a dimension no variable has written, a
/// coordinate variable, a subgroup's own unlimited dimension. Compared with
/// what netCDF4-python reports for the same file.
#[test]
fn unlimited_dimension_lengths_match_netcdf4_python() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("unlimited.nc");
    let script = format!(
        r#"
import netCDF4 as nc
import numpy as np
with nc.Dataset({path:?}, "w", format="NETCDF4") as f:
    f.createDimension("time", None)
    f.createDimension("empty", None)
    f.createDimension("rec", None)
    f.createDimension("x", 3)
    f.createVariable("t", "f8", ("time", "x"))[0:2, :] = np.ones((2, 3))
    f.createVariable("a", "i4", ("time",))[0:4] = np.arange(4)
    f.createVariable("e", "i4", ("empty",))
    f.createVariable("rec", "f4", ("rec",))[0:3] = [1, 2, 3]
    f.createVariable("r", "f4", ("x", "rec"))[:, 0:5] = np.ones((3, 5))
    g = f.createGroup("sub")
    g.createVariable("c", "i4", ("time",))[0:6] = np.arange(6)
    g.createDimension("srec", None)
    g.createVariable("s", "i4", ("srec", "x"))[0:1, :] = np.ones((1, 3))
with nc.Dataset({path:?}) as f:
    for grp in (f, f.groups["sub"]):
        for name, d in grp.dimensions.items():
            print(grp.path, name, len(d), d.isunlimited())
"#,
        path = path.display().to_string()
    );
    let out = Command::new(python())
        .args(["-c", &script])
        .output()
        .expect("failed to run python3");
    assert!(
        out.status.success(),
        "{}",
        String::from_utf8_lossy(&out.stderr)
    );
    let expected: Vec<String> = String::from_utf8(out.stdout)
        .unwrap()
        .lines()
        .map(str::to_string)
        .collect();
    // time: t has 2 records, a 4 and sub/c 6; rec: the coordinate variable
    // has 3, r 5; empty: nothing written.
    for want in [
        "/ time 6 True",
        "/ empty 0 True",
        "/ rec 5 True",
        "/ x 3 False",
        "/sub srec 1 True",
    ] {
        assert!(
            expected.iter().any(|l| l == want),
            "netCDF4 reports {expected:?}"
        );
    }

    let file = NetCDF4File::open(&path).unwrap();
    let sub = file.group("sub").unwrap();
    let mut got = Vec::new();
    for (path, dims) in [
        ("/", file.dimensions().unwrap()),
        ("/sub", sub.dimensions().unwrap()),
    ] {
        for d in dims {
            let unlimited = if d.is_unlimited { "True" } else { "False" };
            got.push(format!("{path} {} {} {unlimited}", d.name, d.size));
        }
    }
    assert_eq!(got, expected);
}

// ===========================================================================
// Variables' dimensions, shapes and values as netCDF4-python reports them
// ===========================================================================

/// Whether python can import `module`.
fn python_has(module: &str) -> bool {
    Command::new(python())
        .args(["-c", &format!("import {module}")])
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

/// h5netcdf is not in every interop environment (CI installs it; a local
/// `.venv` may not have it), so its tests skip without it even under
/// `CLAWHDF5_REQUIRE_INTEROP=1`.
macro_rules! skip_if_no_h5netcdf {
    () => {
        if !python_has("h5netcdf") {
            eprintln!("SKIP: python3 with h5netcdf not available");
            return;
        }
    };
}

/// Every variable of the file at `path`, in every group, as netCDF4-python
/// reports it: `"<group path> <name> (<dims>) (<shape>)"` and its values
/// (numeric variables; element by element with masking off, so unwritten
/// records are the fill value), sorted by the description.
///
/// Values are read one element at a time because netCDF-C 4.9.3 lays out a
/// whole-variable read of a variable shorter than an unlimited dimension
/// that is not its first wrongly (the written values first, then the fill);
/// element reads, and reads of one index of the leading axis, are right.
fn netcdf4_view(path: &std::path::Path) -> Vec<(String, Vec<f64>)> {
    let script = r#"
import sys
import numpy as np
import netCDF4 as nc
def walk(g):
    for name, v in g.variables.items():
        v.set_auto_mask(False)
        head = "%s %s (%s) (%s)" % (g.path, name, ",".join(v.dimensions), ",".join(map(str, v.shape)))
        vals = []
        if v.dtype != str and v.dtype.kind in "iuf":
            vals = [repr(float(v[i])) for i in np.ndindex(v.shape)]
        print(head + "|" + " ".join(vals))
    for sub in g.groups.values():
        walk(sub)
with nc.Dataset(sys.argv[1]) as f:
    walk(f)
"#;
    let out = Command::new(python())
        .args(["-c", script, &path.display().to_string()])
        .output()
        .expect("failed to run python3");
    assert!(
        out.status.success(),
        "{}",
        String::from_utf8_lossy(&out.stderr)
    );
    let mut view: Vec<(String, Vec<f64>)> = String::from_utf8(out.stdout)
        .unwrap()
        .lines()
        .map(|line| {
            let (head, vals) = line.split_once('|').unwrap();
            let vals = vals
                .split_whitespace()
                .map(|v| v.parse().unwrap())
                .collect();
            (head.to_string(), vals)
        })
        .collect();
    view.sort_by(|a, b| a.0.cmp(&b.0));
    view
}

/// The same view of the file through clawhdf5-netcdf4.
fn clawhdf5_view(path: &std::path::Path) -> Vec<(String, Vec<f64>)> {
    fn describe(
        group_path: &str,
        vars: Vec<clawhdf5_netcdf4::Variable<'_>>,
    ) -> Vec<(String, Vec<f64>)> {
        vars.into_iter()
            .map(|v| {
                let dims: Vec<&str> = v.dimensions().iter().map(|d| d.name.as_str()).collect();
                let shape: Vec<String> = v.shape().unwrap().iter().map(u64::to_string).collect();
                let head = format!(
                    "{group_path} {} ({}) ({})",
                    v.name(),
                    dims.join(","),
                    shape.join(",")
                );
                let vals = match v.nc_type().unwrap() {
                    NcType::String | NcType::Char => Vec::new(),
                    _ => v.read_raw_f64().unwrap(),
                };
                (head, vals)
            })
            .collect()
    }
    fn walk(
        group_path: &str,
        group: &clawhdf5_netcdf4::NetCDF4Group<'_>,
        out: &mut Vec<(String, Vec<f64>)>,
    ) {
        out.extend(describe(group_path, group.variables().unwrap()));
        for name in group.group_names().unwrap() {
            walk(
                &format!("{group_path}/{name}"),
                &group.group(&name).unwrap(),
                out,
            );
        }
    }
    let file = NetCDF4File::open(path).unwrap();
    let mut view = describe("/", file.variables().unwrap());
    for name in file.group_names().unwrap() {
        walk(&format!("/{name}"), &file.group(&name).unwrap(), &mut view);
    }
    view.sort_by(|a, b| a.0.cmp(&b.0));
    view
}

/// clawhdf5-netcdf4 reports the same variables, dimensions, shapes and
/// values (bit for bit, NaN equal to NaN) as netCDF4-python.
fn assert_same_view(path: &std::path::Path) {
    let want = netcdf4_view(path);
    let got = clawhdf5_view(path);
    let heads = |v: &[(String, Vec<f64>)]| v.iter().map(|(h, _)| h.clone()).collect::<Vec<_>>();
    assert_eq!(heads(&got), heads(&want), "variables differ from netCDF4's");
    for ((head, got), (_, want)) in got.iter().zip(&want) {
        let same = got.len() == want.len()
            && got
                .iter()
                .zip(want)
                .all(|(a, b)| a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()));
        assert!(same, "{head}: got {got:?}, netCDF4 reads {want:?}");
    }
}

/// The reproducer of the known-issues entry: `a` is on the unlimited `time`
/// (5 long through `b`) with 2 records, not on an anonymous `dim_2`; the
/// pure dimension scales `time` and `empty` are not variables; `a` has
/// shape (5,) and reads its 3 unwritten records as the fill value.
#[test]
fn variable_dimensions_come_from_the_file() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("repro.nc");
    run_python(&format!(
        r#"
import netCDF4 as nc
import numpy as np
with nc.Dataset({path:?}, "w") as f:
    f.createDimension("time", None)
    f.createDimension("empty", None)
    f.createDimension("x", 3)
    f.createVariable("a", "i4", ("time",))[0:2] = [1, 2]
    f.createVariable("b", "f4", ("time", "x"))[0:5, :] = np.arange(15).reshape(5, 3)
    f.createVariable("e", "i4", ("empty",))
    f.createVariable("c", "i4", ("x",))[:] = [7, 8, 9]
"#,
        path = path.display().to_string()
    ));
    assert_same_view(&path);

    let file = NetCDF4File::open(&path).unwrap();
    let mut names = file.variable_names().unwrap();
    names.sort();
    assert_eq!(names, ["a", "b", "c", "e"]);
    assert!(matches!(
        file.variable("time"),
        Err(clawhdf5_netcdf4::Error::VariableNotFound(_))
    ));
    let a = file.variable("a").unwrap();
    assert_eq!(a.dimensions()[0].name, "time");
    assert_eq!(a.shape().unwrap(), [5]);
    assert_eq!(a.stored_shape().unwrap(), [2]);
    assert_eq!(
        a.read_raw_i32().unwrap(),
        [1, 2, -2_147_483_647, -2_147_483_647, -2_147_483_647]
    );
}

/// Dimensions of one size are told apart by the file, not by order: `p`
/// and `q` are both 2 long, and `v(q, p)`, `same(p, p)` (one dimension
/// twice), a scalar, `q`'s coordinate variable, a variable called `p` that
/// is not `p`'s coordinate variable (stored as `_nc4_non_coord_p`), and
/// variables in a subgroup and a sub-subgroup on dimensions of their
/// ancestors.
#[test]
fn equal_size_and_inherited_dimensions_match_netcdf4_python() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("dims.nc");
    run_python(&format!(
        r#"
import netCDF4 as nc
import numpy as np
with nc.Dataset({path:?}, "w") as f:
    f.createDimension("p", 2)
    f.createDimension("q", 2)
    f.createVariable("v", "i4", ("q", "p"))[:] = np.array([[1, 2], [3, 4]])
    f.createVariable("same", "i4", ("p", "p"))[:] = np.array([[5, 6], [7, 8]])
    f.createVariable("s", "f8", ())[...] = 3.5
    f.createVariable("q", "f4", ("q",))[:] = [0, 1]
    f.createVariable("p", "f4", ("q", "p"))[:] = np.array([[0, 1], [2, 3]])
    g = f.createGroup("g")
    g.createDimension("r", 2)
    g.createVariable("w", "i4", ("r", "q", "p"))[:] = np.arange(8).reshape(2, 2, 2)
    h = g.createGroup("h")
    h.createVariable("z", "i4", ("p", "r"))[:] = np.array([[1, 2], [3, 4]])
"#,
        path = path.display().to_string()
    ));
    assert_same_view(&path);

    let file = NetCDF4File::open(&path).unwrap();
    let v = file.variable("v").unwrap();
    let dims: Vec<&str> = v.dimensions().iter().map(|d| d.name.as_str()).collect();
    assert_eq!(dims, ["q", "p"]);
    let p = file.variable("p").unwrap();
    assert!(!p.is_coordinate());
    assert!(file.variable("q").unwrap().is_coordinate());
    let s = file.variable("s").unwrap();
    assert!(s.dimensions().is_empty());
    assert_eq!(s.shape().unwrap(), Vec::<u64>::new());
    let z = file
        .group("g")
        .unwrap()
        .group("h")
        .unwrap()
        .variable("z")
        .unwrap();
    let dims: Vec<&str> = z.dimensions().iter().map(|d| d.name.as_str()).collect();
    assert_eq!(dims, ["p", "r"]);
}

/// Variables shorter than their unlimited dimension have its length and
/// read the fill value (`_FillValue`, else netCDF's default for the type)
/// where nothing was written — also when the unlimited dimension is not
/// the first; `read_f64` gives NaN there.
#[test]
fn unwritten_records_read_as_fill_like_netcdf4_python() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("pad.nc");
    run_python(&format!(
        r#"
import netCDF4 as nc
import numpy as np
with nc.Dataset({path:?}, "w") as f:
    f.createDimension("t", None)
    f.createDimension("x", 2)
    f.createVariable("a", "i4", ("t",))[0:2] = [1, 2]
    f.createVariable("f", "f4", ("x", "t"), fill_value=-5.0)[:, 0:1] = np.array([[1], [2]])
    f.createVariable("d", "f8", ("t",))[0:4] = [1, 2, 3, 4]
    f.createVariable("u", "u8", ("t",))[0:1] = [1]
    f.createVariable("b", "i1", ("t", "x"))[0:3, :] = np.ones((3, 2))
    f.createVariable("st", str, ("t",))[0] = "hi"
    g = f.createGroup("g")
    g.createVariable("k", "f4", ("t",))[0:1] = [9]
"#,
        path = path.display().to_string()
    ));
    assert_same_view(&path);

    let file = NetCDF4File::open(&path).unwrap();
    let mut f = file.variable("f").unwrap();
    assert_eq!(f.shape().unwrap(), [2, 4]);
    assert_eq!(f.stored_shape().unwrap(), [2, 1]);
    assert_eq!(
        f.read_raw_f32().unwrap(),
        [1.0, -5.0, -5.0, -5.0, 2.0, -5.0, -5.0, -5.0]
    );
    let read = f.read_f64().unwrap();
    assert_eq!(read[0], 1.0);
    assert!(read[1].is_nan() && read[7].is_nan());
    let st = file.variable("st").unwrap();
    assert_eq!(st.read_string().unwrap(), ["hi", "", "", ""]);
    assert_eq!(st.shape().unwrap(), [4]);
}

/// A file with HDF5 dimension scales but none of netCDF's own attributes
/// (h5py's `dims` API): the dimensions come from `DIMENSION_LIST`, so
/// `v(q, p)` is not `v(p, q)` although both are 2 long; with two scales
/// attached to one axis (`w`), netCDF-C takes the last.
#[test]
fn h5py_dimension_scales_match_netcdf4_python() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("scales.h5");
    run_python(&format!(
        r#"
import h5py
import numpy as np
with h5py.File({path:?}, "w") as f:
    f["p"] = np.arange(2.0)
    f["q"] = np.arange(2.0) + 10
    f["p"].make_scale("p")
    f["q"].make_scale("q")
    f["v"] = np.arange(4).reshape(2, 2)
    f["v"].dims[0].attach_scale(f["q"])
    f["v"].dims[1].attach_scale(f["p"])
    f["w"] = np.arange(2)
    f["w"].dims[0].attach_scale(f["p"])
    f["w"].dims[0].attach_scale(f["q"])
"#,
        path = path.display().to_string()
    ));
    assert_same_view(&path);
}

/// Files h5netcdf writes (its own implementation of the netCDF-4
/// conventions over h5py): an unlimited dimension, equal sizes, a subgroup
/// on inherited dimensions, a scalar.
#[test]
fn h5netcdf_file_matches_netcdf4_python() {
    skip_if_no_netcdf4!();
    skip_if_no_h5netcdf!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("h5netcdf.nc");
    run_python(&format!(
        r#"
import h5netcdf
import numpy as np
with h5netcdf.File({path:?}, "w") as f:
    f.dimensions = {{"p": 2, "q": 2, "t": None}}
    f.create_variable("v", ("q", "p"), "i4")[...] = np.array([[1, 2], [3, 4]])
    f.create_variable("q", ("q",), "f4")[...] = [0, 1]
    f.create_variable("same", ("p", "p"), "i4")[...] = np.array([[5, 6], [7, 8]])
    a = f.create_variable("a", ("t", "p"), "f8")
    f.resize_dimension("t", 3)
    a[...] = np.ones((3, 2))
    f.create_variable("short", ("t",), "i4")
    g = f.create_group("g")
    g.dimensions = {{"r": 2}}
    g.create_variable("w", ("r", "q", "p"), "i4")[...] = np.arange(8).reshape(2, 2, 2)
    g.create_variable("s", (), "f8")[...] = 2.5
"#,
        path = path.display().to_string()
    ));
    assert_same_view(&path);
}

/// Files xarray writes, through netCDF4 and (when installed) h5netcdf:
/// coordinates, two dimensions of one size, an unlimited dimension.
#[test]
fn xarray_files_match_netcdf4_python() {
    skip_if_no_netcdf4!();
    skip_if_no_xarray!();
    let dir = tempfile::tempdir().unwrap();
    let mut engines = vec!["netcdf4"];
    if python_has("h5netcdf") {
        engines.push("h5netcdf");
    } else {
        eprintln!("SKIP: xarray with engine h5netcdf (h5netcdf not available)");
    }
    for engine in engines {
        let path = dir.path().join(format!("xarray_{engine}.nc"));
        run_python(&format!(
            r#"
import numpy as np
import xarray as xr
ds = xr.Dataset(
    {{
        "temp": (("time", "lat", "lon"), np.arange(12.0).reshape(3, 2, 2)),
        "grid": (("lon", "lat"), np.array([[1, 2], [3, 4]], dtype="i4")),
        "scalar": ((), 1.5),
    }},
    coords={{"time": [0.0, 6.0, 12.0], "lat": [10.0, 20.0], "lon": [5.0, 6.0]}},
)
ds.to_netcdf({path:?}, engine={engine:?}, unlimited_dims=["time"])
"#,
            path = path.display().to_string()
        ));
        assert_same_view(&path);
    }
}

// ===========================================================================
// Compared with netCDF-C itself (tests/netcdf_c_view.py): groups,
// dimensions, variables, types and values, in netCDF-C's order
// ===========================================================================

/// The dimensions of the variable `name` (a path) of `file`.
fn dim_names(file: &NetCDF4File, name: &str) -> Vec<String> {
    file.variable(name)
        .unwrap()
        .dimensions()
        .iter()
        .map(|d| d.name.clone())
        .collect()
}

/// An h5py file without dimension scales: netCDF-C's phony dimensions,
/// numbered file-wide (subgroups before their parent's variables, each
/// group's datasets in name order, as h5py does not track creation order),
/// shared by length within a group — but not between two axes of one
/// variable, nor between a fixed and an unlimited axis — with a length of 0
/// always unlimited and never shared with a fixed axis, and the real
/// dimension of a scale taken by length too.
#[test]
fn phony_dimensions_match_netcdf_c() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("phony.h5");
    run_python(&format!(
        r#"
import h5py
import numpy as np
with h5py.File({path:?}, "w") as f:
    f["zz"] = np.arange(12.0).reshape(2, 2, 3)
    f["aa"] = np.arange(6, dtype="i4").reshape(3, 2)
    f.create_dataset("un", data=np.ones((2, 3), "f4"), maxshape=(None, 3))
    f.create_dataset("un2", data=np.ones(2, "f4"), maxshape=(None,))
    f["zero"] = np.zeros((0,))
    f.create_dataset("zero_un", (0,), "f4", maxshape=(None,))
    f["zero2"] = np.zeros((0, 2))
    f["s"] = 1.5
    g = f.create_group("g")
    g["x"] = np.arange(5, dtype="i2")
    g["y"] = np.arange(2, dtype="u1")
    g.create_group("h")["q"] = np.arange(7.0)
    f.create_group("b")["w"] = np.arange(18, dtype="i8").reshape(2, 9)
    f["sc"] = np.arange(6.0)
    f["sc"].make_scale("sc")
    f["second_only"] = np.zeros((2, 6))
    f["second_only"].dims[1].attach_scale(f["sc"])
"#,
        path = path.display().to_string()
    ));
    common::assert_matches_netcdf_c(&path);

    let file = NetCDF4File::open(&path).unwrap();
    // `sc` is dimension 0; /b gets 1 and 2, /g/h 3, /g 4 and 5, / from 6.
    assert_eq!(dim_names(&file, "b/w"), ["phony_dim_1", "phony_dim_2"]);
    assert_eq!(dim_names(&file, "g/h/q"), ["phony_dim_3"]);
    let h = file.group("g/h").unwrap();
    assert_eq!(h.variable_names().unwrap(), ["q"]);
    assert_eq!(h.dimensions().unwrap()[0].name, "phony_dim_3");
    assert_eq!(dim_names(&file, "aa"), ["phony_dim_6", "phony_dim_7"]);
    assert_eq!(
        dim_names(&file, "zz"),
        ["phony_dim_7", "phony_dim_11", "phony_dim_6"]
    );
    assert_eq!(dim_names(&file, "second_only"), ["phony_dim_7", "sc"]);
    assert_eq!(dim_names(&file, "un"), ["phony_dim_8", "phony_dim_6"]);
    assert_eq!(dim_names(&file, "un2"), ["phony_dim_8"]);
    assert_eq!(dim_names(&file, "zero"), ["phony_dim_9"]);
    assert_eq!(dim_names(&file, "zero_un"), ["phony_dim_9"]);
    assert_eq!(dim_names(&file, "zero2"), ["phony_dim_10", "phony_dim_7"]);
    let root: Vec<(String, u64, bool)> = file
        .dimensions()
        .unwrap()
        .into_iter()
        .map(|d| (d.name, d.size, d.is_unlimited))
        .collect();
    assert_eq!(root[0], ("sc".to_string(), 6, false));
    assert!(root.contains(&("phony_dim_8".to_string(), 2, true)));
    assert!(root.contains(&("phony_dim_9".to_string(), 0, true)));
    assert!(root.contains(&("phony_dim_10".to_string(), 0, true)));
    assert_eq!(
        file.variable_names().unwrap(),
        [
            "aa",
            "s",
            "sc",
            "second_only",
            "un",
            "un2",
            "zero",
            "zero2",
            "zero_un",
            "zz"
        ]
    );
}

/// Datasets of types netCDF-C cannot represent are not variables:
/// references, bit fields, array types, a compound with a reference member
/// or a half-float member, a compound nesting a compound not seen before;
/// enum, compound, variable-length and opaque types are variables of those
/// classes. netCDF-C also remembers a type it failed to read, so the second
/// dataset of a compound with a reference member is a variable, and a
/// nested compound is accepted once a dataset of the inner type was read.
#[test]
fn types_netcdf_c_skips_are_not_variables() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("types.h5");
    run_python(&format!(
        r#"
import h5py
import numpy as np
inner = np.dtype([("x", "i2"), ("y", "f4")])
with h5py.File({path:?}, "w") as f:
    f["i4"] = np.arange(3, dtype="i4")
    f["f2"] = np.arange(3, dtype="f2")
    f["s1"] = np.array([b"a", b"b"], dtype="S1")
    f["s5"] = np.array([b"abc", b"b"], dtype="S5")
    f["vs"] = np.array(["x", "yy"], dtype=h5py.string_dtype())
    f["bool"] = np.array([True, False])
    f["cmp"] = np.array([(1, 2.0)], dtype=[("a", "i4"), ("b", "f8")])
    f["en"] = np.array([0, 1], dtype=h5py.enum_dtype({{"A": 0, "B": 1}}, basetype="i1"))
    d = f.create_dataset("vl", (2,), dtype=h5py.vlen_dtype("i4"))
    d[0] = [1, 2]
    d[1] = [3]
    f["op"] = np.array([b"ab", b"cd"], dtype="V2")
    f.create_dataset("ref", (1,), dtype=h5py.ref_dtype)[0] = f["i4"].ref
    f["cref1"] = np.array([(1, f["i4"].ref)], dtype=[("a", "i4"), ("r", h5py.ref_dtype)])
    f["cref2"] = np.array([(2, f["i4"].ref)], dtype=[("a", "i4"), ("r", h5py.ref_dtype)])
    f["cmp_f2"] = np.zeros(2, dtype=[("a", "f2")])
    f["in1"] = np.zeros(2, dtype=inner)
    f["nested"] = np.zeros(2, dtype=[("a", "i4"), ("in", inner)])
    f["a_nested"] = np.zeros(2, dtype=[("b", "i4"), ("in", np.dtype([("p", "i1")]))])
    sid = h5py.h5s.create_simple((2,))
    h5py.h5d.create(f.id, b"bitf", h5py.h5t.STD_B8LE.copy(), sid)
    h5py.h5d.create(f.id, b"arr", h5py.h5t.array_create(h5py.h5t.NATIVE_INT32, (3,)), sid)
"#,
        path = path.display().to_string()
    ));
    common::assert_matches_netcdf_c(&path);

    let file = NetCDF4File::open(&path).unwrap();
    assert_eq!(
        file.variable_names().unwrap(),
        [
            "bool", "cmp", "cref2", "en", "f2", "i4", "in1", "nested", "op", "s1", "s5", "vl", "vs"
        ]
    );
    let nc_type = |name: &str| file.variable(name).unwrap().nc_type().unwrap();
    assert_eq!(nc_type("bool"), NcType::Enum);
    assert_eq!(nc_type("cmp"), NcType::Compound);
    assert_eq!(nc_type("vl"), NcType::VLen);
    assert_eq!(nc_type("op"), NcType::Opaque);
    assert_eq!(nc_type("s1"), NcType::Char);
    assert_eq!(nc_type("s5"), NcType::String);
    // netCDF-C 4.9.3 on libhdf5 1.14.6 says NC_STRING (deliberate
    // difference, see the README).
    assert_eq!(nc_type("f2"), NcType::Float);
    assert_eq!(
        file.variable("f2").unwrap().read_raw_f32().unwrap(),
        [0.0, 1.0, 2.0]
    );
    assert!(matches!(
        file.variable("ref"),
        Err(clawhdf5_netcdf4::Error::VariableNotFound(_))
    ));
}

/// The order of groups and variables: creation order where the group
/// tracks it (every netCDF-4 file; also h5py with `track_order`), compact
/// or dense (more than 8 links), else name order (h5py by default).
#[test]
fn group_and_variable_order_match_netcdf_c() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let nc = dir.path().join("order.nc");
    let h5 = dir.path().join("order.h5");
    run_python(&format!(
        r#"
import h5py
import netCDF4 as nc
import numpy as np
names = ["zeta", "alpha", "mid", "beta", "omega", "gamma", "k", "a", "zz", "c"]
with nc.Dataset({nc:?}, "w") as f:
    f.createDimension("x", 2)
    for i, n in enumerate(names):
        f.createVariable(n, "i4", ("x",))[:] = [i, i + 1]
    for n in ["gz", "ga", "gm"]:
        f.createGroup(n).createVariable("v", "f8", ("x",))[:] = [1, 2]
    small = f.createGroup("small")
    for n in ["q", "b", "p"]:
        small.createVariable(n, "i2", ("x",))[:] = [3, 4]
with h5py.File({h5:?}, "w") as f:
    for i, n in enumerate(names):
        f[n] = np.arange(i + 1)
    t = f.create_group("tracked", track_order=True)
    for i, n in enumerate(names):
        t[n] = np.arange(3, dtype="i1")
    for n in ["gz", "ga"]:
        f.create_group(n)["v"] = np.arange(4.0)
"#,
        nc = nc.display().to_string(),
        h5 = h5.display().to_string()
    ));
    common::assert_matches_netcdf_c(&nc);
    common::assert_matches_netcdf_c(&h5);

    let file = NetCDF4File::open(&nc).unwrap();
    assert_eq!(
        file.variable_names().unwrap(),
        [
            "zeta", "alpha", "mid", "beta", "omega", "gamma", "k", "a", "zz", "c"
        ]
    );
    assert_eq!(file.group_names().unwrap(), ["gz", "ga", "gm", "small"]);
    let file = NetCDF4File::open(&h5).unwrap();
    assert_eq!(file.group_names().unwrap(), ["ga", "gz", "tracked"]);
    assert_eq!(
        file.variable_names().unwrap(),
        [
            "a", "alpha", "beta", "c", "gamma", "k", "mid", "omega", "zeta", "zz"
        ]
    );
    assert_eq!(
        file.group("tracked").unwrap().variable_names().unwrap(),
        [
            "zeta", "alpha", "mid", "beta", "omega", "gamma", "k", "a", "zz", "c"
        ]
    );
}

/// The files of the earlier tests, compared with netCDF-C itself too:
/// dimension scales of h5py, netCDF4-python files with groups, unlimited
/// dimensions and non-coordinate variables named like a dimension.
#[test]
fn netcdf4_python_files_match_netcdf_c() {
    skip_if_no_netcdf4!();
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("mixed.nc");
    run_python(&format!(
        r#"
import netCDF4 as nc
import numpy as np
with nc.Dataset({path:?}, "w") as f:
    f.createDimension("time", None)
    f.createDimension("p", 2)
    f.createDimension("q", 2)
    f.createVariable("a", "i4", ("time",))[0:2] = [1, 2]
    f.createVariable("b", "f4", ("time", "q"))[0:5, :] = np.arange(10).reshape(5, 2)
    f.createVariable("q", "f4", ("q",))[:] = [0, 1]
    f.createVariable("p", "f4", ("q", "p"))[:] = np.array([[0, 1], [2, 3]])
    g = f.createGroup("g")
    g.createDimension("r", 3)
    g.createVariable("w", "i4", ("r", "time"))[:, 0:1] = np.ones((3, 1))
    g.createGroup("h").createVariable("z", "i8", ("p", "r"))[:] = np.arange(6).reshape(2, 3)
"#,
        path = path.display().to_string()
    ));
    common::assert_matches_netcdf_c(&path);
}