clawhdf5-netcdf4: variables' dimensions come from the file

Variables got the first unused dimension of equal size, so a variable on
an unlimited dimension with fewer records got an anonymous dim_<n>, and
dimensions of one size could be swapped. Resolve them as netCDF-C does
(libhdf5/hdf5open.c): _Netcdf4Coordinates ids, else the scales
DIMENSION_LIST references (the last one attached to an axis), searched in
the variable's group and its parents; a coordinate variable is on its own
scale. Size matching remains only for axes the file names nothing for.

variables()/variable_names() leave out dimension scales that are only
dimensions, and _nc4_non_coord_<name> is the variable <name>.
Variable::shape is the netCDF shape (an unlimited dimension's length) and
the reads pad unwritten records with the fill value (_FillValue, else
NC_FILL_*; NaN from read_f64); Variable::stored_shape is the HDF5 extent.
New NetCDF4File::variable_names.

Tests compare with netCDF4-python variable by variable: the known-issues
reproducer, equal sizes, (p, p), scalars, inherited dimensions, unwritten
records, h5py dimension scales, h5netcdf and xarray files. CI installs
h5netcdf. known-issues entry moved to Fixed (history); stale open-table
row for the unlimited-size fix removed.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

crates/clawhdf5-netcdf4/tests/interop_tests.rs

@@ -4,7 +4,7 @@
 
 use std::process::Command;
 
-use clawhdf5_netcdf4::{AttrValue, NetCDF4File};
+use clawhdf5_netcdf4::{AttrValue, NcType, NetCDF4File};
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -461,3 +461,384 @@ with nc.Dataset({path:?}) as f:
     }
     assert_eq!(got, expected);
 }
+
+// ===========================================================================
+// Variables' dimensions, shapes and values as netCDF4-python reports them
+// ===========================================================================
+
+/// Whether python can import `module`.
+fn python_has(module: &str) -> bool {
+    Command::new(python())
+        .args(["-c", &format!("import {module}")])
+        .output()
+        .map(|o| o.status.success())
+        .unwrap_or(false)
+}
+
+/// h5netcdf is not in every interop environment (CI installs it; a local
+/// `.venv` may not have it), so its tests skip without it even under
+/// `CLAWHDF5_REQUIRE_INTEROP=1`.
+macro_rules! skip_if_no_h5netcdf {
+    () => {
+        if !python_has("h5netcdf") {
+            eprintln!("SKIP: python3 with h5netcdf not available");
+            return;
+        }
+    };
+}
+
+/// Every variable of the file at `path`, in every group, as netCDF4-python
+/// reports it: `"<group path> <name> (<dims>) (<shape>)"` and its values
+/// (numeric variables; element by element with masking off, so unwritten
+/// records are the fill value), sorted by the description.
+///
+/// Values are read one element at a time because netCDF-C 4.9.3 lays out a
+/// whole-variable read of a variable shorter than an unlimited dimension
+/// that is not its first wrongly (the written values first, then the fill);
+/// element reads, and reads of one index of the leading axis, are right.
+fn netcdf4_view(path: &std::path::Path) -> Vec<(String, Vec<f64>)> {
+    let script = r#"
+import sys
+import numpy as np
+import netCDF4 as nc
+def walk(g):
+    for name, v in g.variables.items():
+        v.set_auto_mask(False)
+        head = "%s %s (%s) (%s)" % (g.path, name, ",".join(v.dimensions), ",".join(map(str, v.shape)))
+        vals = []
+        if v.dtype != str and v.dtype.kind in "iuf":
+            vals = [repr(float(v[i])) for i in np.ndindex(v.shape)]
+        print(head + "|" + " ".join(vals))
+    for sub in g.groups.values():
+        walk(sub)
+with nc.Dataset(sys.argv[1]) as f:
+    walk(f)
+"#;
+    let out = Command::new(python())
+        .args(["-c", script, &path.display().to_string()])
+        .output()
+        .expect("failed to run python3");
+    assert!(
+        out.status.success(),
+        "{}",
+        String::from_utf8_lossy(&out.stderr)
+    );
+    let mut view: Vec<(String, Vec<f64>)> = String::from_utf8(out.stdout)
+        .unwrap()
+        .lines()
+        .map(|line| {
+            let (head, vals) = line.split_once('|').unwrap();
+            let vals = vals
+                .split_whitespace()
+                .map(|v| v.parse().unwrap())
+                .collect();
+            (head.to_string(), vals)
+        })
+        .collect();
+    view.sort_by(|a, b| a.0.cmp(&b.0));
+    view
+}
+
+/// The same view of the file through clawhdf5-netcdf4.
+fn clawhdf5_view(path: &std::path::Path) -> Vec<(String, Vec<f64>)> {
+    fn describe(
+        group_path: &str,
+        vars: Vec<clawhdf5_netcdf4::Variable<'_>>,
+    ) -> Vec<(String, Vec<f64>)> {
+        vars.into_iter()
+            .map(|v| {
+                let dims: Vec<&str> = v.dimensions().iter().map(|d| d.name.as_str()).collect();
+                let shape: Vec<String> = v.shape().unwrap().iter().map(u64::to_string).collect();
+                let head = format!(
+                    "{group_path} {} ({}) ({})",
+                    v.name(),
+                    dims.join(","),
+                    shape.join(",")
+                );
+                let vals = match v.nc_type().unwrap() {
+                    NcType::String | NcType::Char => Vec::new(),
+                    _ => v.read_raw_f64().unwrap(),
+                };
+                (head, vals)
+            })
+            .collect()
+    }
+    fn walk(
+        group_path: &str,
+        group: &clawhdf5_netcdf4::NetCDF4Group<'_>,
+        out: &mut Vec<(String, Vec<f64>)>,
+    ) {
+        out.extend(describe(group_path, group.variables().unwrap()));
+        for name in group.group_names().unwrap() {
+            walk(
+                &format!("{group_path}/{name}"),
+                &group.group(&name).unwrap(),
+                out,
+            );
+        }
+    }
+    let file = NetCDF4File::open(path).unwrap();
+    let mut view = describe("/", file.variables().unwrap());
+    for name in file.group_names().unwrap() {
+        walk(&format!("/{name}"), &file.group(&name).unwrap(), &mut view);
+    }
+    view.sort_by(|a, b| a.0.cmp(&b.0));
+    view
+}
+
+/// clawhdf5-netcdf4 reports the same variables, dimensions, shapes and
+/// values (bit for bit, NaN equal to NaN) as netCDF4-python.
+fn assert_same_view(path: &std::path::Path) {
+    let want = netcdf4_view(path);
+    let got = clawhdf5_view(path);
+    let heads = |v: &[(String, Vec<f64>)]| v.iter().map(|(h, _)| h.clone()).collect::<Vec<_>>();
+    assert_eq!(heads(&got), heads(&want), "variables differ from netCDF4's");
+    for ((head, got), (_, want)) in got.iter().zip(&want) {
+        let same = got.len() == want.len()
+            && got
+                .iter()
+                .zip(want)
+                .all(|(a, b)| a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()));
+        assert!(same, "{head}: got {got:?}, netCDF4 reads {want:?}");
+    }
+}
+
+/// The reproducer of the known-issues entry: `a` is on the unlimited `time`
+/// (5 long through `b`) with 2 records, not on an anonymous `dim_2`; the
+/// pure dimension scales `time` and `empty` are not variables; `a` has
+/// shape (5,) and reads its 3 unwritten records as the fill value.
+#[test]
+fn variable_dimensions_come_from_the_file() {
+    skip_if_no_netcdf4!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("repro.nc");
+    run_python(&format!(
+        r#"
+import netCDF4 as nc
+import numpy as np
+with nc.Dataset({path:?}, "w") as f:
+    f.createDimension("time", None)
+    f.createDimension("empty", None)
+    f.createDimension("x", 3)
+    f.createVariable("a", "i4", ("time",))[0:2] = [1, 2]
+    f.createVariable("b", "f4", ("time", "x"))[0:5, :] = np.arange(15).reshape(5, 3)
+    f.createVariable("e", "i4", ("empty",))
+    f.createVariable("c", "i4", ("x",))[:] = [7, 8, 9]
+"#,
+        path = path.display().to_string()
+    ));
+    assert_same_view(&path);
+
+    let file = NetCDF4File::open(&path).unwrap();
+    let mut names = file.variable_names().unwrap();
+    names.sort();
+    assert_eq!(names, ["a", "b", "c", "e"]);
+    assert!(matches!(
+        file.variable("time"),
+        Err(clawhdf5_netcdf4::Error::VariableNotFound(_))
+    ));
+    let a = file.variable("a").unwrap();
+    assert_eq!(a.dimensions()[0].name, "time");
+    assert_eq!(a.shape().unwrap(), [5]);
+    assert_eq!(a.stored_shape().unwrap(), [2]);
+    assert_eq!(
+        a.read_raw_i32().unwrap(),
+        [1, 2, -2_147_483_647, -2_147_483_647, -2_147_483_647]
+    );
+}
+
+/// Dimensions of one size are told apart by the file, not by order: `p`
+/// and `q` are both 2 long, and `v(q, p)`, `same(p, p)` (one dimension
+/// twice), a scalar, `q`'s coordinate variable, a variable called `p` that
+/// is not `p`'s coordinate variable (stored as `_nc4_non_coord_p`), and
+/// variables in a subgroup and a sub-subgroup on dimensions of their
+/// ancestors.
+#[test]
+fn equal_size_and_inherited_dimensions_match_netcdf4_python() {
+    skip_if_no_netcdf4!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("dims.nc");
+    run_python(&format!(
+        r#"
+import netCDF4 as nc
+import numpy as np
+with nc.Dataset({path:?}, "w") as f:
+    f.createDimension("p", 2)
+    f.createDimension("q", 2)
+    f.createVariable("v", "i4", ("q", "p"))[:] = np.array([[1, 2], [3, 4]])
+    f.createVariable("same", "i4", ("p", "p"))[:] = np.array([[5, 6], [7, 8]])
+    f.createVariable("s", "f8", ())[...] = 3.5
+    f.createVariable("q", "f4", ("q",))[:] = [0, 1]
+    f.createVariable("p", "f4", ("q", "p"))[:] = np.array([[0, 1], [2, 3]])
+    g = f.createGroup("g")
+    g.createDimension("r", 2)
+    g.createVariable("w", "i4", ("r", "q", "p"))[:] = np.arange(8).reshape(2, 2, 2)
+    h = g.createGroup("h")
+    h.createVariable("z", "i4", ("p", "r"))[:] = np.array([[1, 2], [3, 4]])
+"#,
+        path = path.display().to_string()
+    ));
+    assert_same_view(&path);
+
+    let file = NetCDF4File::open(&path).unwrap();
+    let v = file.variable("v").unwrap();
+    let dims: Vec<&str> = v.dimensions().iter().map(|d| d.name.as_str()).collect();
+    assert_eq!(dims, ["q", "p"]);
+    let p = file.variable("p").unwrap();
+    assert!(!p.is_coordinate());
+    assert!(file.variable("q").unwrap().is_coordinate());
+    let s = file.variable("s").unwrap();
+    assert!(s.dimensions().is_empty());
+    assert_eq!(s.shape().unwrap(), Vec::<u64>::new());
+    let z = file
+        .group("g")
+        .unwrap()
+        .group("h")
+        .unwrap()
+        .variable("z")
+        .unwrap();
+    let dims: Vec<&str> = z.dimensions().iter().map(|d| d.name.as_str()).collect();
+    assert_eq!(dims, ["p", "r"]);
+}
+
+/// Variables shorter than their unlimited dimension have its length and
+/// read the fill value (`_FillValue`, else netCDF's default for the type)
+/// where nothing was written — also when the unlimited dimension is not
+/// the first; `read_f64` gives NaN there.
+#[test]
+fn unwritten_records_read_as_fill_like_netcdf4_python() {
+    skip_if_no_netcdf4!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("pad.nc");
+    run_python(&format!(
+        r#"
+import netCDF4 as nc
+import numpy as np
+with nc.Dataset({path:?}, "w") as f:
+    f.createDimension("t", None)
+    f.createDimension("x", 2)
+    f.createVariable("a", "i4", ("t",))[0:2] = [1, 2]
+    f.createVariable("f", "f4", ("x", "t"), fill_value=-5.0)[:, 0:1] = np.array([[1], [2]])
+    f.createVariable("d", "f8", ("t",))[0:4] = [1, 2, 3, 4]
+    f.createVariable("u", "u8", ("t",))[0:1] = [1]
+    f.createVariable("b", "i1", ("t", "x"))[0:3, :] = np.ones((3, 2))
+    f.createVariable("st", str, ("t",))[0] = "hi"
+    g = f.createGroup("g")
+    g.createVariable("k", "f4", ("t",))[0:1] = [9]
+"#,
+        path = path.display().to_string()
+    ));
+    assert_same_view(&path);
+
+    let file = NetCDF4File::open(&path).unwrap();
+    let mut f = file.variable("f").unwrap();
+    assert_eq!(f.shape().unwrap(), [2, 4]);
+    assert_eq!(f.stored_shape().unwrap(), [2, 1]);
+    assert_eq!(
+        f.read_raw_f32().unwrap(),
+        [1.0, -5.0, -5.0, -5.0, 2.0, -5.0, -5.0, -5.0]
+    );
+    let read = f.read_f64().unwrap();
+    assert_eq!(read[0], 1.0);
+    assert!(read[1].is_nan() && read[7].is_nan());
+    let st = file.variable("st").unwrap();
+    assert_eq!(st.read_string().unwrap(), ["hi", "", "", ""]);
+    assert_eq!(st.shape().unwrap(), [4]);
+}
+
+/// A file with HDF5 dimension scales but none of netCDF's own attributes
+/// (h5py's `dims` API): the dimensions come from `DIMENSION_LIST`, so
+/// `v(q, p)` is not `v(p, q)` although both are 2 long; with two scales
+/// attached to one axis (`w`), netCDF-C takes the last.
+#[test]
+fn h5py_dimension_scales_match_netcdf4_python() {
+    skip_if_no_netcdf4!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("scales.h5");
+    run_python(&format!(
+        r#"
+import h5py
+import numpy as np
+with h5py.File({path:?}, "w") as f:
+    f["p"] = np.arange(2.0)
+    f["q"] = np.arange(2.0) + 10
+    f["p"].make_scale("p")
+    f["q"].make_scale("q")
+    f["v"] = np.arange(4).reshape(2, 2)
+    f["v"].dims[0].attach_scale(f["q"])
+    f["v"].dims[1].attach_scale(f["p"])
+    f["w"] = np.arange(2)
+    f["w"].dims[0].attach_scale(f["p"])
+    f["w"].dims[0].attach_scale(f["q"])
+"#,
+        path = path.display().to_string()
+    ));
+    assert_same_view(&path);
+}
+
+/// Files h5netcdf writes (its own implementation of the netCDF-4
+/// conventions over h5py): an unlimited dimension, equal sizes, a subgroup
+/// on inherited dimensions, a scalar.
+#[test]
+fn h5netcdf_file_matches_netcdf4_python() {
+    skip_if_no_netcdf4!();
+    skip_if_no_h5netcdf!();
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("h5netcdf.nc");
+    run_python(&format!(
+        r#"
+import h5netcdf
+import numpy as np
+with h5netcdf.File({path:?}, "w") as f:
+    f.dimensions = {{"p": 2, "q": 2, "t": None}}
+    f.create_variable("v", ("q", "p"), "i4")[...] = np.array([[1, 2], [3, 4]])
+    f.create_variable("q", ("q",), "f4")[...] = [0, 1]
+    f.create_variable("same", ("p", "p"), "i4")[...] = np.array([[5, 6], [7, 8]])
+    a = f.create_variable("a", ("t", "p"), "f8")
+    f.resize_dimension("t", 3)
+    a[...] = np.ones((3, 2))
+    f.create_variable("short", ("t",), "i4")
+    g = f.create_group("g")
+    g.dimensions = {{"r": 2}}
+    g.create_variable("w", ("r", "q", "p"), "i4")[...] = np.arange(8).reshape(2, 2, 2)
+    g.create_variable("s", (), "f8")[...] = 2.5
+"#,
+        path = path.display().to_string()
+    ));
+    assert_same_view(&path);
+}
+
+/// Files xarray writes, through netCDF4 and (when installed) h5netcdf:
+/// coordinates, two dimensions of one size, an unlimited dimension.
+#[test]
+fn xarray_files_match_netcdf4_python() {
+    skip_if_no_netcdf4!();
+    skip_if_no_xarray!();
+    let dir = tempfile::tempdir().unwrap();
+    let mut engines = vec!["netcdf4"];
+    if python_has("h5netcdf") {
+        engines.push("h5netcdf");
+    } else {
+        eprintln!("SKIP: xarray with engine h5netcdf (h5netcdf not available)");
+    }
+    for engine in engines {
+        let path = dir.path().join(format!("xarray_{engine}.nc"));
+        run_python(&format!(
+            r#"
+import numpy as np
+import xarray as xr
+ds = xr.Dataset(
+    {{
+        "temp": (("time", "lat", "lon"), np.arange(12.0).reshape(3, 2, 2)),
+        "grid": (("lon", "lat"), np.array([[1, 2], [3, 4]], dtype="i4")),
+        "scalar": ((), 1.5),
+    }},
+    coords={{"time": [0.0, 6.0, 12.0], "lat": [10.0, 20.0], "lon": [5.0, 6.0]}},
+)
+ds.to_netcdf({path:?}, engine={engine:?}, unlimited_dims=["time"])
+"#,
+            path = path.display().to_string()
+        ));
+        assert_same_view(&path);
+    }
+}