clawhdf5/crates/clawhdf5-netcdf4/tests/common/mod.rs

//! clawhdf5-netcdf4 compared with netCDF-C itself: `tests/netcdf_c_view.py`
//! prints what netCDF-C (the libnetcdf netCDF4-python bundles) reports for
//! a file, and [`differences`] lists how clawhdf5-netcdf4's reading differs.
#![allow(dead_code)]

use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicUsize, Ordering};

use clawhdf5_netcdf4::{NcType, NetCDF4File, NetCDF4Group, Variable};

/// The Python with netCDF4-python (`CLAWHDF5_PYTHON`, else `python3`).
pub fn python() -> String {
    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
}

/// netCDF-C's view of one file.
#[derive(Default)]
pub struct View {
    /// `G`, `D` and `V` lines, in order.
    pub meta: Vec<String>,
    /// `(group, variable)` → values, from the `X` lines.
    pub values: BTreeMap<(String, String), Vec<f64>>,
}

/// netCDF-C's view of each file (`None`: netCDF-C cannot open it), from
/// `tests/netcdf_c_view.py`.
pub fn netcdf_c_views(files: &[PathBuf]) -> BTreeMap<PathBuf, Option<View>> {
    let dir = tempfile::tempdir().unwrap();
    let list = dir.path().join("files.txt");
    let mut paths = String::new();
    for f in files {
        paths.push_str(&f.to_string_lossy());
        paths.push('\n');
    }
    std::fs::write(&list, paths).unwrap();
    let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/netcdf_c_view.py");
    let out = Command::new(python())
        .arg(&script)
        .arg("--list")
        .arg(&list)
        .output()
        .expect("failed to run python");
    assert!(
        out.status.success(),
        "netcdf_c_view.py failed: {}",
        String::from_utf8_lossy(&out.stderr)
    );
    parse_views(&String::from_utf8_lossy(&out.stdout))
}

/// The views in the output of `tests/netcdf_c_view.py`.
pub fn parse_views(text: &str) -> BTreeMap<PathBuf, Option<View>> {
    let mut views = BTreeMap::new();
    let mut current: Option<(PathBuf, Option<View>)> = None;
    for line in text.lines() {
        let fields: Vec<&str> = line.split('\t').collect();
        match fields[0] {
            "FILE" => current = Some((PathBuf::from(fields[1]), Some(View::default()))),
            "END" => {
                let (path, view) = current.take().expect("END without FILE");
                views.insert(path, view);
            }
            "ERROR" => current.as_mut().expect("ERROR without FILE").1 = None,
            "X" => {
                let view = current.as_mut().and_then(|c| c.1.as_mut()).unwrap();
                let vals = if fields[3] == "-" {
                    Vec::new()
                } else {
                    fields[3]
                        .split(' ')
                        .map(|v| v.parse().expect("value"))
                        .collect()
                };
                view.values
                    .insert((fields[1].to_string(), fields[2].to_string()), vals);
            }
            _ => {
                let view = current.as_mut().and_then(|c| c.1.as_mut()).unwrap();
                view.meta.push(line.to_string());
            }
        }
    }
    views
}

/// Variables whose type is labelled as netCDF-C labels it rather than as
/// clawhdf5-netcdf4 does (see [`nc_type_label`]).
pub static RELABELLED: AtomicUsize = AtomicUsize::new(0);
/// Values not compared because this build of the HDF5 reader lacks a
/// filter (a cargo feature).
pub static FILTER_SKIPPED: AtomicUsize = AtomicUsize::new(0);

/// The same `G`/`D`/`V` lines from clawhdf5-netcdf4.
pub fn our_meta(file: &NetCDF4File) -> Result<Vec<String>, String> {
    let e = |e: clawhdf5_netcdf4::Error| e.to_string();
    let mut out = Vec::new();
    out.push("G\t/".to_string());
    push_group(
        &mut out,
        file.hdf5_file(),
        "/",
        file.dimensions().map_err(e)?,
        file.variables().map_err(e)?,
    )?;
    for name in file.group_names().map_err(e)? {
        walk(
            &mut out,
            file.hdf5_file(),
            &format!("/{name}"),
            &file.group(&name).map_err(e)?,
        )?;
    }
    Ok(out)
}

fn walk(
    out: &mut Vec<String>,
    hdf5: &clawhdf5::File,
    path: &str,
    group: &NetCDF4Group<'_>,
) -> Result<(), String> {
    let e = |e: clawhdf5_netcdf4::Error| e.to_string();
    out.push(format!("G\t{path}"));
    push_group(
        out,
        hdf5,
        path,
        group.dimensions().map_err(e)?,
        group.variables().map_err(e)?,
    )?;
    for name in group.group_names().map_err(e)? {
        walk(
            out,
            hdf5,
            &format!("{path}/{name}"),
            &group.group(&name).map_err(e)?,
        )?;
    }
    Ok(())
}

/// The type of variable `name` of the group at `path` as the `V` line
/// shows it: its `NcType`, except for the one deliberate difference from
/// netCDF-C — a float of other than 4 or 8 bytes (half, bfloat16, the 4-,
/// 6- and 8-bit floats, `long double`), `NC_FLOAT`/`NC_DOUBLE` here, is
/// labelled `NC_STRING` by netCDF-C 4.9.3 on libhdf5 1.14.6; such a
/// variable is shown as netCDF-C shows it, and counted.
fn nc_type_label(hdf5: &clawhdf5::File, path: &str, var: &Variable<'_>) -> Result<String, String> {
    let nc_type = var.nc_type().map_err(|e| e.to_string())?;
    if matches!(nc_type, NcType::Float | NcType::Double) {
        let dataset = format!("{}/{}", path.trim_end_matches('/'), var.name());
        if let Ok(ds) = hdf5.dataset(&dataset)
            && let Ok(clawhdf5_format::datatype::Datatype::FloatingPoint { size, .. }) =
                ds.raw_datatype()
            && size != 4
            && size != 8
        {
            RELABELLED.fetch_add(1, Ordering::Relaxed);
            return Ok("NC_STRING".to_string());
        }
    }
    Ok(nc_type.to_string())
}

fn push_group(
    out: &mut Vec<String>,
    hdf5: &clawhdf5::File,
    path: &str,
    dims: Vec<clawhdf5_netcdf4::Dimension>,
    vars: Vec<Variable<'_>>,
) -> Result<(), String> {
    for d in dims {
        out.push(format!(
            "D\t{path}\t{}\t{}\t{}",
            d.name,
            d.size,
            u8::from(d.is_unlimited)
        ));
    }
    for v in vars {
        let dims: Vec<&str> = v.dimensions().iter().map(|d| d.name.as_str()).collect();
        let shape: Vec<String> = v
            .shape()
            .map_err(|e| e.to_string())?
            .iter()
            .map(u64::to_string)
            .collect();
        let or_dash = |s: String| if s.is_empty() { "-".to_string() } else { s };
        out.push(format!(
            "V\t{path}\t{}\t{}\t{}\t{}",
            v.name(),
            nc_type_label(hdf5, path, &v)?,
            or_dash(dims.join(",")),
            or_dash(shape.join(","))
        ));
    }
    Ok(())
}

/// The values of variable `name` of group `group`.
fn our_values(file: &NetCDF4File, group: &str, name: &str) -> Result<Vec<f64>, String> {
    let var = if group == "/" {
        file.variable(name)
    } else {
        file.variable(&format!("{}/{name}", group.trim_start_matches('/')))
    }
    .map_err(|e| e.to_string())?;
    match var.nc_type().map_err(|e| e.to_string())? {
        NcType::String | NcType::Char => Err("not numeric".into()),
        _ => var.read_raw_f64().map_err(|e| e.to_string()),
    }
}

/// How clawhdf5-netcdf4's reading of `path` differs from `want`; empty
/// when it does not.
pub fn differences(path: &Path, want: &View) -> Vec<String> {
    let file = match NetCDF4File::open(path) {
        Ok(f) => f,
        Err(e) => return vec![format!("netCDF-C opens it, clawhdf5-netcdf4 does not: {e}")],
    };
    let meta = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| our_meta(&file))) {
        Ok(Ok(meta)) => meta,
        Ok(Err(e)) => return vec![format!("error: {e}")],
        Err(_) => return vec!["panic".to_string()],
    };
    let mut diffs = Vec::new();
    if meta != want.meta {
        let ours: BTreeSet<&String> = meta.iter().collect();
        let theirs: BTreeSet<&String> = want.meta.iter().collect();
        for line in want.meta.iter().filter(|l| !ours.contains(l)) {
            diffs.push(format!("netCDF-C: {line}"));
        }
        for line in meta.iter().filter(|l| !theirs.contains(l)) {
            diffs.push(format!("ours:     {line}"));
        }
        if diffs.is_empty() {
            diffs.push("same lines, different order".to_string());
        }
    }
    for ((group, name), want) in &want.values {
        let got = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
            our_values(&file, group, name)
        }))
        .unwrap_or_else(|_| Err("panic".to_string()));
        match got {
            Ok(got) => {
                let same = got.len() == want.len()
                    && got
                        .iter()
                        .zip(want)
                        .all(|(a, b)| a.to_bits() == b.to_bits() || (a.is_nan() && b.is_nan()));
                if !same {
                    diffs.push(format!("values of {group} {name} differ"));
                }
            }
            Err(e) if e.contains("this build lacks the") => {
                FILTER_SKIPPED.fetch_add(1, Ordering::Relaxed);
            }
            Err(e) => diffs.push(format!("values of {group} {name}: {e}")),
        }
    }
    diffs
}

/// clawhdf5-netcdf4 reads the file at `path` as netCDF-C does: the same
/// groups, dimensions, variables and values (see [`differences`]).
pub fn assert_matches_netcdf_c(path: &Path) {
    let views = netcdf_c_views(&[path.to_path_buf()]);
    let Some(Some(want)) = views.get(path) else {
        panic!("netCDF-C cannot open {}", path.display());
    };
    let diffs = differences(path, want);
    assert!(
        diffs.is_empty(),
        "{} differs from netCDF-C:\n{}",
        path.display(),
        diffs.join("\n")
    );
}