rustytorch/crates/specialized/rtx-fsi/tests/fsi2_overset_plane.rs

//! PERF-2 P3-iii (`docs/perf2_campaign.md` in omni-cortex): the
//! `MarchPlane` — K overset FSI marches in ONE process, one lane thread
//! each, the device V-cycles of every lane inside a CG gathered into one
//! batched launch set by the plane rendezvous in `rtx-cfd`'s Poisson
//! driver. Every lane runs the unchanged `run_march_overset`.
//!
//! Knobs (all the `RTX_FSI2O_*` knobs of `fsi2_on_the_overset` apply to
//! every lane; these add the plane):
//! - `RTX_FSI2O_PLANE_FILE`: the points, one per line, `u_mean e_s tag`
//!   (the `p7_points.txt` format; `#` comments and blank lines skipped).
//!   Without it the test returns.
//! - `RTX_FSI2O_PLANE_DIR`: each lane's CSV goes to `<dir>/<tag>.csv`.
//! - `RTX_FSI2O_PLANE_STACK_MB`: the lane threads' stack (default 64).
//! Requires `RTX_FSI2O_MG_RB=1 RTX_FSI2O_MG_DEVICE=1` on a
//! `--features rtx-cfd/cuda` build; the test fails loudly otherwise (a
//! CPU plane is K separate processes, not this driver).
//!
//! `RTX_FSI2O_PLANE_FILE=points.txt RTX_FSI2O_PLANE_DIR=$L RTX_FSI2O_MG_RB=1 RTX_FSI2O_MG_DEVICE=1 RTX_CUDA_ARCH=sm_120 <binary> fsi2_overset_plane --nocapture`

mod fsi2_harness;

use fsi2_harness::overset_march::{OversetMarchConfig, run_march_overset};
use fsi2_harness::{BenchmarkCase, FSI2, FSI3};
use rtx_cfd::solvers::incompressible::{plane_counters, set_plane_lane};
use std::time::Instant;

struct Point {
    u_mean: f64,
    e_s: f64,
    tag: String,
}

fn read_points(path: &str) -> Vec<Point> {
    let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}"));
    text.lines()
        .map(str::trim)
        .filter(|l| !l.is_empty() && !l.starts_with('#'))
        .map(|l| {
            let f: Vec<&str> = l.split_whitespace().collect();
            assert!(
                f.len() >= 3,
                "plane point line needs `u_mean e_s tag`: {l:?}"
            );
            Point {
                u_mean: f[0].parse().expect("u_mean"),
                e_s: f[1].parse().expect("e_s"),
                tag: f[2].to_string(),
            }
        })
        .collect()
}

struct LaneSummary {
    tag: String,
    wall: f64,
    steps: usize,
    subit: f64,
    retries: usize,
    uy_amp_mm: f64,
    uy_mid_mm: f64,
    frequency: Option<f64>,
    drag_mid: f64,
    lift_amp: f64,
    death: Option<(usize, f64, String)>,
}

#[test]
fn fsi2_overset_plane() {
    let Ok(points_path) = std::env::var("RTX_FSI2O_PLANE_FILE") else {
        return;
    };
    let dir = std::env::var("RTX_FSI2O_PLANE_DIR").unwrap_or_else(|_| ".".into());
    let stack_mb: usize = std::env::var("RTX_FSI2O_PLANE_STACK_MB")
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(64);
    let device_on = std::env::var("RTX_FSI2O_MG_DEVICE").is_ok_and(|v| v == "1")
        && std::env::var("RTX_FSI2O_MG_RB").is_ok_and(|v| v == "1");
    assert!(
        device_on,
        "the plane needs RTX_FSI2O_MG_RB=1 RTX_FSI2O_MG_DEVICE=1 on a cuda build"
    );
    let points = read_points(&points_path);
    let k = points.len();
    assert!(k >= 1, "no plane points in {points_path}");

    let config = OversetMarchConfig::from_env(
        "FSI2O",
        OversetMarchConfig {
            ny: 41,
            flag_nx: 35,
            t_release: 6.0,
            t_end: 7.0,
            subcycle: 1,
            tol_floor: 2e-4,
            rtol: 1e-2,
            stall_accept: 5.0,
            max_subiterations: 12,
            coupler: "aitken".into(),
            reuse: 2,
            initial_relaxation: 0.5,
            c1_interface: false,
            predictor: "structure".into(),
            sweeps: 100,
            max_rounds: 3,
            csv_path: None,
            trace_steps: 0,
        },
    );
    let base = if std::env::var("RTX_FSI2O_CASE").as_deref() == Ok("FSI3") {
        FSI3
    } else {
        FSI2
    };
    println!(
        "  MARCH PLANE: {k} lanes in one process ({} base, ny {}, t_end {}, coupler {}, s = {}); points from {points_path}, CSVs under {dir}",
        base.name, config.ny, config.t_end, config.coupler, config.subcycle
    );
    for (i, p) in points.iter().enumerate() {
        println!(
            "  lane {i}: {} u_mean {} (Re {:.0}) e_s {:.4e} (E/E0 {:.4})",
            p.tag,
            p.u_mean,
            100.0 * p.u_mean,
            p.e_s,
            p.e_s / base.e_s
        );
    }

    let t_plane = Instant::now();
    let handles: Vec<_> = points
        .into_iter()
        .enumerate()
        .map(|(i, p)| {
            let mut cfg = config.clone();
            cfg.csv_path = Some(format!("{dir}/{}.csv", p.tag));
            let case = BenchmarkCase {
                u_mean: p.u_mean,
                e_s: p.e_s,
                ..base
            };
            std::thread::Builder::new()
                .name(format!("lane-{i}-{}", p.tag))
                .stack_size(stack_mb << 20)
                .spawn(move || {
                    set_plane_lane(Some(i));
                    let t0 = Instant::now();
                    let r = run_march_overset(case, &cfg);
                    let m = &r.result;
                    let w = m.window(3.0);
                    LaneSummary {
                        tag: p.tag,
                        wall: t0.elapsed().as_secs_f64(),
                        steps: m.coupled_steps,
                        subit: m.mean_subiterations,
                        retries: m.retried_steps,
                        uy_amp_mm: w.uy_amp * 1e3,
                        uy_mid_mm: w.uy_mid * 1e3,
                        frequency: w.frequency,
                        drag_mid: w.drag_mid,
                        lift_amp: w.lift_amp,
                        death: r.death,
                    }
                })
                .expect("lane thread")
        })
        .collect();
    let mut summaries = Vec::with_capacity(k);
    let mut failures = Vec::new();
    for (i, h) in handles.into_iter().enumerate() {
        match h.join() {
            Ok(s) => summaries.push(s),
            Err(_) => failures.push(i),
        }
    }
    let plane_wall = t_plane.elapsed().as_secs_f64();
    let (batches, served) = plane_counters();
    println!(
        "  MARCH PLANE done: {k} lanes in {plane_wall:.0} s wall; {batches} batched V-cycle launches served {served} lane calls ({:.2} lanes per batch)",
        if batches > 0 {
            served as f64 / batches as f64
        } else {
            0.0
        }
    );
    for s in &summaries {
        println!(
            "  lane {}: {:.0} s wall, {} coupled steps, {:.1} subit/step, {} retries; last 3 s: uy(A) = {:.3} ± {:.3} mm, f = {:?} Hz, drag mid {:.2}, lift amp {:.2}; death {:?}",
            s.tag,
            s.wall,
            s.steps,
            s.subit,
            s.retries,
            s.uy_mid_mm,
            s.uy_amp_mm,
            s.frequency,
            s.drag_mid,
            s.lift_amp,
            s.death
        );
    }
    assert!(failures.is_empty(), "lane threads panicked: {failures:?}");
    let dead: Vec<&str> = summaries
        .iter()
        .filter(|s| s.death.is_some())
        .map(|s| s.tag.as_str())
        .collect();
    assert!(dead.is_empty(), "lanes died: {dead:?}");
}