rtx-cfd/rtx-fsi PERF-2 P3-iii: the MarchPlane — lane-table V-cycle kernels (ml_*), the batched apply over K lanes' own operators, the plane rendezvous inside the CG's preconditioner (lane thread-local, CG guard), set_plane_lane/plane_counters; the fsi2_overset_plane driver (K lane threads, one process); G-P0 unit tests: K distinct operators batched = solo apply bit for bit, uneven lanes served exactly

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/specialized/rtx-fsi/tests/fsi2_overset_plane.rs

@@ -0,0 +1,209 @@
+//! PERF-2 P3-iii (`docs/perf2_campaign.md` in omni-cortex): the
+//! `MarchPlane` — K overset FSI marches in ONE process, one lane thread
+//! each, the device V-cycles of every lane inside a CG gathered into one
+//! batched launch set by the plane rendezvous in `rtx-cfd`'s Poisson
+//! driver. Every lane runs the unchanged `run_march_overset`.
+//!
+//! Knobs (all the `RTX_FSI2O_*` knobs of `fsi2_on_the_overset` apply to
+//! every lane; these add the plane):
+//! - `RTX_FSI2O_PLANE_FILE`: the points, one per line, `u_mean e_s tag`
+//!   (the `p7_points.txt` format; `#` comments and blank lines skipped).
+//!   Without it the test returns.
+//! - `RTX_FSI2O_PLANE_DIR`: each lane's CSV goes to `<dir>/<tag>.csv`.
+//! - `RTX_FSI2O_PLANE_STACK_MB`: the lane threads' stack (default 64).
+//! Requires `RTX_FSI2O_MG_RB=1 RTX_FSI2O_MG_DEVICE=1` on a
+//! `--features rtx-cfd/cuda` build; the test fails loudly otherwise (a
+//! CPU plane is K separate processes, not this driver).
+//!
+//! `RTX_FSI2O_PLANE_FILE=points.txt RTX_FSI2O_PLANE_DIR=$L RTX_FSI2O_MG_RB=1 RTX_FSI2O_MG_DEVICE=1 RTX_CUDA_ARCH=sm_120 <binary> fsi2_overset_plane --nocapture`
+
+mod fsi2_harness;
+
+use fsi2_harness::overset_march::{OversetMarchConfig, run_march_overset};
+use fsi2_harness::{BenchmarkCase, FSI2, FSI3};
+use rtx_cfd::solvers::incompressible::{plane_counters, set_plane_lane};
+use std::time::Instant;
+
+struct Point {
+    u_mean: f64,
+    e_s: f64,
+    tag: String,
+}
+
+fn read_points(path: &str) -> Vec<Point> {
+    let text = std::fs::read_to_string(path).unwrap_or_else(|e| panic!("{path}: {e}"));
+    text.lines()
+        .map(str::trim)
+        .filter(|l| !l.is_empty() && !l.starts_with('#'))
+        .map(|l| {
+            let f: Vec<&str> = l.split_whitespace().collect();
+            assert!(
+                f.len() >= 3,
+                "plane point line needs `u_mean e_s tag`: {l:?}"
+            );
+            Point {
+                u_mean: f[0].parse().expect("u_mean"),
+                e_s: f[1].parse().expect("e_s"),
+                tag: f[2].to_string(),
+            }
+        })
+        .collect()
+}
+
+struct LaneSummary {
+    tag: String,
+    wall: f64,
+    steps: usize,
+    subit: f64,
+    retries: usize,
+    uy_amp_mm: f64,
+    uy_mid_mm: f64,
+    frequency: Option<f64>,
+    drag_mid: f64,
+    lift_amp: f64,
+    death: Option<(usize, f64, String)>,
+}
+
+#[test]
+fn fsi2_overset_plane() {
+    let Ok(points_path) = std::env::var("RTX_FSI2O_PLANE_FILE") else {
+        return;
+    };
+    let dir = std::env::var("RTX_FSI2O_PLANE_DIR").unwrap_or_else(|_| ".".into());
+    let stack_mb: usize = std::env::var("RTX_FSI2O_PLANE_STACK_MB")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(64);
+    let device_on = std::env::var("RTX_FSI2O_MG_DEVICE").is_ok_and(|v| v == "1")
+        && std::env::var("RTX_FSI2O_MG_RB").is_ok_and(|v| v == "1");
+    assert!(
+        device_on,
+        "the plane needs RTX_FSI2O_MG_RB=1 RTX_FSI2O_MG_DEVICE=1 on a cuda build"
+    );
+    let points = read_points(&points_path);
+    let k = points.len();
+    assert!(k >= 1, "no plane points in {points_path}");
+
+    let config = OversetMarchConfig::from_env(
+        "FSI2O",
+        OversetMarchConfig {
+            ny: 41,
+            flag_nx: 35,
+            t_release: 6.0,
+            t_end: 7.0,
+            subcycle: 1,
+            tol_floor: 2e-4,
+            rtol: 1e-2,
+            stall_accept: 5.0,
+            max_subiterations: 12,
+            coupler: "aitken".into(),
+            reuse: 2,
+            initial_relaxation: 0.5,
+            c1_interface: false,
+            predictor: "structure".into(),
+            sweeps: 100,
+            max_rounds: 3,
+            csv_path: None,
+            trace_steps: 0,
+        },
+    );
+    let base = if std::env::var("RTX_FSI2O_CASE").as_deref() == Ok("FSI3") {
+        FSI3
+    } else {
+        FSI2
+    };
+    println!(
+        "  MARCH PLANE: {k} lanes in one process ({} base, ny {}, t_end {}, coupler {}, s = {}); points from {points_path}, CSVs under {dir}",
+        base.name, config.ny, config.t_end, config.coupler, config.subcycle
+    );
+    for (i, p) in points.iter().enumerate() {
+        println!(
+            "  lane {i}: {} u_mean {} (Re {:.0}) e_s {:.4e} (E/E0 {:.4})",
+            p.tag,
+            p.u_mean,
+            100.0 * p.u_mean,
+            p.e_s,
+            p.e_s / base.e_s
+        );
+    }
+
+    let t_plane = Instant::now();
+    let handles: Vec<_> = points
+        .into_iter()
+        .enumerate()
+        .map(|(i, p)| {
+            let mut cfg = config.clone();
+            cfg.csv_path = Some(format!("{dir}/{}.csv", p.tag));
+            let case = BenchmarkCase {
+                u_mean: p.u_mean,
+                e_s: p.e_s,
+                ..base
+            };
+            std::thread::Builder::new()
+                .name(format!("lane-{i}-{}", p.tag))
+                .stack_size(stack_mb << 20)
+                .spawn(move || {
+                    set_plane_lane(Some(i));
+                    let t0 = Instant::now();
+                    let r = run_march_overset(case, &cfg);
+                    let m = &r.result;
+                    let w = m.window(3.0);
+                    LaneSummary {
+                        tag: p.tag,
+                        wall: t0.elapsed().as_secs_f64(),
+                        steps: m.coupled_steps,
+                        subit: m.mean_subiterations,
+                        retries: m.retried_steps,
+                        uy_amp_mm: w.uy_amp * 1e3,
+                        uy_mid_mm: w.uy_mid * 1e3,
+                        frequency: w.frequency,
+                        drag_mid: w.drag_mid,
+                        lift_amp: w.lift_amp,
+                        death: r.death,
+                    }
+                })
+                .expect("lane thread")
+        })
+        .collect();
+    let mut summaries = Vec::with_capacity(k);
+    let mut failures = Vec::new();
+    for (i, h) in handles.into_iter().enumerate() {
+        match h.join() {
+            Ok(s) => summaries.push(s),
+            Err(_) => failures.push(i),
+        }
+    }
+    let plane_wall = t_plane.elapsed().as_secs_f64();
+    let (batches, served) = plane_counters();
+    println!(
+        "  MARCH PLANE done: {k} lanes in {plane_wall:.0} s wall; {batches} batched V-cycle launches served {served} lane calls ({:.2} lanes per batch)",
+        if batches > 0 {
+            served as f64 / batches as f64
+        } else {
+            0.0
+        }
+    );
+    for s in &summaries {
+        println!(
+            "  lane {}: {:.0} s wall, {} coupled steps, {:.1} subit/step, {} retries; last 3 s: uy(A) = {:.3} ± {:.3} mm, f = {:?} Hz, drag mid {:.2}, lift amp {:.2}; death {:?}",
+            s.tag,
+            s.wall,
+            s.steps,
+            s.subit,
+            s.retries,
+            s.uy_mid_mm,
+            s.uy_amp_mm,
+            s.frequency,
+            s.drag_mid,
+            s.lift_amp,
+            s.death
+        );
+    }
+    assert!(failures.is_empty(), "lane threads panicked: {failures:?}");
+    let dead: Vec<&str> = summaries
+        .iter()
+        .filter(|s| s.death.is_some())
+        .map(|s| s.tag.as_str())
+        .collect();
+    assert!(dead.is_empty(), "lanes died: {dead:?}");
+}