From 172a26dee4dea325b94aea63783b13686dae0750 Mon Sep 17 00:00:00 2001 From: Omar Sobh Date: Tue, 15 Sep 2026 22:25:52 -0500 Subject: [PATCH] =?UTF-8?q?PERF-2=20P0:=20intra-step=20profiler=20?= =?UTF-8?q?=E2=80=94=20StepTimers=20on=20the=20overset=20solver=20(RTX=5FP?= =?UTF-8?q?ROFILE;=20overlap=20build=20/=20predictors=20/=20patch=20BiCGST?= =?UTF-8?q?AB=20/=20background=20Poisson=20with=20its=20setup-vs-iterate?= =?UTF-8?q?=20split=20/=20round=20exchange=20/=20apply=20/=20end=20exchang?= =?UTF-8?q?e),=20PoissonSolution=20carries=20setup=5Fns=20and=20iterate=5F?= =?UTF-8?q?ns,=20the=20harness=20times=20advance=20/=20restore=20/=20snaps?= =?UTF-8?q?hot=20/=20load=20sampling=20/=20force=20/=20FEA=20predictor=20a?= =?UTF-8?q?nd=20prints=20the=20wall=20split=20of=20the=20coupled=20phase;?= =?UTF-8?q?=20no=20clock=20is=20read=20when=20profiling=20is=20off;=20the?= =?UTF-8?q?=20reclassified/step=20progress=20denominator=20fixed=20(was=20?= =?UTF-8?q?=C3=974)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YJPeT6WA2e7YvAnS875AHL --- .../incompressible/curvilinear/projection.rs | 2 +- .../solvers/incompressible/embedded/mod.rs | 11 ++- .../incompressible/embedded/projection.rs | 31 +++---- .../solvers/incompressible/embedded_body.rs | 6 +- .../rtx-cfd/src/solvers/incompressible/mod.rs | 6 +- .../src/solvers/incompressible/overset/mod.rs | 71 ++++++++++++++++ .../src/solvers/incompressible/piso.rs | 2 +- .../src/solvers/incompressible/poisson.rs | 11 +++ .../src/solvers/incompressible/polygon_sdf.rs | 6 +- .../rtx-fsi/tests/fsi2_harness/overset.rs | 80 ++++++++++++++++++- .../tests/fsi2_harness/overset_march.rs | 37 ++++++++- 11 files changed, 223 insertions(+), 40 deletions(-) diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/curvilinear/projection.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/curvilinear/projection.rs index 156e38c..21b372f 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/curvilinear/projection.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/curvilinear/projection.rs @@ -6,7 +6,7 @@ use super::{CurvilinearPisoSolver, PatchField, PressureSystem, SideBc, StepGeometry}; use crate::mesh::PatchSide; use crate::solvers::incompressible::sparse_bicgstab::{ - bicgstab_jacobi, project_mean, BicgstabResult, CsrMatrix, + BicgstabResult, CsrMatrix, bicgstab_jacobi, project_mean, }; impl CurvilinearPisoSolver { diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/mod.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/mod.rs index 1d9735b..0ecefec 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/mod.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/mod.rs @@ -41,7 +41,7 @@ mod projection; use super::ale::{AleBoundaries, SideBoundary}; use super::embedded_body::{EmbeddedBody, EmbeddedMask, FaceKind}; use super::poisson::{ - solve_multigrid_pcg, MgPrecision, MultigridParameters, PoissonProblem, PoissonSolverKind, + MgPrecision, MultigridParameters, PoissonProblem, PoissonSolverKind, solve_multigrid_pcg, }; use super::simple::ConvectionScheme; use super::{FlowField, SolverResult}; @@ -168,6 +168,9 @@ pub struct EmbeddedPisoSolver { /// (measured: at 1e-2 the first corrector's rounds never settled below /// a 1e-3 relative change — 20/20 rounds every step). inner_stop_factor: f64, + /// PERF-2 P0: Poisson `(setup ns, iterate ns, calls)` summed over + /// every multigrid-PCG solve (`docs/perf2_campaign.md`). + poisson_profile: std::cell::Cell<(u64, u64, u64)>, moving: bool, /// Mask hysteresis band in multiples of the min cell size (0 = off). mask_hysteresis: f64, @@ -194,6 +197,7 @@ impl EmbeddedPisoSolver { fringe: None, fringe_correction: Vec::new(), inner_stop_factor: 1e-2, + poisson_profile: std::cell::Cell::new((0, 0, 0)), moving: false, mask_hysteresis: 0.0, time: 0.0, @@ -290,6 +294,11 @@ impl EmbeddedPisoSolver { } /// Relative part of the pressure solve's inner stop (see the field). + /// The Poisson solves' `(setup ns, iterate ns, calls)` so far. + pub fn poisson_profile(&self) -> (u64, u64, u64) { + self.poisson_profile.get() + } + pub fn set_inner_stop_factor(&mut self, factor: f64) { self.inner_stop_factor = factor; } diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/projection.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/projection.rs index 89a8e70..967d7f8 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/projection.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded/projection.rs @@ -4,12 +4,12 @@ //! meshes before applying the correction once. use super::EmbeddedPisoSolver; +use crate::CfdResult; use crate::solvers::incompressible::ale::SideBoundary; use crate::solvers::incompressible::poisson::{ - solve_multigrid_pcg, MultigridParameters, PoissonProblem, PoissonSolverKind, + MultigridParameters, PoissonProblem, PoissonSolverKind, solve_multigrid_pcg, }; use crate::solvers::incompressible::{EmbeddedMask, FlowField}; -use crate::CfdResult; impl EmbeddedPisoSolver { /// The pressure-correction system of one projection as a @@ -304,6 +304,9 @@ impl EmbeddedPisoSolver { inner_stop, anchor_cell, ); + let (s0, i0, c0) = self.poisson_profile.get(); + self.poisson_profile + .set((s0 + solution.setup_ns, i0 + solution.iterate_ns, c0 + 1)); // Unconverged: fall back to the SOR sweeps for this projection // rather than apply a correction that did not reach the stop. multigrid_converged = solution.converged; @@ -337,44 +340,28 @@ impl EmbeddedPisoSolver { // prescribed: a domain side with velocity data, or a // non-fluid interior face. let ae = if i + 1 == nx { - if b.right == outlet { - ae_outlet - } else { - 0.0 - } + if b.right == outlet { ae_outlet } else { 0.0 } } else if self.u_is_fluid(j, i + 1) { ae_interior } else { 0.0 }; let aw = if i == 0 { - if b.left == outlet { - ae_outlet - } else { - 0.0 - } + if b.left == outlet { ae_outlet } else { 0.0 } } else if self.u_is_fluid(j, i) { ae_interior } else { 0.0 }; let an = if j + 1 == ny { - if b.top == outlet { - an_outlet - } else { - 0.0 - } + if b.top == outlet { an_outlet } else { 0.0 } } else if self.v_is_fluid(j + 1, i) { an_interior } else { 0.0 }; let as_ = if j == 0 { - if b.bottom == outlet { - an_outlet - } else { - 0.0 - } + if b.bottom == outlet { an_outlet } else { 0.0 } } else if self.v_is_fluid(j, i) { an_interior } else { diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded_body.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded_body.rs index 20ef8db..7e01a98 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded_body.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/embedded_body.rs @@ -322,11 +322,7 @@ pub fn polygon_signed_distance(vertices: &[(f64, f64)], x: f64, y: f64) -> f64 { } } let dist = dist2.sqrt(); - if inside { - -dist - } else { - dist - } + if inside { -dist } else { dist } } /// Velocity of the point on a closed polygon nearest to `(x, y)`, where diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/mod.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/mod.rs index c5a9fee..fee8162 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/mod.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/mod.rs @@ -53,13 +53,13 @@ pub use curvilinear::{ }; pub use embedded::{EmbeddedParameters, EmbeddedPisoSolver, EmbeddedResult, EmbeddedSolverState}; pub use embedded_body::{ - polygon_interface_velocity, polygon_signed_distance, EmbeddedBody, EmbeddedMask, FaceKind, - SurfaceForce, SurfaceSample, + EmbeddedBody, EmbeddedMask, FaceKind, SurfaceForce, SurfaceSample, polygon_interface_velocity, + polygon_signed_distance, }; pub use flow_field::FlowField; pub use overset::{ CellClass, MomentumResidual, OverlapMap, OversetField, OversetParameters, OversetPisoSolver, - OversetResult, OversetSolverState, ResidualBucket, + OversetResult, OversetSolverState, ResidualBucket, StepTimers, }; pub use piso::{PisoParameters, PisoResult, PisoSolver}; #[cfg(feature = "cuda")] diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/overset/mod.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/overset/mod.rs index e59280b..e2486b8 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/overset/mod.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/overset/mod.rs @@ -155,6 +155,35 @@ pub struct OversetSolverState { } /// The composite solver. +/// PERF-2 P0 (`docs/perf2_campaign.md`): where one composite step's wall +/// time goes, in nanoseconds, summed over the steps since construction. +/// Present only when `RTX_PROFILE` is set at construction; the phase +/// boundaries are the ones of [`OversetPisoSolver::advance`]. +#[derive(Debug, Clone, Default)] +pub struct StepTimers { + /// The overlap rebuild + reclassification + fringe re-stamp (phase 1). + pub overlap_build_ns: u64, + /// The patch predictor (phase 1b: operators, predict, pressure matrix). + pub predictor_patch_ns: u64, + /// The background predictor (phase 2). + pub predictor_bg_ns: u64, + /// Patch pressure solves (BiCGSTAB), summed over rounds. + pub patch_solve_ns: u64, + /// Background pressure solves (rhs assembly + multigrid-PCG), summed. + pub bg_solve_ns: u64, + /// The rest of a round: gathers, scatters, mean removal, Anderson. + pub round_exchange_ns: u64, + /// The correctors' applications on both meshes. + pub apply_ns: u64, + /// The end-of-step exchange, defects and clocks (phase 4). + pub end_exchange_ns: u64, + /// The whole of `advance`. + pub advance_ns: u64, + /// Steps and Schwarz rounds counted. + pub steps: u64, + pub rounds: u64, +} + pub struct OversetPisoSolver { background: EmbeddedPisoSolver, patch: CurvilinearPisoSolver, @@ -162,6 +191,8 @@ pub struct OversetPisoSolver { params: OversetParameters, grid: (usize, usize, f64, f64), pending: Option, + /// PERF-2 P0 timers (`RTX_PROFILE`), or `None` — no clock is read then. + timers: Option>, /// The first corrector's converged acceptor `p'` of the previous step: /// the temporal warm start (the correction is correlated step to step). acceptor_warm: Vec, @@ -293,10 +324,32 @@ impl OversetPisoSolver { params, grid, pending: None, + timers: std::env::var_os("RTX_PROFILE").map(|_| Box::default()), acceptor_warm: Vec::new(), }) } + /// The step timers, when profiling (`RTX_PROFILE`). + pub fn timers(&self) -> Option<&StepTimers> { + self.timers.as_deref() + } + + /// Charge the time since `start` to `slot` and return a fresh start; + /// `None` in, `None` out when not profiling (no clock is read). + fn lap( + &mut self, + start: Option, + slot: fn(&mut StepTimers) -> &mut u64, + ) -> Option { + match (start, self.timers.as_deref_mut()) { + (Some(t), Some(s)) => { + *slot(s) += t.elapsed().as_nanos() as u64; + Some(std::time::Instant::now()) + } + _ => None, + } + } + /// The background solver. pub fn background(&self) -> &EmbeddedPisoSolver { &self.background @@ -512,6 +565,8 @@ impl OversetPisoSolver { /// Advance both meshes one step of `dt`. pub async fn advance(&mut self, field: &mut OversetField, dt: f64) -> CfdResult { let (nx, ny, dx, dy) = self.grid; + let t_advance = self.timers.as_ref().map(|_| std::time::Instant::now()); + let mut t_lap = t_advance; // 1. If the patch moves, the overlap follows the NEXT mesh before any // predictor runs: the background is reclassified (fresh cells @@ -663,14 +718,17 @@ impl OversetPisoSolver { self.overlap.stamp_fringe_cells(&mut field.background.p, &p); } + t_lap = self.lap(t_lap, |s| &mut s.overlap_build_ns); // 1b. Patch predictor (swaps in the pending mesh). let patch_start = self.patch.begin_step(&mut field.patch, dt)?; + t_lap = self.lap(t_lap, |s| &mut s.predictor_patch_ns); // 2. Background predictor. let bg_start = self.background.begin_step(&mut field.background, dt)?; if let Some(old) = &old_class { self.trace_reclassification_source(field, old, dt); } + t_lap = self.lap(t_lap, |s| &mut s.predictor_bg_ns); // 3. Correctors: alternating Schwarz on the acceptor p' vector `a` // (patch solve with Dirichlet a → fringe p' → background solve → @@ -721,6 +779,7 @@ impl OversetPisoSolver { } None => patch_pc.iter_mut().for_each(|v| *v = 0.0), } + t_lap = self.lap(t_lap, |s| &mut s.patch_solve_ns); // Background with the fringe p' from the patch. let fringe_vals = self.overlap.fringe_cell_values(&patch_pc); self.background @@ -730,6 +789,7 @@ impl OversetPisoSolver { dt, corrector == 0 || round > 0, )?; + t_lap = self.lap(t_lap, |s| &mut s.bg_solve_ns); // Pin the composite level: the coupled p' problem is pure // Neumann (walls everywhere), so its constant mode is // undamped by the rounds and the warm start hands each @@ -744,6 +804,10 @@ impl OversetPisoSolver { step_scale = step_scale.max(g_max); let scale = step_scale.max(1e-300); let change = r.iter().fold(0.0_f64, |m, v| m.max(v.abs())); + t_lap = self.lap(t_lap, |s| &mut s.round_exchange_ns); + if let Some(s) = self.timers.as_deref_mut() { + s.rounds += 1; + } if trace_rounds { println!( " corrector {corrector} round {round}: |G(a) − a| {change:.3e} / step scale {scale:.3e} = {:.3e} (max|G(a)| {g_max:.3e})", @@ -785,6 +849,7 @@ impl OversetPisoSolver { } else { next }; + t_lap = self.lap(t_lap, |s| &mut s.round_exchange_ns); } schwarz_converged &= done; rounds.push(used); @@ -800,6 +865,7 @@ impl OversetPisoSolver { if corrector + 1 < self.params.corrector_steps.max(1) { field.background.copy_to_starred(); } + t_lap = self.lap(t_lap, |s| &mut s.apply_ns); } let patch_max_div = self.patch.max_divergence_pub(&field.patch.flux); @@ -822,6 +888,11 @@ impl OversetPisoSolver { patch_iterations, patch_converged, ); + let _ = self.lap(t_lap, |s| &mut s.end_exchange_ns); + if let (Some(t0), Some(s)) = (t_advance, self.timers.as_deref_mut()) { + s.advance_ns += t0.elapsed().as_nanos() as u64; + s.steps += 1; + } if (self.patch.time() - self.background.time()).abs() > 1e-12 * self.patch.time().abs().max(1.0) { diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/piso.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/piso.rs index 5ed4a87..28f53d2 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/piso.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/piso.rs @@ -38,7 +38,7 @@ //! instead of using the prescribed boundary faces that exist there. use super::poisson::{ - solve_multigrid_pcg, MgPrecision, MultigridParameters, PoissonProblem, PoissonSolverKind, + MgPrecision, MultigridParameters, PoissonProblem, PoissonSolverKind, solve_multigrid_pcg, }; use super::{BoundaryConditions, FlowField, IncompressibleSolver, SolverResult}; use crate::{CfdConfig, CfdResult}; diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/poisson.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/poisson.rs index a178b32..dbc41d9 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/poisson.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/poisson.rs @@ -310,6 +310,10 @@ pub struct PoissonSolution { pub residual: f64, /// `residual < tolerance` at exit. pub converged: bool, + /// Wall time of the setup (hierarchy, fine level, components) [ns]. + pub setup_ns: u64, + /// Wall time of the CG iteration (including the V-cycles) [ns]. + pub iterate_ns: u64, } /// Which inner solver a projection uses for its pressure-correction system. @@ -761,6 +765,7 @@ fn solve_pcg_with( "invalid PoissonProblem: {:?}", problem.validate() ); + let t_entry = std::time::Instant::now(); let mut hier = Hierarchy::::build(problem, params); let fine = Level::::new(problem.clone()); @@ -771,6 +776,8 @@ fn solve_pcg_with( iterations: 0, residual: 0.0, converged: true, + setup_ns: t_entry.elapsed().as_nanos() as u64, + iterate_ns: 0, }; } @@ -818,6 +825,8 @@ fn solve_pcg_with( |p: &[f64], r: &mut [f64], fine: &Level| -> f64 { fine.residual(&b, p, r) }; let anchor = anchor.filter(|&a| a < n && fine.active[a]); + let setup_ns = t_entry.elapsed().as_nanos() as u64; + let t_iter = std::time::Instant::now(); let finish = |p: &mut [f64], iterations: usize, residual: f64| { // Level of each singular component: the anchor's component is // shifted so p[anchor] == 0, every other singular component to mean @@ -839,6 +848,8 @@ fn solve_pcg_with( iterations, residual, converged: residual < tolerance, + setup_ns, + iterate_ns: t_iter.elapsed().as_nanos() as u64, } }; diff --git a/crates/specialized/rtx-cfd/src/solvers/incompressible/polygon_sdf.rs b/crates/specialized/rtx-cfd/src/solvers/incompressible/polygon_sdf.rs index 4f2b907..b8679e9 100644 --- a/crates/specialized/rtx-cfd/src/solvers/incompressible/polygon_sdf.rs +++ b/crates/specialized/rtx-cfd/src/solvers/incompressible/polygon_sdf.rs @@ -206,11 +206,7 @@ impl PolygonSdf { } let dist = dist2.sqrt(); - if inside { - -dist - } else { - dist - } + if inside { -dist } else { dist } } } diff --git a/crates/specialized/rtx-fsi/tests/fsi2_harness/overset.rs b/crates/specialized/rtx-fsi/tests/fsi2_harness/overset.rs index 56afef3..c4a0b91 100644 --- a/crates/specialized/rtx-fsi/tests/fsi2_harness/overset.rs +++ b/crates/specialized/rtx-fsi/tests/fsi2_harness/overset.rs @@ -218,6 +218,14 @@ pub struct OversetFluid { /// Patch regenerations and their wall time. pub regen_count: Cell, pub regen_seconds: Cell, + /// PERF-2 P0 wall-time buckets [s]: the composite step (`advance`), the + /// per-pass restore, the per-step snapshot, the load sampling and the + /// force measurement (`docs/perf2_campaign.md`). + pub t_advance: Cell, + pub t_restore: Cell, + pub t_snapshot: Cell, + pub t_sample: Cell, + pub t_force: Cell, /// Background cells reclassified, summed over every fluid step. pub reclassified_total: Cell, pub fresh_total: Cell, @@ -441,6 +449,11 @@ impl OversetFluid { sweeps, regen_count: Cell::new(0), regen_seconds: Cell::new(0.0), + t_advance: Cell::new(0.0), + t_restore: Cell::new(0.0), + t_snapshot: Cell::new(0.0), + t_sample: Cell::new(0.0), + t_force: Cell::new(0.0), reclassified_total: Cell::new(0), fresh_total: Cell::new(0), rounds_total: Cell::new(0), @@ -895,6 +908,7 @@ impl OversetFluid { /// One fluid step at the current wall. pub fn step(&mut self) -> CfdResult { + let t0 = std::time::Instant::now(); let r = match futures::executor::block_on( self.solver.advance(&mut self.field, self.dt_fluid), ) { @@ -919,6 +933,8 @@ impl OversetFluid { .set(self.rounds_total.get() + r.rounds.iter().sum::()); self.correctors_total .set(self.correctors_total.get() + r.rounds.len()); + self.t_advance + .set(self.t_advance.get() + t0.elapsed().as_secs_f64()); Ok(r) } @@ -987,11 +1003,14 @@ impl OversetFluid { /// Drag and lift on cylinder + flag from the patch's wall stress. pub fn measure_force(&self) -> (f64, f64) { + let t0 = std::time::Instant::now(); let f = self .solver .patch() .surface_force(&self.field.patch, PatchSide::Inner, self.solver.time()) .total(); + self.t_force + .set(self.t_force.get() + t0.elapsed().as_secs_f64()); (f[0], f[1]) } @@ -1000,6 +1019,14 @@ impl OversetFluid { /// Faces on the cylinder proper are skipped; the fillets' load goes /// to the nearest (clamped) root nodes. pub fn sample_load(&self, d: &[f64]) -> (Vec<(NodeId, Vector3)>, f64, usize) { + let t0 = std::time::Instant::now(); + let out = self.sample_load_inner(d); + self.t_sample + .set(self.t_sample.get() + t0.elapsed().as_secs_f64()); + out + } + + fn sample_load_inner(&self, d: &[f64]) -> (Vec<(NodeId, Vector3)>, f64, usize) { let mut faces = Vec::new(); let mut tractions: Vec> = Vec::new(); for (centre, normal, len, traction) in self.solver.patch().wall_tractions( @@ -1117,12 +1144,63 @@ impl OversetFluid { } pub fn snapshot(&self) -> (OversetSolverState, OversetField) { - (self.solver.snapshot(), self.field.clone()) + let t0 = std::time::Instant::now(); + let out = (self.solver.snapshot(), self.field.clone()); + self.t_snapshot + .set(self.t_snapshot.get() + t0.elapsed().as_secs_f64()); + out } pub fn restore(&mut self, saved: &(OversetSolverState, OversetField)) { + let t0 = std::time::Instant::now(); self.solver.restore(&saved.0); self.field = saved.1.clone(); + self.t_restore + .set(self.t_restore.get() + t0.elapsed().as_secs_f64()); + } + + /// PERF-2 P0: the solver's own split of `advance` (when `RTX_PROFILE` + /// is set) as one printable line, with the background Poisson's + /// setup / iterate split. + pub fn profile_line(&self) -> Option { + let t = self.solver.timers()?; + let (ps, pi, pc) = self.solver.background().poisson_profile(); + let s = |ns: u64| ns as f64 * 1e-9; + let adv = s(t.advance_ns).max(1e-300); + let pct = |ns: u64| 100.0 * s(ns) / adv; + Some(format!( + " advance split over {} steps ({} rounds), {:.0} s: overlap build {:.0} s ({:.1}%), predictor bg {:.0} s ({:.1}%), predictor patch {:.0} s ({:.1}%), bg Poisson {:.0} s ({:.1}%) [setup {:.0} s, iterate {:.0} s, {} solves], patch BiCGSTAB {:.0} s ({:.1}%), round exchange {:.0} s ({:.1}%), apply {:.0} s ({:.1}%), end exchange {:.0} s ({:.1}%), other {:.0} s", + t.steps, + t.rounds, + adv, + s(t.overlap_build_ns), + pct(t.overlap_build_ns), + s(t.predictor_bg_ns), + pct(t.predictor_bg_ns), + s(t.predictor_patch_ns), + pct(t.predictor_patch_ns), + s(t.bg_solve_ns), + pct(t.bg_solve_ns), + s(ps), + s(pi), + pc, + s(t.patch_solve_ns), + pct(t.patch_solve_ns), + s(t.round_exchange_ns), + pct(t.round_exchange_ns), + s(t.apply_ns), + pct(t.apply_ns), + s(t.end_exchange_ns), + pct(t.end_exchange_ns), + adv - s(t.overlap_build_ns + + t.predictor_bg_ns + + t.predictor_patch_ns + + t.bg_solve_ns + + t.patch_solve_ns + + t.round_exchange_ns + + t.apply_ns + + t.end_exchange_ns), + )) } pub fn time(&self) -> f64 { diff --git a/crates/specialized/rtx-fsi/tests/fsi2_harness/overset_march.rs b/crates/specialized/rtx-fsi/tests/fsi2_harness/overset_march.rs index 24602a8..e810a7f 100644 --- a/crates/specialized/rtx-fsi/tests/fsi2_harness/overset_march.rs +++ b/crates/specialized/rtx-fsi/tests/fsi2_harness/overset_march.rs @@ -377,6 +377,8 @@ pub fn run_march_overset(case: BenchmarkCase, config: &OversetMarchConfig) -> Ov f }); let (t_fluid, t_structure) = (std::cell::Cell::new(0.0_f64), std::cell::Cell::new(0.0_f64)); + // PERF-2 P0: the FEA predictor step (outside both buckets before). + let t_predictor = std::cell::Cell::new(0.0_f64); let prev_area = std::cell::Cell::new(fluid.borrow().shared.read().unwrap().area()); let save_from: f64 = std::env::var("RTX_FSI2O_SAVE_FROM") .ok() @@ -395,9 +397,11 @@ pub fn run_march_overset(case: BenchmarkCase, config: &OversetMarchConfig) -> Ov let v = extract_velocity(&flag_state); d_n.iter().zip(&v).map(|(d, v)| d + dt * v).collect() } else { + let ps = std::time::Instant::now(); flag.borrow_mut() .set_nodal_forces(&with_fict(&committed_nodal, &extract_accel(&flag_state))); let (predicted, _) = flag.borrow_mut().step(&flag_state).unwrap(); + t_predictor.set(t_predictor.get() + ps.elapsed().as_secs_f64()); extract(&predicted) }; if robin_alpha > 0.0 { @@ -582,7 +586,7 @@ pub fn run_march_overset(case: BenchmarkCase, config: &OversetMarchConfig) -> Ov total_subiterations as f64 / (step + 1) as f64, fl.rounds_total.get() as f64 / fl.correctors_total.get().max(1) as f64, fl.reclassified_total.get() as f64 - / (rigid_steps + (step + 1) * cfg.subcycle * 4).max(1) as f64, + / (rigid_steps + (step + 1) * cfg.subcycle).max(1) as f64, fl.regen_seconds.get(), t_fluid.get(), phase_start.elapsed().as_secs_f64() @@ -590,6 +594,37 @@ pub fn run_march_overset(case: BenchmarkCase, config: &OversetMarchConfig) -> Ov } } let fl = fluid.borrow(); + // PERF-2 P0: the wall split of the coupled phase (every bucket the + // harness can see; `other` is what none of them caught). + { + let wall = phase_start.elapsed().as_secs_f64().max(1e-300); + let pct = |x: f64| 100.0 * x / wall; + let (adv, res, snap, samp, force, regen) = ( + fl.t_advance.get(), + fl.t_restore.get(), + fl.t_snapshot.get(), + fl.t_sample.get(), + fl.t_force.get(), + fl.regen_seconds.get(), + ); + let (st, pr) = (t_structure.get(), t_predictor.get()); + let other = wall - (adv + res + snap + samp + force + regen + st + pr); + println!( + " wall split over the coupled phase ({wall:.0} s): fluid advance {adv:.0} s ({:.1}%), patch regeneration {regen:.0} s ({:.1}%), restore {res:.0} s ({:.1}%), snapshot {snap:.0} s ({:.1}%), load sampling {samp:.0} s ({:.1}%), force {force:.0} s ({:.1}%), structure {st:.0} s ({:.1}%), FEA predictor {pr:.0} s ({:.1}%), other {other:.0} s ({:.1}%)", + pct(adv), + pct(regen), + pct(res), + pct(snap), + pct(samp), + pct(force), + pct(st), + pct(pr), + pct(other) + ); + if let Some(line) = fl.profile_line() { + println!("{line}"); + } + } let final_state_finite = flag_state.displacement.iter().all(|v| v.is_finite()); let steps_done = times.len(); OversetMarchResult {