embedded3 PERF-3 P1-2: the predictor's device tables derived from the projection's (shared buffers move over; 282 -> 48 ms), the operator key lazy after a refresh (240 -> 0 ms), parallel active / owner maps — slab flag CSV byte-identical; RTX_E3_TABLES_REBUILD=1 = the full-rebuild reference

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/poisson/device_cg.rs

@@ -55,7 +55,9 @@ fn kernels() -> &'static CgKernels {
 
 /// The prepared operator on the device and the CG's work vectors.
 pub struct DeviceCg {
-    key: OperatorKey,
+    /// `None` after a `refresh` (PERF-3 P1-2): the key copies and hashes the whole operator, and
+    /// the refresh path never consults it — a refreshed solver simply never matches the cache.
+    key: Option<OperatorKey>,
     n: usize,
     nx: i32,
     n_cells: usize,
@@ -139,7 +141,7 @@ impl DeviceCg {
             link_ptr.push(link_idx.len() as u32);
         }
         Self {
-            key: OperatorKey::of(problem, params),
+            key: Some(OperatorKey::of(problem, params)),
             n,
             nx: problem.nx as i32,
             n_cells,
@@ -178,7 +180,9 @@ impl DeviceCg {
 
     /// Whether this prepared operator is the one for `problem` + `params`.
     pub fn matches(&self, problem: &Problem, params: &MultigridParameters) -> bool {
-        self.key.matches(problem, params)
+        self.key
+            .as_ref()
+            .is_some_and(|k| k.matches(problem, params))
     }
 
     /// Refresh the FINE operator (cells, coefficients, links, components)
@@ -251,7 +255,7 @@ impl DeviceCg {
         self.singular = singular_count > 0;
         self.active_host = fine.active.clone();
         let l_upload = lap.elapsed();
-        self.key = OperatorKey::of(problem, params);
+        self.key = None;
         let l_key = lap.elapsed();
         // The V-cycle's finest level follows the operator (its coarser
         // levels stay): the fine level alone, no hierarchy build.

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/step/device/cut.rs

@@ -184,7 +184,9 @@ impl DeviceCut {
             mask.wall_flux_table(body, t).0
         };
         let nc = g.cells();
+        use rayon::prelude::*;
         let active: Vec<i32> = (0..nc)
+            .into_par_iter()
             .map(|i| {
                 i32::from(if projection {
                     mask.cell_active(i)
@@ -202,6 +204,7 @@ impl DeviceCut {
         let ap_v: &[f64] = step.map_or(&cut.a_v, |a| &a.1);
         let ap_w: &[f64] = step.map_or(&cut.a_w, |a| &a.2);
         let owner: Vec<u32> = (0..nc)
+            .into_par_iter()
             .map(|i| mask.master(i).unwrap_or(i) as u32)
             .collect();
         // The slaves of every cell as a CSR by counting (no per-cell lists).
@@ -246,6 +249,51 @@ impl DeviceCut {
         })
     }
 
+    /// The PREDICTOR tables from the PROJECTION tables of the same mask and
+    /// time (PERF-3 P1-2): the wall distances, the centroid shifts, the
+    /// surface velocities, the wall flux and the merged-cell maps are the
+    /// same arrays — their device buffers move over; only the open flags, the
+    /// activity flags and the apertures (end-of-step instead of the step's)
+    /// are recomputed and uploaded. Identical to a fresh `build_with` in
+    /// every table.
+    pub(super) fn predictor_from(prev: Self, solver: &Solver, g: Grid) -> Option<Self> {
+        use rayon::prelude::*;
+        let mask = solver.mask()?;
+        let cut = mask.cut()?;
+        let rt = runtime();
+        let up_f = |v: &[f64]| -> CudaSlice<f64> {
+            rt.stream
+                .memcpy_stod(if v.is_empty() { &[0.0f64][..] } else { v })
+                .expect("upload")
+        };
+        let up_i = |v: &[i32]| -> CudaSlice<i32> { rt.stream.memcpy_stod(v).expect("upload") };
+        let (nx, ny, nz) = (g.nx, g.ny, g.nz);
+        let counts = [(nx + 1) * ny * nz, nx * (ny + 1) * nz, nx * ny * (nz + 1)];
+        let open = |c: usize| -> Vec<i32> {
+            (0..counts[c])
+                .into_par_iter()
+                .map(|idx| {
+                    let kind = match c {
+                        0 => mask.u_kind(idx),
+                        1 => mask.v_kind(idx),
+                        _ => mask.w_kind(idx),
+                    };
+                    i32::from(kind == FaceKind::Fluid)
+                })
+                .collect()
+        };
+        let active: Vec<i32> = (0..g.cells())
+            .into_par_iter()
+            .map(|i| i32::from(mask.is_fluid_cell(i)))
+            .collect();
+        Some(Self {
+            a: [up_f(&cut.a_u), up_f(&cut.a_v), up_f(&cut.a_w)],
+            open: [up_i(&open(0)), up_i(&open(1)), up_i(&open(2))],
+            active: up_i(&active),
+            ..prev
+        })
+    }
+
     fn ptrs(&self) -> E3CutPtrs {
         let rt = runtime();
         let s = &rt.stream;
@@ -526,9 +574,23 @@ impl DeviceStep {
         // The prescribed faces take the surface velocity (the host's
         // end-of-step impose), and the next predictor's tables.
         if self.solver.is_moving() {
-            let shared = self.cut.take().map(|c| c.ub_host);
-            self.cut =
-                DeviceCut::build_with(&self.solver, g, Phase::Predictor, t_new, shared.as_ref());
+            let lap_tables = Instant::now();
+            self.cut = match self.cut.take() {
+                Some(prev) if std::env::var("RTX_E3_TABLES_REBUILD").is_err() => {
+                    DeviceCut::predictor_from(prev, &self.solver, g)
+                }
+                // `RTX_E3_TABLES_REBUILD=1`: the full rebuild (the identity reference).
+                prev => {
+                    let shared = prev.map(|c| c.ub_host);
+                    DeviceCut::build_with(&self.solver, g, Phase::Predictor, t_new, shared.as_ref())
+                }
+            };
+            if std::env::var("RTX_E3_MOVING_PROFILE").is_ok() {
+                eprintln!(
+                    "    moving laps: predictor tables {:.0} ms",
+                    lap_tables.elapsed().as_secs_f64() * 1e3
+                );
+            }
             let cptrs = self.cut.as_ref().expect("cut").ptrs();
             for c in 0..3i32 {
                 unsafe {