embedded3 S2-2b-ii: DeviceCg::refresh (fine operator per step, hierarchy every K steps, z zeroed before the scatter) + gate test; the flag driver's correctors 3 / inner 1e-3 (residual remedy measured 7.6e-9 on the moving circle); residual study test

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/poisson/device_cg.rs

@@ -177,6 +177,73 @@ impl DeviceCg {
         self.key.matches(problem, params)
     }
 
+    /// Refresh the FINE operator (cells, coefficients, links, components)
+    /// for `problem`, keeping the multigrid hierarchy as it was: the CG's
+    /// operator is exact, the preconditioner stale (a moving body's
+    /// operator changes little per step; the hierarchy's rebuild is the
+    /// cost). `z` is zeroed before every V-cycle scatter, so cells absent
+    /// from the stale hierarchy get no correction rather than a stale one.
+    pub fn refresh(&mut self, problem: &Problem, params: &MultigridParameters) {
+        let rt = runtime();
+        let fine = Level::<f64>::new(problem.clone());
+        let components = Components::find(problem, &fine.cells);
+        let singular_count = components.singular.iter().filter(|&&s| s).count();
+        assert!(
+            singular_count == 0 || components.members.len() == 1,
+            "DeviceCg::refresh: {} components with {} singular",
+            components.members.len(),
+            singular_count
+        );
+        let to_u32 = |v: &[usize]| {
+            v.iter()
+                .map(|&i| if i == usize::MAX { u32::MAX } else { i as u32 })
+                .collect::<Vec<u32>>()
+        };
+        let up_u = |v: &[u32]| -> CudaSlice<u32> {
+            rt.stream
+                .memcpy_stod(if v.is_empty() { &[0u32][..] } else { v })
+                .expect("upload")
+        };
+        let up_f = |v: &[f64]| -> CudaSlice<f64> { rt.stream.memcpy_stod(v).expect("upload") };
+        let lists = problem.link_lists();
+        let mut link_ptr = Vec::with_capacity(self.n + 1);
+        let mut link_idx = Vec::new();
+        let mut link_coef = Vec::new();
+        link_ptr.push(0u32);
+        for list in &lists {
+            for &(other, c) in list {
+                link_idx.push(other as u32);
+                link_coef.push(c);
+            }
+            link_ptr.push(link_idx.len() as u32);
+        }
+        self.n_cells = fine.cells.len();
+        self.n_blocks = self.n_cells.div_ceil(256).max(1);
+        if self.partial.len() < self.n_blocks {
+            self.partial = rt.stream.alloc_zeros::<f64>(self.n_blocks).expect("alloc");
+        }
+        self.cells = up_u(&to_u32(&fine.cells));
+        self.top = up_u(&to_u32(&fine.top));
+        self.bot = up_u(&to_u32(&fine.bot));
+        self.ae = up_f(&fine.ae);
+        self.aw = up_f(&fine.aw);
+        self.an = up_f(&fine.an);
+        self.as_ = up_f(&fine.as_);
+        self.at = up_f(&fine.at);
+        self.ab = up_f(&fine.ab);
+        self.ap = up_f(&fine.ap);
+        self.link_ptr = up_u(&link_ptr);
+        self.link_idx = up_u(&link_idx);
+        self.link_coef = up_f(if link_coef.is_empty() {
+            &[0.0]
+        } else {
+            &link_coef
+        });
+        self.singular = singular_count > 0;
+        self.active_host = fine.active.clone();
+        self.key = OperatorKey::of(problem, params);
+    }
+
     pub fn n_cells(&self) -> usize {
         self.n_cells
     }
@@ -367,6 +434,7 @@ impl DeviceCg {
         let rt = runtime();
         let k = kernels();
         let n_i = self.n_cells as i32;
+        rt.stream.memset_zeros(&mut self.z).expect("z = 0");
         unsafe {
             rt.stream
                 .launch_builder(&k.gather_f32)

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/step/device.rs

@@ -155,6 +155,8 @@ pub struct DeviceStep {
     initialized: bool,
     /// A static cut-cell mask's tables (item 9b), when the solver has one.
     cut: Option<cut::DeviceCut>,
+    /// Steps since the multigrid hierarchy was last rebuilt (moving bodies).
+    steps_since_hierarchy: usize,
 }
 
 impl DeviceStep {
@@ -210,6 +212,7 @@ impl DeviceStep {
             timers,
             initialized: false,
             cut,
+            steps_since_hierarchy: 0,
         }
     }
 

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/step/device/cut.rs

@@ -329,7 +329,26 @@ impl DeviceStep {
             fresh_cells = self.solver.rebuild_moving_mask(&mut field, dt, t_new);
             self.upload(&field);
             self.cut = DeviceCut::build(&self.solver, g, Phase::Projection, t_new);
-            self.cg = None;
+            // The operator: refreshed every step, the hierarchy every
+            // `RTX_E3_PRECOND_REFRESH` steps (default 10; 1 = rebuild always).
+            let every: usize = std::env::var("RTX_E3_PRECOND_REFRESH")
+                .ok()
+                .and_then(|v| v.parse().ok())
+                .unwrap_or(10)
+                .max(1);
+            self.steps_since_hierarchy += 1;
+            if self.steps_since_hierarchy >= every || self.cg_dt != dt {
+                self.cg = None;
+                self.steps_since_hierarchy = 0;
+            } else if let Some(cg) = self.cg.as_mut() {
+                let problem = self.solver.poisson_operator(g, dt);
+                let params = MultigridParameters {
+                    precision: self.solver.params.poisson_precision,
+                    smoother: self.solver.params.poisson_smoother,
+                    ..MultigridParameters::default()
+                };
+                cg.refresh(&problem, &params);
+            }
             rt.stream
                 .memcpy_dtod(&self.u, &mut self.u_star)
                 .expect("u*");