embedded3 S2-2a/S2-3: moving bodies on the device (phased tables, host rebuild per step, impose kernel; gate test), the per-span cut wall route, the flag-wake driver and its geometry pre-flight

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/specialized/rtx-cfd/src/kernels/cuda/e3_cut.cu

@@ -307,3 +307,23 @@ extern "C" __global__ void e3_cut_add_p(E3Params g, E3Ptrs f, E3Cut m)
     if (!m.active[t]) return;
     f.p[t] += f.pp[m.owner[t]];
 }
+
+/* The prescribed interior faces of component c take the surface velocity
+ * (the host `impose` for a cut mask; `open` = the instantaneous kinds). */
+extern "C" __global__ void e3_cut_impose(E3Params g, E3Ptrs f, E3Cut m, int c)
+{
+    int t = blockIdx.x * blockDim.x + threadIdx.x;
+    int ni = c == 0 ? g.nx + 1 : g.nx;
+    int nj = c == 1 ? g.ny + 1 : g.ny;
+    int nk = c == 2 ? g.nz + 1 : g.nz;
+    if (t >= ni * nj * nk) return;
+    int i = t % ni; int j = (t / ni) % nj; int k = t / (ni * nj);
+    if (c == 0 && (i == 0 || i == g.nx)) return;
+    if (c == 1 && (j == 0 || j == g.ny)) return;
+    if (c == 2) { if (g.periodic_z) { if (k == g.nz) return; } else if (k == 0 || k == g.nz) return; }
+    const int* open = c == 0 ? m.open_u : (c == 1 ? m.open_v : m.open_w);
+    if (open[t]) return;
+    const double* ubt = c == 0 ? m.ub_u : (c == 1 ? m.ub_v : m.ub_w);
+    double* out = c == 0 ? f.u : (c == 1 ? f.v : f.w);
+    out[t] = ubt[t];
+}

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/cutwall.rs

@@ -472,6 +472,22 @@ impl Mask {
         Some([p[0] + s[0], p[1] + s[1], p[2] + s[2]])
     }
 
+    /// The cut-cell load route restricted to the cells (and faces) of the
+    /// planes `k0..k1`, divided by the slab's thickness: the load per unit
+    /// span on a body's mid-section.
+    pub fn cut_wall_force_per_span(
+        &self,
+        body: &Body,
+        f: &Field,
+        mu: f64,
+        t: f64,
+        (k0, k1): (usize, usize),
+    ) -> Option<[f64; 3]> {
+        let (p, s) = self.cut_wall_force_parts_in(body, f, mu, t, Some((k0, k1)))?;
+        let lz = (k1 - k0) as f64 * self.grid.dz;
+        Some([(p[0] + s[0]) / lz, (p[1] + s[1]) / lz, (p[2] + s[2]) / lz])
+    }
+
     /// The cut-cell load route split into its pressure and shear parts.
     pub fn cut_wall_force_parts(
         &self,
@@ -479,14 +495,27 @@ impl Mask {
         f: &Field,
         mu: f64,
         t: f64,
+    ) -> Option<([f64; 3], [f64; 3])> {
+        self.cut_wall_force_parts_in(body, f, mu, t, None)
+    }
+
+    fn cut_wall_force_parts_in(
+        &self,
+        body: &Body,
+        f: &Field,
+        mu: f64,
+        t: f64,
+        planes: Option<(usize, usize)>,
     ) -> Option<([f64; 3], [f64; 3])> {
         let cut = self.cut.as_ref()?;
         let g = self.grid;
         let (nx, ny, nz) = (g.nx, g.ny, g.nz);
+        let (k0, k1) = planes.unwrap_or((0, nz));
         let mut pressure = [0.0; 3];
         let mut force = [0.0; 3];
         for (idx, w) in cut.wall.iter().enumerate() {
-            if self.cell_fluid[idx] {
+            let k = g.kji(idx).0;
+            if self.cell_fluid[idx] && k >= k0 && k < k1 {
                 for c in 0..3 {
                     pressure[c] += f.p[idx] * w[c];
                 }
@@ -497,9 +526,9 @@ impl Mask {
         let w_range = if self.periodic_z { 0..nz } else { 1..nz };
         for c in 0..3 {
             let (ir, jr, kr) = match c {
-                0 => (1..nx, 0..ny, 0..nz),
-                1 => (0..nx, 1..ny, 0..nz),
-                _ => (0..nx, 0..ny, w_range.clone()),
+                0 => (1..nx, 0..ny, k0..k1),
+                1 => (0..nx, 1..ny, k0..k1),
+                _ => (0..nx, 0..ny, w_range.start.max(k0)..w_range.end.min(k1)),
             };
             for k in kr {
                 for j in jr.clone() {

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/step/device.rs

@@ -121,6 +121,7 @@ pub struct StepTimers {
     pub predictor_ns: u64,
     pub poisson_ns: u64,
     pub apply_ns: u64,
+    /// The moving body's host rebuild (mask, tables, transfers) per step.
     pub transfer_ns: u64,
     pub steps: u64,
     pub cg_iterations: u64,
@@ -181,7 +182,7 @@ impl DeviceStep {
         let timers = std::env::var("RTX_PROFILE")
             .is_ok()
             .then(StepTimers::default);
-        let cut = cut::DeviceCut::build(&solver, grid);
+        let cut = cut::DeviceCut::build(&solver, grid, cut::Phase::Predictor, solver.time());
         Self {
             solver,
             grid,

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/step/device/cut.rs

@@ -6,9 +6,11 @@
 
 use super::{DeviceStep, E3Params, E3Ptrs, StepResult};
 use crate::solvers::incompressible::embedded3::Grid;
+use crate::solvers::incompressible::embedded3::field::Field;
 use crate::solvers::incompressible::embedded3::poisson::device::{cfg, load_module, runtime};
 use crate::solvers::incompressible::embedded3::poisson::device_cg::DeviceCg;
 use crate::solvers::incompressible::embedded3::step::Solver;
+use crate::solvers::incompressible::embedded3::wall::FaceKind;
 use crate::solvers::incompressible::poisson::MultigridParameters;
 use cudarc::driver::{
     CudaFunction, CudaModule, CudaSlice, DevicePtr, DeviceRepr, PushKernelArg, ValidAsZeroBits,
@@ -28,6 +30,7 @@ struct CutKernels {
     fold: CudaFunction,
     correct: CudaFunction,
     add_p: CudaFunction,
+    impose: CudaFunction,
 }
 
 static CUT_KERNELS_ONCE: OnceLock<CutKernels> = OnceLock::new();
@@ -42,6 +45,7 @@ fn cut_kernels() -> &'static CutKernels {
             fold: f("e3_cut_fold"),
             correct: f("e3_cut_correct"),
             add_p: f("e3_cut_add_p"),
+            impose: f("e3_cut_impose"),
             _module: module,
         }
     })
@@ -71,12 +75,23 @@ pub(super) struct DeviceCut {
     pub(super) merged: usize,
 }
 
+/// Which phase the tables serve: the predictor reads the instantaneous
+/// apertures and kinds of the mask at its time; the projection reads the
+/// step-averaged apertures and the space-time classification.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub(super) enum Phase {
+    Predictor,
+    Projection,
+}
+
 impl DeviceCut {
-    /// The tables of the solver's cut mask (`None` without one).
-    pub(super) fn build(solver: &Solver, g: Grid) -> Option<Self> {
+    /// The tables of the solver's cut mask (`None` without one) for
+    /// `phase`, the surface velocities at the mask's time `t`.
+    pub(super) fn build(solver: &Solver, g: Grid, phase: Phase, t: f64) -> Option<Self> {
         let mask = solver.mask()?;
         let cut = mask.cut()?;
         let body = solver.body()?;
+        let projection = phase == Phase::Projection;
         let rt = runtime();
         let (nx, ny, nz) = (g.nx, g.ny, g.nz);
         let h = [g.dx, g.dy, g.dz];
@@ -112,11 +127,20 @@ impl DeviceCut {
                             (j as f64 + if c == 1 { 0.0 } else { 0.5 }) * h[1],
                             (k as f64 + if c == 2 { 0.0 } else { 0.5 }) * h[2],
                         ];
-                        ubc[idx] = mask.surface_velocity_at(body, x, c, 0.0);
-                        opc[idx] = i32::from(match c {
-                            0 => mask.u_open(idx),
-                            1 => mask.v_open(idx),
-                            _ => mask.w_open(idx),
+                        ubc[idx] = mask.surface_velocity_at(body, x, c, t);
+                        opc[idx] = i32::from(if projection {
+                            match c {
+                                0 => mask.u_open(idx),
+                                1 => mask.v_open(idx),
+                                _ => mask.w_open(idx),
+                            }
+                        } else {
+                            let kind = match c {
+                                0 => mask.u_kind(idx),
+                                1 => mask.v_kind(idx),
+                                _ => mask.w_kind(idx),
+                            };
+                            kind == FaceKind::Fluid
                         });
                     }
                 }
@@ -124,9 +148,31 @@ impl DeviceCut {
             ub[c] = ubc;
             open[c] = opc;
         }
-        let (wall_flux, _) = mask.wall_flux_table(body, 0.0);
+        // The solver's current table (the GCL table on a moving body) when
+        // it has one, else the static porous table.
+        let wall_flux: Vec<f64> = if solver.wall_fluxes().len() == g.cells() {
+            solver.wall_fluxes().to_vec()
+        } else {
+            mask.wall_flux_table(body, t).0
+        };
         let nc = g.cells();
-        let active: Vec<i32> = (0..nc).map(|i| i32::from(mask.cell_active(i))).collect();
+        let active: Vec<i32> = (0..nc)
+            .map(|i| {
+                i32::from(if projection {
+                    mask.cell_active(i)
+                } else {
+                    mask.is_fluid_cell(i)
+                })
+            })
+            .collect();
+        let step = if projection {
+            mask.step_apertures()
+        } else {
+            None
+        };
+        let ap_u: &[f64] = step.map_or(&cut.a_u, |a| &a.0);
+        let ap_v: &[f64] = step.map_or(&cut.a_v, |a| &a.1);
+        let ap_w: &[f64] = step.map_or(&cut.a_w, |a| &a.2);
         let owner: Vec<u32> = (0..nc)
             .map(|i| mask.master(i).unwrap_or(i) as u32)
             .collect();
@@ -146,7 +192,7 @@ impl DeviceCut {
             fold_ptr.push(fold_idx.len() as u32);
         }
         Some(Self {
-            a: [up_f(&cut.a_u), up_f(&cut.a_v), up_f(&cut.a_w)],
+            a: [up_f(ap_u), up_f(ap_v), up_f(ap_w)],
             d: [up_f(&cut.d_u), up_f(&cut.d_v), up_f(&cut.d_w)],
             ub: [up_f(&ub[0]), up_f(&ub[1]), up_f(&ub[2])],
             wall_flux: up_f(&wall_flux),
@@ -246,6 +292,30 @@ impl DeviceStep {
             .expect("w*");
         rt.stream.synchronize().expect("sync");
         let t_pred = t0.elapsed();
+        // A moving body: the mask at the end-of-step geometry on the host
+        // (the predicted field down, the rebuilt one up), the projection
+        // tables and the operator rebuilt.
+        let t_rebuild = Instant::now();
+        let mut fresh_cells = 0;
+        if self.solver.is_moving() {
+            let mut field = Field::new(g);
+            self.download(&mut field);
+            fresh_cells = self.solver.rebuild_moving_mask(&mut field, dt, t_new);
+            self.upload(&field);
+            self.cut = DeviceCut::build(&self.solver, g, Phase::Projection, t_new);
+            self.cg = None;
+            rt.stream
+                .memcpy_dtod(&self.u, &mut self.u_star)
+                .expect("u*");
+            rt.stream
+                .memcpy_dtod(&self.v, &mut self.v_star)
+                .expect("v*");
+            rt.stream
+                .memcpy_dtod(&self.w, &mut self.w_star)
+                .expect("w*");
+        }
+        let cptrs = self.cut.as_ref().expect("cut").ptrs();
+        let rebuild = t_rebuild.elapsed();
         if self.cg.is_none() || self.cg_dt != dt {
             let problem = self.solver.poisson_operator(g, dt);
             let params = MultigridParameters {
@@ -361,16 +431,39 @@ impl DeviceStep {
                 .memcpy_dtod(&self.w, &mut self.w_star)
                 .expect("w*");
         }
+        // The prescribed faces take the surface velocity (the host's
+        // end-of-step impose), and the next predictor's tables.
+        if self.solver.is_moving() {
+            self.cut = DeviceCut::build(&self.solver, g, Phase::Predictor, t_new);
+            let cptrs = self.cut.as_ref().expect("cut").ptrs();
+            for c in 0..3i32 {
+                unsafe {
+                    rt.stream
+                        .launch_builder(&k.impose)
+                        .arg(&prm)
+                        .arg(&ptrs)
+                        .arg(&cptrs)
+                        .arg(&c)
+                        .launch(cfg(counts[c as usize]))
+                        .expect("e3_cut_impose");
+                }
+            }
+            if prm.periodic_z != 0 {
+                self.launch_sides(prm, &ptrs, 0);
+            }
+            rt.stream.synchronize().expect("sync");
+        }
         self.solver.set_time(t_new);
         if let Some(tm) = self.timers.as_mut() {
             tm.predictor_ns += t_pred.as_nanos() as u64;
             tm.poisson_ns += t_poisson.as_nanos() as u64;
             tm.apply_ns += t_apply.as_nanos() as u64;
+            tm.transfer_ns += rebuild.as_nanos() as u64;
             tm.steps += 1;
             tm.cg_iterations += cg_iterations as u64;
         }
         StepResult {
-            fresh_cells: 0,
+            fresh_cells,
             converged: final_residual < self.solver.params.tolerance,
             corrector_steps_performed: total,
             final_residual,

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/step/mod.rs

@@ -463,6 +463,56 @@ impl Solver {
         self.initialized = true;
     }
 
+    /// Whether the body moves (the mask is rebuilt every step).
+    #[must_use]
+    pub fn is_moving(&self) -> bool {
+        self.moving
+    }
+
+    /// The current step's compatible wall-flux table (cut wall).
+    #[must_use]
+    pub fn wall_fluxes(&self) -> &[f64] {
+        &self.wall_fluxes
+    }
+
+    /// The moving body's mask at the end-of-step geometry `t_new`: the
+    /// pressure of the cells that just became fluid refilled from their
+    /// neighbours (fluid in both masks), the new mask's prescribed and
+    /// ghost values imposed from the previous corrected field, the
+    /// step-averaged apertures and the GCL wall-flux table (cut wall).
+    /// Returns the fresh-cell count. `field` holds the predicted field.
+    pub fn rebuild_moving_mask(&mut self, field: &mut Field, dt: f64, t_new: f64) -> usize {
+        let Some(body) = &self.body else {
+            return 0;
+        };
+        let mut fresh_cells = 0;
+        let mut new_mask = self.build_mask(body, field.grid, t_new);
+        if let Some(old_mask) = &self.mask {
+            fresh_cells = refill_fresh_cells(old_mask, &new_mask, field);
+            new_mask.set_step_apertures(old_mask);
+        }
+        new_mask.impose_from(
+            body,
+            &field.u_old,
+            &field.v_old,
+            &field.w_old,
+            &mut field.u,
+            &mut field.v,
+            &mut field.w,
+            t_new,
+        );
+        if new_mask.cut().is_some() {
+            let (table, correction) = match &self.mask {
+                Some(old_mask) => new_mask.gcl_flux_table(old_mask, dt),
+                None => new_mask.wall_flux_table(body, t_new),
+            };
+            self.wall_fluxes = table;
+            self.last_ghost_correction = correction;
+        }
+        self.mask = Some(new_mask);
+        fresh_cells
+    }
+
     /// One step of `dt`: predictor, correctors, clock.
     pub fn advance(&mut self, field: &mut Field, dt: f64) -> StepResult {
         assert!(
@@ -477,39 +527,12 @@ impl Solver {
         field.update_old_values();
         self.momentum_predictor(field, dt, t_old);
         self.apply_boundary_normals(field, t_new);
-        // A moving body: the mask at the end-of-step geometry, the pressure
-        // of the cells that just became fluid refilled from their
-        // neighbours (fluid in both masks), the new mask's prescribed and
-        // ghost values imposed from the previous corrected field.
-        let mut fresh_cells = 0;
-        if self.moving {
-            if let Some(body) = &self.body {
-                let mut new_mask = self.build_mask(body, field.grid, t_new);
-                if let Some(old_mask) = &self.mask {
-                    fresh_cells = refill_fresh_cells(old_mask, &new_mask, field);
-                    new_mask.set_step_apertures(old_mask);
-                }
-                new_mask.impose_from(
-                    body,
-                    &field.u_old,
-                    &field.v_old,
-                    &field.w_old,
-                    &mut field.u,
-                    &mut field.v,
-                    &mut field.w,
-                    t_new,
-                );
-                if new_mask.cut().is_some() {
-                    let (table, correction) = match &self.mask {
-                        Some(old_mask) => new_mask.gcl_flux_table(old_mask, dt),
-                        None => new_mask.wall_flux_table(body, t_new),
-                    };
-                    self.wall_fluxes = table;
-                    self.last_ghost_correction = correction;
-                }
-                self.mask = Some(new_mask);
-            }
-        }
+        // A moving body: the mask at the end-of-step geometry.
+        let fresh_cells = if self.moving {
+            self.rebuild_moving_mask(field, dt, t_new)
+        } else {
+            0
+        };
         field.copy_to_starred();
         let mut cut_correction = None;
         if let (Some(body), Some(mask)) = (&self.body, &self.mask) {

crates/specialized/rtx-cfd/src/solvers/incompressible/embedded3/wall.rs

@@ -505,6 +505,12 @@ impl Mask {
         })
     }
 
+    /// The step-averaged apertures of a moving cut wall (`None` at rest).
+    #[must_use]
+    pub fn step_apertures(&self) -> Option<&(Vec<f64>, Vec<f64>, Vec<f64>)> {
+        self.step_apertures.as_ref()
+    }
+
     /// The master of a virtually merged small cell.
     #[inline]
     #[must_use]