embedded3 R6-2: the moving mask's classification on the device (RTX_E3_MASK_DEVICE=1, needs RTX_E3_GEOM_DEVICE=1)

- e3_mask.cu: the changed set (touched by either build, dilated), the faces of the changed cells, per changed cell the fluid flag / space-time activity / GCL entry and the merging master, per face the kind, the step aperture and open flag and the open-part centroid shift (compute_face_shifts' cv geometry), the GCL table's wall cells / areas / table with the correction, the merged cells' CSR (slave ranks from the master's distinct face neighbours), the imposition band's faces; block scans for ascending stream compaction and the CSR's exclusive scan. fp64, FMA contraction off. - step/device/mask.rs DeviceMask: snapshots the previous apertures / volumes / evaluated cells before the R6-1 geometry kernels, runs the classification into DeviceCut's persistent tables (a, open, open_pred, active, active_pred, owner, shift, wall_flux, fold_ptr/fold_idx) and returns the compact values (MaskUpdate); the GCL sums in ascending order on the host. DeviceCut::update_after_device_mask: only the band's surface velocities remain (band list compacted on the device). - maskupdate.rs: Mask::from_update — the host mirror from the old mask's step arrays and merging map (moved) and the mask retired a step earlier (instantaneous arrays, pooled), the compact values scattered; incremental fluid count / anchor / side check; the fresh-cell refill over the changed set. - RTX_E3_BAND_CHECK=1 with the knob: the host rebuild on copies (from_cut, face shifts, step apertures, merging, GCL, refill) against the mirror, bit for bit, and every device table against a full build. - Knob off: the host path unchanged (Mask::from_parts, Solver::configure_mask and retire_mask are the same statements); profile laps under RTX_E3_MOVING_PROFILE only. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-24 21:00:25 -05:00
parent f5f0ffdd2a
commit 0150f79a4a
10 changed files with 2024 additions and 116 deletions
@@ -13,13 +13,13 @@
 //! the computed one as the aperture closes (the shear coefficient grows as
 //! `1/A_f`), which is what makes the wall smooth in the interface position.

-use super::Grid;
 use super::body::Body;
 use super::cut::CutGeometry;
 use super::exchange::in_load_window;
 use super::field::Field;
 use super::step::{Boundaries, Side};
 use super::wall::{FaceKind, Mask};
+use super::Grid;

 /// The inertia floor: the momentum volume's fraction in the time
 /// derivative is at least this.
@@ -32,6 +32,12 @@ pub(super) const DISTANCE_FLOOR_FINE: f64 = 0.01;
 /// step) stays below this shares its pressure unknown with a neighbour.
 pub(super) const MERGE_FRACTION: f64 = 0.1;

+/// `RTX_E3_MOVING_PROFILE` (read once).
+pub(super) fn profile_on() -> bool {
+    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ON.get_or_init(|| std::env::var("RTX_E3_MOVING_PROFILE").is_ok())
+}
+
 /// Lattice addressing of faces and cells with the periodic wrap in z as
 /// data: a face of component `c` at `p = [i, j, k]` (its own coordinate is
 /// the face index, the others the cell's), a cell at `[i, j, k]`.
@@ -138,6 +144,7 @@ impl Mask {
    /// R6-1: the mask of a cut geometry built elsewhere (the device's
    /// `DeviceGeom` mirror); `build_cut_from` is this after `CutGeometry::build_from`.
    pub fn from_cut(cut: CutGeometry, g: Grid, b: Boundaries) -> Result<Self, String> {
+        let lap = std::time::Instant::now();
        let (nx, ny, nz) = (g.nx, g.ny, g.nz);
        let periodic = b.z0 == Side::Periodic;
        let allowed = |side: Side| matches!(side, Side::Velocity | Side::Periodic | Side::SlipWall);
@@ -148,7 +155,9 @@ impl Mask {
        let nxy = nx * ny;
        let cell_fluid: Vec<bool> = cut.vol.par_iter().map(|&v| v > 0.0).collect();
        let fluid_cells = cell_fluid.par_iter().filter(|&&f| f).count();
-        let anchor = (0..g.cells()).into_par_iter().find_first(|&idx| cell_fluid[idx]);
+        let anchor = (0..g.cells())
+            .into_par_iter()
+            .find_first(|&idx| cell_fluid[idx]);
        let touching = (0..g.cells()).into_par_iter().find_first(|&idx| {
            if cell_fluid[idx] {
                return false;
@@ -182,24 +191,69 @@ impl Mask {
            .into_par_iter()
            .map(|f| {
                let i = f % (nx + 1);
-                if i == 0 || i == nx { FaceKind::Fluid } else { kind(cut.a_u[f]) }
+                if i == 0 || i == nx {
+                    FaceKind::Fluid
+                } else {
+                    kind(cut.a_u[f])
+                }
            })
            .collect();
        let v_kind: Vec<FaceKind> = (0..g.n_vfaces())
            .into_par_iter()
            .map(|f| {
                let j = (f / nx) % (ny + 1);
-                if j == 0 || j == ny { FaceKind::Fluid } else { kind(cut.a_v[f]) }
+                if j == 0 || j == ny {
+                    FaceKind::Fluid
+                } else {
+                    kind(cut.a_v[f])
+                }
            })
            .collect();
        let w_kind: Vec<FaceKind> = (0..g.n_wfaces())
            .into_par_iter()
            .map(|f| {
                let k = f / nxy;
-                if !periodic && (k == 0 || k == nz) { FaceKind::Fluid } else { kind(cut.a_w[f]) }
+                if !periodic && (k == 0 || k == nz) {
+                    FaceKind::Fluid
+                } else {
+                    kind(cut.a_w[f])
+                }
            })
            .collect();
-        let mut mask = Self {
+        let mut mask = Self::from_parts(
+            g,
+            periodic,
+            cell_fluid,
+            [u_kind, v_kind, w_kind],
+            anchor,
+            fluid_cells,
+            cut,
+        );
+        let l_class = lap.elapsed();
+        mask.compute_merging(None);
+        if profile_on() {
+            eprintln!(
+                "    from_cut laps: classification {:.0} ms, merging {:.0} ms",
+                l_class.as_secs_f64() * 1e3,
+                (lap.elapsed() - l_class).as_secs_f64() * 1e3
+            );
+        }
+        Ok(mask)
+    }
+
+    /// The mask struct of a cut classification (every closure flag at its
+    /// default; the caller sets them).
+    pub(super) fn from_parts(
+        g: Grid,
+        periodic: bool,
+        cell_fluid: Vec<bool>,
+        kinds: [Vec<FaceKind>; 3],
+        anchor: usize,
+        fluid_cells: usize,
+        cut: CutGeometry,
+    ) -> Self {
+        let [u_kind, v_kind, w_kind] = kinds;
+        Self {
            grid: g,
            periodic_z: periodic,
            cell_fluid,
@@ -234,9 +288,7 @@ impl Mask {
            grad_weights: None,
            diffusion_centroid: false,
            face_shifts: None,
-        };
-        mask.compute_merging(None);
-        Ok(mask)
+        }
    }

    /// The virtual merging map: a small cell (fraction < `MERGE_FRACTION`
@@ -306,6 +358,7 @@ impl Mask {
        let (Some(cut), Some(old_cut)) = (self.cut.as_ref(), old.cut.as_ref()) else {
            return;
        };
+        let lap = std::time::Instant::now();
        let n = inner.len() + 1;
        let w_end = 0.5 / n as f64;
        let w_in = 1.0 / n as f64;
@@ -336,12 +389,17 @@ impl Mask {
                    || (periodic && nz > 1 && k == 0 && t(idx + (nz - 1) * nxy))
            })
            .collect();
+        let l_changed = lap.elapsed();
        // P1-5 (j): with the trapezoid alone and a previous step's arrays, a
        // face of no changed cell keeps its step aperture (its corners were
        // untouched in both builds, so αⁿ⁻¹ = αⁿ = αⁿ⁺¹): the old mask's
        // arrays move over and only the changed cells' faces and the
        // changed cells' activity are recomputed, with the same expressions.
-        let prev = if inner.is_empty() { old.step_apertures.take().zip(old.step_open.take()) } else { None };
+        let prev = if inner.is_empty() {
+            old.step_apertures.take().zip(old.step_open.take())
+        } else {
+            None
+        };
        if let Some(((mut au, mut av, mut aw), (mut ou, mut ov, mut ow, mut active))) = prev {
            let mix = |a: &[f64], b: &[f64], f: usize| w_end * (a[f] + b[f]);
            for &idx in &changed {
@@ -387,7 +445,16 @@ impl Mask {
            self.step_open = Some((open(&au), open(&av), open(&aw), active));
            self.step_apertures = Some((au, av, aw));
        }
+        let l_apert = lap.elapsed();
        self.compute_merging(Some(old));
+        if profile_on() {
+            eprintln!(
+                "    step-aperture laps: changed set {:.0} ms, apertures {:.0} ms, merging {:.0} ms",
+                l_changed.as_secs_f64() * 1e3,
+                (l_apert - l_changed).as_secs_f64() * 1e3,
+                (lap.elapsed() - l_apert).as_secs_f64() * 1e3
+            );
+        }
        self.changed_cells = Some(changed);
    }

@@ -449,7 +516,11 @@ impl Mask {
        }
        // S2-7: the sides' own apertures instead of the whole-face averages.
        if self.cv_sides_exact {
-            if let Some(exact) = self.cut.as_ref().and_then(|cut| self.exact_cv_sides(cut, c, p)) {
+            if let Some(exact) = self
+                .cut
+                .as_ref()
+                .and_then(|cut| self.exact_cv_sides(cut, c, p))
+            {
                ap = exact;
            }
        }
@@ -715,6 +786,7 @@ impl Mask {
        let (Some(cut), Some(old_cut)) = (self.cut.as_ref(), old.cut.as_ref()) else {
            return (table, 0.0);
        };
+        let lap = std::time::Instant::now();
        let g = self.grid;
        let dv = g.dx * g.dy * g.dz;
        let (mut net, mut area) = (0.0, 0.0);
@@ -759,6 +831,7 @@ impl Mask {
                vn1 - vn
            );
        }
+        let l_sums = lap.elapsed();
        let correction = if area > 0.0 { net / area } else { 0.0 };
        if correction != 0.0 {
            // Every cell with a wall (the others subtract exactly 0).
@@ -770,6 +843,13 @@ impl Mask {
                }
            }
        }
+        if profile_on() {
+            eprintln!(
+                "    gcl laps: lists + sums {:.0} ms, correction {:.0} ms",
+                l_sums.as_secs_f64() * 1e3,
+                (lap.elapsed() - l_sums).as_secs_f64() * 1e3
+            );
+        }
        (table, correction)
    }