embedded3 item 9b: the device step carries a static cut-cell mask (e3_cut.cu: cut predictor, apertured merged continuity with the fold, owner-read corrections; device CG off-stencil links) — host = device to 2e-10 (CFD1 cylinder nz 4) and 4e-14 (sphere) under tight tolerances

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/specialized/rtx-cfd/src/kernels/cuda/e3_cg.cu

@@ -7,9 +7,12 @@
  */
 #define NONE 0xFFFFFFFFu
 
+/* The seven-point neighbour sum plus the cell's off-stencil links (CSR:
+ * link_ptr[g] .. link_ptr[g + 1]; the virtually merged small cells). */
 __device__ __forceinline__ double nb_sum3d(
     int g, int nx, const double* ae, const double* aw, const double* an, const double* as_,
     const double* at, const double* ab, const unsigned int* top, const unsigned int* bot,
+    const unsigned int* link_ptr, const unsigned int* link_idx, const double* link_coef,
     const double* x)
 {
     double s = 0.0;
@@ -19,6 +22,7 @@ __device__ __forceinline__ double nb_sum3d(
     double ss = as_[g]; if (ss != 0.0) s += ss * x[g - nx];
     double t = at[g]; if (t != 0.0) s += t * x[top[g]];
     double b = ab[g]; if (b != 0.0) s += b * x[bot[g]];
+    for (unsigned int l = link_ptr[g]; l < link_ptr[g + 1]; ++l) s += link_coef[l] * x[link_idx[l]];
     return s;
 }
 
@@ -42,12 +46,14 @@ extern "C" __global__ void e3_cg_spmv(
     const double* __restrict__ an, const double* __restrict__ as_,
     const double* __restrict__ at, const double* __restrict__ ab,
     const unsigned int* __restrict__ top, const unsigned int* __restrict__ bot,
+    const unsigned int* __restrict__ link_ptr, const unsigned int* __restrict__ link_idx,
+    const double* __restrict__ link_coef,
     const double* __restrict__ ap, const double* __restrict__ d, double* __restrict__ q, int nx)
 {
     int t = blockIdx.x * blockDim.x + threadIdx.x;
     if (t >= n_cells) return;
     int g = cells[t];
-    q[g] = ap[g] * d[g] - nb_sum3d(g, nx, ae, aw, an, as_, at, ab, top, bot, d);
+    q[g] = ap[g] * d[g] - nb_sum3d(g, nx, ae, aw, an, as_, at, ab, top, bot, link_ptr, link_idx, link_coef, d);
 }
 
 /* r = b − A p on the cells; partial[block] = Σ |r| over the block's cells. */
@@ -57,6 +63,8 @@ extern "C" __global__ void e3_cg_residual(
     const double* __restrict__ an, const double* __restrict__ as_,
     const double* __restrict__ at, const double* __restrict__ ab,
     const unsigned int* __restrict__ top, const unsigned int* __restrict__ bot,
+    const unsigned int* __restrict__ link_ptr, const unsigned int* __restrict__ link_idx,
+    const double* __restrict__ link_coef,
     const double* __restrict__ ap, const double* __restrict__ b,
     const double* __restrict__ p, double* __restrict__ r, double* __restrict__ partial, int nx)
 {
@@ -64,7 +72,7 @@ extern "C" __global__ void e3_cg_residual(
     double v = 0.0;
     if (t < n_cells) {
         int g = cells[t];
-        v = b[g] - (ap[g] * p[g] - nb_sum3d(g, nx, ae, aw, an, as_, at, ab, top, bot, p));
+        v = b[g] - (ap[g] * p[g] - nb_sum3d(g, nx, ae, aw, an, as_, at, ab, top, bot, link_ptr, link_idx, link_coef, p));
         r[g] = v;
         v = fabs(v);
     }