feat(flash-attention): add varlen packed-sequence support

Implements variable-length (varlen) FlashAttention that processes
mixed-length batches without padding waste:

- New CUDA kernel flash_attention_varlen_forward with BLOCK_Q=64 /
  BLOCK_K=64 tiling; grid=(ceil(max_seqlen_q/64), num_heads, 1).
  Each block uses a linear scan over cu_seqlens_q to identify its
  owning sequence and exits early when past sequence end.
- New Rust module flash_varlen_forward: always-compiled CPU simulation
  (varlen_attention_cpu) for testing + #[cfg(cuda)] FlashVarlenKernel.
- SdpaBackend::VarLen variant added to backend_selector.
- 8 new CPU-only tests; total test count: 50.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-27 01:09:39 +00:00
co-authored by Claude Sonnet 4.6
parent 311eb23dbd
commit 80d7c7fb6c
5 changed files with 2121 additions and 0 deletions
@@ -51,6 +51,8 @@ pub enum SdpaBackend {
Metal,
/// Fallback CPU implementation
Cpu,
/// Variable-length packed-sequence attention (no padding)
VarLen,
}
impl std::fmt::Display for SdpaBackend {
@@ -63,6 +65,7 @@ impl std::fmt::Display for SdpaBackend {
SdpaBackend::CuDnn => write!(f, "cuDNN"),
SdpaBackend::Metal => write!(f, "Metal"),
SdpaBackend::Cpu => write!(f, "CPU"),
SdpaBackend::VarLen => write!(f, "VarLen"),
}
}
}
@@ -212,6 +215,7 @@ impl HardwareCapabilities {
SdpaBackend::CuDnn => self.supports_cudnn_attention,
SdpaBackend::Metal => self.device_type == DeviceType::Metal,
SdpaBackend::Cpu => true,
SdpaBackend::VarLen => true, // CPU simulation always available; CUDA variant when feature is on
}
}
}
@@ -575,6 +579,7 @@ impl SdpaBackendSelector {
SdpaBackend::Math,
SdpaBackend::Metal,
SdpaBackend::Cpu,
SdpaBackend::VarLen,
]
.iter()
.filter(|&&b| self.is_backend_available(b, input))
@@ -643,6 +648,9 @@ impl SdpaBackendSelector {
SdpaBackend::Cpu => {
(0.1, "CPU fallback".to_string())
}
SdpaBackend::VarLen => {
(0.75, "VarLen eliminates padding waste for mixed-length batches".to_string())
}
}
}
@@ -674,6 +682,7 @@ impl SdpaBackendSelector {
SdpaBackend::Metal => 2.5,
SdpaBackend::Math => 1.0,
SdpaBackend::Cpu => 0.1,
SdpaBackend::VarLen => 2.0, // avoids padding overhead for mixed-length batches
}
}