feat(flash-attention): add varlen packed-sequence support
Implements variable-length (varlen) FlashAttention that processes mixed-length batches without padding waste: - New CUDA kernel flash_attention_varlen_forward with BLOCK_Q=64 / BLOCK_K=64 tiling; grid=(ceil(max_seqlen_q/64), num_heads, 1). Each block uses a linear scan over cu_seqlens_q to identify its owning sequence and exits early when past sequence end. - New Rust module flash_varlen_forward: always-compiled CPU simulation (varlen_attention_cpu) for testing + #[cfg(cuda)] FlashVarlenKernel. - SdpaBackend::VarLen variant added to backend_selector. - 8 new CPU-only tests; total test count: 50. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
311eb23dbd
commit
80d7c7fb6c
@@ -51,6 +51,8 @@ pub enum SdpaBackend {
|
||||
Metal,
|
||||
/// Fallback CPU implementation
|
||||
Cpu,
|
||||
/// Variable-length packed-sequence attention (no padding)
|
||||
VarLen,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SdpaBackend {
|
||||
@@ -63,6 +65,7 @@ impl std::fmt::Display for SdpaBackend {
|
||||
SdpaBackend::CuDnn => write!(f, "cuDNN"),
|
||||
SdpaBackend::Metal => write!(f, "Metal"),
|
||||
SdpaBackend::Cpu => write!(f, "CPU"),
|
||||
SdpaBackend::VarLen => write!(f, "VarLen"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,6 +215,7 @@ impl HardwareCapabilities {
|
||||
SdpaBackend::CuDnn => self.supports_cudnn_attention,
|
||||
SdpaBackend::Metal => self.device_type == DeviceType::Metal,
|
||||
SdpaBackend::Cpu => true,
|
||||
SdpaBackend::VarLen => true, // CPU simulation always available; CUDA variant when feature is on
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -575,6 +579,7 @@ impl SdpaBackendSelector {
|
||||
SdpaBackend::Math,
|
||||
SdpaBackend::Metal,
|
||||
SdpaBackend::Cpu,
|
||||
SdpaBackend::VarLen,
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&b| self.is_backend_available(b, input))
|
||||
@@ -643,6 +648,9 @@ impl SdpaBackendSelector {
|
||||
SdpaBackend::Cpu => {
|
||||
(0.1, "CPU fallback".to_string())
|
||||
}
|
||||
SdpaBackend::VarLen => {
|
||||
(0.75, "VarLen eliminates padding waste for mixed-length batches".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -674,6 +682,7 @@ impl SdpaBackendSelector {
|
||||
SdpaBackend::Metal => 2.5,
|
||||
SdpaBackend::Math => 1.0,
|
||||
SdpaBackend::Cpu => 0.1,
|
||||
SdpaBackend::VarLen => 2.0, // avoids padding overhead for mixed-length batches
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user