fix(gpu): stop gpu_tests hanging under the parallel test runner
Every test created its own wgpu instance and device (with adapter-maximum limits) concurrently, which could wedge the driver and hang the suite indefinitely. Tests now hold a process-wide lock while they own a device, and GpuAccelerator readback waits are bounded at 30s so a stuck driver surfaces as GpuError::BufferMap instead of blocking forever. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
926dc457e0
commit
706189c3ef
@@ -6,6 +6,9 @@ use crate::shaders;
|
||||
use bytemuck::Pod;
|
||||
use wgpu::util::DeviceExt;
|
||||
|
||||
/// Upper bound on a single GPU→CPU readback wait.
|
||||
const READBACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
|
||||
/// GPU-accelerated vector search engine.
|
||||
///
|
||||
/// Upload vectors once, then run many searches against them.
|
||||
@@ -1033,10 +1036,12 @@ impl GpuAccelerator {
|
||||
slice.map_async(wgpu::MapMode::Read, move |result| {
|
||||
let _ = tx.send(result);
|
||||
});
|
||||
// Bounded wait: a wedged driver must surface as an error, not hang
|
||||
// the caller forever.
|
||||
self.device
|
||||
.poll(wgpu::PollType::Wait {
|
||||
submission_index: None,
|
||||
timeout: None,
|
||||
timeout: Some(READBACK_TIMEOUT),
|
||||
})
|
||||
.map_err(|e| GpuError::BufferMap(format!("device poll failed: {e}")))?;
|
||||
rx.recv()
|
||||
|
||||
@@ -6,9 +6,41 @@
|
||||
mod tests {
|
||||
use clawhdf5_gpu::{GpuAccelerator, GpuError};
|
||||
|
||||
fn skip_if_no_gpu() -> Option<GpuAccelerator> {
|
||||
/// Serialises GPU access across tests. The harness runs tests on many
|
||||
/// threads; letting each create its own wgpu instance + device (with
|
||||
/// adapter-maximum limits) at the same time can wedge the driver and hang
|
||||
/// the whole suite, so every test holds this lock while it owns a device.
|
||||
static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
fn gpu_lock() -> std::sync::MutexGuard<'static, ()> {
|
||||
// A panicking test poisons the lock; the guarded state is `()`.
|
||||
GPU_LOCK.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// A `GpuAccelerator` plus the lock that keeps other tests off the GPU.
|
||||
/// Field order matters: the device is dropped before the lock is released.
|
||||
struct LockedGpu {
|
||||
gpu: GpuAccelerator,
|
||||
_guard: std::sync::MutexGuard<'static, ()>,
|
||||
}
|
||||
|
||||
impl std::ops::Deref for LockedGpu {
|
||||
type Target = GpuAccelerator;
|
||||
fn deref(&self) -> &GpuAccelerator {
|
||||
&self.gpu
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::DerefMut for LockedGpu {
|
||||
fn deref_mut(&mut self) -> &mut GpuAccelerator {
|
||||
&mut self.gpu
|
||||
}
|
||||
}
|
||||
|
||||
fn skip_if_no_gpu() -> Option<LockedGpu> {
|
||||
let guard = gpu_lock();
|
||||
match GpuAccelerator::new() {
|
||||
Ok(gpu) => Some(gpu),
|
||||
Ok(gpu) => Some(LockedGpu { gpu, _guard: guard }),
|
||||
Err(_) => {
|
||||
eprintln!("SKIPPED: no GPU available");
|
||||
None
|
||||
@@ -69,6 +101,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_gpu_availability_detection() {
|
||||
// Should not panic regardless of GPU presence
|
||||
let _guard = gpu_lock();
|
||||
let available = GpuAccelerator::is_available();
|
||||
eprintln!("GPU available: {available}");
|
||||
}
|
||||
@@ -425,6 +458,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_graceful_no_gpu_fallback() {
|
||||
// This test just demonstrates the pattern — it always passes
|
||||
let _guard = gpu_lock();
|
||||
match GpuAccelerator::new() {
|
||||
Ok(gpu) => {
|
||||
eprintln!("GPU found: {}", gpu.device_info());
|
||||
|
||||
Reference in New Issue
Block a user