fix(gpu): stop gpu_tests hanging under the parallel test runner

Every test created its own wgpu instance and device (with adapter-maximum
limits) concurrently, which could wedge the driver and hang the suite
indefinitely. Tests now hold a process-wide lock while they own a device, and
GpuAccelerator readback waits are bounded at 30s so a stuck driver surfaces as
GpuError::BufferMap instead of blocking forever.

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
osobh
2026-09-19 05:36:22 -07:00
co-authored by Claude Fable 5.1
parent 926dc457e0
commit 706189c3ef
2 changed files with 42 additions and 3 deletions
+6 -1
View File
@@ -6,6 +6,9 @@ use crate::shaders;
use bytemuck::Pod;
use wgpu::util::DeviceExt;
/// Upper bound on a single GPU→CPU readback wait.
const READBACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
/// GPU-accelerated vector search engine.
///
/// Upload vectors once, then run many searches against them.
@@ -1033,10 +1036,12 @@ impl GpuAccelerator {
slice.map_async(wgpu::MapMode::Read, move |result| {
let _ = tx.send(result);
});
// Bounded wait: a wedged driver must surface as an error, not hang
// the caller forever.
self.device
.poll(wgpu::PollType::Wait {
submission_index: None,
timeout: None,
timeout: Some(READBACK_TIMEOUT),
})
.map_err(|e| GpuError::BufferMap(format!("device poll failed: {e}")))?;
rx.recv()
+36 -2
View File
@@ -6,9 +6,41 @@
mod tests {
use clawhdf5_gpu::{GpuAccelerator, GpuError};
fn skip_if_no_gpu() -> Option<GpuAccelerator> {
/// Serialises GPU access across tests. The harness runs tests on many
/// threads; letting each create its own wgpu instance + device (with
/// adapter-maximum limits) at the same time can wedge the driver and hang
/// the whole suite, so every test holds this lock while it owns a device.
static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn gpu_lock() -> std::sync::MutexGuard<'static, ()> {
// A panicking test poisons the lock; the guarded state is `()`.
GPU_LOCK.lock().unwrap_or_else(|e| e.into_inner())
}
/// A `GpuAccelerator` plus the lock that keeps other tests off the GPU.
/// Field order matters: the device is dropped before the lock is released.
struct LockedGpu {
gpu: GpuAccelerator,
_guard: std::sync::MutexGuard<'static, ()>,
}
impl std::ops::Deref for LockedGpu {
type Target = GpuAccelerator;
fn deref(&self) -> &GpuAccelerator {
&self.gpu
}
}
impl std::ops::DerefMut for LockedGpu {
fn deref_mut(&mut self) -> &mut GpuAccelerator {
&mut self.gpu
}
}
fn skip_if_no_gpu() -> Option<LockedGpu> {
let guard = gpu_lock();
match GpuAccelerator::new() {
Ok(gpu) => Some(gpu),
Ok(gpu) => Some(LockedGpu { gpu, _guard: guard }),
Err(_) => {
eprintln!("SKIPPED: no GPU available");
None
@@ -69,6 +101,7 @@ mod tests {
#[test]
fn test_gpu_availability_detection() {
// Should not panic regardless of GPU presence
let _guard = gpu_lock();
let available = GpuAccelerator::is_available();
eprintln!("GPU available: {available}");
}
@@ -425,6 +458,7 @@ mod tests {
#[test]
fn test_graceful_no_gpu_fallback() {
// This test just demonstrates the pattern — it always passes
let _guard = gpu_lock();
match GpuAccelerator::new() {
Ok(gpu) => {
eprintln!("GPU found: {}", gpu.device_info());