fix(gpu): stop gpu_tests hanging under the parallel test runner
Every test created its own wgpu instance and device (with adapter-maximum limits) concurrently, which could wedge the driver and hang the suite indefinitely. Tests now hold a process-wide lock while they own a device, and GpuAccelerator readback waits are bounded at 30s so a stuck driver surfaces as GpuError::BufferMap instead of blocking forever. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
926dc457e0
commit
706189c3ef
@@ -6,6 +6,9 @@ use crate::shaders;
|
|||||||
use bytemuck::Pod;
|
use bytemuck::Pod;
|
||||||
use wgpu::util::DeviceExt;
|
use wgpu::util::DeviceExt;
|
||||||
|
|
||||||
|
/// Upper bound on a single GPU→CPU readback wait.
|
||||||
|
const READBACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
||||||
|
|
||||||
/// GPU-accelerated vector search engine.
|
/// GPU-accelerated vector search engine.
|
||||||
///
|
///
|
||||||
/// Upload vectors once, then run many searches against them.
|
/// Upload vectors once, then run many searches against them.
|
||||||
@@ -1033,10 +1036,12 @@ impl GpuAccelerator {
|
|||||||
slice.map_async(wgpu::MapMode::Read, move |result| {
|
slice.map_async(wgpu::MapMode::Read, move |result| {
|
||||||
let _ = tx.send(result);
|
let _ = tx.send(result);
|
||||||
});
|
});
|
||||||
|
// Bounded wait: a wedged driver must surface as an error, not hang
|
||||||
|
// the caller forever.
|
||||||
self.device
|
self.device
|
||||||
.poll(wgpu::PollType::Wait {
|
.poll(wgpu::PollType::Wait {
|
||||||
submission_index: None,
|
submission_index: None,
|
||||||
timeout: None,
|
timeout: Some(READBACK_TIMEOUT),
|
||||||
})
|
})
|
||||||
.map_err(|e| GpuError::BufferMap(format!("device poll failed: {e}")))?;
|
.map_err(|e| GpuError::BufferMap(format!("device poll failed: {e}")))?;
|
||||||
rx.recv()
|
rx.recv()
|
||||||
|
|||||||
@@ -6,9 +6,41 @@
|
|||||||
mod tests {
|
mod tests {
|
||||||
use clawhdf5_gpu::{GpuAccelerator, GpuError};
|
use clawhdf5_gpu::{GpuAccelerator, GpuError};
|
||||||
|
|
||||||
fn skip_if_no_gpu() -> Option<GpuAccelerator> {
|
/// Serialises GPU access across tests. The harness runs tests on many
|
||||||
|
/// threads; letting each create its own wgpu instance + device (with
|
||||||
|
/// adapter-maximum limits) at the same time can wedge the driver and hang
|
||||||
|
/// the whole suite, so every test holds this lock while it owns a device.
|
||||||
|
static GPU_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||||
|
|
||||||
|
fn gpu_lock() -> std::sync::MutexGuard<'static, ()> {
|
||||||
|
// A panicking test poisons the lock; the guarded state is `()`.
|
||||||
|
GPU_LOCK.lock().unwrap_or_else(|e| e.into_inner())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A `GpuAccelerator` plus the lock that keeps other tests off the GPU.
|
||||||
|
/// Field order matters: the device is dropped before the lock is released.
|
||||||
|
struct LockedGpu {
|
||||||
|
gpu: GpuAccelerator,
|
||||||
|
_guard: std::sync::MutexGuard<'static, ()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::ops::Deref for LockedGpu {
|
||||||
|
type Target = GpuAccelerator;
|
||||||
|
fn deref(&self) -> &GpuAccelerator {
|
||||||
|
&self.gpu
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::ops::DerefMut for LockedGpu {
|
||||||
|
fn deref_mut(&mut self) -> &mut GpuAccelerator {
|
||||||
|
&mut self.gpu
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn skip_if_no_gpu() -> Option<LockedGpu> {
|
||||||
|
let guard = gpu_lock();
|
||||||
match GpuAccelerator::new() {
|
match GpuAccelerator::new() {
|
||||||
Ok(gpu) => Some(gpu),
|
Ok(gpu) => Some(LockedGpu { gpu, _guard: guard }),
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
eprintln!("SKIPPED: no GPU available");
|
eprintln!("SKIPPED: no GPU available");
|
||||||
None
|
None
|
||||||
@@ -69,6 +101,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_gpu_availability_detection() {
|
fn test_gpu_availability_detection() {
|
||||||
// Should not panic regardless of GPU presence
|
// Should not panic regardless of GPU presence
|
||||||
|
let _guard = gpu_lock();
|
||||||
let available = GpuAccelerator::is_available();
|
let available = GpuAccelerator::is_available();
|
||||||
eprintln!("GPU available: {available}");
|
eprintln!("GPU available: {available}");
|
||||||
}
|
}
|
||||||
@@ -425,6 +458,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_graceful_no_gpu_fallback() {
|
fn test_graceful_no_gpu_fallback() {
|
||||||
// This test just demonstrates the pattern — it always passes
|
// This test just demonstrates the pattern — it always passes
|
||||||
|
let _guard = gpu_lock();
|
||||||
match GpuAccelerator::new() {
|
match GpuAccelerator::new() {
|
||||||
Ok(gpu) => {
|
Ok(gpu) => {
|
||||||
eprintln!("GPU found: {}", gpu.device_info());
|
eprintln!("GPU found: {}", gpu.device_info());
|
||||||
|
|||||||
Reference in New Issue
Block a user