clawhdf5/crates/clawhdf5/tests/busy_decode_pool.rs

//! Full reads of chunked datasets must not wait for a busy rayon pool of
//! any size.
//!
//! A full read handed its chunks to the rayon pool (`par_iter`) and the
//! calling thread — not a pool worker — slept until the pool had decoded
//! them. With a small pool (2-4 threads) and more reading threads than
//! workers, every reader queued behind the same few workers
//! (`docs/known-issues.md`, "Concurrent and contiguous read performance").
//! Now the calling thread decodes too, and pool workers only help when they
//! are free. The test keeps both workers of a two-thread pool busy and
//! requires reads to finish anyway, with the right values.
//!
//! One test in its own binary: it configures the process-wide rayon pool.

#![cfg(feature = "parallel")]

use std::sync::mpsc;
use std::time::Duration;

use clawhdf5::{File, FileBuilder};

const N: usize = 4096; // 64 chunks of 64 elements

fn values() -> Vec<f64> {
    (0..N).map(|i| i as f64 * 0.25 - 7.0).collect()
}

fn build() -> File {
    let mut b = FileBuilder::new();
    b.create_dataset("data")
        .with_f64_data(&values())
        .with_shape(&[N as u64])
        .with_chunks(&[64])
        .with_deflate(1)
        .with_provenance("test-suite", "2026-09-26T00:00:00Z", None);
    File::from_bytes(b.finish().unwrap()).unwrap()
}

/// Run `f` on a fresh thread; `None` if it has not finished within `limit`.
fn finishes_within<T: Send + 'static>(
    limit: Duration,
    f: impl FnOnce() -> T + Send + 'static,
) -> Option<T> {
    let (tx, rx) = mpsc::channel();
    std::thread::spawn(move || {
        let _ = tx.send(f());
    });
    rx.recv_timeout(limit).ok()
}

#[test]
fn full_reads_do_not_wait_for_a_busy_small_pool() {
    const WORKERS: usize = 2;
    rayon::ThreadPoolBuilder::new()
        .num_threads(WORKERS)
        .build_global()
        .expect("this test binary configures the global pool first");

    // Built first: the writer compresses on the pool too.
    let file = std::sync::Arc::new(build());

    // Occupy every worker of the pool until the reads are done.
    let (started_tx, started_rx) = mpsc::channel();
    let (release_tx, release_rx) = mpsc::channel::<()>();
    let release_rx = std::sync::Arc::new(std::sync::Mutex::new(release_rx));
    for _ in 0..WORKERS {
        let (started_tx, release_rx) = (started_tx.clone(), release_rx.clone());
        rayon::spawn(move || {
            started_tx.send(()).unwrap();
            let _ = release_rx.lock().unwrap().recv();
        });
    }
    for _ in 0..WORKERS {
        started_rx.recv().unwrap();
    }

    let limit = Duration::from_secs(20);
    // Several readers at once, as in the `concurrent_read` benchmark: the
    // cached full read (`read_*`), the typed one and the uncached reader
    // behind `verify_provenance`.
    let readers: Vec<_> = (0..4)
        .map(|_| {
            let file = std::sync::Arc::clone(&file);
            std::thread::spawn(move || {
                finishes_within(limit, move || {
                    let ds = file.dataset("data").unwrap();
                    (
                        ds.read_f64().unwrap(),
                        ds.read_f32().unwrap(),
                        ds.verify_provenance().unwrap(),
                    )
                })
            })
        })
        .collect();
    let results: Vec<_> = readers.into_iter().map(|h| h.join().unwrap()).collect();
    // Free the workers before asserting, so a failure does not hang the
    // blocked reader threads forever.
    for _ in 0..WORKERS {
        release_tx.send(()).unwrap();
    }

    let want = values();
    let want_f32: Vec<f32> = want.iter().map(|&v| v as f32).collect();
    for result in results {
        let (f64s, f32s, verified) =
            result.expect("a full read waited for the busy two-thread rayon pool");
        assert_eq!(f64s, want);
        assert_eq!(f32s, want_f32);
        assert_eq!(verified, clawhdf5::provenance::VerifyResult::Ok);
    }
}