Chunked reads beat an h5py process pool; unlimited writer B-trees; Blosc2; 599/697 conformance #16
@@ -183,8 +183,11 @@ enum Stage {
|
||||
}
|
||||
|
||||
impl DecodeScratch {
|
||||
/// Largest buffer [`trim`](Self::trim) keeps (4 MiB).
|
||||
pub const RETAIN_BYTES: usize = 4 << 20;
|
||||
/// Largest buffer [`trim`](Self::trim) keeps (1 MiB): enough for common
|
||||
/// chunk sizes (a 256 x 256 `f32` chunk is 256 KiB) while bounding what
|
||||
/// every long-lived thread (rayon's workers never exit) holds on to, at
|
||||
/// two buffers each.
|
||||
pub const RETAIN_BYTES: usize = 1 << 20;
|
||||
|
||||
/// An empty scratch; buffers are allocated on first use.
|
||||
pub fn new() -> Self {
|
||||
|
||||
@@ -47,6 +47,12 @@ pub fn pool_can_parallelise() -> bool {
|
||||
/// item beyond the caller's first.
|
||||
pub(crate) fn helper_count(items: usize) -> usize {
|
||||
let pool = rayon::current_num_threads();
|
||||
// A one-thread pool means "decode on the calling thread" (the setting
|
||||
// benchmarks use to compare with h5py, where each call decodes on its
|
||||
// caller): no helper, so one read never uses two cores.
|
||||
if pool <= 1 {
|
||||
return 0;
|
||||
}
|
||||
let others = if rayon::current_thread_index().is_some() {
|
||||
pool.saturating_sub(1)
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user