73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""h5py mmap-like and parallel read benchmarks."""
|
|
import time, os, tempfile, numpy as np, h5py
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
N = 1_000_000
|
|
ITERS = 50
|
|
data = np.arange(N, dtype=np.float64)
|
|
|
|
def bench(name, fn, iters=ITERS):
|
|
for _ in range(3): fn()
|
|
times = []
|
|
for _ in range(iters):
|
|
t0 = time.perf_counter()
|
|
fn()
|
|
times.append(time.perf_counter() - t0)
|
|
times.sort()
|
|
median = times[len(times)//2]
|
|
print(f"{name}: {median*1000:.3f} ms (median of {iters})")
|
|
|
|
# --- Mmap-style reads (rdcc disabled = no chunk cache, driver='core' = in-memory) ---
|
|
tmp = tempfile.NamedTemporaryFile(suffix='.h5', delete=False)
|
|
with h5py.File(tmp.name, 'w') as hf:
|
|
hf.create_dataset('data', data=data)
|
|
hf.create_dataset('chunked', data=data, chunks=(10000,))
|
|
hf.create_dataset('deflate', data=data, chunks=(10000,), compression='gzip', compression_opts=6)
|
|
|
|
def read_core_driver():
|
|
with h5py.File(tmp.name, 'r', driver='core', backing_store=False) as hf:
|
|
_ = hf['data'][:]
|
|
|
|
def read_default():
|
|
with h5py.File(tmp.name, 'r') as hf:
|
|
_ = hf['data'][:]
|
|
|
|
# --- Parallel reads: 10 datasets, read all with thread pool ---
|
|
tmp_par = tempfile.NamedTemporaryFile(suffix='.h5', delete=False)
|
|
with h5py.File(tmp_par.name, 'w') as hf:
|
|
for i in range(10):
|
|
hf.create_dataset(f'ds_{i}', data=data)
|
|
|
|
def read_10ds_sequential():
|
|
with h5py.File(tmp_par.name, 'r') as hf:
|
|
for i in range(10):
|
|
_ = hf[f'ds_{i}'][:]
|
|
|
|
def read_10ds_threaded():
|
|
def read_one(i):
|
|
with h5py.File(tmp_par.name, 'r') as hf:
|
|
return hf[f'ds_{i}'][:]
|
|
with ThreadPoolExecutor(max_workers=10) as ex:
|
|
list(ex.map(read_one, range(10)))
|
|
|
|
# --- 100 datasets, read 1 (lazy access pattern) ---
|
|
tmp_100 = tempfile.NamedTemporaryFile(suffix='.h5', delete=False)
|
|
with h5py.File(tmp_100.name, 'w') as hf:
|
|
for i in range(100):
|
|
hf.create_dataset(f'ds_{i:04}', data=np.array([float(i)] * 10))
|
|
|
|
def read_1_of_100():
|
|
with h5py.File(tmp_100.name, 'r') as hf:
|
|
_ = hf['ds_0050'][:]
|
|
|
|
print("=== h5py mmap/parallel benchmarks ===\n")
|
|
bench("read_default_driver", read_default)
|
|
bench("read_core_driver (in-memory)", read_core_driver)
|
|
bench("read_10ds_sequential", read_10ds_sequential)
|
|
bench("read_10ds_threaded_10w", read_10ds_threaded, iters=20)
|
|
bench("read_1_of_100_datasets", read_1_of_100)
|
|
|
|
for f in [tmp, tmp_par, tmp_100]:
|
|
os.unlink(f.name)
|