docs: fix untraceable benchmark claims, add dual-audience framing, validate on second machine
- README's "HDF5 Core I/O" table claimed 19ns/2,080µs labeled 308× (real ratio ~109,000×) and a 313ns zero-copy mmap figure — neither traced to any dated benchmark in BENCHMARKS.md. Replaced the table wholesale with the existing "vs libhdf5 Summary" figures, relabeled from "h5py/C HDF5" to "libhdf5" (BENCHMARKS.md never benchmarks against h5py, only libhdf5 directly). - Added two new Criterion benchmarks to close the coverage gaps that produced the untraceable numbers: metadata_open_from_disk (I/O-inclusive, fair clawhdf5-vs-libhdf5 file-open comparison) and metadata_parse_in_memory (clawhdf5-only, explicitly labeled as excluding I/O) in h5bench_meta.rs; read_zerocopy_mmap in h5bench_read.rs (forces real page-ins by summing elements rather than just returning a slice length — the mmap path turns out to be slower than a plain copy at these sizes, an honest, unflattering but real result now documented instead of a fabricated 313ns). - Re-ran the full existing benchmark suite plus the two new ones on a second, independently administered machine (tank: Ryzen 7 7800X3D) to validate the numbers before publishing them. 5 of 6 rows landed within ~15% of the original i7-12650H figures; recorded both in BENCHMARKS.md's new "Independent Validation" section. README now cites the tank numbers. - Added a short top-of-file README callout naming both halves of the project (general-purpose HDF5 library vs. agent memory layer) with links to BENCHMARKS.md and the Crate Map, so a data-infra reader isn't 60% through a memory-store pitch before finding the part relevant to them. - Added one factual, no-names line noting benchmark numbers are being validated in collaboration with HDF5 Group engineers. - Fixed the same untraceable "2-300x faster than h5py/C HDF5" / "313 ns" claims in docs/QUICKSTART.md, one click from the README's own "New here?" link. Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
6b1ea450f5
commit
88195d1c33
@@ -220,6 +220,104 @@ fn bench_metadata_string_attrs(c: &mut Criterion) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workload: metadata_open_from_disk
|
||||
// Open a small pre-built file from disk and resolve one attribute. Both
|
||||
// sides pay the OS open()/read() cost plus header-parse cost, so this is a
|
||||
// fair, I/O-inclusive "open a file and touch its metadata" comparison — the
|
||||
// honest version of the "metadata parse" claim this benchmark replaces.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn bench_metadata_open_from_disk(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("metadata_open_from_disk");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let clawhdf5_path = tmp.path().join("open_clawhdf5.h5");
|
||||
{
|
||||
let mut fb = FileBuilder::new();
|
||||
let ds = fb
|
||||
.create_dataset("data")
|
||||
.with_f64_data(&[1.0, 2.0, 3.0])
|
||||
.with_shape(&[3]);
|
||||
ds.set_attr("label", AttrValue::I64(42));
|
||||
fb.write(&clawhdf5_path).unwrap();
|
||||
}
|
||||
|
||||
group.bench_function("clawhdf5", |b| {
|
||||
b.iter(|| {
|
||||
let raw = std::fs::read(&clawhdf5_path).unwrap();
|
||||
let file = File::from_bytes(raw).unwrap();
|
||||
let ds = file.dataset("data").unwrap();
|
||||
ds.attrs().unwrap()
|
||||
});
|
||||
});
|
||||
|
||||
#[cfg(feature = "libhdf5-compare")]
|
||||
{
|
||||
let libhdf5_path = tmp.path().join("open_libhdf5.h5");
|
||||
{
|
||||
let file = hdf5::File::create(&libhdf5_path).unwrap();
|
||||
let ds = file
|
||||
.new_dataset::<f64>()
|
||||
.shape([3])
|
||||
.create("data")
|
||||
.unwrap();
|
||||
ds.write(&[1.0f64, 2.0, 3.0]).unwrap();
|
||||
ds.new_attr::<i64>()
|
||||
.create("label")
|
||||
.unwrap()
|
||||
.write_scalar(&42i64)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
group.bench_function("libhdf5", |b| {
|
||||
b.iter(|| {
|
||||
let file = hdf5::File::open(&libhdf5_path).unwrap();
|
||||
let ds = file.dataset("data").unwrap();
|
||||
let _: i64 = ds.attr("label").unwrap().read_scalar().unwrap();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workload: metadata_parse_in_memory (clawhdf5-only)
|
||||
// Times File::from_bytes() alone on bytes already resident in memory — i.e.
|
||||
// the header-parse cost with disk I/O excluded. There is no fair libhdf5
|
||||
// equivalent (its API has no "parse from an in-memory buffer" path that
|
||||
// skips the OS open), so this is reported standalone, not as a speedup
|
||||
// multiple against libhdf5. See metadata_open_from_disk above for the
|
||||
// I/O-inclusive, directly comparable number.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn bench_metadata_parse_in_memory(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("metadata_parse_in_memory");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
|
||||
let bytes = {
|
||||
let mut fb = FileBuilder::new();
|
||||
let ds = fb
|
||||
.create_dataset("data")
|
||||
.with_f64_data(&[1.0, 2.0, 3.0])
|
||||
.with_shape(&[3]);
|
||||
ds.set_attr("label", AttrValue::I64(42));
|
||||
fb.finish().unwrap()
|
||||
};
|
||||
|
||||
group.bench_with_input(BenchmarkId::new("clawhdf5", "in_memory"), &bytes, |b, raw| {
|
||||
b.iter(|| {
|
||||
let file = File::from_bytes(raw.clone()).unwrap();
|
||||
let ds = file.dataset("data").unwrap();
|
||||
ds.attrs().unwrap()
|
||||
});
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
meta_benches,
|
||||
bench_metadata_attrs_write,
|
||||
@@ -227,5 +325,7 @@ criterion_group!(
|
||||
bench_metadata_groups_create,
|
||||
bench_metadata_groups_traverse,
|
||||
bench_metadata_string_attrs,
|
||||
bench_metadata_open_from_disk,
|
||||
bench_metadata_parse_in_memory,
|
||||
);
|
||||
criterion_main!(meta_benches);
|
||||
|
||||
Reference in New Issue
Block a user