Omar Sobh
55959b4920
CI / test (push) Failing after 15s
- Add .gitea/workflows/ci.yml running scripts/ci-test.sh (fmt, clippy,
test, no_std check) on push/PR to main.
- Fix stale rustyhdf5-py/rustyhdf5-format package names in
ci-test.sh/check-nostd.sh, which had been silently no-op'ing those
checks (cargo warns but doesn't fail on an unknown --exclude/-p
target).
- With those checks actually running, fix the real issues they surface:
- clippy: useless_conversion in chunked_write.rs, byte_char_slices in
global_heap.rs/object_header.rs.
- cargo fmt: apply formatting across the workspace (whitespace only).
- no_std (thumbv7em-none-eabihf) build errors in clawhdf5-format:
core::sync::atomic::AtomicU64 doesn't exist on that target (no
native 64-bit atomics) — switch profiling.rs's counters to
portable-atomic, which falls back to a CAS-based emulation there
and is a no-op wrapper elsewhere. Add missing alloc imports for
Box (filters.rs), Vec (filters_szip.rs), and format! (dict_encoding.rs)
on no_std paths. Replace f64::powi (std/libm-only) with a small
local exponentiation-by-squaring helper in the scale-offset filter.
328 lines
12 KiB
Rust
328 lines
12 KiB
Rust
//! h5bench-equivalent metadata workloads for clawhdf5.
|
|
//!
|
|
//! Measures attribute creation/read throughput and group traversal latency —
|
|
//! the workloads that h5bench's `metadata` mode targets against libhdf5.
|
|
|
|
use clawhdf5::{AttrValue, File, FileBuilder};
|
|
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
|
|
use tempfile::TempDir;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_attrs_write
|
|
// Create K attributes on a single dataset.
|
|
// Exercises attribute message allocation and compact → dense header transition.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_attrs_write(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_attrs_write");
|
|
|
|
for &k in &[4usize, 16, 64, 128] {
|
|
group.throughput(Throughput::Elements(k as u64));
|
|
|
|
group.bench_with_input(BenchmarkId::new("clawhdf5", k), &k, |b, &k| {
|
|
let tmp = TempDir::new().unwrap();
|
|
let path = tmp.path().join("attrs_write.h5");
|
|
b.iter(|| {
|
|
let mut fb = FileBuilder::new();
|
|
let ds = fb
|
|
.create_dataset("data")
|
|
.with_f64_data(&[1.0, 2.0, 3.0])
|
|
.with_shape(&[3]);
|
|
for i in 0..k {
|
|
ds.set_attr(&format!("attr_{i:04}"), AttrValue::I64(i as i64));
|
|
}
|
|
fb.write(&path).unwrap();
|
|
});
|
|
});
|
|
|
|
#[cfg(feature = "libhdf5-compare")]
|
|
group.bench_with_input(BenchmarkId::new("libhdf5", k), &k, |b, &k| {
|
|
let tmp = TempDir::new().unwrap();
|
|
let path = tmp.path().join("attrs_libhdf5.h5");
|
|
b.iter(|| {
|
|
let file = hdf5::File::create(&path).unwrap();
|
|
let ds = file.new_dataset::<f64>().shape([3]).create("data").unwrap();
|
|
ds.write(&[1.0f64, 2.0, 3.0]).unwrap();
|
|
for i in 0..k {
|
|
ds.new_attr::<i64>()
|
|
.create(format!("attr_{i:04}").as_str())
|
|
.unwrap()
|
|
.write_scalar(&(i as i64))
|
|
.unwrap();
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_attrs_read
|
|
// Open a pre-built file and read all K attributes back.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_attrs_read(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_attrs_read");
|
|
|
|
for &k in &[4usize, 16, 64, 128] {
|
|
// Build the reference file in memory.
|
|
let bytes = {
|
|
let mut fb = FileBuilder::new();
|
|
let ds = fb
|
|
.create_dataset("data")
|
|
.with_f64_data(&[1.0, 2.0, 3.0])
|
|
.with_shape(&[3]);
|
|
for i in 0..k {
|
|
ds.set_attr(&format!("attr_{i:04}"), AttrValue::I64(i as i64));
|
|
}
|
|
fb.finish().unwrap()
|
|
};
|
|
|
|
group.throughput(Throughput::Elements(k as u64));
|
|
|
|
group.bench_with_input(BenchmarkId::new("clawhdf5", k), &bytes, |b, raw| {
|
|
b.iter(|| {
|
|
let file = File::from_bytes(raw.clone()).unwrap();
|
|
let ds = file.dataset("data").unwrap();
|
|
ds.attrs().unwrap()
|
|
});
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_groups_create
|
|
// Create K top-level groups (no datasets inside).
|
|
// Measures link-storage allocation: compact → dense B-tree transition.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_groups_create(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_groups_create");
|
|
|
|
for &k in &[4usize, 16, 32, 64] {
|
|
group.throughput(Throughput::Elements(k as u64));
|
|
|
|
group.bench_with_input(BenchmarkId::new("clawhdf5", k), &k, |b, &k| {
|
|
let tmp = TempDir::new().unwrap();
|
|
let path = tmp.path().join("groups_create.h5");
|
|
b.iter(|| {
|
|
let mut fb = FileBuilder::new();
|
|
for i in 0..k {
|
|
let mut g = fb.create_group(&format!("group_{i:04}"));
|
|
// Minimal dataset inside each group to make it non-trivial.
|
|
g.create_dataset("x").with_f64_data(&[0.0]);
|
|
let finished = g.finish();
|
|
fb.add_group(finished);
|
|
}
|
|
fb.write(&path).unwrap();
|
|
});
|
|
});
|
|
|
|
#[cfg(feature = "libhdf5-compare")]
|
|
group.bench_with_input(BenchmarkId::new("libhdf5", k), &k, |b, &k| {
|
|
let tmp = TempDir::new().unwrap();
|
|
let path = tmp.path().join("groups_libhdf5.h5");
|
|
b.iter(|| {
|
|
let file = hdf5::File::create(&path).unwrap();
|
|
for i in 0..k {
|
|
let g = file.create_group(&format!("group_{i:04}")).unwrap();
|
|
g.new_dataset::<f64>()
|
|
.shape([1])
|
|
.create("x")
|
|
.unwrap()
|
|
.write(&[0.0f64])
|
|
.unwrap();
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_groups_traverse
|
|
// Open a pre-built file with K groups and traverse (list) the root group.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_groups_traverse(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_groups_traverse");
|
|
|
|
for &k in &[4usize, 16, 32, 64] {
|
|
// Pre-build.
|
|
let bytes = {
|
|
let mut fb = FileBuilder::new();
|
|
for i in 0..k {
|
|
let mut g = fb.create_group(&format!("group_{i:04}"));
|
|
g.create_dataset("x").with_f64_data(&[0.0]);
|
|
let finished = g.finish();
|
|
fb.add_group(finished);
|
|
}
|
|
fb.finish().unwrap()
|
|
};
|
|
|
|
group.throughput(Throughput::Elements(k as u64));
|
|
|
|
group.bench_with_input(BenchmarkId::new("clawhdf5", k), &bytes, |b, raw| {
|
|
b.iter(|| {
|
|
let file = File::from_bytes(raw.clone()).unwrap();
|
|
let root = file.root();
|
|
root.groups().unwrap()
|
|
});
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_roundtrip_string_attrs
|
|
// Write and read back K variable-length string attributes.
|
|
// String attrs require a dedicated VL heap entry — distinct from numeric ones.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_string_attrs(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_string_attrs");
|
|
|
|
for &k in &[4usize, 16, 32] {
|
|
group.throughput(Throughput::Elements(k as u64));
|
|
|
|
group.bench_with_input(BenchmarkId::new("clawhdf5", k), &k, |b, &k| {
|
|
b.iter(|| {
|
|
let mut fb = FileBuilder::new();
|
|
let ds = fb
|
|
.create_dataset("data")
|
|
.with_f64_data(&[1.0])
|
|
.with_shape(&[1]);
|
|
for i in 0..k {
|
|
ds.set_attr(
|
|
&format!("label_{i:04}"),
|
|
AttrValue::String(format!("value-{i}-some-longer-string-payload")),
|
|
);
|
|
}
|
|
let bytes = fb.finish().unwrap();
|
|
|
|
// Immediately read back to exercise both directions.
|
|
let file = File::from_bytes(bytes).unwrap();
|
|
let ds_r = file.dataset("data").unwrap();
|
|
ds_r.attrs().unwrap()
|
|
});
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_open_from_disk
|
|
// Open a small pre-built file from disk and resolve one attribute. Both
|
|
// sides pay the OS open()/read() cost plus header-parse cost, so this is a
|
|
// fair, I/O-inclusive "open a file and touch its metadata" comparison — the
|
|
// honest version of the "metadata parse" claim this benchmark replaces.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_open_from_disk(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_open_from_disk");
|
|
group.throughput(Throughput::Elements(1));
|
|
|
|
let tmp = TempDir::new().unwrap();
|
|
let clawhdf5_path = tmp.path().join("open_clawhdf5.h5");
|
|
{
|
|
let mut fb = FileBuilder::new();
|
|
let ds = fb
|
|
.create_dataset("data")
|
|
.with_f64_data(&[1.0, 2.0, 3.0])
|
|
.with_shape(&[3]);
|
|
ds.set_attr("label", AttrValue::I64(42));
|
|
fb.write(&clawhdf5_path).unwrap();
|
|
}
|
|
|
|
group.bench_function("clawhdf5", |b| {
|
|
b.iter(|| {
|
|
let raw = std::fs::read(&clawhdf5_path).unwrap();
|
|
let file = File::from_bytes(raw).unwrap();
|
|
let ds = file.dataset("data").unwrap();
|
|
ds.attrs().unwrap()
|
|
});
|
|
});
|
|
|
|
#[cfg(feature = "libhdf5-compare")]
|
|
{
|
|
let libhdf5_path = tmp.path().join("open_libhdf5.h5");
|
|
{
|
|
let file = hdf5::File::create(&libhdf5_path).unwrap();
|
|
let ds = file.new_dataset::<f64>().shape([3]).create("data").unwrap();
|
|
ds.write(&[1.0f64, 2.0, 3.0]).unwrap();
|
|
ds.new_attr::<i64>()
|
|
.create("label")
|
|
.unwrap()
|
|
.write_scalar(&42i64)
|
|
.unwrap();
|
|
}
|
|
|
|
group.bench_function("libhdf5", |b| {
|
|
b.iter(|| {
|
|
let file = hdf5::File::open(&libhdf5_path).unwrap();
|
|
let ds = file.dataset("data").unwrap();
|
|
let _: i64 = ds.attr("label").unwrap().read_scalar().unwrap();
|
|
});
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workload: metadata_parse_in_memory (clawhdf5-only)
|
|
// Times File::from_bytes() alone on bytes already resident in memory — i.e.
|
|
// the header-parse cost with disk I/O excluded. There is no fair libhdf5
|
|
// equivalent (its API has no "parse from an in-memory buffer" path that
|
|
// skips the OS open), so this is reported standalone, not as a speedup
|
|
// multiple against libhdf5. See metadata_open_from_disk above for the
|
|
// I/O-inclusive, directly comparable number.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn bench_metadata_parse_in_memory(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("metadata_parse_in_memory");
|
|
group.throughput(Throughput::Elements(1));
|
|
|
|
let bytes = {
|
|
let mut fb = FileBuilder::new();
|
|
let ds = fb
|
|
.create_dataset("data")
|
|
.with_f64_data(&[1.0, 2.0, 3.0])
|
|
.with_shape(&[3]);
|
|
ds.set_attr("label", AttrValue::I64(42));
|
|
fb.finish().unwrap()
|
|
};
|
|
|
|
group.bench_with_input(
|
|
BenchmarkId::new("clawhdf5", "in_memory"),
|
|
&bytes,
|
|
|b, raw| {
|
|
b.iter(|| {
|
|
let file = File::from_bytes(raw.clone()).unwrap();
|
|
let ds = file.dataset("data").unwrap();
|
|
ds.attrs().unwrap()
|
|
});
|
|
},
|
|
);
|
|
|
|
group.finish();
|
|
}
|
|
|
|
criterion_group!(
|
|
meta_benches,
|
|
bench_metadata_attrs_write,
|
|
bench_metadata_attrs_read,
|
|
bench_metadata_groups_create,
|
|
bench_metadata_groups_traverse,
|
|
bench_metadata_string_attrs,
|
|
bench_metadata_open_from_disk,
|
|
bench_metadata_parse_in_memory,
|
|
);
|
|
criterion_main!(meta_benches);
|