Chunked reads beat an h5py process pool; unlimited writer B-trees; Blosc2; 599/697 conformance #16

Merged
osobh merged 48 commits from feat/p2b-scale into main 2026-09-26 17:42:16 +00:00
Showing only changes of commit f512bf3d09 - Show all commits
+26 -6
View File
@@ -712,18 +712,35 @@ fn main() {
return;
}
};
// libhdf5 decodes the superblock extension at open, and loads a
// metadata cache image over the file's own metadata.
let view = match guarded(|| {
clawhdf5_format::superblock_ext::metadata_view(hdf5, &sb).map_err(e)
}) {
Ok(v) => v,
// libhdf5 decodes the superblock extension at open (File::open does
// the same), and loads a metadata cache image over the file's own
// metadata. It loads the image only when it first reads metadata — the
// root group — so a file whose image it cannot load still opens and
// every object fails; File::open refuses such a file outright. The
// probe records the image's error where libhdf5 reports it.
use clawhdf5_format::superblock_ext;
let ext = match guarded(|| superblock_ext::read_superblock_extension(hdf5, &sb).map_err(e)) {
Ok(x) => x,
Err(msg) => {
top.insert("open_error".into(), Value::String(msg));
println!("{}", Value::Object(top));
return;
}
};
let mut image_error = None;
let view = match ext.and_then(|x| x.cache_image) {
None => None,
Some(loc) => match guarded(|| {
superblock_ext::apply_cache_image(hdf5, loc, sb.offset_size, sb.length_size)
.map_err(e)
}) {
Ok(v) => Some(v),
Err(msg) => {
image_error = Some(msg);
None
}
},
};
let hdf5: &[u8] = view.as_deref().unwrap_or(hdf5);
top.insert("superblock_version".into(), json!(sb.version));
let ctx = Ctx {
@@ -752,6 +769,9 @@ fn main() {
let mut rec = Map::new();
rec.insert("path".into(), Value::String(p.clone()));
let r = guarded(|| {
if let Some(msg) = &image_error {
return Err(msg.clone());
}
let h = ctx.header(addr)?;
Ok(h)
});