quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

py: remote files (clawhdf5.File(url), File.open_url) through File::storage()

Browse Source 
The Python bindings could not open a remote file: they parsed through
File::as_bytes() in eight places (path lookups, object headers,
dataspaces, attributes, group listings, the global heap of
variable-length data), which a storage-backed file does not have.

- Every object of a File now shares one handle (src/handle.rs) that
  runs all file access, metadata included, with the GIL released and
  parses through File::storage() and the clawhdf5_format *_in functions.
  Local files take the same path (their storage is the mmap).
- clawhdf5.File(url) opens any scheme://... through
  clawhdf5_remote::storage_for_url (read-only; another mode is a
  ValueError). File.open_url(url, **options) takes the cache and HTTP
  options (block_size, cache_size, headers, retries, timeout,
  allow_full_download, max_full_download, require_validator,
  max_redirects, max_parallel); File.remote_stats gives the block
  cache's counters.
- Default build: plain HTTP only, no C. https (rustls/ring) and
  s3/gcs/azure (aws-lc-rs) are opt-in features of clawhdf5-py, and
  ci-test.sh's no-C check now covers the crate.
- A failed storage read (network error, file changed on the server) is an
  OSError, never KeyError/ValueError and never data; `key in group`
  raises it instead of answering False.

Tests: the read-vs-h5py suite runs locally and over HTTP (1 MiB and
1 KiB blocks) against a range-capable http.server in the test process
(conftest.RangeServer); test_remote.py covers request counts, cache
hits, a server without Range support, a changed file, a server that
hangs up, 16 threads, and a spinning thread that keeps running while a
read waits on 0.2 s requests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-27 06:40:44 -05:00
co-authored by Claude Opus 5.5
parent a4c2aced55
commit 910d81904c
18 changed files with 1146 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-py/src/group.rs
+67 -84
View File
@@ -3,11 +3,12 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex, OnceLock};
use pyo3::exceptions::{PyIOError, PyKeyError, PyValueError};
use pyo3::exceptions::{PyIOError, PyKeyError, PyOSError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::PyList;
use crate::attrs::PyAttrs;
use crate::handle::Handle;
use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node};
/// Shared state for a group being written.
@@ -34,9 +35,9 @@ enum GroupInner {
}
impl PyGroup {
pub(crate) fn from_read(file: Arc<clawhdf5_rs::File>, path: String, addr: u64) -> Self {
pub(crate) fn from_read(handle: Arc<Handle>, path: String, addr: u64) -> Self {
Self {
inner: GroupInner::Read(ReadGroup::new(file, path, addr)),
inner: GroupInner::Read(ReadGroup::new(handle, path, addr)),
}
}
@@ -60,9 +61,10 @@ impl PyGroup {
/// h5py). It keeps its own address and, once listed, its links, so looking
/// up a child neither resolves the path from the root nor scans the group's
/// links again: visiting every member of a large group is linear, not
/// quadratic.
/// quadratic. (Edits never add or remove links, so these stay valid in a
/// file open for editing.)
pub(crate) struct ReadGroup {
pub file: Arc<clawhdf5_rs::File>,
pub handle: Arc<Handle>,
pub path: String,
pub addr: u64,
/// Link name -> object address (soft links resolved), filled on first use.
@@ -72,9 +74,9 @@ pub(crate) struct ReadGroup {
}
impl ReadGroup {
pub(crate) fn new(file: Arc<clawhdf5_rs::File>, path: String, addr: u64) -> Self {
pub(crate) fn new(handle: Arc<Handle>, path: String, addr: u64) -> Self {
Self {
file,
handle,
path,
addr,
links: OnceLock::new(),
@@ -82,17 +84,14 @@ impl ReadGroup {
}
}
fn links(&self) -> PyResult<&HashMap<String, u64>> {
fn links(&self, py: Python<'_>) -> PyResult<&HashMap<String, u64>> {
if let Some(links) = self.links.get() {
return Ok(links);
}
let entries = crate::no_panic(|| {
clawhdf5_format::group_v2::resolve_group_children(
self.file.as_bytes(),
self.file.superblock(),
self.addr,
)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", node::name(&self.path))))
let (addr, path) = (self.addr, &self.path);
let entries = self.handle.with(py, |f| {
clawhdf5_format::group_v2::resolve_group_children_in(f.storage(), f.superblock(), addr)
.map_err(|e| node::format_err(path, e, PyValueError::new_err))
})?;
let map = entries
.into_iter()
@@ -102,7 +101,7 @@ impl ReadGroup {
}
/// The path and address of `key` (a name, a relative or an absolute path).
fn locate(&self, key: &str) -> PyResult<(String, u64)> {
fn locate(&self, py: Python<'_>, key: &str) -> PyResult<(String, u64)> {
let path = node::join(&self.path, key);
let rel = if self.path.is_empty() {
Some(path.as_str())
@@ -112,24 +111,24 @@ impl ReadGroup {
path.strip_prefix(self.path.as_str())
.and_then(|r| r.strip_prefix('/'))
};
let addr = match rel {
// A direct child: the link table, when it has the name.
Some(name) if !name.is_empty() && !name.contains('/') => {
match self.links()?.get(name) {
Some(&a) => a,
None => node::resolve_from(&self.file, self.addr, name, &path)?,
}
}
Some(rel) => node::resolve_from(&self.file, self.addr, rel, &path)?,
None => node::address(&self.file, &path)?,
};
Ok((path, addr))
// A direct child: the link table, when it has the name.
if let Some(name) = rel.filter(|n| !n.is_empty() && !n.contains('/'))
&& let Some(&a) = self.links(py)?.get(name)
{
return Ok((path, a));
}
let addr = self.addr;
let found = self.handle.with(py, |f| match rel {
Some(rel) => node::resolve_from(f, addr, rel, &path),
None => node::address(f, &path),
})?;
Ok((path, found))
}
/// `group[key]`.
pub(crate) fn get_item(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
let (path, addr) = self.locate(key)?;
node::open(py, &self.file, path, addr)
let (path, addr) = self.locate(py, key)?;
node::open(py, &self.handle, path, addr)
}
/// `group.get(key, default)`.
@@ -148,48 +147,57 @@ impl ReadGroup {
}
/// Names of the group's datasets and subgroups, sorted (h5py's order).
pub(crate) fn member_names(&self) -> PyResult<&[String]> {
pub(crate) fn member_names(&self, py: Python<'_>) -> PyResult<&[String]> {
if let Some(m) = self.members.get() {
return Ok(m);
}
let mut names = Vec::new();
for (name, &addr) in self.links()? {
let hdr = node::header_at(&self.file, addr, &node::join(&self.path, name))?;
if matches!(
node::kind(&hdr),
Some(node::Kind::Dataset | node::Kind::Group)
) {
names.push(name.clone());
let links = self.links(py)?;
let path = &self.path;
let mut names = self.handle.with(py, |f| {
let mut names = Vec::new();
for (name, &addr) in links {
if matches!(
node::kind_at(f, addr, &node::join(path, name))?,
Some(node::Kind::Dataset | node::Kind::Group)
) {
names.push(name.clone());
}
}
}
Ok(names)
})?;
names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
Ok(self.members.get_or_init(|| names))
}
pub(crate) fn contains(&self, key: &str) -> bool {
self.locate(key)
.and_then(|(path, addr)| node::header_at(&self.file, addr, &path))
.ok()
.and_then(|h| node::kind(&h))
.is_some_and(|k| k != node::Kind::Datatype)
/// `key in group`: whether `key` names a dataset or group. A failed
/// read of the file (a network error) is raised, not `False`.
pub(crate) fn contains(&self, py: Python<'_>, key: &str) -> PyResult<bool> {
let found = self
.locate(py, key)
.and_then(|(path, addr)| self.handle.with(py, |f| node::kind_at(f, addr, &path)));
match found {
Ok(kind) => Ok(kind.is_some_and(|k| k != node::Kind::Datatype)),
Err(e) if e.is_instance_of::<PyOSError>(py) => Err(e),
Err(_) => Ok(false),
}
}
pub(crate) fn values(&self, py: Python<'_>) -> PyResult<Vec<Py<PyAny>>> {
self.member_names()?
self.member_names(py)?
.iter()
.map(|n| self.get_item(py, n))
.collect()
}
pub(crate) fn items(&self, py: Python<'_>) -> PyResult<Vec<(String, Py<PyAny>)>> {
self.member_names()?
self.member_names(py)?
.iter()
.map(|n| Ok((n.clone(), self.get_item(py, n)?)))
.collect()
}
pub(crate) fn attrs(&self) -> PyResult<PyAttrs> {
PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path)
pub(crate) fn attrs(&self, py: Python<'_>) -> PyResult<PyAttrs> {
PyAttrs::read(py, Arc::clone(&self.handle), self.addr, &self.path)
}
}
@@ -210,7 +218,7 @@ impl PyGroup {
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
match &self.inner {
GroupInner::Read(g) => {
let list = PyList::new(py, g.member_names()?)?;
let list = PyList::new(py, g.member_names(py)?)?;
Ok(list.into_any().unbind())
}
GroupInner::Write(state) => {
@@ -236,9 +244,9 @@ impl PyGroup {
self.keys(py)?.call_method0(py, "__iter__")
}
fn __len__(&self) -> PyResult<usize> {
fn __len__(&self, py: Python<'_>) -> PyResult<usize> {
match &self.inner {
GroupInner::Read(g) => Ok(g.member_names()?.len()),
GroupInner::Read(g) => Ok(g.member_names(py)?.len()),
GroupInner::Write(state) => Ok(state.lock().unwrap().datasets.len()),
}
}
@@ -301,9 +309,9 @@ impl PyGroup {
/// Attribute access.
#[getter]
fn attrs(&self) -> PyResult<PyAttrs> {
fn attrs(&self, py: Python<'_>) -> PyResult<PyAttrs> {
match &self.inner {
GroupInner::Read(g) => g.attrs(),
GroupInner::Read(g) => g.attrs(py),
GroupInner::Write(state) => {
let store = Arc::clone(&state.lock().unwrap().attrs);
Ok(PyAttrs::from_write(store))
@@ -311,10 +319,10 @@ impl PyGroup {
}
}
fn __repr__(&self) -> String {
fn __repr__(&self, py: Python<'_>) -> String {
match &self.inner {
GroupInner::Read(g) => {
let n = g.member_names().map_or(0, |m| m.len());
let n = g.member_names(py).map_or(0, |m| m.len());
format!("<HDF5 group \"{}\" ({n} members)>", node::name(&g.path))
}
GroupInner::Write(state) => {
@@ -324,9 +332,9 @@ impl PyGroup {
}
}
fn __contains__(&self, key: &str) -> PyResult<bool> {
fn __contains__(&self, py: Python<'_>, key: &str) -> PyResult<bool> {
match &self.inner {
GroupInner::Read(g) => Ok(g.contains(key)),
GroupInner::Read(g) => g.contains(py, key),
GroupInner::Write(state) => {
let guard = state.lock().unwrap();
Ok(guard.datasets.iter().any(|d| d.name == key))
@@ -357,31 +365,6 @@ pub(crate) fn finalize_write_group(
mod tests {
use super::*;
#[test]
fn member_names_are_sorted() {
let mut b = clawhdf5_rs::FileBuilder::new();
b.create_dataset("zeta").with_f64_data(&[1.0]);
b.create_dataset("alpha").with_f64_data(&[1.0]);
let mut g = b.create_group("mid");
g.create_dataset("x").with_f64_data(&[1.0]);
let finished = g.finish();
b.add_group(finished);
let bytes = b.finish().unwrap();
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap());
let root = file.superblock().root_group_address;
let top = ReadGroup::new(Arc::clone(&file), String::new(), root);
assert_eq!(top.member_names().unwrap(), ["alpha", "mid", "zeta"]);
let (path, addr) = top.locate("mid").unwrap();
assert_eq!(path, "mid");
let mid = ReadGroup::new(Arc::clone(&file), path, addr);
assert_eq!(mid.member_names().unwrap(), ["x"]);
assert!(top.contains("mid/x"));
assert!(mid.contains("/alpha"));
assert!(mid.contains("x") && mid.contains("./x"));
assert!(!top.contains("nope"));
assert!(!mid.contains("alpha"));
}
#[test]
fn finalize_group() {
let state = WriteGroupState {