quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

perf(py): datasets and groups keep their address; groups their links

Browse Source 
Every ds[...] and g[k] resolved the path from the root again, two or
three times per open, and resolving a name in a large group scans its
links: visiting a group was O(n^2). 4000 scalar datasets in one group
took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now
0.3 s each. A Dataset keeps its object address, a Group (and the file's
root) its address and, after the first lookup, its link table.

New facade API File::dataset_at(address), tested in integration_tests.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-26 09:01:54 -05:00
co-authored by Claude Opus 5.5
parent b43bd2e67f
commit f0ecae38b6
9 changed files with 349 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-py/src/attrs.rs
+4 -3
View File
@@ -34,9 +34,10 @@ pub struct PyAttrs {
}
impl PyAttrs {
/// The attributes of the object at `path` in a file opened for reading.
pub(crate) fn read(file: Arc<clawhdf5_rs::File>, path: &str) -> PyResult<Self> {
let attrs = node::attributes(&file, path)?;
/// The attributes of the object at `addr` (whose path is `path`) in a
/// file opened for reading.
pub(crate) fn read(file: Arc<clawhdf5_rs::File>, addr: u64, path: &str) -> PyResult<Self> {
let attrs = node::attributes(&file, addr, path)?;
Ok(Self {
inner: AttrsInner::Read { file, attrs },
})
crates/clawhdf5-py/src/dataset.rs
+14 -8
View File
@@ -9,6 +9,7 @@
use std::sync::Arc;
use clawhdf5_format::datatype::Datatype;
use clawhdf5_format::object_header::ObjectHeader;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyList, PyTuple};
@@ -29,6 +30,9 @@ use crate::{PyEmpty, node, to_py_err};
pub struct PyDataset {
file: Arc<clawhdf5_rs::File>,
path: String,
/// Where the dataset's object header is: reads open it from here rather
/// than resolve `path` again.
addr: u64,
/// `None` for a dataset with a null dataspace (h5py's `Empty`).
shape: Option<Vec<u64>>,
/// The chunk shape, for a chunked dataset.
@@ -43,12 +47,13 @@ impl PyDataset {
py: Python<'_>,
file: Arc<clawhdf5_rs::File>,
path: String,
addr: u64,
hdr: &ObjectHeader,
) -> PyResult<Self> {
crate::no_panic(|| {
let hdr = node::header(&file, &path)?;
let null = node::is_null(&node::dataspace(&file, &hdr)?);
let null = node::is_null(&node::dataspace(&file, hdr)?);
let (shape, datatype) = {
let ds = file.dataset(&path).map_err(to_py_err)?;
let ds = file.dataset_at(addr).map_err(to_py_err)?;
let shape = if null {
None
} else {
@@ -60,10 +65,11 @@ impl PyDataset {
.map_err(|e| e.value(py).to_string());
let chunks = shape
.as_ref()
.and_then(|s| node::chunk_shape(&file, &hdr, s.len()));
.and_then(|s| node::chunk_shape(&file, hdr, s.len()));
Ok(Self {
file,
path,
addr,
shape,
chunks,
datatype,
@@ -96,10 +102,10 @@ impl PyDataset {
let (reads, list_axis) = plan.reads(dims, chunk_len, elem_size);
let read_shape = plan.read_shape();
let file = &*self.file;
let path = self.path.as_str();
let addr = self.addr;
// Everything below touches only Rust data: release the GIL.
let read = || -> Result<Elements, ReadError> {
let ds = file.dataset(path)?;
let ds = file.dataset_at(addr)?;
let mut blocks = Vec::with_capacity(reads.len());
for read in reads {
let raw = ds.read_selection(&read.sel)?;
@@ -250,7 +256,7 @@ impl PyDataset {
};
let max = self
.file
.dataset(&self.path)
.dataset_at(self.addr)
.and_then(|ds| ds.max_dimensions())
.map_err(to_py_err)?
.unwrap_or_else(|| shape.clone());
@@ -288,7 +294,7 @@ impl PyDataset {
/// The dataset's attributes (read-only, dict-like).
#[getter]
fn attrs(&self) -> PyResult<PyAttrs> {
PyAttrs::read(Arc::clone(&self.file), &self.path)
PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path)
}
/// Read with h5py indexing: integers, slices with positive steps,
crates/clawhdf5-py/src/file.rs
+21 -20
View File
@@ -3,12 +3,11 @@
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use pyo3::exceptions::PyKeyError;
use pyo3::prelude::*;
use pyo3::types::PyList;
use crate::attrs::PyAttrs;
use crate::group::{self, PyGroup, WriteGroupState, finalize_write_group};
use crate::group::{PyGroup, ReadGroup, WriteGroupState, finalize_write_group};
use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err};
/// Internal state for write mode.
@@ -40,7 +39,8 @@ pub struct PyFile {
}
enum FileInner {
Read(Arc<clawhdf5_rs::File>),
/// The root group; it holds the file.
Read(ReadGroup),
Write(WriteState),
}
@@ -61,7 +61,7 @@ impl PyFile {
crate::no_panic(|| clawhdf5_rs::File::open(path).map_err(to_py_err))
})?;
Ok(Self {
inner: Some(FileInner::Read(Arc::new(file))),
inner: Some(FileInner::Read(root_group(Arc::new(file)))),
filename,
})
}
@@ -110,33 +110,28 @@ impl PyFile {
/// Get a child object (dataset or group) by path; `f['/']` is the root.
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
group::get_item(py, self.read_file()?, "", key)
self.read_file()?.get_item(py, key)
}
/// `f.get(key, default=None)`.
#[pyo3(signature = (key, default=None))]
fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
match group::get_item(py, self.read_file()?, "", key) {
Err(e) if e.is_instance_of::<PyKeyError>(py) => {
Ok(default.unwrap_or_else(|| py.None()))
}
other => other,
}
self.read_file()?.get(py, key, default)
}
/// List the names of all children in the root group.
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let names = group::member_names(self.read_file()?, "")?;
let names = self.read_file()?.member_names()?;
Ok(PyList::new(py, names)?.into_any().unbind())
}
fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let vals = group::values(py, self.read_file()?, "")?;
let vals = self.read_file()?.values(py)?;
Ok(PyList::new(py, vals)?.into_any().unbind())
}
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let items = group::items(py, self.read_file()?, "")?;
let items = self.read_file()?.items(py)?;
Ok(PyList::new(py, items)?.into_any().unbind())
}
@@ -145,7 +140,7 @@ impl PyFile {
}
fn __len__(&self) -> PyResult<usize> {
Ok(group::member_names(self.read_file()?, "")?.len())
Ok(self.read_file()?.member_names()?.len())
}
/// The root group's name, `/`.
@@ -211,7 +206,7 @@ impl PyFile {
#[getter]
fn attrs(&self) -> PyResult<PyAttrs> {
match self.inner.as_ref() {
Some(FileInner::Read(file)) => PyAttrs::read(Arc::clone(file), ""),
Some(FileInner::Read(root)) => root.attrs(),
Some(FileInner::Write(state)) => Ok(PyAttrs::from_write(Arc::clone(&state.root_attrs))),
None => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
"file is closed",
@@ -221,8 +216,8 @@ impl PyFile {
fn __repr__(&self) -> String {
match &self.inner {
Some(FileInner::Read(f)) => {
format!("<HDF5 File (read, {} bytes)>", f.as_bytes().len())
Some(FileInner::Read(root)) => {
format!("<HDF5 File (read, {} bytes)>", root.file.as_bytes().len())
}
Some(FileInner::Write(s)) => {
format!("<HDF5 File (write, \"{}\")>", s.path.display())
@@ -232,12 +227,13 @@ impl PyFile {
}
fn __contains__(&self, key: &str) -> PyResult<bool> {
Ok(group::contains(self.read_file()?, "", key))
Ok(self.read_file()?.contains(key))
}
}
impl PyFile {
fn read_file(&self) -> PyResult<&Arc<clawhdf5_rs::File>> {
/// The root group of a file opened for reading.
fn read_file(&self) -> PyResult<&ReadGroup> {
match &self.inner {
Some(FileInner::Read(f)) => Ok(f),
Some(FileInner::Write(_)) => Err(PyErr::new::<pyo3::exceptions::PyIOError, _>(
@@ -275,6 +271,11 @@ fn parse_compression(
}
}
fn root_group(file: Arc<clawhdf5_rs::File>) -> ReadGroup {
let root = file.superblock().root_group_address;
ReadGroup::new(file, String::new(), root)
}
/// Build and write the HDF5 file from accumulated write state.
fn finalize_write(state: WriteState) -> PyResult<()> {
crate::no_panic(|| {
crates/clawhdf5-py/src/group.rs
+174 -102
View File
@@ -1,13 +1,14 @@
//! PyGroup — navigable HDF5 group with read and write support.
use std::sync::{Arc, Mutex};
use std::collections::HashMap;
use std::sync::{Arc, Mutex, OnceLock};
use pyo3::exceptions::{PyIOError, PyKeyError};
use pyo3::exceptions::{PyIOError, PyKeyError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::PyList;
use crate::attrs::PyAttrs;
use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node, to_py_err};
use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node};
/// Shared state for a group being written.
pub(crate) struct WriteGroupState {
@@ -28,17 +29,14 @@ pub struct PyGroup {
}
enum GroupInner {
Read {
file: Arc<clawhdf5_rs::File>,
path: String,
},
Read(ReadGroup),
Write(Arc<Mutex<WriteGroupState>>),
}
impl PyGroup {
pub(crate) fn from_read(file: Arc<clawhdf5_rs::File>, path: String) -> Self {
pub(crate) fn from_read(file: Arc<clawhdf5_rs::File>, path: String, addr: u64) -> Self {
Self {
inner: GroupInner::Read { file, path },
inner: GroupInner::Read(ReadGroup::new(file, path, addr)),
}
}
@@ -48,9 +46,9 @@ impl PyGroup {
}
}
fn read_parts(&self, what: &str) -> PyResult<(&Arc<clawhdf5_rs::File>, &str)> {
fn read_group(&self, what: &str) -> PyResult<&ReadGroup> {
match &self.inner {
GroupInner::Read { file, path } => Ok((file, path)),
GroupInner::Read(g) => Ok(g),
GroupInner::Write(_) => Err(PyIOError::new_err(format!(
"cannot {what} a group opened for writing"
))),
@@ -58,77 +56,90 @@ impl PyGroup {
}
}
// Read-mode operations shared by `Group` and `File` (a file is its root
// group, as in h5py).
/// `group[key]`.
pub(crate) fn get_item(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: &str,
key: &str,
) -> PyResult<Py<PyAny>> {
node::open(py, file, node::join(path, key))
/// A group in a file opened for reading (a file is its root group, as in
/// h5py). It keeps its own address and, once listed, its links, so looking
/// up a child neither resolves the path from the root nor scans the group's
/// links again: visiting every member of a large group is linear, not
/// quadratic.
pub(crate) struct ReadGroup {
pub file: Arc<clawhdf5_rs::File>,
pub path: String,
pub addr: u64,
/// Link name -> object address (soft links resolved), filled on first use.
links: OnceLock<HashMap<String, u64>>,
/// Names of the datasets and subgroups, sorted (h5py's order).
members: OnceLock<Vec<String>>,
}
/// Names of the group's datasets and subgroups, sorted (h5py's order).
pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<String>> {
crate::no_panic(|| {
let group = if path.is_empty() {
file.root()
} else {
file.group(path).map_err(to_py_err)?
};
let mut names = group.datasets().map_err(to_py_err)?;
names.extend(group.groups().map_err(to_py_err)?);
names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
names.dedup();
Ok(names)
})
}
pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool {
node::exists(file, &node::join(path, key))
}
pub(crate) fn values(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: &str,
) -> PyResult<Vec<Py<PyAny>>> {
member_names(file, path)?
.iter()
.map(|n| get_item(py, file, path, n))
.collect()
}
pub(crate) fn items(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: &str,
) -> PyResult<Vec<(String, Py<PyAny>)>> {
member_names(file, path)?
.into_iter()
.map(|n| {
let v = get_item(py, file, path, &n)?;
Ok((n, v))
})
.collect()
}
#[pymethods]
impl PyGroup {
/// Get a child object (dataset or subgroup) by name or path.
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
get_item(py, file, path, key)
impl ReadGroup {
pub(crate) fn new(file: Arc<clawhdf5_rs::File>, path: String, addr: u64) -> Self {
Self {
file,
path,
addr,
links: OnceLock::new(),
members: OnceLock::new(),
}
}
/// `group.get(key, default=None)`.
#[pyo3(signature = (key, default=None))]
fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
match get_item(py, file, path, key) {
fn links(&self) -> PyResult<&HashMap<String, u64>> {
if let Some(links) = self.links.get() {
return Ok(links);
}
let entries = crate::no_panic(|| {
clawhdf5_format::group_v2::resolve_group_children(
self.file.as_bytes(),
self.file.superblock(),
self.addr,
)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", node::name(&self.path))))
})?;
let map = entries
.into_iter()
.map(|e| (e.name, e.object_header_address))
.collect();
Ok(self.links.get_or_init(|| map))
}
/// The path and address of `key` (a name, a relative or an absolute path).
fn locate(&self, key: &str) -> PyResult<(String, u64)> {
let path = node::join(&self.path, key);
let rel = if self.path.is_empty() {
Some(path.as_str())
} else if path == self.path {
Some("")
} else {
path.strip_prefix(self.path.as_str())
.and_then(|r| r.strip_prefix('/'))
};
let addr = match rel {
// A direct child: the link table, when it has the name.
Some(name) if !name.is_empty() && !name.contains('/') => {
match self.links()?.get(name) {
Some(&a) => a,
None => node::resolve_from(&self.file, self.addr, name, &path)?,
}
}
Some(rel) => node::resolve_from(&self.file, self.addr, rel, &path)?,
None => node::address(&self.file, &path)?,
};
Ok((path, addr))
}
/// `group[key]`.
pub(crate) fn get_item(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
let (path, addr) = self.locate(key)?;
node::open(py, &self.file, path, addr)
}
/// `group.get(key, default)`.
pub(crate) fn get(
&self,
py: Python<'_>,
key: &str,
default: Option<Py<PyAny>>,
) -> PyResult<Py<PyAny>> {
match self.get_item(py, key) {
Err(e) if e.is_instance_of::<PyKeyError>(py) => {
Ok(default.unwrap_or_else(|| py.None()))
}
@@ -136,11 +147,70 @@ impl PyGroup {
}
}
/// Names of the group's datasets and subgroups, sorted (h5py's order).
pub(crate) fn member_names(&self) -> PyResult<&[String]> {
if let Some(m) = self.members.get() {
return Ok(m);
}
let mut names = Vec::new();
for (name, &addr) in self.links()? {
let hdr = node::header_at(&self.file, addr, &node::join(&self.path, name))?;
if matches!(
node::kind(&hdr),
Some(node::Kind::Dataset | node::Kind::Group)
) {
names.push(name.clone());
}
}
names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
Ok(self.members.get_or_init(|| names))
}
pub(crate) fn contains(&self, key: &str) -> bool {
self.locate(key)
.and_then(|(path, addr)| node::header_at(&self.file, addr, &path))
.ok()
.and_then(|h| node::kind(&h))
.is_some_and(|k| k != node::Kind::Datatype)
}
pub(crate) fn values(&self, py: Python<'_>) -> PyResult<Vec<Py<PyAny>>> {
self.member_names()?
.iter()
.map(|n| self.get_item(py, n))
.collect()
}
pub(crate) fn items(&self, py: Python<'_>) -> PyResult<Vec<(String, Py<PyAny>)>> {
self.member_names()?
.iter()
.map(|n| Ok((n.clone(), self.get_item(py, n)?)))
.collect()
}
pub(crate) fn attrs(&self) -> PyResult<PyAttrs> {
PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path)
}
}
#[pymethods]
impl PyGroup {
/// Get a child object (dataset or subgroup) by name or path.
fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult<Py<PyAny>> {
self.read_group("read children from")?.get_item(py, key)
}
/// `group.get(key, default=None)`.
#[pyo3(signature = (key, default=None))]
fn get(&self, py: Python<'_>, key: &str, default: Option<Py<PyAny>>) -> PyResult<Py<PyAny>> {
self.read_group("read children from")?.get(py, key, default)
}
/// List the names of all children (datasets and subgroups).
fn keys(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
match &self.inner {
GroupInner::Read { file, path } => {
let list = PyList::new(py, member_names(file, path)?)?;
GroupInner::Read(g) => {
let list = PyList::new(py, g.member_names()?)?;
Ok(list.into_any().unbind())
}
GroupInner::Write(state) => {
@@ -153,15 +223,13 @@ impl PyGroup {
}
fn values(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
Ok(PyList::new(py, values(py, file, path)?)?
.into_any()
.unbind())
let g = self.read_group("read children from")?;
Ok(PyList::new(py, g.values(py)?)?.into_any().unbind())
}
fn items(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
let (file, path) = self.read_parts("read children from")?;
Ok(PyList::new(py, items(py, file, path)?)?.into_any().unbind())
let g = self.read_group("read children from")?;
Ok(PyList::new(py, g.items(py)?)?.into_any().unbind())
}
fn __iter__(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
@@ -170,7 +238,7 @@ impl PyGroup {
fn __len__(&self) -> PyResult<usize> {
match &self.inner {
GroupInner::Read { file, path } => Ok(member_names(file, path)?.len()),
GroupInner::Read(g) => Ok(g.member_names()?.len()),
GroupInner::Write(state) => Ok(state.lock().unwrap().datasets.len()),
}
}
@@ -179,7 +247,7 @@ impl PyGroup {
#[getter]
fn name(&self) -> String {
match &self.inner {
GroupInner::Read { path, .. } => node::name(path),
GroupInner::Read(g) => node::name(&g.path),
GroupInner::Write(state) => node::name(&state.lock().unwrap().name),
}
}
@@ -235,7 +303,7 @@ impl PyGroup {
#[getter]
fn attrs(&self) -> PyResult<PyAttrs> {
match &self.inner {
GroupInner::Read { file, path } => PyAttrs::read(Arc::clone(file), path),
GroupInner::Read(g) => g.attrs(),
GroupInner::Write(state) => {
let store = Arc::clone(&state.lock().unwrap().attrs);
Ok(PyAttrs::from_write(store))
@@ -245,9 +313,9 @@ impl PyGroup {
fn __repr__(&self) -> String {
match &self.inner {
GroupInner::Read { file, path } => {
let n = member_names(file, path).map_or(0, |m| m.len());
format!("<HDF5 group \"{}\" ({n} members)>", node::name(path))
GroupInner::Read(g) => {
let n = g.member_names().map_or(0, |m| m.len());
format!("<HDF5 group \"{}\" ({n} members)>", node::name(&g.path))
}
GroupInner::Write(state) => {
let name = &state.lock().unwrap().name;
@@ -258,7 +326,7 @@ impl PyGroup {
fn __contains__(&self, key: &str) -> PyResult<bool> {
match &self.inner {
GroupInner::Read { file, path } => Ok(contains(file, path, key)),
GroupInner::Read(g) => Ok(g.contains(key)),
GroupInner::Write(state) => {
let guard = state.lock().unwrap();
Ok(guard.datasets.iter().any(|d| d.name == key))
@@ -299,15 +367,19 @@ mod tests {
let finished = g.finish();
b.add_group(finished);
let bytes = b.finish().unwrap();
let file = clawhdf5_rs::File::from_bytes(bytes).unwrap();
assert_eq!(
member_names(&file, "").unwrap(),
vec!["alpha", "mid", "zeta"]
);
assert_eq!(member_names(&file, "mid").unwrap(), vec!["x"]);
assert!(contains(&file, "", "mid/x"));
assert!(contains(&file, "mid", "/alpha"));
assert!(!contains(&file, "", "nope"));
let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap());
let root = file.superblock().root_group_address;
let top = ReadGroup::new(Arc::clone(&file), String::new(), root);
assert_eq!(top.member_names().unwrap(), ["alpha", "mid", "zeta"]);
let (path, addr) = top.locate("mid").unwrap();
assert_eq!(path, "mid");
let mid = ReadGroup::new(Arc::clone(&file), path, addr);
assert_eq!(mid.member_names().unwrap(), ["x"]);
assert!(top.contains("mid/x"));
assert!(mid.contains("/alpha"));
assert!(mid.contains("x") && mid.contains("./x"));
assert!(!top.contains("nope"));
assert!(!mid.contains("alpha"));
}
#[test]
crates/clawhdf5-py/src/node.rs
+44 -30
View File
@@ -33,24 +33,40 @@ pub(crate) fn name(path: &str) -> String {
format!("/{path}")
}
/// The object header of the object at `path`.
pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult<ObjectHeader> {
/// The address of the object at `path`, resolved from the root group.
pub(crate) fn address(file: &clawhdf5_rs::File, path: &str) -> PyResult<u64> {
resolve_from(file, file.superblock().root_group_address, path, path)
}
/// The address of `rel` resolved from the group at `group` (`full` is the
/// resulting path, for the error message).
pub(crate) fn resolve_from(
file: &clawhdf5_rs::File,
group: u64,
rel: &str,
full: &str,
) -> PyResult<u64> {
if rel.is_empty() {
return Ok(group);
}
crate::no_panic(|| {
let sb = file.superblock();
let data = file.as_bytes();
let addr = if path.is_empty() {
sb.root_group_address
} else {
clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| {
clawhdf5_format::group_v2::resolve_path_from(file.as_bytes(), file.superblock(), group, rel)
.map_err(|e| {
PyKeyError::new_err(format!(
"Unable to open object (object '{}' doesn't exist): {e}",
name(path)
name(full)
))
})?
};
let addr = usize::try_from(addr)
})
})
}
/// The object header at `addr` (the object at `path`).
pub(crate) fn header_at(file: &clawhdf5_rs::File, addr: u64, path: &str) -> PyResult<ObjectHeader> {
crate::no_panic(|| {
let sb = file.superblock();
let at = usize::try_from(addr)
.map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?;
ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size)
ObjectHeader::parse(file.as_bytes(), at, sb.offset_size, sb.length_size)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))
})
}
@@ -80,19 +96,21 @@ pub(crate) fn kind(hdr: &ObjectHeader) -> Option<Kind> {
}
}
/// Open the object at `path` as a `Dataset` or `Group`.
/// Open the object at `addr` (whose path is `path`) as a `Dataset` or
/// `Group`. Both keep the address, so later reads resolve nothing.
pub(crate) fn open(
py: Python<'_>,
file: &Arc<clawhdf5_rs::File>,
path: String,
addr: u64,
) -> PyResult<Py<PyAny>> {
let hdr = header(file, &path)?;
let hdr = header_at(file, addr, &path)?;
match kind(&hdr) {
Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path)?
Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path, addr, &hdr)?
.into_pyobject(py)?
.into_any()
.unbind()),
Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path)
Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path, addr)
.into_pyobject(py)?
.into_any()
.unbind()),
@@ -107,14 +125,6 @@ pub(crate) fn open(
}
}
/// Whether `path` names a dataset or group.
pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool {
header(file, path)
.ok()
.and_then(|h| kind(&h))
.is_some_and(|k| k != Kind::Datatype)
}
/// The dataspace message of an object header.
pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult<Dataspace> {
crate::no_panic(|| {
@@ -166,12 +176,16 @@ pub(crate) fn is_null(space: &Dataspace) -> bool {
space.space_type == DataspaceType::Null
}
/// The attributes of the object at `path`, sorted by name (h5py's order).
/// Attributes whose messages cannot be parsed are left out, as the facade's
/// `attrs()` does.
pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<AttributeMessage>> {
/// The attributes of the object at `addr` (whose path is `path`), sorted by
/// name (h5py's order). Attributes whose messages cannot be parsed are left
/// out, as the facade's `attrs()` does.
pub(crate) fn attributes(
file: &clawhdf5_rs::File,
addr: u64,
path: &str,
) -> PyResult<Vec<AttributeMessage>> {
let hdr = header_at(file, addr, path)?;
crate::no_panic(|| {
let hdr = header(file, path)?;
let sb = file.superblock();
let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
file.as_bytes(),