Fast contiguous and concurrent reads, VL data, nested groups and links, Python bindings #15

Merged
osobh merged 41 commits from feat/p2-perf-coverage into main 2026-09-26 14:57:01 +00:00
8 changed files with 289 additions and 176 deletions
Showing only changes of commit 24412a0e59 - Show all commits
+6
View File
@@ -54,6 +54,12 @@
h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's: h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's:
it returns variable-length sequences of big-endian floats unswapped; this it returns variable-length sequences of big-endian floats unswapped; this
package returns the stored values. package returns the stored values.
- **A panic in the library is an ordinary Python exception.** PyO3 turns a
Rust panic into `PanicException`, a `BaseException` that `except
Exception` does not catch. Every call from the bindings into the library
is now guarded and a panic becomes `clawhdf5.InternalError` (a
`RuntimeError`) naming the object; with the implicit-index panic above
restored, `ds[0:30]` raises it.
- **CI builds and tests the Python package.** It was excluded from CI. - **CI builds and tests the Python package.** It was excluded from CI.
`scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with
maturin, unpacks it under `target/` and runs the pytest suite; skipped maturin, unpacks it under `target/` and runs the pytest suite; skipped
+41 -39
View File
@@ -191,46 +191,48 @@ fn attr_to_py<'py>(
file: &clawhdf5_rs::File, file: &clawhdf5_rs::File,
attr: &AttributeMessage, attr: &AttributeMessage,
) -> PyResult<Bound<'py, PyAny>> { ) -> PyResult<Bound<'py, PyAny>> {
let sb = file.superblock(); crate::no_panic(|| {
let conv = Converter::new(py, &attr.datatype, sb.offset_size) let sb = file.superblock();
.map_err(|e| prefix_err(py, &attr.name, e))?; let conv = Converter::new(py, &attr.datatype, sb.offset_size)
if node::is_null(&attr.dataspace) { .map_err(|e| prefix_err(py, &attr.name, e))?;
return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any()); if node::is_null(&attr.dataspace) {
} return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any());
let shape: Vec<usize> = attr
.dataspace
.dimensions
.iter()
.map(|&d| d as usize)
.collect();
let n: usize = shape.iter().product();
let data = if conv.is_vl() {
let want = n * conv.elem_size;
if attr.raw_data.len() < want {
return Err(PyValueError::new_err(format!(
"attribute {}: {} bytes of variable-length references, expected {want}",
attr.name,
attr.raw_data.len(),
)));
} }
let raw = &attr.raw_data[..want]; let shape: Vec<usize> = attr
let file_data = file.as_bytes(); .dataspace
let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit); .dimensions
Elements::Vl( .iter()
py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit)) .map(|&d| d as usize)
.map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?, .collect();
) let n: usize = shape.iter().product();
} else { let data = if conv.is_vl() {
Elements::Bytes(attr.raw_data.clone()) let want = n * conv.elem_size;
}; if attr.raw_data.len() < want {
let arr = conv return Err(PyValueError::new_err(format!(
.to_array(py, data, &shape, true) "attribute {}: {} bytes of variable-length references, expected {want}",
.map_err(|e| prefix_err(py, &attr.name, e))?; attr.name,
if shape.is_empty() { attr.raw_data.len(),
// A scalar dataspace: h5py returns the element itself. )));
return arr.get_item(PyTuple::empty(py)); }
} let raw = &attr.raw_data[..want];
Ok(arr) let file_data = file.as_bytes();
let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit);
Elements::Vl(
py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit))
.map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?,
)
} else {
Elements::Bytes(attr.raw_data.clone())
};
let arr = conv
.to_array(py, data, &shape, true)
.map_err(|e| prefix_err(py, &attr.name, e))?;
if shape.is_empty() {
// A scalar dataspace: h5py returns the element itself.
return arr.get_item(PyTuple::empty(py));
}
Ok(arr)
})
} }
fn prefix_err(py: Python<'_>, name: &str, e: PyErr) -> PyErr { fn prefix_err(py: Python<'_>, name: &str, e: PyErr) -> PyErr {
+77 -64
View File
@@ -42,25 +42,27 @@ impl PyDataset {
file: Arc<clawhdf5_rs::File>, file: Arc<clawhdf5_rs::File>,
path: String, path: String,
) -> PyResult<Self> { ) -> PyResult<Self> {
let hdr = node::header(&file, &path)?; crate::no_panic(|| {
let null = node::is_null(&node::dataspace(&file, &hdr)?); let hdr = node::header(&file, &path)?;
let (shape, datatype) = { let null = node::is_null(&node::dataspace(&file, &hdr)?);
let ds = file.dataset(&path).map_err(to_py_err)?; let (shape, datatype) = {
let shape = if null { let ds = file.dataset(&path).map_err(to_py_err)?;
None let shape = if null {
} else { None
Some(ds.shape().map_err(to_py_err)?) } else {
Some(ds.shape().map_err(to_py_err)?)
};
(shape, ds.raw_datatype().map_err(to_py_err)?)
}; };
(shape, ds.raw_datatype().map_err(to_py_err)?) let conv = Converter::new(py, &datatype, file.superblock().offset_size)
}; .map_err(|e| e.value(py).to_string());
let conv = Converter::new(py, &datatype, file.superblock().offset_size) Ok(Self {
.map_err(|e| e.value(py).to_string()); file,
Ok(Self { path,
file, shape,
path, datatype,
shape, conv,
datatype, })
conv,
}) })
} }
@@ -84,39 +86,43 @@ impl PyDataset {
let path = self.path.as_str(); let path = self.path.as_str();
let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit); let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit);
// Everything below touches only Rust data: release the GIL. // Everything below touches only Rust data: release the GIL.
let blocks: Vec<(Elements, Vec<usize>)> = py let read = || -> Result<_, ReadError> {
.detach(|| -> Result<_, ReadError> { let ds = file.dataset(path)?;
let ds = file.dataset(path)?; let sb = file.superblock();
let sb = file.superblock(); let mut blocks = Vec::with_capacity(reads.len());
let mut blocks = Vec::with_capacity(reads.len()); for (sel, shape) in reads {
for (sel, shape) in reads { let raw = ds.read_selection(&sel)?;
let raw = ds.read_selection(&sel)?; let n: usize = shape.iter().product();
let n: usize = shape.iter().product(); let data = if vl {
let data = if vl { if raw.len() != n * elem_size {
if raw.len() != n * elem_size { return Err(ReadError::Other(format!(
return Err(ReadError::Other(format!( "read {} bytes of variable-length references, expected {}",
"read {} bytes of variable-length references, expected {}", raw.len(),
raw.len(), n * elem_size
n * elem_size )));
))); }
} Elements::Vl(
Elements::Vl( resolve_vl(
resolve_vl( file.as_bytes(),
file.as_bytes(), &raw,
&raw, n,
n, sb.offset_size,
sb.offset_size, sb.length_size,
sb.length_size, unit,
unit,
)
.map_err(ReadError::Other)?,
) )
} else { .map_err(ReadError::Other)?,
Elements::Bytes(raw) )
}; } else {
blocks.push((data, shape)); Elements::Bytes(raw)
} };
Ok(blocks) blocks.push((data, shape));
}
Ok(blocks)
};
let blocks: Vec<(Elements, Vec<usize>)> = py
.detach(|| {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(read))
.unwrap_or_else(|p| Err(ReadError::Panic(crate::panic_text(&*p))))
}) })
.map_err(|e| e.into_py(&self.path))?; .map_err(|e| e.into_py(&self.path))?;
@@ -160,6 +166,7 @@ impl PyDataset {
enum ReadError { enum ReadError {
Lib(clawhdf5_rs::Error), Lib(clawhdf5_rs::Error),
Other(String), Other(String),
Panic(String),
} }
impl From<clawhdf5_rs::Error> for ReadError { impl From<clawhdf5_rs::Error> for ReadError {
@@ -173,6 +180,10 @@ impl ReadError {
match self { match self {
ReadError::Lib(e) => to_py_err(e), ReadError::Lib(e) => to_py_err(e),
ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))), ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))),
ReadError::Panic(msg) => crate::InternalError::new_err(format!(
"{}: clawhdf5 internal error (please report it): {msg}",
node::name(path)
)),
} }
} }
} }
@@ -223,20 +234,22 @@ impl PyDataset {
/// The maximum shape (`None` per unlimited dimension), like h5py. /// The maximum shape (`None` per unlimited dimension), like h5py.
#[getter] #[getter]
fn maxshape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> { fn maxshape<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let Some(shape) = &self.shape else { crate::no_panic(|| {
return Ok(py.None().into_bound(py)); let Some(shape) = &self.shape else {
}; return Ok(py.None().into_bound(py));
let max = self };
.file let max = self
.dataset(&self.path) .file
.and_then(|ds| ds.max_dimensions()) .dataset(&self.path)
.map_err(to_py_err)? .and_then(|ds| ds.max_dimensions())
.unwrap_or_else(|| shape.clone()); .map_err(to_py_err)?
let items: Vec<Option<u64>> = max .unwrap_or_else(|| shape.clone());
.into_iter() let items: Vec<Option<u64>> = max
.map(|d| (d != u64::MAX).then_some(d)) .into_iter()
.collect(); .map(|d| (d != u64::MAX).then_some(d))
Ok(PyTuple::new(py, items)?.into_any()) .collect();
Ok(PyTuple::new(py, items)?.into_any())
})
} }
/// The dataset's numpy dtype, as h5py reports it. /// The dataset's numpy dtype, as h5py reports it.
+24 -22
View File
@@ -57,9 +57,9 @@ impl PyFile {
let filename = path.to_string(); let filename = path.to_string();
match mode { match mode {
"r" => { "r" => {
let file = py let file = py.detach(|| {
.detach(|| clawhdf5_rs::File::open(path)) crate::no_panic(|| clawhdf5_rs::File::open(path).map_err(to_py_err))
.map_err(to_py_err)?; })?;
Ok(Self { Ok(Self {
inner: Some(FileInner::Read(Arc::new(file))), inner: Some(FileInner::Read(Arc::new(file))),
filename, filename,
@@ -277,29 +277,31 @@ fn parse_compression(
/// Build and write the HDF5 file from accumulated write state. /// Build and write the HDF5 file from accumulated write state.
fn finalize_write(state: WriteState) -> PyResult<()> { fn finalize_write(state: WriteState) -> PyResult<()> {
let mut builder = clawhdf5_rs::FileBuilder::new(); crate::no_panic(|| {
let mut builder = clawhdf5_rs::FileBuilder::new();
// Root attributes // Root attributes
let root_attrs = state.root_attrs.lock().unwrap_or_else(|e| e.into_inner()); let root_attrs = state.root_attrs.lock().unwrap_or_else(|e| e.into_inner());
for (name, val) in root_attrs.iter() { for (name, val) in root_attrs.iter() {
builder.set_attr(name, val.clone().into()); builder.set_attr(name, val.clone().into());
} }
drop(root_attrs); drop(root_attrs);
// Root datasets // Root datasets
for spec in &state.root_datasets { for spec in &state.root_datasets {
let db = builder.create_dataset(&spec.name); let db = builder.create_dataset(&spec.name);
apply_dataset_spec(db, spec); apply_dataset_spec(db, spec);
} }
// Groups // Groups
for group_arc in &state.groups { for group_arc in &state.groups {
let guard = group_arc.lock().unwrap(); let guard = group_arc.lock().unwrap();
finalize_write_group(&mut builder, &guard); finalize_write_group(&mut builder, &guard);
} }
builder.write(&state.path).map_err(to_py_err)?; builder.write(&state.path).map_err(to_py_err)?;
Ok(()) Ok(())
})
} }
#[cfg(test)] #[cfg(test)]
+12 -10
View File
@@ -73,16 +73,18 @@ pub(crate) fn get_item(
/// Names of the group's datasets and subgroups, sorted (h5py's order). /// Names of the group's datasets and subgroups, sorted (h5py's order).
pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<String>> { pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<String>> {
let group = if path.is_empty() { crate::no_panic(|| {
file.root() let group = if path.is_empty() {
} else { file.root()
file.group(path).map_err(to_py_err)? } else {
}; file.group(path).map_err(to_py_err)?
let mut names = group.datasets().map_err(to_py_err)?; };
names.extend(group.groups().map_err(to_py_err)?); let mut names = group.datasets().map_err(to_py_err)?;
names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); names.extend(group.groups().map_err(to_py_err)?);
names.dedup(); names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
Ok(names) names.dedup();
Ok(names)
})
} }
pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool { pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool {
+38
View File
@@ -24,6 +24,42 @@ pub(crate) use dataset::PyDataset;
pub(crate) use file::PyFile; pub(crate) use file::PyFile;
pub(crate) use group::PyGroup; pub(crate) use group::PyGroup;
pyo3::create_exception!(
clawhdf5,
InternalError,
pyo3::exceptions::PyRuntimeError,
"A bug in clawhdf5 met while reading or writing a file (a Rust panic, \
caught). Derived from RuntimeError, so `except Exception` handles it."
);
/// The text of a caught panic.
pub(crate) fn panic_text(payload: &(dyn std::any::Any + Send)) -> String {
payload
.downcast_ref::<&str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "unknown panic".to_string())
}
/// Run `f`, turning a panic in the library into [`InternalError`] instead of
/// PyO3's `PanicException` (a `BaseException`, which `except Exception`
/// does not catch). Wraps every call into the library.
pub(crate) fn no_panic<T>(f: impl FnOnce() -> PyResult<T>) -> PyResult<T> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).unwrap_or_else(|p| {
Err(InternalError::new_err(format!(
"clawhdf5 internal error (please report it): {}",
panic_text(&*p)
)))
})
}
/// A test hook: panics inside [`no_panic`], so the tests can check that a
/// library panic reaches Python as an ordinary exception.
#[pyfunction]
fn _panic_for_test() -> PyResult<()> {
no_panic(|| panic!("deliberate panic for the test suite"))
}
/// Convert a `clawhdf5_rs::Error` into a `PyErr`. /// Convert a `clawhdf5_rs::Error` into a `PyErr`.
/// ///
/// Maps different error variants to more specific Python exception types: /// Maps different error variants to more specific Python exception types:
@@ -276,6 +312,8 @@ fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyGroup>()?; m.add_class::<PyGroup>()?;
m.add_class::<PyAttrs>()?; m.add_class::<PyAttrs>()?;
m.add_class::<PyEmpty>()?; m.add_class::<PyEmpty>()?;
m.add("InternalError", m.py().get_type::<InternalError>())?;
m.add_function(wrap_pyfunction!(_panic_for_test, m)?)?;
Ok(()) Ok(())
} }
+47 -41
View File
@@ -35,22 +35,24 @@ pub(crate) fn name(path: &str) -> String {
/// The object header of the object at `path`. /// The object header of the object at `path`.
pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult<ObjectHeader> { pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult<ObjectHeader> {
let sb = file.superblock(); crate::no_panic(|| {
let data = file.as_bytes(); let sb = file.superblock();
let addr = if path.is_empty() { let data = file.as_bytes();
sb.root_group_address let addr = if path.is_empty() {
} else { sb.root_group_address
clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| { } else {
PyKeyError::new_err(format!( clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| {
"Unable to open object (object '{}' doesn't exist): {e}", PyKeyError::new_err(format!(
name(path) "Unable to open object (object '{}' doesn't exist): {e}",
)) name(path)
})? ))
}; })?
let addr = usize::try_from(addr) };
.map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?; let addr = usize::try_from(addr)
ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size) .map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?;
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path)))) ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size)
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))
})
} }
/// What an object header describes. /// What an object header describes.
@@ -115,20 +117,22 @@ pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool {
/// The dataspace message of an object header. /// The dataspace message of an object header.
pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult<Dataspace> { pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult<Dataspace> {
let sb = file.superblock(); crate::no_panic(|| {
let msg = hdr let sb = file.superblock();
.messages let msg = hdr
.iter() .messages
.find(|m| m.msg_type == MessageType::Dataspace) .iter()
.ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?; .find(|m| m.msg_type == MessageType::Dataspace)
let data = clawhdf5_format::shared_message::message_data( .ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?;
file.as_bytes(), let data = clawhdf5_format::shared_message::message_data(
msg, file.as_bytes(),
sb.offset_size, msg,
sb.length_size, sb.offset_size,
) sb.length_size,
.map_err(|e| PyValueError::new_err(e.to_string()))?; )
Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string())) .map_err(|e| PyValueError::new_err(e.to_string()))?;
Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string()))
})
} }
pub(crate) fn is_null(space: &Dataspace) -> bool { pub(crate) fn is_null(space: &Dataspace) -> bool {
@@ -139,17 +143,19 @@ pub(crate) fn is_null(space: &Dataspace) -> bool {
/// Attributes whose messages cannot be parsed are left out, as the facade's /// Attributes whose messages cannot be parsed are left out, as the facade's
/// `attrs()` does. /// `attrs()` does.
pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<AttributeMessage>> { pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult<Vec<AttributeMessage>> {
let hdr = header(file, path)?; crate::no_panic(|| {
let sb = file.superblock(); let hdr = header(file, path)?;
let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant( let sb = file.superblock();
file.as_bytes(), let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant(
&hdr, file.as_bytes(),
sb.offset_size, &hdr,
sb.length_size, sb.offset_size,
) sb.length_size,
.map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?; )
attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes())); .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?;
Ok(attrs) attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes()));
Ok(attrs)
})
} }
#[cfg(test)] #[cfg(test)]
@@ -467,3 +467,47 @@ def test_threads_read_the_same_file(pair):
with ThreadPoolExecutor(8) as pool: with ThreadPoolExecutor(8) as pool:
list(pool.map(work, range(8))) list(pool.map(work, range(8)))
assert not errors, errors[:3] assert not errors, errors[:3]
def _v4_index_fixture(h5py, path):
"""One 2-D dataset per v4 chunk index (HDF5 1.10+ layout, libver='latest')."""
data = (np.arange(37 * 23, dtype="<i4") * 7 - 1000).reshape(37, 23)
early = h5py.h5p.create(h5py.h5p.DATASET_CREATE)
early.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY)
with h5py.File(path, "w", libver="latest") as f:
f.create_dataset("implicit", data=data, chunks=(5, 4), dcpl=early)
f.create_dataset("fixed_array", data=data, chunks=(5, 4), compression="gzip")
f.create_dataset("extensible_array", data=data, chunks=(5, 4), maxshape=(None, 23), compression="gzip")
f.create_dataset("btree2", data=data, chunks=(5, 4), maxshape=(None, None), compression="gzip")
f.create_dataset("single_chunk", data=data, chunks=(37, 23), compression="gzip")
V4_KEYS = [
slice(0, 3), slice(0, 30), (slice(7, 16), slice(3, 9)), (36, 22), (slice(None, None, 3), slice(1, None, 4)),
(slice(1, None, 2), Ellipsis), (Ellipsis, slice(2, 22)), [0, 5, 6, 36], (slice(None), [0, 3, 22]), -1, (),
]
def test_every_v4_chunk_index_matches_h5py(h5py, tmp_path):
"""Partial reads of each v4 chunk index. The implicit index (early
allocation, no filters) used to panic in the library for any selection
covering more than half the dataset, e.g. ds[0:30]."""
path = str(tmp_path / "v4.h5")
_v4_index_fixture(h5py, path)
with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours:
for name in theirs:
for key in V4_KEYS:
assert_same(ours[name][key], theirs[name][key], f"{name}[{key!r}]")
def test_a_library_panic_is_an_ordinary_exception():
"""PyO3 turns a Rust panic into PanicException, a BaseException that
`except Exception` does not catch. Every call into the library is
guarded, so a panic surfaces as clawhdf5.InternalError instead."""
assert issubclass(clawhdf5.InternalError, RuntimeError)
with pytest.raises(clawhdf5.InternalError, match="deliberate panic"):
clawhdf5._panic_for_test()
try:
clawhdf5._panic_for_test()
except Exception: # noqa: BLE001 - the point of the test
pass