Fast contiguous and concurrent reads, VL data, nested groups and links, Python bindings #15

Merged
osobh merged 41 commits from feat/p2-perf-coverage into main 2026-09-26 14:57:01 +00:00
6 changed files with 59 additions and 45 deletions
Showing only changes of commit 006bf3b131 - Show all commits
+2
View File
@@ -5,3 +5,5 @@ benchmarks/longmemeval/*.json
# Local model weights (MiniLM etc.) — large, not committed
weights/
.venv
__pycache__/
.pytest_cache/
+8
View File
@@ -2,6 +2,14 @@
## Unreleased
### Python bindings (2026-09-26)
- **`pip install` / `maturin develop` now gives `import clawhdf5`.** The
distribution in `crates/clawhdf5-py/pyproject.toml` was still called
`rustyhdf5` while the extension module was `clawhdf5`, and the package's
tests imported `rustyhdf5`, so they failed at collection. Distribution,
module and tests now all say `clawhdf5`, and the module has
`__version__`.
### Plugin filters (2026-09-26)
- **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files
written by h5py with `compression="lzf"`, or with hdf5plugin's
+1 -1
View File
@@ -3,7 +3,7 @@ name = "clawhdf5-py"
version = "2.7.0"
edition = "2024"
rust-version.workspace = true
description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library"
description = "Python bindings for clawhdf5 — a pure-Rust HDF5 library"
license = "MIT"
repository = "https://git.redclaw.dev/quantumclaw/clawhdf5"
readme = "README.md"
+5 -2
View File
@@ -3,12 +3,15 @@ requires = ["maturin>=1.0,<2.0"]
build-backend = "maturin"
[project]
name = "rustyhdf5"
name = "clawhdf5"
version = "2.7.0"
description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library"
description = "Python bindings for clawhdf5 — a pure-Rust HDF5 library"
requires-python = ">=3.8"
license = { text = "MIT" }
dependencies = ["numpy"]
[tool.maturin]
features = ["extension-module"]
# The extension module is `clawhdf5` (the cdylib's [lib] name): the
# distribution, the import name and the #[pymodule] all agree.
module-name = "clawhdf5"
+1
View File
@@ -219,6 +219,7 @@ pub(crate) fn extract_numpy_data(
/// The clawhdf5 Python module.
#[pymodule]
fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add("__version__", env!("CARGO_PKG_VERSION"))?;
m.add_class::<PyFile>()?;
m.add_class::<PyDataset>()?;
m.add_class::<PyGroup>()?;
@@ -1,4 +1,4 @@
"""Tests for rustyhdf5 Python bindings."""
"""Tests for clawhdf5 Python bindings."""
import os
import tempfile
@@ -6,7 +6,7 @@ import tempfile
import numpy as np
import pytest
import rustyhdf5
import clawhdf5
@pytest.fixture
@@ -18,7 +18,7 @@ def tmp_h5(tmp_path):
@pytest.fixture
def sample_read_file(tmp_h5):
"""Create a sample HDF5 file for reading tests."""
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("temperatures", data=np.array([22.5, 23.1, 21.8]))
f.create_dataset("counts", data=np.array([10, 20, 30], dtype=np.int32))
f.attrs["version"] = 1
@@ -29,7 +29,7 @@ def sample_read_file(tmp_h5):
@pytest.fixture
def grouped_read_file(tmp_h5):
"""Create an HDF5 file with groups for reading tests."""
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("root_data", data=np.array([0.0, 1.0]))
grp = f.create_group("sensors")
grp.create_dataset("temperature", data=np.array([22.5, 23.1, 21.8]))
@@ -46,7 +46,7 @@ def grouped_read_file(tmp_h5):
def test_open_and_read_f64(sample_read_file):
f = rustyhdf5.File(sample_read_file, "r")
f = clawhdf5.File(sample_read_file, "r")
ds = f["temperatures"]
data = ds[:]
np.testing.assert_array_almost_equal(data, [22.5, 23.1, 21.8])
@@ -54,7 +54,7 @@ def test_open_and_read_f64(sample_read_file):
def test_open_and_read_i32(sample_read_file):
f = rustyhdf5.File(sample_read_file, "r")
f = clawhdf5.File(sample_read_file, "r")
ds = f["counts"]
data = ds[:]
np.testing.assert_array_equal(data, [10, 20, 30])
@@ -68,13 +68,13 @@ def test_open_and_read_i32(sample_read_file):
def test_dataset_shape(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
ds = f["temperatures"]
assert ds.shape == (3,)
def test_dataset_dtype(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
assert f["temperatures"].dtype == "float64"
assert f["counts"].dtype == "int32"
@@ -85,24 +85,24 @@ def test_dataset_dtype(sample_read_file):
def test_read_root_attrs(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
assert f.attrs["version"] == 1
assert f.attrs["description"] == "test file"
def test_attrs_len(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
assert len(f.attrs) >= 2
def test_attrs_contains(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
assert "version" in f.attrs
assert "nonexistent" not in f.attrs
def test_attrs_keys(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
keys = f.attrs.keys()
assert "version" in keys
assert "description" in keys
@@ -114,7 +114,7 @@ def test_attrs_keys(sample_read_file):
def test_read_group_keys(grouped_read_file):
with rustyhdf5.File(grouped_read_file, "r") as f:
with clawhdf5.File(grouped_read_file, "r") as f:
keys = f.keys()
assert "sensors" in keys
assert "metadata" in keys
@@ -122,7 +122,7 @@ def test_read_group_keys(grouped_read_file):
def test_read_group_dataset(grouped_read_file):
with rustyhdf5.File(grouped_read_file, "r") as f:
with clawhdf5.File(grouped_read_file, "r") as f:
grp = f["sensors"]
ds = grp["temperature"]
data = ds[:]
@@ -130,14 +130,14 @@ def test_read_group_dataset(grouped_read_file):
def test_read_group_attrs(grouped_read_file):
with rustyhdf5.File(grouped_read_file, "r") as f:
with clawhdf5.File(grouped_read_file, "r") as f:
grp = f["sensors"]
assert grp.attrs["location"] == "lab"
def test_nested_path_access(grouped_read_file):
"""Test f['group/dataset'] path navigation."""
with rustyhdf5.File(grouped_read_file, "r") as f:
with clawhdf5.File(grouped_read_file, "r") as f:
ds = f["sensors/temperature"]
data = ds[:]
np.testing.assert_array_almost_equal(data, [22.5, 23.1, 21.8])
@@ -149,7 +149,7 @@ def test_nested_path_access(grouped_read_file):
def test_context_manager(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
data = f["temperatures"][:]
np.testing.assert_array_almost_equal(data, [22.5, 23.1, 21.8])
# File should be closed after with block
@@ -162,30 +162,30 @@ def test_context_manager(sample_read_file):
def test_write_simple(tmp_h5):
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("data", data=np.array([1.0, 2.0, 3.0]))
# Verify by reading back
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
data = f["data"][:]
np.testing.assert_array_almost_equal(data, [1.0, 2.0, 3.0])
def test_write_with_attrs(tmp_h5):
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("values", data=np.array([10, 20], dtype=np.int32))
f.attrs["author"] = "test"
f.attrs["count"] = 42
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
assert f.attrs["author"] == "test"
assert f.attrs["count"] == 42
def test_write_with_group(tmp_h5):
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
grp = f.create_group("experiment")
grp.create_dataset("results", data=np.array([3.14, 2.72]))
grp.attrs["version"] = 1
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
ds = f["experiment/results"]
np.testing.assert_array_almost_equal(ds[:], [3.14, 2.72])
grp = f["experiment"]
@@ -199,9 +199,9 @@ def test_write_with_group(tmp_h5):
def test_roundtrip_float64(tmp_h5):
original = np.array([1.1, 2.2, 3.3], dtype=np.float64)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("data", data=original)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
result = f["data"][:]
np.testing.assert_array_almost_equal(result, original)
assert result.dtype == np.float64
@@ -209,9 +209,9 @@ def test_roundtrip_float64(tmp_h5):
def test_roundtrip_float32(tmp_h5):
original = np.array([1.5, 2.5, 3.5], dtype=np.float32)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("data", data=original)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
result = f["data"][:]
np.testing.assert_array_almost_equal(result, original)
assert result.dtype == np.float32
@@ -219,9 +219,9 @@ def test_roundtrip_float32(tmp_h5):
def test_roundtrip_int32(tmp_h5):
original = np.array([-10, 0, 10, 100], dtype=np.int32)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("data", data=original)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
result = f["data"][:]
np.testing.assert_array_equal(result, original)
assert result.dtype == np.int32
@@ -229,9 +229,9 @@ def test_roundtrip_int32(tmp_h5):
def test_roundtrip_int64(tmp_h5):
original = np.array([-1, 0, 1, 2**40], dtype=np.int64)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("data", data=original)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
result = f["data"][:]
np.testing.assert_array_equal(result, original)
assert result.dtype == np.int64
@@ -239,9 +239,9 @@ def test_roundtrip_int64(tmp_h5):
def test_roundtrip_uint8(tmp_h5):
original = np.array([0, 127, 255], dtype=np.uint8)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("data", data=original)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
result = f["data"][:]
np.testing.assert_array_equal(result, original)
assert result.dtype == np.uint8
@@ -254,7 +254,7 @@ def test_roundtrip_uint8(tmp_h5):
def test_chunked_gzip(tmp_h5):
original = np.arange(100, dtype=np.float64)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset(
"compressed",
data=original,
@@ -262,7 +262,7 @@ def test_chunked_gzip(tmp_h5):
compression="gzip",
compression_opts=6,
)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
result = f["compressed"][:]
np.testing.assert_array_equal(result, original)
@@ -276,7 +276,7 @@ def test_h5py_can_read_our_file(tmp_h5):
"""Verify that h5py can read files we create."""
import h5py
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("values", data=np.array([1.0, 2.0, 3.0]))
f.attrs["meta"] = "hello"
with h5py.File(tmp_h5, "r") as f:
@@ -292,7 +292,7 @@ def test_we_can_read_h5py_file(tmp_h5):
with h5py.File(tmp_h5, "w") as f:
f.create_dataset("data", data=np.array([10.0, 20.0, 30.0]))
f.attrs["version"] = 2
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
data = f["data"][:]
np.testing.assert_array_equal(data, [10.0, 20.0, 30.0])
assert f.attrs["version"] == 2
@@ -305,9 +305,9 @@ def test_we_can_read_h5py_file(tmp_h5):
def test_2d_array_roundtrip(tmp_h5):
original = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float64)
with rustyhdf5.File(tmp_h5, "w") as f:
with clawhdf5.File(tmp_h5, "w") as f:
f.create_dataset("matrix", data=original)
with rustyhdf5.File(tmp_h5, "r") as f:
with clawhdf5.File(tmp_h5, "r") as f:
ds = f["matrix"]
assert ds.shape == (2, 3)
result = ds[:]
@@ -321,15 +321,15 @@ def test_2d_array_roundtrip(tmp_h5):
def test_open_nonexistent_file():
with pytest.raises(OSError):
rustyhdf5.File("/nonexistent/path.h5", "r")
clawhdf5.File("/nonexistent/path.h5", "r")
def test_invalid_mode(tmp_h5):
with pytest.raises(ValueError):
rustyhdf5.File(tmp_h5, "x")
clawhdf5.File(tmp_h5, "x")
def test_key_error_on_missing_dataset(sample_read_file):
with rustyhdf5.File(sample_read_file, "r") as f:
with clawhdf5.File(sample_read_file, "r") as f:
with pytest.raises(KeyError):
f["nonexistent"]