From 8cbbef3fae6a11286aa096f47f0168ddfcd96463 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:08:31 -0500 Subject: [PATCH 01/36] fix(format): write a Group Info message in every group libhdf5 reads a group's Group Info message before it inserts a link, and FileWriter wrote none, so h5py in "r+" mode could not add a link to any group we wrote: "Unable to create link (message type not found)". Each group header now carries a version 0 Group Info message with the default link-phase thresholds, as libhdf5 writes for a new group. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++ crates/clawhdf5-format/src/file_writer.rs | 7 + .../clawhdf5/tests/writer_groups_interop.rs | 136 ++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 crates/clawhdf5/tests/writer_groups_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4bd2f..cc1010f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### Writer: groups and links (2026-09-26) +- **libhdf5 could not add links to groups we wrote.** h5py in `"r+"` mode + failed with "Unable to create link (message type not found)" on every + group `FileWriter` wrote: libhdf5 reads a group's Group Info message before + inserting a link, and none was written. Every group now carries one + (version 0, default thresholds: 6 more bytes per group header, so files + are not byte-identical to earlier versions). Regression test: + `crates/clawhdf5/tests/writer_groups_interop.rs`. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index dbf59f2..36c3fbb 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -196,6 +196,13 @@ pub(crate) fn build_group_oh( li.extend_from_slice(&u64::MAX.to_le_bytes()); // fractal heap addr = UNDEF li.extend_from_slice(&u64::MAX.to_le_bytes()); // btree name index addr = UNDEF w.add_message(MessageType::LinkInfo, li); + } + // Group Info (version 0, default link-phase thresholds, no estimates). + // Readers don't need it, but libhdf5 reads it before inserting a link: + // without one, adding a link to a group we wrote (h5py in "r+" mode) + // failed with "message type not found". + w.add_message(MessageType::GroupInfo, vec![0, 0]); + if dense_link_info.is_none() { for link in links { w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE)); } diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs new file mode 100644 index 0000000..9ea3053 --- /dev/null +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -0,0 +1,136 @@ +//! Groups and links written by `FileBuilder`, read back by h5py (libhdf5) +//! and h5dump, and by clawhdf5 itself. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{File, FileBuilder}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn h5dump_available() -> bool { + Command::new("h5dump") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Run `body` under h5py with `path` bound to the file's path. +fn h5py(path: &str, body: &str) -> String { + run_python(&format!( + "import h5py, numpy as np, json\npath = r'{path}'\n{body}" + )) +} + +fn write(dir: &tempfile::TempDir, name: &str, b: FileBuilder) -> String { + let path = dir.path().join(name).display().to_string(); + b.write(&path).unwrap(); + path +} + +/// h5dump must read the whole file without an error. +fn h5dump_ok(path: &str) -> String { + if !h5dump_available() { + assert!(!interop_required(), "h5dump is not available"); + return String::new(); + } + let o = Command::new("h5dump").arg(path).output().unwrap(); + let out = String::from_utf8_lossy(&o.stdout).to_string(); + assert!( + o.status.success(), + "h5dump failed:\n{out}{}", + String::from_utf8_lossy(&o.stderr) + ); + out +} + +// ---- libhdf5 can modify the groups we write ---- + +#[test] +fn h5py_can_add_links_to_groups_we_wrote() { + skip_if_no_python!(); + // Measured before the fix: h5py in "r+" mode could not add a link to any + // group we wrote ("Unable to create link (message type not found)"): + // libhdf5 reads a group's Group Info message before inserting a link, and + // the writer wrote none. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.create_dataset("x").with_f64_data(&[1.0, 2.0]); + let mut g = b.create_group("small"); + g.create_dataset("a").with_i32_data(&[1]); + b.add_group(g.finish()); + let mut g = b.create_group("big"); // dense link storage + for i in 0..20 { + g.create_dataset(&format!("d{i:02}")).with_i32_data(&[i]); + } + b.add_group(g.finish()); + let path = write(&dir, "modify.h5", b); + + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 f['alias'] = f['x']\n\ + \x20 f['small']['new'] = np.arange(3)\n\ + \x20 f['big']['new'] = np.arange(4)\n\ + \x20 f.create_group('added/deeper')\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([sorted(f), sorted(f['small']), len(f['big']),\n\ + \x20 f['alias'][()].tolist(), f['big/new'][()].tolist(), f['big/d07'][()].tolist()]))", + ); + assert_eq!( + out, + r#"[["added", "alias", "big", "small", "x"], ["a", "new"], 21, [1.0, 2.0], [0, 1, 2, 3], [7]]"# + ); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("big/new").unwrap().read_i64().unwrap(), + [0, 1, 2, 3] + ); + assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]); +} -- 2.54.0 From 006bf3b13136a18c9bc48b66fb09df2797c694d2 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:09:49 -0500 Subject: [PATCH 02/36] fix(py): one name, clawhdf5, for the Python distribution and module pyproject.toml named the distribution rustyhdf5 while the extension module is clawhdf5, and the package's tests imported rustyhdf5, so pytest failed at collection. Distribution, module-name and tests now agree; the module gains __version__. maturin develop + pytest: 28 pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitignore | 2 + CHANGELOG.md | 8 ++ crates/clawhdf5-py/Cargo.toml | 2 +- crates/clawhdf5-py/pyproject.toml | 7 +- crates/clawhdf5-py/src/lib.rs | 1 + .../{test_rustyhdf5.py => test_write_read.py} | 84 +++++++++---------- 6 files changed, 59 insertions(+), 45 deletions(-) rename crates/clawhdf5-py/tests/{test_rustyhdf5.py => test_write_read.py} (82%) diff --git a/.gitignore b/.gitignore index 029ea7a..cf90c2c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ benchmarks/longmemeval/*.json # Local model weights (MiniLM etc.) — large, not committed weights/ .venv +__pycache__/ +.pytest_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4bd2f..fb0a5f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +### Python bindings (2026-09-26) +- **`pip install` / `maturin develop` now gives `import clawhdf5`.** The + distribution in `crates/clawhdf5-py/pyproject.toml` was still called + `rustyhdf5` while the extension module was `clawhdf5`, and the package's + tests imported `rustyhdf5`, so they failed at collection. Distribution, + module and tests now all say `clawhdf5`, and the module has + `__version__`. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-py/Cargo.toml b/crates/clawhdf5-py/Cargo.toml index 0d54a4f..3513a62 100644 --- a/crates/clawhdf5-py/Cargo.toml +++ b/crates/clawhdf5-py/Cargo.toml @@ -3,7 +3,7 @@ name = "clawhdf5-py" version = "2.7.0" edition = "2024" rust-version.workspace = true -description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" +description = "Python bindings for clawhdf5 — a pure-Rust HDF5 library" license = "MIT" repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" diff --git a/crates/clawhdf5-py/pyproject.toml b/crates/clawhdf5-py/pyproject.toml index 87a6f6e..6788adc 100644 --- a/crates/clawhdf5-py/pyproject.toml +++ b/crates/clawhdf5-py/pyproject.toml @@ -3,12 +3,15 @@ requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" [project] -name = "rustyhdf5" +name = "clawhdf5" version = "2.7.0" -description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" +description = "Python bindings for clawhdf5 — a pure-Rust HDF5 library" requires-python = ">=3.8" license = { text = "MIT" } dependencies = ["numpy"] [tool.maturin] features = ["extension-module"] +# The extension module is `clawhdf5` (the cdylib's [lib] name): the +# distribution, the import name and the #[pymodule] all agree. +module-name = "clawhdf5" diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index 5721964..fe31887 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -219,6 +219,7 @@ pub(crate) fn extract_numpy_data( /// The clawhdf5 Python module. #[pymodule] fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add("__version__", env!("CARGO_PKG_VERSION"))?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/clawhdf5-py/tests/test_rustyhdf5.py b/crates/clawhdf5-py/tests/test_write_read.py similarity index 82% rename from crates/clawhdf5-py/tests/test_rustyhdf5.py rename to crates/clawhdf5-py/tests/test_write_read.py index 20c4e19..94ad70c 100644 --- a/crates/clawhdf5-py/tests/test_rustyhdf5.py +++ b/crates/clawhdf5-py/tests/test_write_read.py @@ -1,4 +1,4 @@ -"""Tests for rustyhdf5 Python bindings.""" +"""Tests for clawhdf5 Python bindings.""" import os import tempfile @@ -6,7 +6,7 @@ import tempfile import numpy as np import pytest -import rustyhdf5 +import clawhdf5 @pytest.fixture @@ -18,7 +18,7 @@ def tmp_h5(tmp_path): @pytest.fixture def sample_read_file(tmp_h5): """Create a sample HDF5 file for reading tests.""" - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("temperatures", data=np.array([22.5, 23.1, 21.8])) f.create_dataset("counts", data=np.array([10, 20, 30], dtype=np.int32)) f.attrs["version"] = 1 @@ -29,7 +29,7 @@ def sample_read_file(tmp_h5): @pytest.fixture def grouped_read_file(tmp_h5): """Create an HDF5 file with groups for reading tests.""" - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("root_data", data=np.array([0.0, 1.0])) grp = f.create_group("sensors") grp.create_dataset("temperature", data=np.array([22.5, 23.1, 21.8])) @@ -46,7 +46,7 @@ def grouped_read_file(tmp_h5): def test_open_and_read_f64(sample_read_file): - f = rustyhdf5.File(sample_read_file, "r") + f = clawhdf5.File(sample_read_file, "r") ds = f["temperatures"] data = ds[:] np.testing.assert_array_almost_equal(data, [22.5, 23.1, 21.8]) @@ -54,7 +54,7 @@ def test_open_and_read_f64(sample_read_file): def test_open_and_read_i32(sample_read_file): - f = rustyhdf5.File(sample_read_file, "r") + f = clawhdf5.File(sample_read_file, "r") ds = f["counts"] data = ds[:] np.testing.assert_array_equal(data, [10, 20, 30]) @@ -68,13 +68,13 @@ def test_open_and_read_i32(sample_read_file): def test_dataset_shape(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: ds = f["temperatures"] assert ds.shape == (3,) def test_dataset_dtype(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: assert f["temperatures"].dtype == "float64" assert f["counts"].dtype == "int32" @@ -85,24 +85,24 @@ def test_dataset_dtype(sample_read_file): def test_read_root_attrs(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: assert f.attrs["version"] == 1 assert f.attrs["description"] == "test file" def test_attrs_len(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: assert len(f.attrs) >= 2 def test_attrs_contains(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: assert "version" in f.attrs assert "nonexistent" not in f.attrs def test_attrs_keys(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: keys = f.attrs.keys() assert "version" in keys assert "description" in keys @@ -114,7 +114,7 @@ def test_attrs_keys(sample_read_file): def test_read_group_keys(grouped_read_file): - with rustyhdf5.File(grouped_read_file, "r") as f: + with clawhdf5.File(grouped_read_file, "r") as f: keys = f.keys() assert "sensors" in keys assert "metadata" in keys @@ -122,7 +122,7 @@ def test_read_group_keys(grouped_read_file): def test_read_group_dataset(grouped_read_file): - with rustyhdf5.File(grouped_read_file, "r") as f: + with clawhdf5.File(grouped_read_file, "r") as f: grp = f["sensors"] ds = grp["temperature"] data = ds[:] @@ -130,14 +130,14 @@ def test_read_group_dataset(grouped_read_file): def test_read_group_attrs(grouped_read_file): - with rustyhdf5.File(grouped_read_file, "r") as f: + with clawhdf5.File(grouped_read_file, "r") as f: grp = f["sensors"] assert grp.attrs["location"] == "lab" def test_nested_path_access(grouped_read_file): """Test f['group/dataset'] path navigation.""" - with rustyhdf5.File(grouped_read_file, "r") as f: + with clawhdf5.File(grouped_read_file, "r") as f: ds = f["sensors/temperature"] data = ds[:] np.testing.assert_array_almost_equal(data, [22.5, 23.1, 21.8]) @@ -149,7 +149,7 @@ def test_nested_path_access(grouped_read_file): def test_context_manager(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: data = f["temperatures"][:] np.testing.assert_array_almost_equal(data, [22.5, 23.1, 21.8]) # File should be closed after with block @@ -162,30 +162,30 @@ def test_context_manager(sample_read_file): def test_write_simple(tmp_h5): - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("data", data=np.array([1.0, 2.0, 3.0])) # Verify by reading back - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: data = f["data"][:] np.testing.assert_array_almost_equal(data, [1.0, 2.0, 3.0]) def test_write_with_attrs(tmp_h5): - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("values", data=np.array([10, 20], dtype=np.int32)) f.attrs["author"] = "test" f.attrs["count"] = 42 - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: assert f.attrs["author"] == "test" assert f.attrs["count"] == 42 def test_write_with_group(tmp_h5): - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: grp = f.create_group("experiment") grp.create_dataset("results", data=np.array([3.14, 2.72])) grp.attrs["version"] = 1 - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: ds = f["experiment/results"] np.testing.assert_array_almost_equal(ds[:], [3.14, 2.72]) grp = f["experiment"] @@ -199,9 +199,9 @@ def test_write_with_group(tmp_h5): def test_roundtrip_float64(tmp_h5): original = np.array([1.1, 2.2, 3.3], dtype=np.float64) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("data", data=original) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: result = f["data"][:] np.testing.assert_array_almost_equal(result, original) assert result.dtype == np.float64 @@ -209,9 +209,9 @@ def test_roundtrip_float64(tmp_h5): def test_roundtrip_float32(tmp_h5): original = np.array([1.5, 2.5, 3.5], dtype=np.float32) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("data", data=original) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: result = f["data"][:] np.testing.assert_array_almost_equal(result, original) assert result.dtype == np.float32 @@ -219,9 +219,9 @@ def test_roundtrip_float32(tmp_h5): def test_roundtrip_int32(tmp_h5): original = np.array([-10, 0, 10, 100], dtype=np.int32) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("data", data=original) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: result = f["data"][:] np.testing.assert_array_equal(result, original) assert result.dtype == np.int32 @@ -229,9 +229,9 @@ def test_roundtrip_int32(tmp_h5): def test_roundtrip_int64(tmp_h5): original = np.array([-1, 0, 1, 2**40], dtype=np.int64) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("data", data=original) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: result = f["data"][:] np.testing.assert_array_equal(result, original) assert result.dtype == np.int64 @@ -239,9 +239,9 @@ def test_roundtrip_int64(tmp_h5): def test_roundtrip_uint8(tmp_h5): original = np.array([0, 127, 255], dtype=np.uint8) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("data", data=original) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: result = f["data"][:] np.testing.assert_array_equal(result, original) assert result.dtype == np.uint8 @@ -254,7 +254,7 @@ def test_roundtrip_uint8(tmp_h5): def test_chunked_gzip(tmp_h5): original = np.arange(100, dtype=np.float64) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset( "compressed", data=original, @@ -262,7 +262,7 @@ def test_chunked_gzip(tmp_h5): compression="gzip", compression_opts=6, ) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: result = f["compressed"][:] np.testing.assert_array_equal(result, original) @@ -276,7 +276,7 @@ def test_h5py_can_read_our_file(tmp_h5): """Verify that h5py can read files we create.""" import h5py - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("values", data=np.array([1.0, 2.0, 3.0])) f.attrs["meta"] = "hello" with h5py.File(tmp_h5, "r") as f: @@ -292,7 +292,7 @@ def test_we_can_read_h5py_file(tmp_h5): with h5py.File(tmp_h5, "w") as f: f.create_dataset("data", data=np.array([10.0, 20.0, 30.0])) f.attrs["version"] = 2 - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: data = f["data"][:] np.testing.assert_array_equal(data, [10.0, 20.0, 30.0]) assert f.attrs["version"] == 2 @@ -305,9 +305,9 @@ def test_we_can_read_h5py_file(tmp_h5): def test_2d_array_roundtrip(tmp_h5): original = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float64) - with rustyhdf5.File(tmp_h5, "w") as f: + with clawhdf5.File(tmp_h5, "w") as f: f.create_dataset("matrix", data=original) - with rustyhdf5.File(tmp_h5, "r") as f: + with clawhdf5.File(tmp_h5, "r") as f: ds = f["matrix"] assert ds.shape == (2, 3) result = ds[:] @@ -321,15 +321,15 @@ def test_2d_array_roundtrip(tmp_h5): def test_open_nonexistent_file(): with pytest.raises(OSError): - rustyhdf5.File("/nonexistent/path.h5", "r") + clawhdf5.File("/nonexistent/path.h5", "r") def test_invalid_mode(tmp_h5): with pytest.raises(ValueError): - rustyhdf5.File(tmp_h5, "x") + clawhdf5.File(tmp_h5, "x") def test_key_error_on_missing_dataset(sample_read_file): - with rustyhdf5.File(sample_read_file, "r") as f: + with clawhdf5.File(sample_read_file, "r") as f: with pytest.raises(KeyError): f["nonexistent"] -- 2.54.0 From 78c769f179dd26b85a675e8de436d9e83138b5ef Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:12:55 -0500 Subject: [PATCH 03/36] perf(format): back large read buffers with transparent huge pages A full read of a contiguous dataset is one memcpy from the mapped file, yet ran at a quarter of h5py's speed on one thread: the fresh output Vec took a page fault and a kernel page clear for every 4 KiB page written, 16384 per 64 MiB, costing several times the copy (the benchmark spent 6.2 s of 8 s in the kernel, 4.3M minor faults). numpy, so h5py, madvises MADV_HUGEPAGE on allocations of 4 MiB or more; the typed readers' output, the raw contiguous read and the chunk assembly buffer now do the same (Linux only, libc as a Linux-only dependency; no-op otherwise). New h5py comparison tests cover full and selection reads of contiguous data for every 1-8-byte integer and float type, both byte orders, ranks 1-4, empty selections, and datasets past the 4 MiB threshold. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 17 + crates/clawhdf5-format/Cargo.toml | 4 + crates/clawhdf5-format/src/bulk_alloc.rs | 78 ++++ crates/clawhdf5-format/src/chunked_read.rs | 2 + crates/clawhdf5-format/src/data_read.rs | 27 +- crates/clawhdf5-format/src/lib.rs | 1 + .../clawhdf5/tests/contiguous_read_interop.rs | 403 ++++++++++++++++++ 7 files changed, 521 insertions(+), 11 deletions(-) create mode 100644 crates/clawhdf5-format/src/bulk_alloc.rs create mode 100644 crates/clawhdf5/tests/contiguous_read_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4bd2f..99f86f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ ## Unreleased +### Contiguous read speed (2026-09-26) +- **Large read buffers are backed by transparent huge pages.** A full read + of a contiguous dataset was one `memcpy` from the mapped file, yet ran at + a quarter of h5py's speed on one thread: the fresh output `Vec` took a + page fault (and a kernel page clear) for every 4 KiB page it was written + to, 16384 of them for 64 MiB, and those cost several times the copy. + numpy, and so h5py, asks for transparent huge pages on every allocation of + 4 MiB or more; clawhdf5-format's read buffers now do too + (`madvise(MADV_HUGEPAGE)` on Linux, `libc` added as a Linux-only + dependency; a no-op elsewhere or when THP is disabled). It applies to the + typed readers' output (`read_f32`, `read_f64`, `read_i32`, `read_i64`, + `read_u64`, both byte orders), the raw contiguous read and the chunk + assembly buffer. Values are unchanged; new h5py comparison + `crates/clawhdf5/tests/contiguous_read_interop.rs` covers every 1-8-byte + integer and float type in both byte orders, ranks 1-4, and datasets past + the 4 MiB threshold. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index 75fd233..1d1998c 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -30,6 +30,10 @@ ruzstd = { version = "0.9", optional = true } bzip2 = { version = "0.6", optional = true } snap = { version = "1", optional = true } +[target.'cfg(target_os = "linux")'.dependencies] +# madvise(MADV_HUGEPAGE) for large read buffers (see src/bulk_alloc.rs). +libc = { version = "0.2", default-features = false } + [dev-dependencies] half = { workspace = true } serde_json = "1" diff --git a/crates/clawhdf5-format/src/bulk_alloc.rs b/crates/clawhdf5-format/src/bulk_alloc.rs new file mode 100644 index 0000000..22a8bf2 --- /dev/null +++ b/crates/clawhdf5-format/src/bulk_alloc.rs @@ -0,0 +1,78 @@ +//! Large output buffers backed by transparent huge pages where the OS offers +//! them. +//! +//! A fresh multi-megabyte `Vec` is mapped lazily by the kernel: the first +//! write to each 4 KiB page takes a page fault, and the kernel zeroes the page +//! before handing it over. For a 64 MiB read that is 16384 faults, and they +//! cost far more than the copy that fills the buffer — single-threaded +//! contiguous reads ran at about a quarter of h5py's speed because of them. +//! numpy (so h5py) avoids this by asking for transparent huge pages +//! (`madvise(MADV_HUGEPAGE)`) on every allocation of 4 MiB or more, which +//! turns 512 faults into one; this module does the same. +//! +//! The advice only changes how the pages are backed, never their contents, so +//! it is harmless when it cannot be honoured (THP disabled, not Linux, a +//! region that is part of the heap): the buffer is then exactly what it would +//! have been without it. + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +/// Buffers smaller than this are left alone (numpy uses the same threshold). +pub(crate) const HUGE_PAGE_THRESHOLD: usize = 4 << 20; + +/// Advise the kernel to back `[ptr, ptr + len)` with transparent huge pages, +/// when `len` is large enough to benefit. Call it before the first write so +/// the faults happen at huge-page granularity. +#[inline] +pub(crate) fn advise_huge_pages(ptr: *const u8, len: usize) { + #[cfg(target_os = "linux")] + if len >= HUGE_PAGE_THRESHOLD { + const PAGE: usize = 4096; + let start = (ptr as usize).next_multiple_of(PAGE); + let end = (ptr as usize + len) & !(PAGE - 1); + if end > start { + // SAFETY: `[start, end)` lies inside an allocation of `len` bytes + // at `ptr` that the caller owns, and is page aligned as madvise + // requires. MADV_HUGEPAGE does not change the memory's contents or + // validity; on failure (EINVAL when THP is compiled out, etc.) the + // region is simply left as it was, so the result is ignored. + unsafe { + libc::madvise(start as *mut libc::c_void, end - start, libc::MADV_HUGEPAGE); + } + } + } + #[cfg(not(target_os = "linux"))] + let _ = (ptr, len); +} + +/// `Vec::with_capacity(count)` for a buffer about to be filled in bulk, with +/// huge-page advice when it is large (see the module docs). +#[inline] +pub(crate) fn vec_for_bulk(count: usize) -> Vec { + let v: Vec = Vec::with_capacity(count); + advise_huge_pages( + v.as_ptr().cast::(), + v.capacity().saturating_mul(core::mem::size_of::()), + ); + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bulk_vec_is_an_ordinary_vec() { + for count in [0usize, 1, 1000, HUGE_PAGE_THRESHOLD / 4 + 3] { + let mut v: Vec = vec_for_bulk(count); + assert!(v.capacity() >= count); + v.extend((0..count as u32).map(|i| i.wrapping_mul(2654435761))); + assert!( + v.iter() + .enumerate() + .all(|(i, &x)| x == (i as u32).wrapping_mul(2654435761)) + ); + } + } +} diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index cbbaa69..0baec25 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -277,6 +277,8 @@ pub(crate) fn alloc_output(len: usize) -> Result, FormatError> { if ptr.is_null() { return Err(failed()); } + // Before anything writes to it, so a large buffer faults in huge pages. + crate::bulk_alloc::advise_huge_pages(ptr, len); // SAFETY: `ptr` came from the global allocator with the layout of // `[u8; len]`, which is exactly what `Vec` with capacity `len` frees; // all `len` bytes are initialised (zero). diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index b8f2ffe..1c53418 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -180,7 +180,9 @@ fn read_raw_data_full_impl( }); } ensure_len(file_data, addr, sz)?; - Ok(file_data[addr..addr + sz].to_vec()) + let mut out = crate::bulk_alloc::vec_for_bulk(sz); + out.extend_from_slice(&file_data[addr..addr + sz]); + Ok(out) } DataLayout::Chunked { .. } => read_chunked_data( file_data, @@ -765,7 +767,7 @@ fn get_size(dt: &Datatype) -> usize { fn native_le_to_vec(raw: &[u8], count: usize) -> Vec { let bytes = count * core::mem::size_of::(); debug_assert!(bytes <= raw.len()); - let mut result: Vec = Vec::with_capacity(count); + let mut result: Vec = crate::bulk_alloc::vec_for_bulk(count); // SAFETY: `result` has capacity for `count` values of `T`, i.e. `bytes` // bytes; `raw` holds at least `bytes` bytes (callers derive `count` from // `raw.len() / size_of::()`); the regions cannot overlap because @@ -803,7 +805,7 @@ pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let order = get_byte_order(datatype); - let mut result = Vec::with_capacity(count); + let mut result = crate::bulk_alloc::vec_for_bulk(count); if let Datatype::FloatingPoint { .. } = datatype { let format = FloatFormat::of(datatype)?; for chunk in raw.chunks_exact(elem_size) { @@ -955,7 +957,7 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let order = get_byte_order(datatype); - let mut result = Vec::with_capacity(count); + let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(decode_scalar(chunk, datatype, &order)?.to_i64()); @@ -985,7 +987,7 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let count = raw.len() / elem_size; let order = get_byte_order(datatype); - let mut result = Vec::with_capacity(count); + let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(decode_scalar(chunk, datatype, &order)?.to_u64()); @@ -1018,14 +1020,17 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr // Little-endian IEEE half precision (numpy float16): widen directly. if is_native_le_float(datatype, FloatFormat::Half) { let (halves, _) = raw[..count * 2].as_chunks::<2>(); - return Ok(halves - .iter() - .map(|&b| f16_bits_to_f32(u16::from_le_bytes(b))) - .collect()); + let mut result = crate::bulk_alloc::vec_for_bulk(count); + result.extend( + halves + .iter() + .map(|&b| f16_bits_to_f32(u16::from_le_bytes(b))), + ); + return Ok(result); } let order = get_byte_order(datatype); - let mut result = Vec::with_capacity(count); + let mut result = crate::bulk_alloc::vec_for_bulk(count); if let Datatype::FloatingPoint { .. } = datatype { let format = FloatFormat::of(datatype)?; for chunk in raw.chunks_exact(elem_size) { @@ -1114,7 +1119,7 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr } let order = get_byte_order(datatype); - let mut result = Vec::with_capacity(count); + let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { let chunk = &raw[i * elem_size..(i + 1) * elem_size]; result.push(decode_scalar(chunk, datatype, &order)?.to_i32()); diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 905ce84..05ebd46 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -61,6 +61,7 @@ pub mod attribute; pub mod attribute_info; pub mod btree_v1; pub mod btree_v2; +mod bulk_alloc; pub mod checksum; pub mod chunk_cache; mod chunk_grid; diff --git a/crates/clawhdf5/tests/contiguous_read_interop.rs b/crates/clawhdf5/tests/contiguous_read_interop.rs new file mode 100644 index 0000000..188856e --- /dev/null +++ b/crates/clawhdf5/tests/contiguous_read_interop.rs @@ -0,0 +1,403 @@ +//! Reads of contiguous datasets — full reads and hyperslab/point selections, +//! through every typed reader — checked against h5py/libhdf5 for every +//! integer and float width, both byte orders, ranks 1 to 4, and datasets +//! larger than the huge-page threshold (4 MiB) the read buffers use. +//! +//! h5py writes the file and, for each selection, reads it with libhdf5's own +//! hyperslab/point selection (`select_hyperslab` with stride and block, +//! `select_elements`) and saves the raw bytes it gets back; the byte-level +//! [`Dataset::read_selection`] must return exactly those bytes, and the typed +//! readers the same values. Skipped when python3 with h5py is unavailable, +//! unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_format::selection::Selection; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// numpy type codes, with the modulus of the value pattern: every value is an +/// integer exactly representable in the type and in every typed reader's +/// output (f16 is exact below 2048, f32 below 2^24). +const DTYPES: [(&str, i64, bool); 11] = [ + ("i1", 201, true), + ("u1", 251, false), + ("i2", 2039, true), + ("u2", 2039, false), + ("i4", 1_000_003, true), + ("u4", 1_000_003, false), + ("i8", 1_000_003, true), + ("u8", 1_000_003, false), + ("f2", 2039, true), + ("f4", 1_000_003, true), + ("f8", 1_000_003, true), +]; + +/// Value of element `i` of a dataset of type `code` (the same formula as the +/// Python side): a permutation-revealing pattern, centred on 0 when signed. +fn value(code: &str, i: u64) -> i64 { + let (_, m, signed) = DTYPES.iter().find(|d| d.0 == code).unwrap(); + let v = ((i as i128 * 7919) % *m as i128) as i64; + if *signed { v - m / 2 } else { v } +} + +const SHAPES: [&[u64]; 4] = [&[1000], &[37, 53], &[7, 11, 13], &[3, 5, 7, 9]]; + +/// Datasets past the 4 MiB huge-page threshold, as (type, shape). +const BIG: [(&str, [u64; 2]); 4] = [ + ("f4", [1100, 1024]), + ("i4", [1100, 1024]), + ("f8", [600, 1024]), + ("i8", [600, 1024]), +]; + +fn datasets() -> Vec<(String, String, Vec)> { + let mut out = Vec::new(); + for (code, _, _) in DTYPES { + for (tag, _) in [("le", '<'), ("be", '>')] { + for shape in SHAPES { + out.push(( + format!("{code}{tag}_r{}", shape.len()), + code.to_string(), + shape.to_vec(), + )); + } + } + } + for (code, shape) in BIG { + for tag in ["le", "be"] { + out.push((format!("{code}{tag}_big"), code.to_string(), shape.to_vec())); + } + } + out +} + +fn write_file(path: &Path) { + let script = format!( + r#" +import h5py, numpy as np +M = {{'i1': 201, 'u1': 251, 'i2': 2039, 'u2': 2039, 'i4': 1000003, 'u4': 1000003, + 'i8': 1000003, 'u8': 1000003, 'f2': 2039, 'f4': 1000003, 'f8': 1000003}} +def values(code, n): + v = (np.arange(n, dtype=np.int64) * 7919) % M[code] + if code[0] != 'u': + v -= M[code] // 2 + return v +shapes = [(1000,), (37, 53), (7, 11, 13), (3, 5, 7, 9)] +big = [('f4', (1100, 1024)), ('i4', (1100, 1024)), ('f8', (600, 1024)), ('i8', (600, 1024))] +with h5py.File("{path}", "w") as f: + for code in M: + for tag, e in (('le', '<'), ('be', '>')): + for shape in shapes: + n = int(np.prod(shape)) + f.create_dataset(f"{{code}}{{tag}}_r{{len(shape)}}", + data=values(code, n).astype(e + code).reshape(shape)) + for code, shape in big: + for tag, e in (('le', '<'), ('be', '>')): + n = int(np.prod(shape)) + f.create_dataset(f"{{code}}{{tag}}_big", + data=values(code, n).astype(e + code).reshape(shape)) +"#, + path = path.display() + ); + run_python(&script); +} + +#[test] +fn full_reads_match_h5py_for_every_type_order_and_size() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("contig.h5"); + write_file(&path); + let file = File::open(&path).unwrap(); + for (name, code, shape) in datasets() { + let ds = file.dataset(&name).unwrap(); + assert!(ds.read_raw_ref().unwrap().is_some(), "{name} is contiguous"); + let n: u64 = shape.iter().product(); + let want: Vec = (0..n).map(|i| value(&code, i)).collect(); + assert_eq!( + ds.read_f64().unwrap(), + want.iter().map(|&v| v as f64).collect::>(), + "{name} read_f64" + ); + assert_eq!( + ds.read_f32().unwrap(), + want.iter().map(|&v| v as f32).collect::>(), + "{name} read_f32" + ); + assert_eq!(ds.read_i64().unwrap(), want, "{name} read_i64"); + assert_eq!( + ds.read_i32().unwrap(), + want.iter().map(|&v| v as i32).collect::>(), + "{name} read_i32" + ); + // libhdf5 saturates negative values to 0 when reading as unsigned. + assert_eq!( + ds.read_u64().unwrap(), + want.iter().map(|&v| v.max(0) as u64).collect::>(), + "{name} read_u64" + ); + } +} + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n.max(1) + } +} + +/// A hyperslab from per-dimension `(start, stride, count, block)`. +fn slab(dims: &[(u64, u64, u64, u64)]) -> Selection { + Selection::Hyperslab { + start: dims.iter().map(|d| d.0).collect(), + stride: dims.iter().map(|d| d.1).collect(), + count: dims.iter().map(|d| d.2).collect(), + block: dims.iter().map(|d| d.3).collect(), + } +} + +/// Selections of every shape the read paths distinguish, all valid for `dims`. +fn selections(rng: &mut Rng, dims: &[u64]) -> Vec { + let mut out = Vec::new(); + // Unit-stride box. + out.push(slab( + &dims + .iter() + .map(|&n| { + let c = 1 + rng.below(n); + (rng.below(n - c + 1), 1, c, 1) + }) + .collect::>(), + )); + // Strided (block 1), blocked (stride > block), and adjacent blocks + // (stride == block, which reads like a box): (block, stride - block). + for (block, gap) in [(1, 1), (2, 1), (2, 0)] { + out.push(slab( + &dims + .iter() + .map(|&n| { + let b = (block + rng.below(2)).min(n); + let st = b + gap + rng.below(2) * gap; + let s = rng.below(n - b + 1); + let c = 1 + rng.below((n - s - b) / st + 1); + (s, st, c, b) + }) + .collect::>(), + )); + } + // Whole inner rows (one run across rows), and the whole dataset. + let r0 = rng.below(dims[0]); + let mut rows = vec![(r0, 1, 1 + rng.below(dims[0] - r0), 1)]; + rows.extend(dims[1..].iter().map(|&n| (0, 1, n, 1))); + out.push(slab(&rows)); + out.push(slab( + &dims.iter().map(|&n| (0, 1, n, 1)).collect::>(), + )); + // One element. + out.push(slab( + &dims + .iter() + .map(|&n| (rng.below(n), 1, 1, 1)) + .collect::>(), + )); + // Distinct points in no particular order. + let mut points: Vec> = Vec::new(); + for _ in 0..1 + rng.below(15) { + let p: Vec = dims.iter().map(|&n| rng.below(n)).collect(); + if !points.contains(&p) { + points.push(p); + } + } + out.push(Selection::Points(points)); + out +} + +fn join(v: &[u64]) -> String { + v.iter().map(u64::to_string).collect::>().join(",") +} + +/// One element of type `code`, given as its raw file-order bytes, as an +/// integer (every value in these files is one). +fn decode(code: &str, big_endian: bool, bytes: &[u8]) -> i64 { + let mut b = bytes.to_vec(); + if big_endian { + b.reverse(); + } + let mut w = [0u8; 8]; + w[..b.len()].copy_from_slice(&b); + let u = u64::from_le_bytes(w); + match code { + "i1" => u as u8 as i8 as i64, + "i2" => u as u16 as i16 as i64, + "i4" => u as u32 as i32 as i64, + "i8" => u as i64, + "u1" | "u2" | "u4" | "u8" => u as i64, + "f2" => clawhdf5_format::float16::f16_bits_to_f32(u as u16) as i64, + "f4" => f32::from_bits(u as u32) as i64, + "f8" => f64::from_bits(u) as i64, + _ => unreachable!(), + } +} + +#[test] +fn selection_reads_match_h5py_for_every_type_order_and_rank() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("contig.h5"); + write_file(&path); + + let mut rng = Rng(2026); + let mut cases: Vec<(String, String, Selection)> = Vec::new(); + for (name, code, shape) in datasets() { + for sel in selections(&mut rng, &shape) { + cases.push((name.clone(), code.clone(), sel)); + } + // Empty: a zero count, and Selection::None. + let mut empty = shape.iter().map(|&n| (0, 1, n, 1)).collect::>(); + empty[shape.len() - 1].2 = 0; + cases.push((name.clone(), code.clone(), slab(&empty))); + cases.push((name, code, Selection::None)); + } + let mut spec = String::new(); + for (k, (name, _, sel)) in cases.iter().enumerate() { + let line = match sel { + Selection::Hyperslab { count, .. } if count.contains(&0) => "N".to_string(), + Selection::Hyperslab { + start, + stride, + count, + block, + } => format!( + "H {};{};{};{}", + join(start), + join(stride), + join(count), + join(block) + ), + Selection::Points(points) => format!( + "P {}", + points.iter().map(|p| join(p)).collect::>().join(";") + ), + Selection::None => "N".to_string(), + Selection::All => unreachable!(), + }; + spec.push_str(&format!("{k} {name} {line}\n")); + } + let spec_path = dir.path().join("cases.txt"); + std::fs::write(&spec_path, spec).unwrap(); + run_python(&format!( + r#" +import h5py, numpy as np +with h5py.File("{path}", "r") as f: + for line in open("{spec}"): + k, name, kind, *rest = line.split() + d = f[name] + space = d.id.get_space() + if kind == 'H': + start, stride, count, block = (tuple(int(x) for x in part.split(',')) + for part in rest[0].split(';')) + space.select_hyperslab(start, count, stride, block) + elif kind == 'P': + pts = np.array([[int(x) for x in p.split(',')] for p in rest[0].split(';')], + dtype=np.uint64) + space.select_elements(pts) + else: + space.select_none() + n = space.get_select_npoints() + out = np.empty(n, dtype=d.dtype) + if n: + d.id.read(h5py.h5s.create_simple((n,)), space, out) + open("{dir}/sel_" + k + ".bin", "wb").write(out.tobytes()) +"#, + path = path.display(), + spec = spec_path.display(), + dir = dir.path().display(), + )); + + let file = File::open(&path).unwrap(); + for (k, (name, code, sel)) in cases.iter().enumerate() { + let ds = file.dataset(name).unwrap(); + let want_bytes = std::fs::read(dir.path().join(format!("sel_{k}.bin"))).unwrap(); + let got_bytes = ds.read_selection(sel).unwrap(); + assert!( + got_bytes == want_bytes, + "{name} {sel:?}: raw bytes differ from libhdf5's ({} vs {} bytes)", + got_bytes.len(), + want_bytes.len() + ); + let size = ds.raw_datatype().unwrap().type_size() as usize; + let want: Vec = want_bytes + .chunks_exact(size) + .map(|e| decode(code, name.contains("be_"), e)) + .collect(); + assert_eq!( + ds.read_f64_selection(sel).unwrap(), + want.iter().map(|&v| v as f64).collect::>(), + "{name} {sel:?} as f64" + ); + assert_eq!( + ds.read_f32_selection(sel).unwrap(), + want.iter().map(|&v| v as f32).collect::>(), + "{name} {sel:?} as f32" + ); + assert_eq!( + ds.read_i64_selection(sel).unwrap(), + want, + "{name} {sel:?} as i64" + ); + assert_eq!( + ds.read_i32_selection(sel).unwrap(), + want.iter().map(|&v| v as i32).collect::>(), + "{name} {sel:?} as i32" + ); + } +} -- 2.54.0 From 10da8f0d09833189cc11ff34384af21d1f3b9e7d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:18:57 -0500 Subject: [PATCH 04/36] fix(format): read VL values in files with 4-byte offsets In a file with sizeof_addr = 4, a VL string attribute came back as AttrValue::Raw, a compound's VL member failed with GlobalHeapObjectNotFound and VL datasets failed with a size mismatch. Two bugs: Datatype::type_size() said 16 for every VL type, while the element is 4 + offset size + 4 bytes (12 here); and the global heap was parsed without the padding libhdf5 puts after its collection and object headers (both round up to 8), so with 4-byte lengths every object was looked up 4 bytes early. Datatype::VariableLength now carries the size its datatype message stores, and writes it back. Checked against h5py in tests/vl_offset4_interop.rs (fails with either fix reverted). Conformance unchanged at 575 of 697; in cve-2024-32608 a VL attribute whose datatype claims 524304-byte elements is now an error (h5py cannot iterate those attributes at all). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +++ crates/clawhdf5-format/src/datatype.rs | 28 ++++- crates/clawhdf5-format/src/global_heap.rs | 19 ++- crates/clawhdf5/tests/vl_offset4_interop.rs | 126 ++++++++++++++++++++ docs/known-issues.md | 5 +- 5 files changed, 184 insertions(+), 9 deletions(-) create mode 100644 crates/clawhdf5/tests/vl_offset4_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4bd2f..3af6328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +### Variable-length data (2026-09-26) +- **VL values in files with 4-byte offsets** (`sizeof_addr = 4`). A VL + string attribute came back as `AttrValue::Raw`, a VL member of a compound + failed with `GlobalHeapObjectNotFound`, and VL datasets failed with a + size mismatch. Two causes: `Datatype::type_size()` reported 16 for every + VL type (the element is 4 + offset size + 4 bytes: 12 here), and the + global heap was parsed without the padding libhdf5 puts after its + collection and object headers (`H5HG_SIZEOF_HDR`/`H5HG_SIZEOF_OBJHDR` + round up to 8), so with 4-byte lengths every object was looked up 4 + bytes early. `Datatype::VariableLength` now carries the element `size` + stored in the datatype message (**breaking** for code that builds or + exhaustively destructures that variant; patterns with `..` are + unaffected), and it is written back as stored. Tested against h5py + (`crates/clawhdf5/tests/vl_offset4_interop.rs`). + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 12e427f..974c249 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -125,6 +125,11 @@ pub enum Datatype { }, /// Class 9: Variable-length type. VariableLength { + /// Size of one element as stored in the file: a sequence length (4 + /// bytes), a global heap collection address (the file's + /// `offset_size`) and an object index (4 bytes) — 16 in a file with + /// 8-byte offsets, 12 with 4-byte offsets. + size: u32, is_string: bool, padding: Option, charset: Option, @@ -771,6 +776,7 @@ impl Datatype { pos += consumed; Ok(( Datatype::VariableLength { + size, is_string, padding, charset, @@ -1017,6 +1023,7 @@ impl Datatype { Self::build_header(3, 1, [bf0, 0, 0], *size) } Datatype::VariableLength { + size, is_string, padding, charset, @@ -1039,7 +1046,7 @@ impl Datatype { } else { 0 }; - let mut buf = Self::build_header(9, 1, [bf0, bf1, 0], 16); + let mut buf = Self::build_header(9, 1, [bf0, bf1, 0], *size); buf.extend_from_slice(&base_type.serialize()); buf } @@ -1208,7 +1215,7 @@ impl Datatype { Datatype::Compound { size, .. } => *size, Datatype::Reference { size, .. } => *size, Datatype::Enumeration { size, .. } => *size, - Datatype::VariableLength { .. } => 16, // typically pointer + length + Datatype::VariableLength { size, .. } => *size, Datatype::Array { base_type, dimensions, @@ -1889,11 +1896,13 @@ mod tests { let (dt, _) = Datatype::parse(&buf).unwrap(); match dt { Datatype::VariableLength { + size, is_string, padding, charset, base_type, } => { + assert_eq!(size, 16); assert!(is_string); assert_eq!(padding, Some(StringPadding::NullTerminate)); assert_eq!(charset, Some(CharacterSet::Utf8)); @@ -1914,11 +1923,13 @@ mod tests { let (dt, _) = Datatype::parse(&buf).unwrap(); match dt { Datatype::VariableLength { + size, is_string, padding, charset, base_type, } => { + assert_eq!(size, 16); assert!(!is_string); assert_eq!(padding, None); assert_eq!(charset, None); @@ -1928,6 +1939,19 @@ mod tests { } } + #[test] + fn variable_length_size_is_the_stored_size() { + // A file with 4-byte offsets stores 12-byte VL elements (length 4 + + // address 4 + index 4); the type used to report 16 regardless, so + // every read laid the elements out 16 bytes apart. + let mut buf = build_dt_header(9, 1, [0x01, 0x00, 0], 12); + buf.extend_from_slice(&build_fixed_point(1, false, false, 0, 8)); + let (dt, _) = Datatype::parse(&buf).unwrap(); + assert_eq!(dt.type_size(), 12); + // And it is written back as stored. + assert_eq!(dt.serialize()[4..8], 12u32.to_le_bytes()); + } + #[test] fn test_array_2d() { // Array [3][4] of i32 LE, version 3 diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index dfba3f0..5eab713 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -64,8 +64,11 @@ impl GlobalHeapCollection { offset: usize, length_size: u8, ) -> Result { - // signature(4) + version(1) + reserved(3) + collection_size(length_size) - let header_size = 8 + length_size as usize; + // signature(4) + version(1) + reserved(3) + collection_size(length_size), + // padded to a multiple of 8 as libhdf5 lays it out (`H5HG_SIZEOF_HDR`). + // With 8-byte lengths the padding is 0; with 4-byte lengths it is 4, + // and reading without it put every object 4 bytes early. + let header_size = pad8(8 + length_size as usize); ensure_len(file_data, offset, header_size)?; if file_data[offset..offset + 4] != GCOL_SIGNATURE { @@ -104,8 +107,9 @@ impl GlobalHeapCollection { break; } - // object_index(2) + reference_count(2) + reserved(4) + object_size(length_size) - let obj_header_size = 8 + length_size as usize; + // object_index(2) + reference_count(2) + reserved(4) + + // object_size(length_size), padded to 8 (`H5HG_SIZEOF_OBJHDR`). + let obj_header_size = pad8(8 + length_size as usize); ensure_len(file_data, pos, obj_header_size)?; let reference_count = u16::from_le_bytes([file_data[pos + 2], file_data[pos + 3]]); @@ -149,10 +153,11 @@ mod tests { let ls = length_size as usize; // Calculate total size - let header_size = 8 + ls; + // libhdf5 pads both headers to a multiple of 8. + let header_size = pad8(8 + ls); let mut obj_size_total = 0usize; for (_, _, data) in objects { - let obj_header = 8 + ls; + let obj_header = pad8(8 + ls); obj_size_total += obj_header + pad8(data.len()); } // Free space marker (2 bytes for index 0) @@ -170,6 +175,7 @@ mod tests { 8 => buf.extend_from_slice(&(collection_size as u64).to_le_bytes()), _ => panic!("unsupported length_size"), } + buf.resize(header_size, 0); // Objects for (index, ref_count, data) in objects { @@ -181,6 +187,7 @@ mod tests { 8 => buf.extend_from_slice(&(data.len() as u64).to_le_bytes()), _ => panic!("unsupported"), } + buf.resize(buf.len() + (pad8(8 + ls) - (8 + ls)), 0); buf.extend_from_slice(data); // Pad to 8 bytes let padded = pad8(data.len()); diff --git a/crates/clawhdf5/tests/vl_offset4_interop.rs b/crates/clawhdf5/tests/vl_offset4_interop.rs new file mode 100644 index 0000000..cc7ce08 --- /dev/null +++ b/crates/clawhdf5/tests/vl_offset4_interop.rs @@ -0,0 +1,126 @@ +//! Variable-length values in files with 4-byte offsets and lengths +//! (`sizeof_addr = 4`), checked against h5py/libhdf5 through the +//! `clawhdf5_format` decoders. +//! +//! These failed with `GlobalHeapObjectNotFound` or came back as +//! `AttrValue::Raw`: the VL datatype claimed 16-byte elements whatever the +//! file's offset size, and the global heap was read without the padding +//! libhdf5 puts after its collection and object headers. Skipped when +//! python3 with h5py is unavailable, unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{AttrValue, File, Selection}; +use clawhdf5_format::data_read::{read_as_i64, read_compound_fields}; +use clawhdf5_format::vl_data::{read_vl_bytes, read_vl_strings}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +#[test] +fn vl_values_in_a_file_with_4_byte_offsets_read_like_h5py() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("offset4.h5"); + let script = format!( + r#" +import h5py, numpy as np +S = h5py.string_dtype() +fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(4, 4) +with h5py.File(h5py.h5f.create({path:?}.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl)) as f: + f.attrs['vlattr'] = 'attr-value' + f.attrs.create('vlattr_arr', np.array(['p', 'qq', ''], dtype=object), dtype=S) + ct = np.dtype([('id', ' = stdout.lines().collect(); + let (vlattr, vlattr_arr, names, seqs) = (lines[0], lines[1], lines[2], lines[3]); + + let file = File::open(&path).unwrap(); + let sb = file.superblock(); + assert_eq!((sb.offset_size, sb.length_size), (4, 4)); + + let attrs = file.root().attrs().unwrap(); + match &attrs["vlattr"] { + AttrValue::String(s) => assert_eq!(s, vlattr), + other => panic!("vlattr: {other:?}"), + } + match &attrs["vlattr_arr"] { + AttrValue::StringArray(s) => assert_eq!(s.join(","), vlattr_arr), + other => panic!("vlattr_arr: {other:?}"), + } + + // The compound's VL string member. + let ds = file.dataset("compound").unwrap(); + let dt = ds.raw_datatype().unwrap(); + assert_eq!(dt.type_size(), 24, "4 + 12-byte VL element + 8"); + let raw = ds.read_selection(&Selection::All).unwrap(); + let fields = read_compound_fields(&raw, &dt).unwrap(); + let name = fields.iter().find(|f| f.name == "name").unwrap(); + assert_eq!(name.datatype.type_size(), 12); + let got = read_vl_strings(file.as_bytes(), &name.raw_data, 3, 4, 4).unwrap(); + assert_eq!(got.join(","), names); + let id = fields.iter().find(|f| f.name == "id").unwrap(); + assert_eq!( + read_as_i64(&id.raw_data, &id.datatype).unwrap(), + vec![1, 2, 3] + ); + + // A VL sequence attribute. + let AttrValue::Raw { datatype, data, .. } = &attrs["vlen_attr"] else { + panic!("vlen_attr is Raw"); + }; + let clawhdf5_format::datatype::Datatype::VariableLength { base_type, .. } = datatype else { + panic!("vlen_attr is VL"); + }; + let got: Vec = read_vl_bytes(file.as_bytes(), data, 2, 4, 4) + .unwrap() + .iter() + .map(|b| { + let v = read_as_i64(b, base_type).unwrap(); + v.iter().map(i64::to_string).collect::>().join(" ") + }) + .collect(); + assert_eq!(got.join(";"), seqs); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index bf8f77a..b9a19b6 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -146,7 +146,10 @@ fill-value item that did is fixed). `GlobalHeapObjectNotFound` or come back as `Raw`: these paths assume the 16-byte element of an 8-byte-offset file. The datatype itself reads (it was refused as "member overlaps with previous member" until - 2026-09-26). + 2026-09-26). **Fixed 2026-09-26:** a VL type's element size is the one + its datatype message stores (12 with 4-byte offsets), and the global + heap is read with libhdf5's header padding + (`crates/clawhdf5/tests/vl_offset4_interop.rs`). - Metadata cache images are not supported. - x87 long double and binary128 are refused. - N-Bit on 64-bit scale-offset data and some N-Bit parameter layouts fail. -- 2.54.0 From 2d4b21152389b7622e73fcd3d5f2e0723583af0c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:19:48 -0500 Subject: [PATCH 05/36] feat(py): h5py-style reads of only the selected elements, GIL released ds[key] read the whole dataset and sliced it in numpy, and knew six dtypes. Keys (ints, positive-step slices, Ellipsis, one increasing index list, compound field names) now map onto hyperslab selections, and the facade's read_selection bytes become the numpy buffer without a copy (PyArray::from_vec viewed as the dtype). dtype mapping follows h5py for all integer/IEEE float widths and byte orders, bool, enum, complex, fixed and variable-length strings, vlen sequences, opaque, array types and (nested, padded) compounds; anything it cannot describe exactly is a TypeError. Attributes return what h5py returns; groups and files gain the rest of the h5py mapping interface. Reads run under py.detach. tests/test_read_vs_h5py.py compares >500 reads with h5py 3.16 on an h5py-written file, checks errors match, that a damaged chunk outside the selection is never touched, and 8 threads reading at once. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 32 + crates/clawhdf5-py/src/attrs.rs | 148 +++-- crates/clawhdf5-py/src/convert.rs | 600 ++++++++++++++++++ crates/clawhdf5-py/src/dataset.rs | 502 +++++++++------ crates/clawhdf5-py/src/file.rs | 97 +-- crates/clawhdf5-py/src/group.rs | 228 ++++--- crates/clawhdf5-py/src/lib.rs | 56 +- crates/clawhdf5-py/src/node.rs | 168 +++++ crates/clawhdf5-py/src/select.rs | 366 +++++++++++ crates/clawhdf5-py/tests/conftest.py | 18 + crates/clawhdf5-py/tests/test_read_vs_h5py.py | 469 ++++++++++++++ crates/clawhdf5-py/tests/test_write_read.py | 6 +- 12 files changed, 2305 insertions(+), 385 deletions(-) create mode 100644 crates/clawhdf5-py/src/convert.rs create mode 100644 crates/clawhdf5-py/src/node.rs create mode 100644 crates/clawhdf5-py/src/select.rs create mode 100644 crates/clawhdf5-py/tests/conftest.py create mode 100644 crates/clawhdf5-py/tests/test_read_vs_h5py.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fb0a5f8..e6a80f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,38 @@ tests imported `rustyhdf5`, so they failed at collection. Distribution, module and tests now all say `clawhdf5`, and the module has `__version__`. +- **h5py-style reads that read only what is selected.** `ds[...]` used to + read the whole dataset and slice it in numpy, and knew six dtypes. Now + integers (negative from the end), slices with positive steps, `...`, one + increasing list of integers per key and compound field names map onto the + facade's hyperslab selection (a list becomes one hyperslab per run of + consecutive indices), with h5py's results (numpy scalar for an all-integer + key, 0-d array for `scalar[...]`) and h5py's errors for everything else + (negative steps, `None`, boolean masks, out-of-range indices). + `Dataset.dtype` is the numpy dtype h5py reports, for every integer and + IEEE float width (incl. `float16`) in either byte order, `bool`, enums + (base integer with `metadata['enum']`), complex (`r`/`i` compounds), + fixed strings (`S`), variable-length strings (`object` of `bytes`, as + h5py), variable-length sequences (`object` of arrays), opaque (`V`), + HDF5 array types and compounds (numpy structured, offsets and padding + kept, nested). The bytes the library returns become the numpy array's + buffer without a copy. Types the mapping cannot describe exactly + (references, bitfields, time, non-IEEE floats, integers with padding + bits, variable-length members inside compounds) raise `TypeError` rather + than return guessed data. Attributes come back as h5py returns them + (numpy scalars and arrays with the stored dtype, `str` for + variable-length strings, `numpy.bytes_` for fixed ones — **a change**: + string attributes written by this package are fixed-length and used to + come back as `str` — and `clawhdf5.Empty` for a null dataspace, which + datasets return too). `Group`/`File` gain `get`, `values`, `items`, + iteration, `len`, `name`, absolute and relative paths (`g['/a/b']`, + `g['c/d']`, `f['/']`); `Dataset` gains `ndim`, `size`, `maxshape`, + `name`, `len()` and `numpy.asarray(ds)`. File access and decoding run + with the GIL released, so Python threads read in parallel. + `crates/clawhdf5-py/tests/test_read_vs_h5py.py` compares every read with + h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's: + it returns variable-length sequences of big-endian floats unswapped; this + package returns the stored values. ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files diff --git a/crates/clawhdf5-py/src/attrs.rs b/crates/clawhdf5-py/src/attrs.rs index a0d270e..ff608ed 100644 --- a/crates/clawhdf5-py/src/attrs.rs +++ b/crates/clawhdf5-py/src/attrs.rs @@ -1,24 +1,31 @@ //! PyAttrs — dict-like access to HDF5 attributes. -use std::collections::HashMap; use std::sync::{Arc, Mutex}; +use clawhdf5_format::attribute::AttributeMessage; +use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::PyList; +use pyo3::types::{PyList, PyTuple}; -use crate::{OwnedAttrValue, attr_value_to_py, py_to_attr_value}; +use crate::convert::{Converter, Elements, resolve_vl}; +use crate::{OwnedAttrValue, PyEmpty, attr_value_to_py, node, py_to_attr_value}; /// Backing storage for attributes. enum AttrsInner { - /// Read-only attributes from an existing HDF5 object. - Read(HashMap), + /// Attributes of an object in a file opened for reading, sorted by name. + Read { + file: Arc, + attrs: Vec, + }, /// Writable attribute list shared with a parent (PyFile or PyGroup). Write(Arc>>), } /// Dict-like access to HDF5 attributes. /// -/// In read mode, provides immutable access to attribute key/value pairs. +/// In read mode, values are what h5py returns: numpy scalars for scalar +/// attributes, numpy arrays otherwise, `str` for variable-length strings, +/// `numpy.bytes_` for fixed-length ones, and `Empty` for a null dataspace. /// In write mode, attributes set here are accumulated and written when /// the parent file is closed. #[pyclass(name = "Attrs")] @@ -27,11 +34,12 @@ pub struct PyAttrs { } impl PyAttrs { - /// Create a read-only attrs from an existing attribute map. - pub(crate) fn from_read(map: HashMap) -> Self { - Self { - inner: AttrsInner::Read(map), - } + /// The attributes of the object at `path` in a file opened for reading. + pub(crate) fn read(file: Arc, path: &str) -> PyResult { + let attrs = node::attributes(&file, path)?; + Ok(Self { + inner: AttrsInner::Read { file, attrs }, + }) } /// Create a writable attrs that shares storage with a parent object. @@ -46,11 +54,11 @@ impl PyAttrs { impl PyAttrs { fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { match &self.inner { - AttrsInner::Read(map) => match map.get(key) { - Some(val) => Ok(attr_value_to_py(py, val)), - None => Err(PyErr::new::( - key.to_string(), - )), + AttrsInner::Read { file, attrs } => match attrs.iter().find(|a| a.name == key) { + Some(attr) => Ok(attr_to_py(py, file, attr)?.unbind()), + None => Err(PyKeyError::new_err(format!( + "Can't open attribute (can't locate attribute: '{key}')" + ))), }, AttrsInner::Write(store) => { let guard = store.lock().unwrap(); @@ -60,16 +68,14 @@ impl PyAttrs { return Ok(attr_value_to_py(py, &attr_val)); } } - Err(PyErr::new::( - key.to_string(), - )) + Err(PyKeyError::new_err(key.to_string())) } } } fn __setitem__(&self, key: &str, value: &Bound<'_, PyAny>) -> PyResult<()> { match &self.inner { - AttrsInner::Read(_) => Err(PyErr::new::( + AttrsInner::Read { .. } => Err(PyErr::new::( "cannot set attributes on a read-only file", )), AttrsInner::Write(store) => { @@ -88,14 +94,14 @@ impl PyAttrs { fn __len__(&self) -> usize { match &self.inner { - AttrsInner::Read(map) => map.len(), + AttrsInner::Read { attrs, .. } => attrs.len(), AttrsInner::Write(store) => store.lock().unwrap().len(), } } fn __contains__(&self, key: &str) -> bool { match &self.inner { - AttrsInner::Read(map) => map.contains_key(key), + AttrsInner::Read { attrs, .. } => attrs.iter().any(|a| a.name == key), AttrsInner::Write(store) => store.lock().unwrap().iter().any(|(k, _)| k == key), } } @@ -111,10 +117,20 @@ impl PyAttrs { format!("") } + /// The value of `key`, or `default` if there is no such attribute. + #[pyo3(signature = (key, default=None))] + fn get(&self, py: Python<'_>, key: &str, default: Option>) -> PyResult> { + if self.__contains__(key) { + self.__getitem__(py, key) + } else { + Ok(default.unwrap_or_else(|| py.None())) + } + } + /// Return attribute names as a list. fn keys(&self, py: Python<'_>) -> PyResult> { let names: Vec = match &self.inner { - AttrsInner::Read(map) => map.keys().cloned().collect(), + AttrsInner::Read { attrs, .. } => attrs.iter().map(|a| a.name.clone()).collect(), AttrsInner::Write(store) => store .lock() .unwrap() @@ -129,7 +145,10 @@ impl PyAttrs { /// Return attribute values as a list. fn values(&self, py: Python<'_>) -> PyResult> { let vals: Vec> = match &self.inner { - AttrsInner::Read(map) => map.values().map(|v| attr_value_to_py(py, v)).collect(), + AttrsInner::Read { file, attrs } => attrs + .iter() + .map(|a| attr_to_py(py, file, a).map(Bound::unbind)) + .collect::>()?, AttrsInner::Write(store) => store .lock() .unwrap() @@ -147,10 +166,10 @@ impl PyAttrs { /// Return attribute (key, value) pairs as a list of tuples. fn items(&self, py: Python<'_>) -> PyResult> { let pairs: Vec<(String, Py)> = match &self.inner { - AttrsInner::Read(map) => map + AttrsInner::Read { file, attrs } => attrs .iter() - .map(|(k, v)| (k.clone(), attr_value_to_py(py, v))) - .collect(), + .map(|a| Ok((a.name.clone(), attr_to_py(py, file, a)?.unbind()))) + .collect::>()?, AttrsInner::Write(store) => store .lock() .unwrap() @@ -166,28 +185,67 @@ impl PyAttrs { } } +/// An attribute's value as h5py returns it. +fn attr_to_py<'py>( + py: Python<'py>, + file: &clawhdf5_rs::File, + attr: &AttributeMessage, +) -> PyResult> { + let sb = file.superblock(); + let conv = Converter::new(py, &attr.datatype, sb.offset_size) + .map_err(|e| prefix_err(py, &attr.name, e))?; + if node::is_null(&attr.dataspace) { + return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any()); + } + let shape: Vec = attr + .dataspace + .dimensions + .iter() + .map(|&d| d as usize) + .collect(); + let n: usize = shape.iter().product(); + let data = if conv.is_vl() { + let want = n * conv.elem_size; + if attr.raw_data.len() < want { + return Err(PyValueError::new_err(format!( + "attribute {}: {} bytes of variable-length references, expected {want}", + attr.name, + attr.raw_data.len(), + ))); + } + let raw = &attr.raw_data[..want]; + let file_data = file.as_bytes(); + let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit); + Elements::Vl( + py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit)) + .map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?, + ) + } else { + Elements::Bytes(attr.raw_data.clone()) + }; + let arr = conv + .to_array(py, data, &shape, true) + .map_err(|e| prefix_err(py, &attr.name, e))?; + if shape.is_empty() { + // A scalar dataspace: h5py returns the element itself. + return arr.get_item(PyTuple::empty(py)); + } + Ok(arr) +} + +fn prefix_err(py: Python<'_>, name: &str, e: PyErr) -> PyErr { + let msg = format!("attribute {name}: {}", e.value(py)); + if e.is_instance_of::(py) { + PyTypeError::new_err(msg) + } else { + PyValueError::new_err(msg) + } +} + #[cfg(test)] mod tests { use super::*; - #[test] - fn read_attrs_len() { - let mut map = HashMap::new(); - map.insert("a".into(), clawhdf5_rs::AttrValue::I64(1)); - map.insert("b".into(), clawhdf5_rs::AttrValue::F64(2.0)); - let attrs = PyAttrs::from_read(map); - assert_eq!(attrs.__len__(), 2); - } - - #[test] - fn read_attrs_contains() { - let mut map = HashMap::new(); - map.insert("x".into(), clawhdf5_rs::AttrValue::String("hello".into())); - let attrs = PyAttrs::from_read(map); - assert!(attrs.__contains__("x")); - assert!(!attrs.__contains__("y")); - } - #[test] fn write_attrs_len() { let store = Arc::new(Mutex::new(Vec::new())); diff --git a/crates/clawhdf5-py/src/convert.rs b/crates/clawhdf5-py/src/convert.rs new file mode 100644 index 0000000..4ca7546 --- /dev/null +++ b/crates/clawhdf5-py/src/convert.rs @@ -0,0 +1,600 @@ +//! HDF5 datatypes as numpy dtypes, and element bytes as numpy arrays. +//! +//! The dtype a file's datatype maps to is the one h5py reports for it +//! (byte order kept, compound offsets and padding kept, `r`/`i` compounds as +//! complex, the `FALSE`/`TRUE` enum as `bool`, fixed strings as `S`, +//! variable-length data as `object`). For every fixed-size type that dtype +//! describes the file's element bytes exactly, so the bytes the library +//! returns become the array's buffer as they are: the `Vec` is handed to +//! numpy without a copy and viewed as the dtype. +//! +//! Anything this mapping cannot describe exactly — non-IEEE floats, integers +//! with padding bits, VAX byte order, references, bitfields, time, and +//! variable-length members inside compounds or arrays — is a `TypeError`, +//! never a best-effort guess. + +use std::collections::HashMap; + +use clawhdf5_format::datatype::{CharacterSet, Datatype, DatatypeByteOrder}; +use clawhdf5_format::global_heap::GlobalHeapCollection; +use numpy::PyArray1; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict, PyList, PyString, PyTuple}; + +/// How the elements of a datatype become Python values. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Layout { + /// Fixed-size elements numpy reads as they are. + Fixed, + /// A top-level HDF5 array type: elements are viewed as the base dtype and + /// the array's dimensions are appended to the shape (as h5py does). + Subarray(Vec), + /// Variable-length string: a global heap reference per element. + VlString { utf8: bool }, + /// Variable-length sequence of a fixed-size base type. + VlSequence, +} + +/// Everything needed to turn a dataset's or attribute's bytes into numpy. +pub(crate) struct Converter { + /// The dtype reported to Python (`Dataset.dtype`). + pub dtype: Py, + /// The dtype the element bytes are viewed as: `dtype` itself, the base + /// of a subarray, or the base of a variable-length sequence. + pub view: Py, + pub layout: Layout, + /// Bytes per element in the raw buffer the library returns. + pub elem_size: usize, + /// For variable-length data, bytes per unit of an element's stored + /// length: 1 for strings, the base type's size for sequences. + pub vl_unit: usize, +} + +fn unsupported(what: impl std::fmt::Display) -> PyErr { + PyTypeError::new_err(format!( + "clawhdf5 cannot read this datatype into numpy: {what}" + )) +} + +fn byte_order_char(order: &DatatypeByteOrder, size: u32) -> PyResult<&'static str> { + if size == 1 { + return Ok("|"); + } + match order { + DatatypeByteOrder::LittleEndian => Ok("<"), + DatatypeByteOrder::BigEndian => Ok(">"), + DatatypeByteOrder::Vax => Err(unsupported("VAX byte order")), + } +} + +/// The numpy format string of an integer type, if it is a plain one. +fn int_format(dt: &Datatype) -> PyResult { + match dt { + Datatype::FixedPoint { + size, + byte_order, + signed, + bit_offset, + bit_precision, + } => { + if !matches!(size, 1 | 2 | 4 | 8) { + return Err(unsupported(format!("{size}-byte integer"))); + } + if *bit_offset != 0 || u32::from(*bit_precision) != size * 8 { + return Err(unsupported(format!( + "integer with {bit_precision} significant bits at offset {bit_offset} in {size} bytes" + ))); + } + let kind = if *signed { 'i' } else { 'u' }; + Ok(format!( + "{}{kind}{size}", + byte_order_char(byte_order, *size)? + )) + } + other => Err(unsupported(format!("{other:?} is not an integer"))), + } +} + +/// The numpy format string of an IEEE 754 binary16/32/64 type. +fn float_format(dt: &Datatype) -> PyResult { + let Datatype::FloatingPoint { + size, + byte_order, + bit_offset, + bit_precision, + exponent_location, + exponent_size, + mantissa_location, + mantissa_size, + exponent_bias, + } = dt + else { + return Err(unsupported(format!("{dt:?} is not a float"))); + }; + // (exponent location, exponent size, mantissa size, bias) of IEEE 754. + let ieee = match size { + 2 => (10, 5, 10, 15), + 4 => (23, 8, 23, 127), + 8 => (52, 11, 52, 1023), + _ => return Err(unsupported(format!("{size}-byte float"))), + }; + let layout = ( + *exponent_location, + *exponent_size, + *mantissa_size, + *exponent_bias, + ); + if *bit_offset != 0 + || u32::from(*bit_precision) != size * 8 + || *mantissa_location != 0 + || layout != ieee + { + return Err(unsupported(format!( + "non-IEEE {size}-byte float (exponent {exponent_size} bits at {exponent_location}, \ + mantissa {mantissa_size} bits at {mantissa_location}, bias {exponent_bias})" + ))); + } + Ok(format!("{}f{size}", byte_order_char(byte_order, *size)?)) +} + +/// `r`/`i` compounds of two identical IEEE floats are complex numbers in h5py. +fn complex_format( + size: u32, + members: &[clawhdf5_format::datatype::CompoundMember], +) -> Option { + let [re, im] = members else { return None }; + if re.name != "r" || im.name != "i" || re.datatype != im.datatype { + return None; + } + let Datatype::FloatingPoint { + size: fsize, + byte_order, + .. + } = &re.datatype + else { + return None; + }; + if !matches!(fsize, 4 | 8) + || re.byte_offset != 0 + || im.byte_offset != u64::from(*fsize) + || size != 2 * fsize + { + return None; + } + float_format(&re.datatype).ok()?; + let order = byte_order_char(byte_order, *fsize).ok()?; + Some(format!("{order}c{}", 2 * fsize)) +} + +/// The members of an enum as `{name: value}`. +fn enum_members<'py>( + py: Python<'py>, + base: &Datatype, + members: &[clawhdf5_format::datatype::EnumMember], +) -> PyResult> { + let signed = matches!(base, Datatype::FixedPoint { signed: true, .. }); + let dict = PyDict::new(py); + for m in members { + let value: Py = if signed { + let v = clawhdf5_format::data_read::read_as_i64(&m.value, base) + .map_err(|e| PyValueError::new_err(format!("enum member {}: {e}", m.name)))?; + let v = *v.first().ok_or_else(|| { + PyValueError::new_err(format!("enum member {} has no value", m.name)) + })?; + v.into_pyobject(py)?.into_any().unbind() + } else { + let v = clawhdf5_format::data_read::read_as_u64(&m.value, base) + .map_err(|e| PyValueError::new_err(format!("enum member {}: {e}", m.name)))?; + let v = *v.first().ok_or_else(|| { + PyValueError::new_err(format!("enum member {} has no value", m.name)) + })?; + v.into_pyobject(py)?.into_any().unbind() + }; + dict.set_item(&m.name, value)?; + } + Ok(dict) +} + +/// Whether an enum is h5py's boolean: a one-byte integer with exactly the +/// members `FALSE` = 0 and `TRUE` = 1. +fn is_h5py_bool(base: &Datatype, members: &[clawhdf5_format::datatype::EnumMember]) -> bool { + if base.type_size() != 1 || members.len() != 2 { + return false; + } + let value = |name: &str| { + members + .iter() + .find(|m| m.name == name) + .and_then(|m| m.value.first().copied()) + }; + value("FALSE") == Some(0) && value("TRUE") == Some(1) +} + +fn np_dtype<'py>(py: Python<'py>, spec: impl IntoPyObject<'py>) -> PyResult> { + py.import("numpy")?.getattr("dtype")?.call1((spec,)) +} + +fn np_dtype_with_metadata<'py>( + py: Python<'py>, + spec: impl IntoPyObject<'py>, + metadata: Bound<'py, PyDict>, +) -> PyResult> { + let kwargs = PyDict::new(py); + kwargs.set_item("metadata", metadata)?; + py.import("numpy")? + .getattr("dtype")? + .call((spec,), Some(&kwargs)) +} + +/// The numpy dtype of a fixed-size datatype, whose element bytes numpy can +/// read as they are. +pub(crate) fn fixed_dtype<'py>(py: Python<'py>, dt: &Datatype) -> PyResult> { + match dt { + Datatype::FixedPoint { .. } => np_dtype(py, int_format(dt)?), + Datatype::FloatingPoint { .. } => np_dtype(py, float_format(dt)?), + Datatype::String { size, charset, .. } => { + if *size == 0 { + return Err(unsupported("zero-length fixed string")); + } + let meta = PyDict::new(py); + let enc = match charset { + CharacterSet::Ascii => "ascii", + CharacterSet::Utf8 => "utf-8", + }; + meta.set_item("h5py_encoding", enc)?; + np_dtype_with_metadata(py, format!("S{size}"), meta) + } + Datatype::Opaque { size, .. } => { + if *size == 0 { + return Err(unsupported("zero-length opaque type")); + } + np_dtype(py, format!("V{size}")) + } + Datatype::Enumeration { + base_type, members, .. + } => { + let base = int_format(base_type)?; + if is_h5py_bool(base_type, members) { + return np_dtype(py, "?"); + } + let meta = PyDict::new(py); + meta.set_item("enum", enum_members(py, base_type, members)?)?; + np_dtype_with_metadata(py, base, meta) + } + Datatype::Compound { size, members } => { + if let Some(c) = complex_format(*size, members) { + return np_dtype(py, c); + } + let names = PyList::empty(py); + let formats = PyList::empty(py); + let offsets = PyList::empty(py); + for m in members { + let end = m.byte_offset.checked_add(u64::from(m.datatype.type_size())); + if end.is_none_or(|end| end > u64::from(*size)) { + return Err(PyValueError::new_err(format!( + "compound member {} lies outside the {size}-byte compound", + m.name + ))); + } + names.append(&m.name)?; + formats.append(fixed_dtype(py, &m.datatype).map_err(|e| { + unsupported(format!("compound member {}: {}", m.name, e.value(py))) + })?)?; + offsets.append(m.byte_offset)?; + } + let spec = PyDict::new(py); + spec.set_item("names", names)?; + spec.set_item("formats", formats)?; + spec.set_item("offsets", offsets)?; + spec.set_item("itemsize", size)?; + np_dtype(py, spec) + } + Datatype::Array { + base_type, + dimensions, + } => { + let base = fixed_dtype(py, base_type)?; + let dims = PyTuple::new(py, dimensions)?; + np_dtype(py, (base, dims)) + } + Datatype::VariableLength { is_string, .. } => Err(unsupported(if *is_string { + "variable-length string inside a compound or array type" + } else { + "variable-length sequence inside a compound or array type" + })), + Datatype::Reference { .. } => Err(unsupported("object/region references")), + Datatype::BitField { .. } => Err(unsupported("bitfield")), + Datatype::Time { .. } => Err(unsupported("time")), + } +} + +impl Converter { + /// The converter for a dataset's or attribute's datatype. + pub(crate) fn new(py: Python<'_>, dt: &Datatype, offset_size: u8) -> PyResult { + match dt { + Datatype::VariableLength { + is_string: true, + charset, + .. + } => { + let utf8 = matches!(charset, Some(CharacterSet::Utf8)); + let meta = PyDict::new(py); + if utf8 { + meta.set_item("vlen", py.get_type::())?; + } else { + meta.set_item("vlen", py.get_type::())?; + } + let dtype = np_dtype_with_metadata(py, "O", meta)?; + Ok(Self { + view: dtype.clone().unbind(), + dtype: dtype.unbind(), + layout: Layout::VlString { utf8 }, + elem_size: vl_ref_size(offset_size)?, + vl_unit: 1, + }) + } + Datatype::VariableLength { + is_string: false, + base_type, + .. + } => { + let base = fixed_dtype(py, base_type)?; + let meta = PyDict::new(py); + meta.set_item("vlen", &base)?; + let dtype = np_dtype_with_metadata(py, "O", meta)?; + Ok(Self { + dtype: dtype.unbind(), + view: base.unbind(), + layout: Layout::VlSequence, + elem_size: vl_ref_size(offset_size)?, + vl_unit: base_type.type_size() as usize, + }) + } + Datatype::Array { + base_type, + dimensions, + } => { + let dtype = fixed_dtype(py, dt)?; + let base = fixed_dtype(py, base_type)?; + Ok(Self { + dtype: dtype.unbind(), + view: base.unbind(), + layout: Layout::Subarray(dimensions.iter().map(|&d| d as usize).collect()), + elem_size: dt.type_size() as usize, + vl_unit: 0, + }) + } + _ => { + let dtype = fixed_dtype(py, dt)?; + Ok(Self { + view: dtype.clone().unbind(), + dtype: dtype.unbind(), + layout: Layout::Fixed, + elem_size: dt.type_size() as usize, + vl_unit: 0, + }) + } + } + } + + pub(crate) fn is_vl(&self) -> bool { + matches!(self.layout, Layout::VlString { .. } | Layout::VlSequence) + } + + /// An empty array of `shape` (some dimension is zero). + pub(crate) fn empty<'py>( + &self, + py: Python<'py>, + shape: &[usize], + ) -> PyResult> { + let np = py.import("numpy")?; + match &self.layout { + Layout::Subarray(dims) => { + let mut full = shape.to_vec(); + full.extend_from_slice(dims); + np.call_method1("empty", (PyTuple::new(py, full)?, self.view.bind(py))) + } + _ => np.call_method1("empty", (PyTuple::new(py, shape)?, self.dtype.bind(py))), + } + } + + /// Turn decoded element data into a numpy array of `shape`. + /// + /// `str_values` decodes variable-length strings to `str` (what h5py + /// does for attributes) instead of `bytes` (what it does for datasets). + pub(crate) fn to_array<'py>( + &self, + py: Python<'py>, + data: Elements, + shape: &[usize], + str_values: bool, + ) -> PyResult> { + let n: usize = shape.iter().product(); + match (data, &self.layout) { + (Elements::Bytes(bytes), Layout::Fixed) => { + bytes_as_array(py, bytes, self.view.bind(py), shape) + } + (Elements::Bytes(bytes), Layout::Subarray(dims)) => { + let mut full = shape.to_vec(); + full.extend_from_slice(dims); + bytes_as_array(py, bytes, self.view.bind(py), &full) + } + (Elements::Vl(items), Layout::VlString { .. }) => { + check_count(items.len(), n)?; + let mut objs: Vec> = Vec::with_capacity(items.len()); + for item in items { + let obj = if str_values { + PyBytes::new(py, &item) + .call_method1("decode", ("utf-8", "surrogateescape"))? + .unbind() + } else { + PyBytes::new(py, &item).into_any().unbind() + }; + objs.push(obj); + } + object_array(py, objs, shape) + } + (Elements::Vl(items), Layout::VlSequence) => { + check_count(items.len(), n)?; + let base = self.view.bind(py); + let itemsize: usize = base.getattr("itemsize")?.extract()?; + let mut objs: Vec> = Vec::with_capacity(items.len()); + for item in items { + if item.len() % itemsize != 0 { + return Err(PyValueError::new_err(format!( + "variable-length element of {} bytes is not a whole number of {itemsize}-byte values", + item.len() + ))); + } + let len = item.len() / itemsize; + objs.push(bytes_as_array(py, item, base, &[len])?.unbind()); + } + object_array(py, objs, shape) + } + _ => Err(PyValueError::new_err( + "internal error: element data does not match the datatype", + )), + } + } +} + +/// Element data as read, before it becomes numpy. +pub(crate) enum Elements { + /// The elements' bytes, back to back. + Bytes(Vec), + /// Each variable-length element's bytes, resolved from the global heap. + Vl(Vec>), +} + +fn vl_ref_size(offset_size: u8) -> PyResult { + // The library sizes a variable-length element as 16 bytes (a length, an + // 8-byte heap address and an index) whatever the file's offset size. + // Refuse the other sizes rather than read misaligned references. + if offset_size != 8 { + return Err(unsupported(format!( + "variable-length data in a file with {offset_size}-byte offsets" + ))); + } + Ok(4 + usize::from(offset_size) + 4) +} + +fn check_count(got: usize, want: usize) -> PyResult<()> { + if got != want { + return Err(PyValueError::new_err(format!( + "read {got} elements, expected {want}" + ))); + } + Ok(()) +} + +/// A numpy array over `bytes` without copying them: the `Vec` becomes the +/// array's buffer and is viewed as `dtype` with `shape`. +pub(crate) fn bytes_as_array<'py>( + py: Python<'py>, + bytes: Vec, + dtype: &Bound<'py, PyAny>, + shape: &[usize], +) -> PyResult> { + let itemsize: usize = dtype.getattr("itemsize")?.extract()?; + let n: usize = shape.iter().product(); + if n.checked_mul(itemsize) != Some(bytes.len()) { + return Err(PyValueError::new_err(format!( + "read {} bytes, expected {n} elements of {itemsize} bytes", + bytes.len() + ))); + } + let shape = PyTuple::new(py, shape)?; + if n == 0 { + return py.import("numpy")?.call_method1("empty", (shape, dtype)); + } + let raw = PyArray1::from_vec(py, bytes); + let arr = raw + .call_method1("view", (dtype,))? + .call_method1("reshape", (shape,))?; + // A `Vec` carries no alignment promise. numpy copes with unaligned + // arrays, but slowly and not in every routine, so hand out an aligned + // copy in the (allocator-dependent, rare) case the buffer is not. + if !arr + .getattr("flags")? + .getattr("aligned")? + .extract::()? + { + return arr.call_method0("copy"); + } + Ok(arr) +} + +fn object_array<'py>( + py: Python<'py>, + objs: Vec>, + shape: &[usize], +) -> PyResult> { + let arr = PyArray1::from_vec(py, objs); + arr.call_method1("reshape", (PyTuple::new(py, shape)?,)) +} + +/// Resolve variable-length elements (global heap references in `raw`) to +/// their bytes: each element's stored length times `unit` (1 for strings, +/// the base type's size for sequences). Pure Rust, so it runs without the +/// GIL. +pub(crate) fn resolve_vl( + file_data: &[u8], + raw: &[u8], + count: usize, + offset_size: u8, + length_size: u8, + unit: usize, +) -> Result>, String> { + let refs = clawhdf5_format::vl_data::parse_vl_references(raw, count as u64, offset_size) + .map_err(|e| e.to_string())?; + let undefined = match offset_size { + 2 => 0xFFFF, + 4 => 0xFFFF_FFFF, + _ => u64::MAX, + }; + let mut collections: HashMap = HashMap::new(); + let mut out = Vec::with_capacity(refs.len()); + for vl in &refs { + if vl.collection_address == 0 || vl.collection_address == undefined { + if vl.length != 0 { + return Err(format!( + "variable-length element of length {} has no heap address", + vl.length + )); + } + out.push(Vec::new()); + continue; + } + let coll = match collections.entry(vl.collection_address) { + std::collections::hash_map::Entry::Occupied(e) => e.into_mut(), + std::collections::hash_map::Entry::Vacant(e) => { + let addr = usize::try_from(vl.collection_address) + .map_err(|_| "global heap address out of range".to_string())?; + e.insert( + GlobalHeapCollection::parse(file_data, addr, length_size) + .map_err(|e| e.to_string())?, + ) + } + }; + let index = u16::try_from(vl.object_index) + .map_err(|_| format!("global heap object index {} out of range", vl.object_index))?; + let obj = coll.get_object(index).ok_or_else(|| { + format!( + "global heap object {index} not found in the collection at {}", + vl.collection_address + ) + })?; + let need = (vl.length as usize) + .checked_mul(unit) + .ok_or("variable-length element too long")?; + if need > obj.data.len() { + return Err(format!( + "variable-length element of {need} bytes in a {}-byte heap object", + obj.data.len() + )); + } + out.push(obj.data[..need].to_vec()); + } + Ok(out) +} diff --git a/crates/clawhdf5-py/src/dataset.rs b/crates/clawhdf5-py/src/dataset.rs index e1b7a38..cbfc68d 100644 --- a/crates/clawhdf5-py/src/dataset.rs +++ b/crates/clawhdf5-py/src/dataset.rs @@ -1,241 +1,341 @@ -//! PyDataset — read access to HDF5 datasets with numpy integration. +//! PyDataset — h5py-style read access to HDF5 datasets. +//! +//! `ds[key]` parses the key into hyperslab selections (see `select`) and +//! reads only those elements through the facade's `read_selection`; the +//! bytes it returns become the numpy array's buffer without a copy (see +//! `convert`). All file access and decoding runs with the GIL released, so +//! Python threads reading the same or different datasets run in parallel. use std::sync::Arc; -use numpy::PyArrayDyn; -use numpy::ndarray::{ArrayD, IxDyn}; +use clawhdf5_format::datatype::Datatype; +use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::PyList; - -use clawhdf5_rs::DType; +use pyo3::types::{PyList, PyTuple}; use crate::attrs::PyAttrs; -use crate::to_py_err; +use crate::convert::{Converter, Elements, resolve_vl}; +use crate::select::{self, Plan}; +use crate::{PyEmpty, node, to_py_err}; -/// A handle to an HDF5 dataset (read mode). +/// A dataset in a file opened for reading. /// -/// Supports numpy-style indexing via `__getitem__`: /// ```python -/// ds = f['dataset_name'] -/// data = ds[:] # read all data as numpy array -/// shape = ds.shape -/// dtype = ds.dtype +/// ds = f['group/dataset'] +/// ds.shape, ds.dtype, ds.attrs['units'] +/// block = ds[10:20, ::2] # reads only the selected elements /// ``` #[pyclass(name = "Dataset")] pub struct PyDataset { file: Arc, path: String, - cached_shape: Vec, - cached_dtype: DType, + /// `None` for a dataset with a null dataspace (h5py's `Empty`). + shape: Option>, + datatype: Datatype, + /// Why the datatype cannot be read into numpy, if it cannot. + conv: Result, } impl PyDataset { - pub fn new(file: Arc, path: String) -> PyResult { - let ds = file.dataset(&path).map_err(to_py_err)?; - let cached_shape = ds.shape().map_err(to_py_err)?; - let cached_dtype = ds.dtype().map_err(to_py_err)?; + pub(crate) fn open( + py: Python<'_>, + file: Arc, + path: String, + ) -> PyResult { + let hdr = node::header(&file, &path)?; + let null = node::is_null(&node::dataspace(&file, &hdr)?); + let (shape, datatype) = { + let ds = file.dataset(&path).map_err(to_py_err)?; + let shape = if null { + None + } else { + Some(ds.shape().map_err(to_py_err)?) + }; + (shape, ds.raw_datatype().map_err(to_py_err)?) + }; + let conv = Converter::new(py, &datatype, file.superblock().offset_size) + .map_err(|e| e.value(py).to_string()); Ok(Self { file, path, - cached_shape, - cached_dtype, + shape, + datatype, + conv, }) } -} -/// Map a `DType` to a numpy dtype string. -fn dtype_to_numpy_str(dt: &DType) -> &'static str { - match dt { - DType::F64 => "float64", - DType::F32 => "float32", - DType::I64 => "int64", - DType::I32 => "int32", - DType::I16 => "int16", - DType::I8 => "int8", - DType::U64 => "uint64", - DType::U32 => "uint32", - DType::U16 => "uint16", - DType::U8 => "uint8", - DType::String | DType::VariableLengthString => "object", - _ => "object", - } -} - -#[pymethods] -impl PyDataset { - /// The shape of the dataset as a tuple. - #[getter] - fn shape(&self, py: Python<'_>) -> PyResult> { - let tuple = pyo3::types::PyTuple::new(py, self.cached_shape.iter().map(|&d| d as usize))?; - Ok(tuple.into_any().unbind()) + fn converter(&self) -> PyResult<&Converter> { + self.conv + .as_ref() + .map_err(|msg| PyTypeError::new_err(format!("{}: {msg}", node::name(&self.path)))) } - /// The numpy dtype string of the dataset. - #[getter] - fn dtype(&self) -> &'static str { - dtype_to_numpy_str(&self.cached_dtype) - } + /// Read the selection described by `plan` into a numpy array. + fn read_plan<'py>(&self, py: Python<'py>, plan: &Plan) -> PyResult> { + let conv = self.converter()?; + let dims = self.shape.as_deref().unwrap_or(&[]); + let out_shape = plan.out_shape(); - /// Attribute access (read-only). - #[getter] - fn attrs(&self) -> PyResult { - let ds = self.file.dataset(&self.path).map_err(to_py_err)?; - let map = ds.attrs().map_err(to_py_err)?; - Ok(PyAttrs::from_read(map)) - } - - /// Read data via indexing. Supports `ds[:]`, `ds[0]`, `ds[0:5]`, etc. - /// - /// The full dataset is always read from the underlying file; the index - /// is then applied on the resulting numpy array. - fn __getitem__<'py>(&self, py: Python<'py>, key: &Bound<'py, PyAny>) -> PyResult> { - let arr = self.read_as_numpy(py)?; - let indexed = arr.get_item(key)?; - Ok(indexed.unbind()) - } - - fn __repr__(&self) -> String { - format!( - "", - self.path, - self.cached_shape, - dtype_to_numpy_str(&self.cached_dtype), - ) - } - - fn __len__(&self) -> usize { - self.cached_shape.first().copied().unwrap_or(0) as usize - } -} - -impl PyDataset { - /// Read the full dataset and return it as a numpy array (or list for strings). - /// - /// For numeric types, the Rust I/O (file reading + decompression) is - /// performed inside `py.detach()` so that the GIL is released - /// during the potentially expensive operation. The numpy array - /// construction still happens with the GIL held. - fn read_as_numpy<'py>(&self, py: Python<'py>) -> PyResult> { - let file = &self.file; - let path = &self.path; - let shape: Vec = self.cached_shape.iter().map(|&d| d as usize).collect(); - - match &self.cached_dtype { - DType::F64 => { - let data = py - .detach(|| file.dataset(path).and_then(|ds| ds.read_f64())) - .map_err(to_py_err)?; - let nd = ArrayD::from_shape_vec(IxDyn(&shape), data) - .map_err(|e| PyErr::new::(e.to_string()))?; - let arr = PyArrayDyn::from_owned_array(py, nd); - Ok(arr.into_any()) - } - DType::F32 => { - let data = py - .detach(|| file.dataset(path).and_then(|ds| ds.read_f32())) - .map_err(to_py_err)?; - let nd = ArrayD::from_shape_vec(IxDyn(&shape), data) - .map_err(|e| PyErr::new::(e.to_string()))?; - let arr = PyArrayDyn::from_owned_array(py, nd); - Ok(arr.into_any()) - } - DType::I32 => { - let data = py - .detach(|| file.dataset(path).and_then(|ds| ds.read_i32())) - .map_err(to_py_err)?; - let nd = ArrayD::from_shape_vec(IxDyn(&shape), data) - .map_err(|e| PyErr::new::(e.to_string()))?; - let arr = PyArrayDyn::from_owned_array(py, nd); - Ok(arr.into_any()) - } - DType::I64 => { - let data = py - .detach(|| file.dataset(path).and_then(|ds| ds.read_i64())) - .map_err(to_py_err)?; - let nd = ArrayD::from_shape_vec(IxDyn(&shape), data) - .map_err(|e| PyErr::new::(e.to_string()))?; - let arr = PyArrayDyn::from_owned_array(py, nd); - Ok(arr.into_any()) - } - DType::U8 => { - // Try zero-copy first (contiguous layout), fall back to - // read_u64 + cast for chunked/compact datasets. - let data: Vec = py - .detach(|| { - let ds = file.dataset(path)?; - match ds.read_u8_zerocopy() { - Ok(slice) => Ok(slice.to_vec()), - Err(_) => { - let raw = ds.read_u64()?; - Ok(raw.iter().map(|&v| v as u8).collect()) + let arr = if plan.is_empty() { + conv.empty(py, &out_shape)? + } else { + let (reads, list_axis) = plan.reads(dims); + let file = &*self.file; + let path = self.path.as_str(); + let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit); + // Everything below touches only Rust data: release the GIL. + let blocks: Vec<(Elements, Vec)> = py + .detach(|| -> Result<_, ReadError> { + let ds = file.dataset(path)?; + let sb = file.superblock(); + let mut blocks = Vec::with_capacity(reads.len()); + for (sel, shape) in reads { + let raw = ds.read_selection(&sel)?; + let n: usize = shape.iter().product(); + let data = if vl { + if raw.len() != n * elem_size { + return Err(ReadError::Other(format!( + "read {} bytes of variable-length references, expected {}", + raw.len(), + n * elem_size + ))); } - } - }) - .map_err(to_py_err)?; - let nd = ArrayD::from_shape_vec(IxDyn(&shape), data) - .map_err(|e| PyErr::new::(e.to_string()))?; - let arr = PyArrayDyn::from_owned_array(py, nd); - Ok(arr.into_any()) + Elements::Vl( + resolve_vl( + file.as_bytes(), + &raw, + n, + sb.offset_size, + sb.length_size, + unit, + ) + .map_err(ReadError::Other)?, + ) + } else { + Elements::Bytes(raw) + }; + blocks.push((data, shape)); + } + Ok(blocks) + }) + .map_err(|e| e.into_py(&self.path))?; + + let mut arrays = Vec::with_capacity(blocks.len()); + for (data, shape) in blocks { + arrays.push(conv.to_array(py, data, &shape, false)?); } - DType::U64 => { - let data = py - .detach(|| file.dataset(path).and_then(|ds| ds.read_u64())) - .map_err(to_py_err)?; - let nd = ArrayD::from_shape_vec(IxDyn(&shape), data) - .map_err(|e| PyErr::new::(e.to_string()))?; - let arr = PyArrayDyn::from_owned_array(py, nd); - Ok(arr.into_any()) + let joined = if arrays.len() == 1 { + arrays.pop().expect("one block") + } else { + let axis = list_axis.expect("several reads only for a list index"); + // Name the dtype: left to itself numpy canonicalises a + // structured dtype here (drops padding, native byte order). + let kwargs = pyo3::types::PyDict::new(py); + kwargs.set_item("axis", axis)?; + kwargs.set_item("dtype", arrays[0].getattr("dtype")?)?; + kwargs.set_item("casting", "no")?; + py.import("numpy")?.call_method( + "concatenate", + (PyList::new(py, arrays)?,), + Some(&kwargs), + )? + }; + // Drop the axes indexed by an integer (length 1 in the blocks). + let mut shape = out_shape.clone(); + if let crate::convert::Layout::Subarray(sub) = &conv.layout { + shape.extend_from_slice(sub); } - DType::String | DType::VariableLengthString => { - // String reads need the GIL for PyList construction, but we - // release it during the Rust I/O portion. - let data = py - .detach(|| file.dataset(path).and_then(|ds| ds.read_string())) - .map_err(to_py_err)?; - let list = PyList::new(py, &data)?; - Ok(list.into_any()) - } - other => Err(PyErr::new::(format!( - "unsupported dataset dtype for reading: {other}" - ))), + joined.call_method1("reshape", (PyTuple::new(py, shape)?,))? + }; + + let arr = select_fields(py, arr, &plan.fields)?; + if plan.scalar { + return arr.get_item(PyTuple::empty(py)); + } + Ok(arr) + } +} + +/// An error from the read closure, turned into a Python error with the GIL. +enum ReadError { + Lib(clawhdf5_rs::Error), + Other(String), +} + +impl From for ReadError { + fn from(e: clawhdf5_rs::Error) -> Self { + ReadError::Lib(e) + } +} + +impl ReadError { + fn into_py(self, path: &str) -> PyErr { + match self { + ReadError::Lib(e) => to_py_err(e), + ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))), } } } -#[cfg(test)] -mod tests { - use super::*; +/// Keep only the named compound fields, as h5py's `ds['x']` / `ds['x', 'y']`. +fn select_fields<'py>( + py: Python<'py>, + arr: Bound<'py, PyAny>, + fields: &[String], +) -> PyResult> { + if fields.is_empty() { + return Ok(arr); + } + let names = arr.getattr("dtype")?.getattr("names")?; + if names.is_none() { + return Err(PyValueError::new_err( + "Field names only allowed for compound types", + )); + } + let names: Vec = names.extract()?; + for f in fields { + if !names.contains(f) { + return Err(PyValueError::new_err(format!( + "Field {f} does not appear in this type." + ))); + } + } + let np = py.import("numpy")?; + if let [one] = fields { + return np.call_method1("ascontiguousarray", (arr.get_item(one)?,)); + } + let picked = arr.get_item(PyList::new(py, fields)?)?; + py.import("numpy.lib.recfunctions")? + .call_method1("repack_fields", (picked,)) +} - #[test] - fn dtype_mapping() { - assert_eq!(dtype_to_numpy_str(&DType::F64), "float64"); - assert_eq!(dtype_to_numpy_str(&DType::F32), "float32"); - assert_eq!(dtype_to_numpy_str(&DType::I32), "int32"); - assert_eq!(dtype_to_numpy_str(&DType::I64), "int64"); - assert_eq!(dtype_to_numpy_str(&DType::U8), "uint8"); - assert_eq!(dtype_to_numpy_str(&DType::String), "object"); +#[pymethods] +impl PyDataset { + /// The shape of the dataset (`None` for an empty/null dataspace). + #[getter] + fn shape<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.shape { + Some(s) => Ok(PyTuple::new(py, s)?.into_any()), + None => Ok(py.None().into_bound(py)), + } } - #[test] - fn dataset_from_file() { - let mut b = clawhdf5_rs::FileBuilder::new(); - b.create_dataset("vals").with_f64_data(&[1.0, 2.0, 3.0]); - let bytes = b.finish().unwrap(); - let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap()); - let ds = PyDataset::new(file, "vals".into()).unwrap(); - assert_eq!(ds.cached_shape, vec![3]); - assert_eq!(ds.cached_dtype, DType::F64); + /// The maximum shape (`None` per unlimited dimension), like h5py. + #[getter] + fn maxshape<'py>(&self, py: Python<'py>) -> PyResult> { + let Some(shape) = &self.shape else { + return Ok(py.None().into_bound(py)); + }; + let max = self + .file + .dataset(&self.path) + .and_then(|ds| ds.max_dimensions()) + .map_err(to_py_err)? + .unwrap_or_else(|| shape.clone()); + let items: Vec> = max + .into_iter() + .map(|d| (d != u64::MAX).then_some(d)) + .collect(); + Ok(PyTuple::new(py, items)?.into_any()) } - #[test] - fn dataset_len() { - let mut b = clawhdf5_rs::FileBuilder::new(); - b.create_dataset("data") - .with_i32_data(&[10, 20, 30, 40]) - .with_shape(&[2, 2]); - let bytes = b.finish().unwrap(); - let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap()); - let ds = PyDataset::new(file, "data".into()).unwrap(); - assert_eq!(ds.__len__(), 2); + /// The dataset's numpy dtype, as h5py reports it. + #[getter] + fn dtype<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self.converter()?.dtype.bind(py).clone()) + } + + #[getter] + fn ndim(&self) -> usize { + self.shape.as_ref().map_or(0, Vec::len) + } + + /// Number of elements (`None` for an empty/null dataspace, as h5py). + #[getter] + fn size(&self) -> Option { + self.shape.as_ref().map(|s| s.iter().product()) + } + + /// The dataset's full name, e.g. `/group/data`. + #[getter] + fn name(&self) -> String { + node::name(&self.path) + } + + /// The dataset's attributes (read-only, dict-like). + #[getter] + fn attrs(&self) -> PyResult { + PyAttrs::read(Arc::clone(&self.file), &self.path) + } + + /// Read with h5py indexing: integers, slices with positive steps, + /// `...`, one increasing list of integers, and compound field names. + /// Only the selected elements are read from the file. + fn __getitem__<'py>( + &self, + py: Python<'py>, + key: &Bound<'py, PyAny>, + ) -> PyResult> { + let Some(dims) = &self.shape else { + let is_empty_tuple = key.cast::().is_ok_and(|t| t.is_empty()); + let is_ellipsis = key.is_instance_of::(); + if is_empty_tuple || is_ellipsis { + let empty = PyEmpty::new(self.converter()?.dtype.clone_ref(py)); + return Ok(empty.into_pyobject(py)?.into_any()); + } + return Err(PyValueError::new_err("Empty datasets cannot be sliced")); + }; + let plan = select::parse(key, dims)?; + self.read_plan(py, &plan) + } + + /// `numpy.asarray(ds)` reads the whole dataset. + #[pyo3(signature = (dtype=None, copy=None))] + fn __array__<'py>( + &self, + py: Python<'py>, + dtype: Option<&Bound<'py, PyAny>>, + copy: Option, + ) -> PyResult> { + let _ = copy; // every read is a fresh array + let Some(dims) = &self.shape else { + return Err(PyValueError::new_err("an empty dataset has no array value")); + }; + let ellipsis = pyo3::types::PyEllipsis::get(py).to_owned().into_any(); + let plan = select::parse(&ellipsis, dims)?; + let arr = self.read_plan(py, &plan)?; + match dtype { + Some(dt) => arr.call_method1("astype", (dt,)), + None => Ok(arr), + } + } + + fn __len__(&self) -> PyResult { + match self.shape.as_deref() { + Some([first, ..]) => Ok(*first as usize), + _ => Err(PyTypeError::new_err( + "Attempt to take len() of scalar dataset", + )), + } + } + + fn __repr__(&self, py: Python<'_>) -> String { + let dtype = match &self.conv { + Ok(c) => c + .dtype + .bind(py) + .str() + .map(|s| s.to_string()) + .unwrap_or_default(), + Err(_) => format!("{:?}", self.datatype), + }; + let shape = match &self.shape { + Some(s) => format!("{s:?}"), + None => "None".to_string(), + }; + format!( + "", + node::name(&self.path) + ) } } diff --git a/crates/clawhdf5-py/src/file.rs b/crates/clawhdf5-py/src/file.rs index 23e5f95..4f6bb7a 100644 --- a/crates/clawhdf5-py/src/file.rs +++ b/crates/clawhdf5-py/src/file.rs @@ -3,11 +3,12 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; +use pyo3::exceptions::PyKeyError; use pyo3::prelude::*; +use pyo3::types::PyList; use crate::attrs::PyAttrs; -use crate::dataset::PyDataset; -use crate::group::{PyGroup, WriteGroupState, finalize_write_group}; +use crate::group::{self, PyGroup, WriteGroupState, finalize_write_group}; use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err}; /// Internal state for write mode. @@ -35,6 +36,7 @@ struct WriteState { #[pyclass(name = "File")] pub struct PyFile { inner: Option, + filename: String, } enum FileInner { @@ -51,15 +53,20 @@ impl PyFile { /// mode: 'r' for read (default), 'w' for write #[new] #[pyo3(signature = (path, mode="r"))] - fn new(path: &str, mode: &str) -> PyResult { + fn new(py: Python<'_>, path: &str, mode: &str) -> PyResult { + let filename = path.to_string(); match mode { "r" => { - let file = clawhdf5_rs::File::open(path).map_err(to_py_err)?; + let file = py + .detach(|| clawhdf5_rs::File::open(path)) + .map_err(to_py_err)?; Ok(Self { inner: Some(FileInner::Read(Arc::new(file))), + filename, }) } "w" => Ok(Self { + filename, inner: Some(FileInner::Write(WriteState { path: PathBuf::from(path), root_datasets: Vec::new(), @@ -101,44 +108,56 @@ impl PyFile { Ok(false) // don't suppress exceptions } - /// Get a child object (dataset or group) by path. + /// Get a child object (dataset or group) by path; `f['/']` is the root. fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { - let file = self.read_file()?; - // Try dataset first - match file.dataset(key) { - Ok(_) => { - let ds = PyDataset::new(Arc::clone(file), key.to_string())?; - Ok(ds.into_pyobject(py)?.into_any().unbind()) - } - Err(clawhdf5_rs::Error::NotADataset(_)) => { - let grp = PyGroup::from_read(Arc::clone(file), key.to_string()); - Ok(grp.into_pyobject(py)?.into_any().unbind()) - } - Err(_) => { - // Could be a group (no DataLayout message, no error) - match file.group(key) { - Ok(_) => { - let grp = PyGroup::from_read(Arc::clone(file), key.to_string()); - Ok(grp.into_pyobject(py)?.into_any().unbind()) - } - Err(e) => Err(PyErr::new::(format!( - "{key}: {e}" - ))), - } + group::get_item(py, self.read_file()?, "", key) + } + + /// `f.get(key, default=None)`. + #[pyo3(signature = (key, default=None))] + fn get(&self, py: Python<'_>, key: &str, default: Option>) -> PyResult> { + match group::get_item(py, self.read_file()?, "", key) { + Err(e) if e.is_instance_of::(py) => { + Ok(default.unwrap_or_else(|| py.None())) } + other => other, } } /// List the names of all children in the root group. fn keys(&self, py: Python<'_>) -> PyResult> { - let file = self.read_file()?; - let root = file.root(); - let mut names = root.datasets().map_err(to_py_err)?; - let groups = root.groups().map_err(to_py_err)?; - names.extend(groups); - names.sort(); - let list = pyo3::types::PyList::new(py, &names)?; - Ok(list.into_any().unbind()) + let names = group::member_names(self.read_file()?, "")?; + Ok(PyList::new(py, names)?.into_any().unbind()) + } + + fn values(&self, py: Python<'_>) -> PyResult> { + let vals = group::values(py, self.read_file()?, "")?; + Ok(PyList::new(py, vals)?.into_any().unbind()) + } + + fn items(&self, py: Python<'_>) -> PyResult> { + let items = group::items(py, self.read_file()?, "")?; + Ok(PyList::new(py, items)?.into_any().unbind()) + } + + fn __iter__(&self, py: Python<'_>) -> PyResult> { + self.keys(py)?.call_method0(py, "__iter__") + } + + fn __len__(&self) -> PyResult { + Ok(group::member_names(self.read_file()?, "")?.len()) + } + + /// The root group's name, `/`. + #[getter] + fn name(&self) -> &'static str { + "/" + } + + /// The path the file was opened with. + #[getter] + fn filename(&self) -> &str { + &self.filename } /// Create a dataset in the root group (write mode only). @@ -192,10 +211,7 @@ impl PyFile { #[getter] fn attrs(&self) -> PyResult { match self.inner.as_ref() { - Some(FileInner::Read(file)) => { - let map = file.root().attrs().map_err(to_py_err)?; - Ok(PyAttrs::from_read(map)) - } + Some(FileInner::Read(file)) => PyAttrs::read(Arc::clone(file), ""), Some(FileInner::Write(state)) => Ok(PyAttrs::from_write(Arc::clone(&state.root_attrs))), None => Err(PyErr::new::( "file is closed", @@ -216,8 +232,7 @@ impl PyFile { } fn __contains__(&self, key: &str) -> PyResult { - let file = self.read_file()?; - Ok(file.dataset(key).is_ok() || file.group(key).is_ok()) + Ok(group::contains(self.read_file()?, "", key)) } } diff --git a/crates/clawhdf5-py/src/group.rs b/crates/clawhdf5-py/src/group.rs index 9f995f8..c97cc17 100644 --- a/crates/clawhdf5-py/src/group.rs +++ b/crates/clawhdf5-py/src/group.rs @@ -2,12 +2,12 @@ use std::sync::{Arc, Mutex}; +use pyo3::exceptions::{PyIOError, PyKeyError}; use pyo3::prelude::*; use pyo3::types::PyList; use crate::attrs::PyAttrs; -use crate::dataset::PyDataset; -use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err}; +use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node, to_py_err}; /// Shared state for a group being written. pub(crate) struct WriteGroupState { @@ -18,15 +18,10 @@ pub(crate) struct WriteGroupState { /// An HDF5 group. /// -/// In read mode, provides `__getitem__` navigation and child listing. -/// In write mode, supports `create_dataset` and `create_group` and -/// attribute setting. -/// -/// ```python -/// grp = f['group_name'] -/// grp.keys() -/// ds = grp['dataset'] -/// ``` +/// In read mode it behaves like an h5py group: `grp['name']`, +/// `grp['sub/path']` and `grp['/absolute/path']`, `keys()`, `values()`, +/// `items()`, iteration, `len()`, `in`, `get()`, `name` and `attrs`. +/// In write mode, supports `create_dataset` and attribute setting. #[pyclass(name = "Group")] pub struct PyGroup { inner: GroupInner, @@ -52,46 +47,90 @@ impl PyGroup { inner: GroupInner::Write(state), } } + + fn read_parts(&self, what: &str) -> PyResult<(&Arc, &str)> { + match &self.inner { + GroupInner::Read { file, path } => Ok((file, path)), + GroupInner::Write(_) => Err(PyIOError::new_err(format!( + "cannot {what} a group opened for writing" + ))), + } + } +} + +// Read-mode operations shared by `Group` and `File` (a file is its root +// group, as in h5py). + +/// `group[key]`. +pub(crate) fn get_item( + py: Python<'_>, + file: &Arc, + path: &str, + key: &str, +) -> PyResult> { + node::open(py, file, node::join(path, key)) +} + +/// Names of the group's datasets and subgroups, sorted (h5py's order). +pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult> { + let group = if path.is_empty() { + file.root() + } else { + file.group(path).map_err(to_py_err)? + }; + let mut names = group.datasets().map_err(to_py_err)?; + names.extend(group.groups().map_err(to_py_err)?); + names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); + names.dedup(); + Ok(names) +} + +pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool { + node::exists(file, &node::join(path, key)) +} + +pub(crate) fn values( + py: Python<'_>, + file: &Arc, + path: &str, +) -> PyResult>> { + member_names(file, path)? + .iter() + .map(|n| get_item(py, file, path, n)) + .collect() +} + +pub(crate) fn items( + py: Python<'_>, + file: &Arc, + path: &str, +) -> PyResult)>> { + member_names(file, path)? + .into_iter() + .map(|n| { + let v = get_item(py, file, path, &n)?; + Ok((n, v)) + }) + .collect() } #[pymethods] impl PyGroup { /// Get a child object (dataset or subgroup) by name or path. fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { - match &self.inner { - GroupInner::Read { file, path } => { - let full_path = if path.is_empty() { - key.to_string() - } else { - format!("{path}/{key}") - }; - // Try dataset first - match file.dataset(&full_path) { - Ok(_) => { - let ds = PyDataset::new(Arc::clone(file), full_path)?; - Ok(ds.into_pyobject(py)?.into_any().unbind()) - } - Err(clawhdf5_rs::Error::NotADataset(_)) => { - let grp = PyGroup::from_read(Arc::clone(file), full_path); - Ok(grp.into_pyobject(py)?.into_any().unbind()) - } - Err(e) => { - // Could be a group without a DataLayout message - match file.group(&full_path) { - Ok(_) => { - let grp = PyGroup::from_read(Arc::clone(file), full_path); - Ok(grp.into_pyobject(py)?.into_any().unbind()) - } - Err(_) => Err(PyErr::new::(format!( - "{key}: {e}" - ))), - } - } - } + let (file, path) = self.read_parts("read children from")?; + get_item(py, file, path, key) + } + + /// `group.get(key, default=None)`. + #[pyo3(signature = (key, default=None))] + fn get(&self, py: Python<'_>, key: &str, default: Option>) -> PyResult> { + let (file, path) = self.read_parts("read children from")?; + match get_item(py, file, path, key) { + Err(e) if e.is_instance_of::(py) => { + Ok(default.unwrap_or_else(|| py.None())) } - GroupInner::Write(_) => Err(PyErr::new::( - "cannot read children from a group opened for writing", - )), + other => other, } } @@ -99,16 +138,7 @@ impl PyGroup { fn keys(&self, py: Python<'_>) -> PyResult> { match &self.inner { GroupInner::Read { file, path } => { - let group = if path.is_empty() { - file.root() - } else { - file.group(path).map_err(to_py_err)? - }; - let mut names = group.datasets().map_err(to_py_err)?; - let groups = group.groups().map_err(to_py_err)?; - names.extend(groups); - names.sort(); - let list = PyList::new(py, &names)?; + let list = PyList::new(py, member_names(file, path)?)?; Ok(list.into_any().unbind()) } GroupInner::Write(state) => { @@ -120,6 +150,38 @@ impl PyGroup { } } + fn values(&self, py: Python<'_>) -> PyResult> { + let (file, path) = self.read_parts("read children from")?; + Ok(PyList::new(py, values(py, file, path)?)? + .into_any() + .unbind()) + } + + fn items(&self, py: Python<'_>) -> PyResult> { + let (file, path) = self.read_parts("read children from")?; + Ok(PyList::new(py, items(py, file, path)?)?.into_any().unbind()) + } + + fn __iter__(&self, py: Python<'_>) -> PyResult> { + self.keys(py)?.call_method0(py, "__iter__") + } + + fn __len__(&self) -> PyResult { + match &self.inner { + GroupInner::Read { file, path } => Ok(member_names(file, path)?.len()), + GroupInner::Write(state) => Ok(state.lock().unwrap().datasets.len()), + } + } + + /// The group's full name, e.g. `/sensors`. + #[getter] + fn name(&self) -> String { + match &self.inner { + GroupInner::Read { path, .. } => node::name(path), + GroupInner::Write(state) => node::name(&state.lock().unwrap().name), + } + } + /// Create a dataset inside this group (write mode only). /// /// Parameters: @@ -161,7 +223,7 @@ impl PyGroup { state.lock().unwrap().datasets.push(spec); Ok(()) } - GroupInner::Read { .. } => Err(PyErr::new::( + GroupInner::Read { .. } => Err(PyIOError::new_err( "cannot create datasets on a read-only group", )), } @@ -171,15 +233,7 @@ impl PyGroup { #[getter] fn attrs(&self) -> PyResult { match &self.inner { - GroupInner::Read { file, path } => { - let group = if path.is_empty() { - file.root() - } else { - file.group(path).map_err(to_py_err)? - }; - let map = group.attrs().map_err(to_py_err)?; - Ok(PyAttrs::from_read(map)) - } + GroupInner::Read { file, path } => PyAttrs::read(Arc::clone(file), path), GroupInner::Write(state) => { let store = Arc::clone(&state.lock().unwrap().attrs); Ok(PyAttrs::from_write(store)) @@ -189,12 +243,9 @@ impl PyGroup { fn __repr__(&self) -> String { match &self.inner { - GroupInner::Read { path, .. } => { - if path.is_empty() { - "".to_string() - } else { - format!("") - } + GroupInner::Read { file, path } => { + let n = member_names(file, path).map_or(0, |m| m.len()); + format!("", node::name(path)) } GroupInner::Write(state) => { let name = &state.lock().unwrap().name; @@ -205,14 +256,7 @@ impl PyGroup { fn __contains__(&self, key: &str) -> PyResult { match &self.inner { - GroupInner::Read { file, path } => { - let full_path = if path.is_empty() { - key.to_string() - } else { - format!("{path}/{key}") - }; - Ok(file.dataset(&full_path).is_ok() || file.group(&full_path).is_ok()) - } + GroupInner::Read { file, path } => Ok(contains(file, path, key)), GroupInner::Write(state) => { let guard = state.lock().unwrap(); Ok(guard.datasets.iter().any(|d| d.name == key)) @@ -244,26 +288,24 @@ mod tests { use super::*; #[test] - fn read_group_construction() { + fn member_names_are_sorted() { let mut b = clawhdf5_rs::FileBuilder::new(); - let mut g = b.create_group("grp"); + b.create_dataset("zeta").with_f64_data(&[1.0]); + b.create_dataset("alpha").with_f64_data(&[1.0]); + let mut g = b.create_group("mid"); g.create_dataset("x").with_f64_data(&[1.0]); let finished = g.finish(); b.add_group(finished); let bytes = b.finish().unwrap(); - let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap()); - let _grp = PyGroup::from_read(file, "grp".into()); - } - - #[test] - fn write_group_state() { - let state = WriteGroupState { - name: "test".into(), - datasets: vec![], - attrs: Arc::new(Mutex::new(vec![])), - }; - let arc = Arc::new(Mutex::new(state)); - let _grp = PyGroup::from_write(arc); + let file = clawhdf5_rs::File::from_bytes(bytes).unwrap(); + assert_eq!( + member_names(&file, "").unwrap(), + vec!["alpha", "mid", "zeta"] + ); + assert_eq!(member_names(&file, "mid").unwrap(), vec!["x"]); + assert!(contains(&file, "", "mid/x")); + assert!(contains(&file, "mid", "/alpha")); + assert!(!contains(&file, "", "nope")); } #[test] diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index fe31887..4c58cd6 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -10,9 +10,12 @@ //! ``` mod attrs; +mod convert; mod dataset; mod file; mod group; +mod node; +mod select; use pyo3::prelude::*; @@ -46,6 +49,54 @@ pub(crate) fn to_py_err(e: clawhdf5_rs::Error) -> PyErr { } } +/// The value of a dataset or attribute with a null dataspace: a type but no +/// data. Mirrors `h5py.Empty`. +#[pyclass(name = "Empty", frozen)] +pub struct PyEmpty { + dtype: Py, +} + +impl PyEmpty { + pub(crate) fn new(dtype: Py) -> Self { + Self { dtype } + } +} + +#[pymethods] +impl PyEmpty { + #[new] + fn py_new(py: Python<'_>, dtype: &Bound<'_, PyAny>) -> PyResult { + let dtype = py.import("numpy")?.getattr("dtype")?.call1((dtype,))?; + Ok(Self::new(dtype.unbind())) + } + + #[getter] + fn dtype(&self, py: Python<'_>) -> Py { + self.dtype.clone_ref(py) + } + + #[getter] + fn shape(&self, py: Python<'_>) -> Py { + py.None() + } + + #[getter] + fn size(&self, py: Python<'_>) -> Py { + py.None() + } + + fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult { + match other.cast::() { + Ok(o) => self.dtype.bind(py).eq(o.get().dtype.bind(py)), + Err(_) => Ok(false), + } + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + Ok(format!("Empty(dtype={})", self.dtype.bind(py).repr()?)) + } +} + /// The data payload for a dataset being written. #[derive(Clone)] pub(crate) enum DatasetData { @@ -224,6 +275,7 @@ fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; Ok(()) } @@ -233,9 +285,9 @@ mod tests { #[test] fn owned_attr_value_roundtrip() { - let val = OwnedAttrValue::F64(3.14); + let val = OwnedAttrValue::F64(2.5); let attr: clawhdf5_rs::AttrValue = val.into(); - assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 3.14).abs() < 1e-10)); + assert!(matches!(attr, clawhdf5_rs::AttrValue::F64(v) if (v - 2.5).abs() < 1e-10)); } #[test] diff --git a/crates/clawhdf5-py/src/node.rs b/crates/clawhdf5-py/src/node.rs new file mode 100644 index 0000000..1c68101 --- /dev/null +++ b/crates/clawhdf5-py/src/node.rs @@ -0,0 +1,168 @@ +//! Resolving paths to objects in a file opened for reading. + +use std::sync::Arc; + +use clawhdf5_format::attribute::AttributeMessage; +use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; +use pyo3::prelude::*; + +use crate::dataset::PyDataset; +use crate::group::PyGroup; + +/// Join `key` onto the group path `base` the way h5py does: an absolute key +/// starts from the root, a relative one from `base`. Paths are kept without +/// a leading `/`; the root is `""`. +pub(crate) fn join(base: &str, key: &str) -> String { + let parts = if key.starts_with('/') { + key.split('/').collect::>() + } else { + base.split('/').chain(key.split('/')).collect() + }; + parts + .into_iter() + .filter(|p| !p.is_empty() && *p != ".") + .collect::>() + .join("/") +} + +/// The HDF5 name (`/a/b`) of a path. +pub(crate) fn name(path: &str) -> String { + format!("/{path}") +} + +/// The object header of the object at `path`. +pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult { + let sb = file.superblock(); + let data = file.as_bytes(); + let addr = if path.is_empty() { + sb.root_group_address + } else { + clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| { + PyKeyError::new_err(format!( + "Unable to open object (object '{}' doesn't exist): {e}", + name(path) + )) + })? + }; + let addr = usize::try_from(addr) + .map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?; + ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size) + .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path)))) +} + +/// What an object header describes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Dataset, + Group, + Datatype, +} + +pub(crate) fn kind(hdr: &ObjectHeader) -> Option { + let has = |t: MessageType| hdr.messages.iter().any(|m| m.msg_type == t); + if has(MessageType::DataLayout) { + Some(Kind::Dataset) + } else if has(MessageType::LinkInfo) + || has(MessageType::Link) + || has(MessageType::SymbolTable) + || has(MessageType::GroupInfo) + { + Some(Kind::Group) + } else if has(MessageType::Datatype) { + Some(Kind::Datatype) + } else { + None + } +} + +/// Open the object at `path` as a `Dataset` or `Group`. +pub(crate) fn open( + py: Python<'_>, + file: &Arc, + path: String, +) -> PyResult> { + let hdr = header(file, &path)?; + match kind(&hdr) { + Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path)? + .into_pyobject(py)? + .into_any() + .unbind()), + Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path) + .into_pyobject(py)? + .into_any() + .unbind()), + Some(Kind::Datatype) => Err(PyTypeError::new_err(format!( + "{}: committed (named) datatypes are not supported by clawhdf5", + name(&path) + ))), + None => Err(PyValueError::new_err(format!( + "{}: not a dataset, group or datatype", + name(&path) + ))), + } +} + +/// Whether `path` names a dataset or group. +pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool { + header(file, path) + .ok() + .and_then(|h| kind(&h)) + .is_some_and(|k| k != Kind::Datatype) +} + +/// The dataspace message of an object header. +pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult { + let sb = file.superblock(); + let msg = hdr + .messages + .iter() + .find(|m| m.msg_type == MessageType::Dataspace) + .ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?; + let data = clawhdf5_format::shared_message::message_data( + file.as_bytes(), + msg, + sb.offset_size, + sb.length_size, + ) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string())) +} + +pub(crate) fn is_null(space: &Dataspace) -> bool { + space.space_type == DataspaceType::Null +} + +/// The attributes of the object at `path`, sorted by name (h5py's order). +/// Attributes whose messages cannot be parsed are left out, as the facade's +/// `attrs()` does. +pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult> { + let hdr = header(file, path)?; + let sb = file.superblock(); + let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant( + file.as_bytes(), + &hdr, + sb.offset_size, + sb.length_size, + ) + .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?; + attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes())); + Ok(attrs) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn join_paths() { + assert_eq!(join("", "a"), "a"); + assert_eq!(join("a", "b/c"), "a/b/c"); + assert_eq!(join("a/b", "/x"), "x"); + assert_eq!(join("a", "/"), ""); + assert_eq!(join("", "/a//b/"), "a/b"); + assert_eq!(join("a", "./b"), "a/b"); + } +} diff --git a/crates/clawhdf5-py/src/select.rs b/crates/clawhdf5-py/src/select.rs new file mode 100644 index 0000000..1b1c796 --- /dev/null +++ b/crates/clawhdf5-py/src/select.rs @@ -0,0 +1,366 @@ +//! h5py-style indexing (`ds[1, 2:10:3, ...]`) mapped onto hyperslab +//! selections, so only the selected elements are read. +//! +//! The rules and error messages follow h5py's `selections.py`: integers +//! (negative from the end) drop their axis, slices must have a positive +//! step, one `Ellipsis` fills the unmentioned axes, a single increasing list +//! of integers may index one axis, and strings name compound fields. +//! Everything else (`None`/`np.newaxis`, boolean masks, several index lists) +//! is refused with the error h5py gives. + +use clawhdf5_format::selection::Selection; +use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyEllipsis, PySlice, PyString, PyTuple}; + +/// The selection along one axis. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Axis { + /// A single index: the axis is dropped from the result. + Index(u64), + /// `start, start + step, ...`, `count` of them. + Slice { start: u64, step: u64, count: u64 }, + /// Increasing, distinct indices. + List(Vec), +} + +impl Axis { + fn len(&self) -> u64 { + match self { + Axis::Index(_) => 1, + Axis::Slice { count, .. } => *count, + Axis::List(v) => v.len() as u64, + } + } +} + +/// A parsed index expression. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Plan { + /// One entry per dataset axis. + pub axes: Vec, + /// Compound field names to keep (empty: all). + pub fields: Vec, + /// For a scalar dataset: `ds[()]` gives a scalar, `ds[...]` a 0-d array. + /// For other datasets: every axis was an integer, so h5py gives a scalar. + pub scalar: bool, +} + +impl Plan { + /// The shape of the result. + pub fn out_shape(&self) -> Vec { + self.axes + .iter() + .filter(|a| !matches!(a, Axis::Index(_))) + .map(|a| a.len() as usize) + .collect() + } + + /// Whether the selection is empty. + pub fn is_empty(&self) -> bool { + self.axes.iter().any(|a| a.len() == 0) + } + + /// The hyperslab reads that make up this selection, each with the shape + /// of its block (index axes kept at length 1). More than one only when an + /// axis is indexed by a list: one read per run of consecutive indices, + /// concatenated along `list_axis` afterwards. + pub fn reads(&self, dims: &[u64]) -> (Vec<(Selection, Vec)>, Option) { + let list_axis = self.axes.iter().position(|a| matches!(a, Axis::List(_))); + let runs: Vec<(u64, u64)> = match list_axis.map(|i| &self.axes[i]) { + Some(Axis::List(idx)) => consecutive_runs(idx), + _ => vec![(0, 0)], + }; + let mut out = Vec::with_capacity(runs.len()); + for (run_start, run_len) in runs { + let mut start = Vec::with_capacity(dims.len()); + let mut stride = Vec::with_capacity(dims.len()); + let mut count = Vec::with_capacity(dims.len()); + for axis in &self.axes { + let (s, st, c) = match axis { + Axis::Index(i) => (*i, 1, 1), + Axis::Slice { start, step, count } => (*start, *step, *count), + Axis::List(_) => (run_start, 1, run_len), + }; + start.push(s); + // A stride only matters between blocks; keep it >= 1. + stride.push(if c <= 1 { 1 } else { st }); + count.push(c); + } + let block_shape: Vec = count.iter().map(|&c| c as usize).collect(); + let whole = start.iter().all(|&s| s == 0) + && stride.iter().all(|&s| s == 1) + && count.as_slice() == dims; + let sel = if whole { + Selection::All + } else { + let block = vec![1; dims.len()]; + Selection::Hyperslab { + start, + stride, + count, + block, + } + }; + out.push((sel, block_shape)); + } + (out, list_axis) + } +} + +fn consecutive_runs(idx: &[u64]) -> Vec<(u64, u64)> { + let mut runs: Vec<(u64, u64)> = Vec::new(); + for &i in idx { + match runs.last_mut() { + Some((s, n)) if *s + *n == i => *n += 1, + _ => runs.push((i, 1)), + } + } + runs +} + +/// Parse `key` for a dataset of shape `dims`. +pub(crate) fn parse(key: &Bound<'_, PyAny>, dims: &[u64]) -> PyResult { + let items: Vec> = match key.cast::() { + Ok(t) => t.iter().collect(), + Err(_) => vec![key.clone()], + }; + let mut fields = Vec::new(); + let mut args = Vec::new(); + for item in items { + if let Ok(s) = item.cast::() { + fields.push(s.to_str()?.to_owned()); + } else { + args.push(item); + } + } + + if args.iter().any(|a| a.is_none()) { + return Err(PyTypeError::new_err( + "Indexing with None (or np.newaxis) is not supported", + )); + } + + let rank = dims.len(); + if rank == 0 { + return match args.as_slice() { + [] => Ok(Plan { + axes: vec![], + fields, + scalar: true, + }), + [a] if a.is_instance_of::() => Ok(Plan { + axes: vec![], + fields, + scalar: false, + }), + _ => Err(PyValueError::new_err( + "Illegal slicing argument for scalar dataspace", + )), + }; + } + + // Expand the ellipsis (at most one) to full slices. + let n_ellipsis = args + .iter() + .filter(|a| a.is_instance_of::()) + .count(); + if n_ellipsis > 1 { + return Err(PyValueError::new_err("Only one ellipsis may be used.")); + } + let explicit = args.len() - n_ellipsis; + if explicit > rank { + return Err(PyValueError::new_err(format!( + "{explicit} indexing arguments for {rank} dimensions" + ))); + } + let py = key.py(); + let mut expanded: Vec>> = Vec::with_capacity(rank); + for a in args { + if a.is_instance_of::() { + for _ in 0..(rank - explicit) { + expanded.push(None); + } + } else { + expanded.push(Some(a)); + } + } + while expanded.len() < rank { + expanded.push(None); + } + + let mut axes = Vec::with_capacity(rank); + for (arg, &n) in expanded.iter().zip(dims) { + axes.push(match arg { + None => Axis::Slice { + start: 0, + step: 1, + count: n, + }, + Some(a) => parse_axis(py, a, n)?, + }); + } + if axes.iter().filter(|a| matches!(a, Axis::List(_))).count() > 1 { + return Err(PyTypeError::new_err( + "Only one indexing vector or array is currently allowed for fancy indexing", + )); + } + let scalar = axes.iter().all(|a| matches!(a, Axis::Index(_))); + Ok(Plan { + axes, + fields, + scalar, + }) +} + +fn parse_axis(py: Python<'_>, a: &Bound<'_, PyAny>, n: u64) -> PyResult { + if a.is_none() { + return Err(PyTypeError::new_err( + "Indexing with None (or np.newaxis) is not supported", + )); + } + if let Ok(s) = a.cast::() { + let n_isize = isize::try_from(n) + .map_err(|_| PyValueError::new_err("dimension too large to slice"))?; + let ind = s.indices(n_isize)?; + if ind.step < 1 { + return Err(PyValueError::new_err(format!( + "Step must be >= 1 (got {})", + ind.step + ))); + } + // `slicelength` is the number of elements selected, >= 0. + let count = ind.slicelength as u64; + let start = if count == 0 { 0 } else { ind.start as u64 }; + return Ok(Axis::Slice { + start, + step: ind.step as u64, + count, + }); + } + let np = py.import("numpy")?; + let is_bool = + a.is_instance_of::() || a.is_instance(&np.getattr("bool_")?)?; + let is_array_like = a.is_instance(&np.getattr("ndarray")?)? + || a.is_instance_of::() + || a.is_instance_of::(); + if !is_bool && !is_array_like && a.hasattr("__index__")? { + let i: i128 = a.call_method0("__index__")?.extract()?; + return Ok(Axis::Index(normalize(i, n)?)); + } + if is_array_like { + let arr = np.call_method1("asarray", (a,))?; + let kind: String = arr.getattr("dtype")?.getattr("kind")?.extract()?; + if kind == "b" { + return Err(PyTypeError::new_err( + "Boolean mask indexing is not supported by clawhdf5", + )); + } + let ndim: usize = arr.getattr("ndim")?.extract()?; + let size: usize = arr.getattr("size")?.extract()?; + if size > 0 && kind != "i" && kind != "u" { + return Err(PyTypeError::new_err( + "Indexing arrays must have integer dtypes", + )); + } + if ndim > 1 { + return Err(PyTypeError::new_err( + "Only 1-D integer lists or arrays can be used for fancy indexing", + )); + } + let vals: Vec = arr.call_method0("tolist")?.extract()?; + let mut idx = Vec::with_capacity(vals.len()); + for v in vals { + idx.push(normalize(v, n)?); + } + if idx.windows(2).any(|w| w[0] >= w[1]) { + return Err(PyTypeError::new_err( + "Indexing elements must be in increasing order", + )); + } + return Ok(Axis::List(idx)); + } + Err(PyTypeError::new_err(format!( + "Illegal index type for clawhdf5 datasets: {}", + a.get_type().name()? + ))) +} + +fn normalize(i: i128, n: u64) -> PyResult { + let n_i = i128::from(n); + let j = if i < 0 { i + n_i } else { i }; + if j < 0 || j >= n_i { + let hi = n_i - 1; + return Err(PyIndexError::new_err(format!( + "Index ({i}) out of range for (0-{hi})" + ))); + } + Ok(j as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn runs_group_consecutive_indices() { + assert_eq!( + consecutive_runs(&[1, 2, 3, 7, 9, 10]), + vec![(1, 3), (7, 1), (9, 2)] + ); + assert_eq!(consecutive_runs(&[]), vec![]); + } + + #[test] + fn full_selection_reads_everything() { + let plan = Plan { + axes: vec![ + Axis::Slice { + start: 0, + step: 1, + count: 4, + }, + Axis::Slice { + start: 0, + step: 1, + count: 3, + }, + ], + fields: vec![], + scalar: false, + }; + let (reads, list) = plan.reads(&[4, 3]); + assert_eq!(list, None); + assert_eq!(reads, vec![(Selection::All, vec![4, 3])]); + } + + #[test] + fn index_and_step_map_to_a_hyperslab() { + let plan = Plan { + axes: vec![ + Axis::Index(2), + Axis::Slice { + start: 1, + step: 3, + count: 2, + }, + ], + fields: vec![], + scalar: false, + }; + let (reads, _) = plan.reads(&[4, 8]); + assert_eq!( + reads, + vec![( + Selection::Hyperslab { + start: vec![2, 1], + stride: vec![1, 3], + count: vec![1, 2], + block: vec![1, 1], + }, + vec![1, 2] + )] + ); + assert_eq!(plan.out_shape(), vec![2]); + } +} diff --git a/crates/clawhdf5-py/tests/conftest.py b/crates/clawhdf5-py/tests/conftest.py new file mode 100644 index 0000000..967f1e0 --- /dev/null +++ b/crates/clawhdf5-py/tests/conftest.py @@ -0,0 +1,18 @@ +"""Shared fixtures for the clawhdf5 Python binding tests.""" + +import os + +import pytest + + +@pytest.fixture(scope="session") +def h5py(): + """h5py, or a skip — unless CLAWHDF5_REQUIRE_INTEROP=1, which makes a + missing h5py a failure (as for the Rust interop suites).""" + try: + import h5py as mod + except ImportError: + if os.environ.get("CLAWHDF5_REQUIRE_INTEROP") == "1": + pytest.fail("h5py is required (CLAWHDF5_REQUIRE_INTEROP=1) but not importable") + pytest.skip("h5py not installed") + return mod diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py new file mode 100644 index 0000000..9a9178d --- /dev/null +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -0,0 +1,469 @@ +"""Every read through clawhdf5 compared against h5py (libhdf5) on files h5py +writes: dtypes, shapes, values and the type of what comes back (array, +numpy scalar, bytes, str, Empty), for every datatype the bindings map and a +spread of index expressions; plus the errors h5py gives for the same keys.""" + +import threading +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pytest + +import clawhdf5 + +# --------------------------------------------------------------------------- +# The generated file +# --------------------------------------------------------------------------- + +NUMERIC = [ + "i2", ">i4", ">i8", ">u2", ">u4", ">u8", + "f2", ">f4", ">f8", +] + + +def _values(dtype, shape, seed): + rng = np.random.default_rng(seed) + dt = np.dtype(dtype) + n = int(np.prod(shape)) + if dt.kind == "f": + return rng.standard_normal(n).astype(dt).reshape(shape) + info = np.iinfo(dt) + return rng.integers(info.min, info.max, size=n, dtype=dt.newbyteorder("=")).astype(dt).reshape(shape) + + +def _compound_dtype(): + return np.dtype( + { + "names": ["id", "pos", "label", "flag", "vec"], + "formats": ["f8", "S6", "u1", ("", "be_") + f.create_dataset(f"num/{name}_1d", data=_values(dt, (37,), i)) + f.create_dataset( + f"num/{name}_2d_gzip", + data=_values(dt, (13, 11), 100 + i), + chunks=(4, 5), + compression="gzip", + ) + f.create_dataset("num/f8_3d", data=_values("f8")) + fseqs = np.empty(3, dtype=object) + fseqs[:] = [np.linspace(0, 1, 5).astype(">f8"), np.array([2.5], dtype=">f8"), np.array([], dtype=">f8")] + f.create_dataset("vlen/f8_be", data=fseqs, dtype=vl_f) + + # Compounds. + cdt = _compound_dtype() + rec = np.zeros(10, dtype=cdt) + rec["id"] = np.arange(10) * 3 + rec["pos"] = np.linspace(-1, 1, 10) + rec["label"] = [f"n{i}".encode() for i in range(10)] + rec["flag"] = np.arange(10) % 2 + rec["vec"] = np.arange(30, dtype=" 1 else [0], (Ellipsis,)] + if n0 > 5: + keys += [[1, 2, 3, 5], [0, 4, 5]] + if len(shape) >= 2: + n1 = shape[1] + keys += [ + (0, 0), + (-1, -1), + (slice(1, 3), slice(None, None, 2)), + (Ellipsis, 1), + (1, Ellipsis), + (slice(None), [0, n1 - 1] if n1 > 1 else [0]), + (slice(None, None, 2), 1), + (slice(0, 2), slice(3, 1)), + ] + if len(shape) >= 3: + keys += [(0, slice(None), -1), (slice(1, None, 2), 2, slice(None, None, 3)), (Ellipsis, 0, 0), (0, Ellipsis, 1)] + return keys + + +# h5py 3.16 (HDF5 2.0) returns the elements of a variable-length sequence of +# big-endian floats unswapped (0.25 comes back as 2.6e-319), so these are +# checked against the values written instead (test_vlen_big_endian). +H5PY_MISREADS = {"vlen/f8_be"} + +ERROR_KEYS_1D = [slice(None, None, -1), 10**6, -(10**6), None, (0, 0, 0, 0, 0), [3, 1], (Ellipsis, Ellipsis), 1.5, "nope"] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_every_dataset_matches_h5py(h5py, pair): + ours, theirs, _ = pair + checked = 0 + for name in _all_datasets(h5py, theirs): + if name.startswith("unsupported/") or name in H5PY_MISREADS or name == "cmp/with_vlen": + continue + t = theirs[name] + o = ours[name] + assert o.shape == t.shape, name + assert o.dtype == t.dtype, f"{name}: {o.dtype} vs {t.dtype}" + assert dict(o.dtype.metadata or {}) == dict(t.dtype.metadata or {}), name + assert o.ndim == t.ndim and o.size == t.size, name + assert o.maxshape == t.maxshape, name + assert o.name == t.name, name + if t.shape is None: + assert_same(o[()], t[()], name) + continue + for key in keys_for(t.shape): + what = f"{name}[{key!r}]" + try: + expected = t[key] + except Exception as e: # noqa: BLE001 - h5py refuses: so must we + with pytest.raises(type(e)): + o[key] + continue + assert_same(o[key], expected, what) + checked += 1 + assert checked > 500 + + +def test_vlen_big_endian(pair): + ours, _, _ = pair + ds = ours["vlen/f8_be"] + assert ds.dtype.metadata["vlen"] == np.dtype(">f8") + got = ds[()] + expected = [np.linspace(0, 1, 5), np.array([2.5]), np.array([])] + assert got.shape == (3,) + for g, e in zip(got, expected): + assert g.dtype == np.dtype(">f8") + np.testing.assert_array_equal(g, e) + np.testing.assert_array_equal(ds[1], [2.5]) + + +def test_errors_match_h5py(h5py, pair): + ours, theirs, _ = pair + for name in ["num/le_i4_1d", "num/be_f8_2d_gzip", "str/vlen", "num/scalar_f8"]: + for key in ERROR_KEYS_1D: + try: + theirs[name][key] + except Exception as e: # noqa: BLE001 + with pytest.raises(type(e)): + ours[name][key] + else: + pass # valid for this shape; covered by the value test + + +def test_compound_fields_match_h5py(pair): + ours, theirs, _ = pair + for name in ["cmp/padded", "cmp/padded_chunked", "cmp/nested_2d"]: + t, o = theirs[name], ours[name] + for field in t.dtype.names: + assert_same(o[field], t[field], f"{name}[{field}]") + assert_same(o[field, 1:3], t[field, 1:3], f"{name}[{field}, 1:3]") + two = list(t.dtype.names[:2]) + expected = t[tuple(two)] + got = o[tuple(two)] + assert got.dtype.names == expected.dtype.names + for field in two: + np.testing.assert_array_equal(got[field], expected[field]) + with pytest.raises(ValueError): + ours["cmp/padded"]["no_such_field"] + with pytest.raises(ValueError): + ours["num/le_i4_1d"]["id"] + + +def test_numpy_asarray_and_len(pair): + ours, theirs, _ = pair + assert_same(np.asarray(ours["num/f8_3d"]), np.asarray(theirs["num/f8_3d"])) + assert len(ours["num/f8_3d"]) == len(theirs["num/f8_3d"]) + with pytest.raises(TypeError): + len(ours["num/scalar_f8"]) + + +def test_attributes_match_h5py(h5py, pair): + ours, theirs, _ = pair + for path in ["/", "num/le_f8_1d", "deep/er/est"]: + t = theirs[path].attrs + o = ours[path].attrs + assert list(o.keys()) == sorted(t.keys()), path + assert len(o) == len(t) + for k in t.keys(): + assert k in o + assert_same(o[k], t[k], f"{path}.attrs[{k}]") + assert o.get("missing", 5) == 5 + with pytest.raises(KeyError): + o["missing"] + assert [k for k, _ in o.items()] == list(o.keys()) + + +def test_groups_match_h5py(h5py, pair): + ours, theirs, _ = pair + assert list(ours.keys()) == list(theirs.keys()) + assert len(ours) == len(theirs) + for path in ["num", "deep", "deep/er", "deep/er/est", "empty_group"]: + o, t = ours[path], theirs[path] + assert isinstance(o, clawhdf5.Group) + assert list(o.keys()) == list(t.keys()), path + assert list(o) == list(t), path + assert len(o) == len(t), path + assert o.name == t.name + g = ours["deep/er"] + assert isinstance(g["est"], clawhdf5.Group) + assert isinstance(g["est/leaf"], clawhdf5.Dataset) + assert g["/deep/er/est/leaf"].name == "/deep/er/est/leaf" + assert "est/leaf" in g and "/num" in g and "nope" not in g + assert ours["/"].name == "/" + assert "deep/er/est/leaf" in ours + assert ours.get("nope") is None + with pytest.raises(KeyError): + ours["deep/nope"] + names = [k for k, v in ours["deep/er/est"].items()] + assert names == ["leaf"] + assert isinstance(ours["deep/er/est"].values()[0], clawhdf5.Dataset) + + +def test_unsupported_types_are_errors_not_data(pair): + ours, _, _ = pair + ds = ours["unsupported/refs"] # opening works + with pytest.raises(TypeError): + ds.dtype + with pytest.raises(TypeError): + ds[()] + with pytest.raises(TypeError): + ours["cmp/with_vlen"][()] + + +def test_boolean_masks_are_refused(pair): + ours, _, _ = pair + with pytest.raises(TypeError): + ours["num/le_i4_1d"][np.ones(37, dtype=bool)] + + +def test_reads_hand_numpy_the_rust_buffer(pair): + """A fixed-size read is a view over the buffer the library filled, not a + copy of it.""" + ours, _, _ = pair + arr = ours["num/le_f8_2d_gzip"][2:9, 1:4] + assert not arr.flags.owndata + assert arr.base is not None + assert arr.flags.aligned and arr.flags.c_contiguous + + +def test_only_the_selected_chunks_are_read(h5py, tmp_path): + """Damage one chunk: a selection that avoids it still reads, one that + touches it fails. Reading everything and slicing afterwards (what the + bindings did before) failed both. (The library reads the whole dataset + anyway when a selection's bounding box covers more than half of it, so + the selections here stay below that.)""" + path = str(tmp_path / "damaged.h5") + data = np.arange(1000, dtype=" Date: Sat, 26 Sep 2026 08:20:19 -0500 Subject: [PATCH 06/36] fix(format): resolve VL elements as libhdf5 does Checked with h5py on a patched file: - a VL string with an embedded NUL reads up to the NUL (libhdf5 converts VL strings to C strings); read_vl_strings returned "a\0b"; - an element whose global heap object is not length x base size bytes is an error ("Expected global heap object size does not match"); we returned the object cut to the length; - a heap address of 0 is a null element whatever its length. vl_data::VlResolver does this, caching each parsed heap collection: read_vl_strings parsed the whole collection again for every element. read_vl_strings and read_vl_bytes use it; check_element_size refuses a VL type whose stored element size is not 4 + offset size + 4. The conformance probe resolves VL values through VlResolver instead of its own lenient copy (575 of 697, unchanged). The new unit tests fail against the old read_vl_strings. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 14 ++ conformance/probe/src/main.rs | 75 ++---- crates/clawhdf5-format/src/vl_data.rs | 338 ++++++++++++++++++++++---- 3 files changed, 329 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3af6328..40781cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,20 @@ exhaustively destructures that variant; patterns with `..` are unaffected), and it is written back as stored. Tested against h5py (`crates/clawhdf5/tests/vl_offset4_interop.rs`). +- **Wrong data: VL strings with an embedded NUL, and VL elements whose heap + object has the wrong size.** libhdf5 hands VL strings over as C strings, + so h5py reads `"a\0b"` as `"a"`; `read_vl_strings` returned the NUL and + what followed. An element whose heap object is not exactly + `length × base size` bytes is refused by libhdf5 ("Expected global heap + object size does not match"); we returned the object cut or padded to + the length. Both now behave as libhdf5, and a heap address of 0 is a null + element (empty) whatever its length. The new + `clawhdf5_format::vl_data::VlResolver` does this and parses each global + heap collection once per read: `read_vl_strings` parsed the whole + collection again for every element. `vl_data::check_element_size` refuses + a VL datatype whose stored size is not 4 + offset size + 4 (libhdf5 + ignores the stored size). The conformance probe resolves VL elements + with `VlResolver` too; conformance unchanged at 575 of 697. ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 87c660d..72f02a1 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -19,9 +19,8 @@ //! with its message, location and the clawhdf5 frames of its backtrace. use std::cell::RefCell; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::panic::{self, AssertUnwindSafe}; -use std::rc::Rc; use clawhdf5_format::attribute::extract_attributes_full; use clawhdf5_format::data_layout::DataLayout; @@ -29,7 +28,6 @@ use clawhdf5_format::data_read; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::global_heap::GlobalHeapCollection; use clawhdf5_format::group_v1::{self, GroupEntry}; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; @@ -37,6 +35,7 @@ use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; use clawhdf5_format::symbol_table::SymbolTableMessage; +use clawhdf5_format::vl_data::{VlResolver, check_element_size}; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; @@ -111,7 +110,10 @@ struct Ctx<'a> { os: u8, ls: u8, base_dir: std::path::PathBuf, - heaps: RefCell, String>>>, + /// Resolves variable-length elements as the library does (null + /// elements, strings cut at a NUL, heap objects of the wrong size + /// refused), caching each heap collection. + vl: RefCell>, } impl<'a> Ctx<'a> { @@ -130,33 +132,6 @@ impl<'a> Ctx<'a> { } } - fn heap_obj(&self, addr: u64, idx: u32) -> Result, String> { - let coll = { - let mut cache = self.heaps.borrow_mut(); - cache - .entry(addr) - .or_insert_with(|| { - GlobalHeapCollection::parse(self.data, addr as usize, self.ls) - .map(Rc::new) - .map_err(e) - }) - .clone()? - }; - coll.get_object(idx as u16) - .map(|o| o.data.clone()) - .ok_or_else(|| { - format!("GlobalHeapObjectNotFound {{ collection_address: {addr}, index: {idx} }}") - }) - } - - fn read_offset(&self, b: &[u8]) -> u64 { - let mut v = 0u64; - for (i, x) in b.iter().take(self.os as usize).enumerate() { - v |= (*x as u64) << (8 * i); - } - v - } - fn canon(&self, dt: &Datatype, b: &[u8], out: &mut Vec) -> Result<(), String> { let size = dt.type_size() as usize; if b.len() < size { @@ -204,41 +179,27 @@ impl<'a> Ctx<'a> { } } Datatype::VariableLength { + size: vl_size, is_string, base_type, .. } => { - let len = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize; - let addr = self.read_offset(&b[4..]); - let idx_off = 4 + self.os as usize; - let idx = u32::from_le_bytes([ - b[idx_off], - b[idx_off + 1], - b[idx_off + 2], - b[idx_off + 3], - ]); - let obj = if len == 0 || addr == 0 || addr == u64::MAX >> (64 - 8 * self.os as u32) - { - Vec::new() - } else { - self.heap_obj(addr, idx)? - }; + check_element_size(*vl_size, self.os).map_err(e)?; + let el = &b[..size]; if *is_string { - let l = len.min(obj.len()); - canon_str(&obj[..l], out); + let s = self.vl.borrow_mut().string_bytes(el).map_err(e)?; + canon_str(&s[0], out); } else { let bs = base_type.type_size() as usize; - if bs == 0 { - return Err("canon: VL base size 0".into()); - } - let need = len.checked_mul(bs).ok_or("canon: VL overflow")?; - if len > 0 && obj.len() < need { - return Err(format!("canon: VL object {} < {need}", obj.len())); - } + // The borrow ends here: the base type may itself be + // variable-length. + let seq = self.vl.borrow_mut().sequences(el, bs).map_err(e)?; + let seq = &seq[0]; + let len = seq.len() / bs; out.push(b'V'); out.extend_from_slice(&(len as u32).to_le_bytes()); for i in 0..len { - self.canon(base_type, &obj[i * bs..], out)?; + self.canon(base_type, &seq[i * bs..], out)?; } } } @@ -744,7 +705,7 @@ fn main() { .parent() .map(|p| p.to_path_buf()) .unwrap_or_default(), - heaps: RefCell::new(HashMap::new()), + vl: RefCell::new(VlResolver::new(hdf5, sb.offset_size, sb.length_size)), }; let mut objects: Vec = Vec::new(); let mut visited = HashSet::new(); diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index 9a50d51..58015a4 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -5,7 +5,9 @@ //! `sequence_length(4 LE) + collection_address(offset_size LE) + object_index(4 LE)`. #[cfg(not(feature = "std"))] -use alloc::{string::String, vec::Vec}; +use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; +#[cfg(feature = "std")] +use std::collections::BTreeMap; use crate::error::FormatError; use crate::global_heap::GlobalHeapCollection; @@ -109,7 +111,174 @@ fn is_undefined_address(addr: u64, offset_size: u8) -> bool { } } +/// The size of one variable-length element in a file with `offset_size`-byte +/// addresses: a sequence length (4), a global heap collection address and an +/// object index (4). libhdf5 computes it this way rather than trusting the +/// datatype message (`H5T_set_loc`). +pub fn element_size(offset_size: u8) -> usize { + 4 + offset_size as usize + 4 +} + +/// Refuse a variable-length datatype whose stored element size is not the +/// one this file's offset size implies. Its elements would be laid out with +/// a stride libhdf5 does not use, so every value after the first would be +/// read from the wrong place. +pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), FormatError> { + let expected = element_size(offset_size); + if stored_size as usize != expected { + return Err(FormatError::VlDataError(format!( + "variable-length datatype stores {stored_size}-byte elements; a file with \ + {offset_size}-byte offsets uses {expected}" + ))); + } + Ok(()) +} + +/// A parsed collection, with its objects indexed for lookup. +struct CachedCollection { + collection: GlobalHeapCollection, + /// `slots[index]` is the position in `collection.objects` of the first + /// object with that index. + slots: Vec>, +} + +impl CachedCollection { + fn new(collection: GlobalHeapCollection) -> Self { + let max = collection + .objects + .iter() + .map(|o| o.index as usize) + .max() + .unwrap_or(0); + let mut slots = vec![None; max + 1]; + for (pos, obj) in collection.objects.iter().enumerate() { + let slot = &mut slots[obj.index as usize]; + if slot.is_none() { + *slot = Some(pos); + } + } + Self { collection, slots } + } + + fn get(&self, index: u32) -> Option<&[u8]> { + let pos = (*self.slots.get(usize::try_from(index).ok()?)?)?; + Some(&self.collection.objects[pos].data) + } +} + +/// Resolves variable-length elements against a file's global heap, parsing +/// each heap collection once however many elements point into it. +/// +/// Values follow libhdf5: an element whose heap address is 0 is null (an +/// empty string or sequence), and an element whose heap object is not +/// exactly `length × base size` bytes is an error ("Expected global heap +/// object size does not match"), not a truncated or padded value. +pub struct VlResolver<'a> { + file_data: &'a [u8], + offset_size: u8, + length_size: u8, + cache: BTreeMap, +} + +impl<'a> VlResolver<'a> { + /// A resolver over `file_data` (the file from its superblock on), with + /// the superblock's offset and length sizes. + pub fn new(file_data: &'a [u8], offset_size: u8, length_size: u8) -> Self { + Self { + file_data, + offset_size, + length_size, + cache: BTreeMap::new(), + } + } + + /// The size of one element in this file (see [`element_size`]). + pub fn element_size(&self) -> usize { + element_size(self.offset_size) + } + + /// Split `raw` into elements; its length must be a whole number of them. + fn elements(&self, raw: &[u8]) -> Result, FormatError> { + let size = self.element_size(); + if !raw.len().is_multiple_of(size) { + return Err(FormatError::VlDataError(format!( + "{} bytes is not a whole number of {size}-byte variable-length elements", + raw.len() + ))); + } + parse_vl_references(raw, (raw.len() / size) as u64, self.offset_size) + } + + /// The bytes of one element: `length × base_size` bytes from the heap, + /// or empty for a null or zero-length element. + fn resolve(&mut self, vl: &VlElement, base_size: usize) -> Result<&[u8], FormatError> { + let addr = vl.collection_address; + if addr == 0 || (vl.length == 0 && is_undefined_address(addr, self.offset_size)) { + return Ok(&[]); + } + let data = self.object(vl)?; + let expected = (vl.length as usize) + .checked_mul(base_size) + .ok_or_else(|| FormatError::Overflow("variable-length element size".into()))?; + if data.len() != expected { + return Err(FormatError::VlDataError(format!( + "global heap object {} in the collection at {addr} holds {} bytes; the element \ + says {} × {base_size}", + vl.object_index, + data.len(), + vl.length + ))); + } + Ok(data) + } + + /// The strings of the variable-length string elements in `raw`, as + /// bytes. A string ends at its first NUL, as libhdf5 returns it (it + /// converts each to a C string); a null element is empty. + pub fn string_bytes(&mut self, raw: &[u8]) -> Result>, FormatError> { + self.elements(raw)? + .iter() + .map(|vl| { + let s = self.resolve(vl, 1)?; + let end = s.iter().position(|&b| b == 0).unwrap_or(s.len()); + Ok(s[..end].to_vec()) + }) + .collect() + } + + /// The strings of the variable-length string elements in `raw`, decoded + /// as UTF-8 with invalid sequences replaced by U+FFFD (see + /// [`string_bytes`](Self::string_bytes) for the exact bytes). + pub fn strings(&mut self, raw: &[u8]) -> Result, FormatError> { + Ok(self + .string_bytes(raw)? + .into_iter() + .map(|b| match String::from_utf8(b) { + Ok(s) => s, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + }) + .collect()) + } + + /// The sequences of the variable-length sequence elements in `raw`, each + /// as its `length × base_size` bytes in the base type's encoding. + pub fn sequences(&mut self, raw: &[u8], base_size: usize) -> Result>, FormatError> { + if base_size == 0 { + return Err(FormatError::VlDataError( + "variable-length sequence of a zero-size base type".into(), + )); + } + self.elements(raw)? + .iter() + .map(|vl| self.resolve(vl, base_size).map(<[u8]>::to_vec)) + .collect() + } +} + /// Resolve VL strings from raw data by looking up each element in the global heap. +/// +/// Reads the first `num_elements` elements of `raw`. Strings end at their +/// first NUL and invalid UTF-8 is replaced, as in [`VlResolver::strings`]. pub fn read_vl_strings( file_data: &[u8], raw_data: &[u8], @@ -117,35 +286,23 @@ pub fn read_vl_strings( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - let refs = parse_vl_references(raw_data, num_elements, offset_size)?; - let mut result = Vec::with_capacity(refs.len()); + let raw = first_elements(raw_data, num_elements, offset_size)?; + VlResolver::new(file_data, offset_size, length_size).strings(raw) +} - for vl in &refs { - if vl.length == 0 && is_undefined_address(vl.collection_address, offset_size) { - result.push(String::new()); - continue; - } - if vl.length == 0 && vl.collection_address == 0 { - result.push(String::new()); - continue; - } - - let coll = - GlobalHeapCollection::parse(file_data, vl.collection_address as usize, length_size)?; - let obj = coll.get_object(vl.object_index as u16).ok_or( - FormatError::GlobalHeapObjectNotFound { - collection_address: vl.collection_address, - index: vl.object_index as u16, - }, - )?; - - // The object data is the raw string bytes - let len = (vl.length as usize).min(obj.data.len()); - let s = String::from_utf8_lossy(&obj.data[..len]).into_owned(); - result.push(s); - } - - Ok(result) +/// The first `num_elements` elements of `raw`, or an error if it is shorter. +fn first_elements(raw: &[u8], num_elements: u64, offset_size: u8) -> Result<&[u8], FormatError> { + let total = usize::try_from(num_elements) + .ok() + .and_then(|n| n.checked_mul(element_size(offset_size))) + .ok_or(FormatError::UnexpectedEof { + expected: usize::MAX, + available: raw.len(), + })?; + raw.get(..total).ok_or(FormatError::UnexpectedEof { + expected: total, + available: raw.len(), + }) } /// Resolve VL sequences from raw data, returning each element's bytes. @@ -153,7 +310,9 @@ pub fn read_vl_strings( /// Each element is the sequence's full encoding — element count × base type /// size bytes, in the base type's byte order — so a sequence of `i32` yields /// four bytes per value. Decode it with the base type (e.g. -/// [`crate::data_read::read_as_i64`]). +/// [`crate::data_read::read_as_i64`]). This does not know the base type, so +/// it returns each heap object whole; [`VlResolver::sequences`] also checks +/// the object's size against the element's length. pub fn read_vl_bytes( file_data: &[u8], raw_data: &[u8], @@ -162,6 +321,7 @@ pub fn read_vl_bytes( length_size: u8, ) -> Result>, FormatError> { let refs = parse_vl_references(raw_data, num_elements, offset_size)?; + let mut resolver = VlResolver::new(file_data, offset_size, length_size); let mut result = Vec::with_capacity(refs.len()); for vl in &refs { @@ -172,25 +332,38 @@ pub fn read_vl_bytes( result.push(Vec::new()); continue; } - - let coll = - GlobalHeapCollection::parse(file_data, vl.collection_address as usize, length_size)?; - let obj = coll.get_object(vl.object_index as u16).ok_or( - FormatError::GlobalHeapObjectNotFound { - collection_address: vl.collection_address, - index: vl.object_index as u16, - }, - )?; - // The heap object holds the whole sequence. `vl.length` counts // elements, not bytes, so it is only the byte length when the base // type is one byte wide. - result.push(obj.data.clone()); + let obj = resolver.object(vl)?; + result.push(obj.to_vec()); } Ok(result) } +impl VlResolver<'_> { + /// The heap object `vl` points to, whatever its size; its collection is + /// parsed on first use. + fn object(&mut self, vl: &VlElement) -> Result<&[u8], FormatError> { + let addr = vl.collection_address; + if !self.cache.contains_key(&addr) { + let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof { + expected: usize::MAX, + available: self.file_data.len(), + })?; + let coll = GlobalHeapCollection::parse(self.file_data, offset, self.length_size)?; + self.cache.insert(addr, CachedCollection::new(coll)); + } + self.cache[&addr] + .get(vl.object_index) + .ok_or(FormatError::GlobalHeapObjectNotFound { + collection_address: addr, + index: vl.object_index as u16, + }) + } +} + #[cfg(test)] mod tests { use super::*; @@ -333,6 +506,89 @@ mod tests { assert_eq!(bytes, vec![vec![0xDE, 0xAD], vec![0xBE, 0xEF, 0xCA]]); } + fn element(length: u32, addr: u64, index: u32, offset_size: u8) -> Vec { + let mut raw = length.to_le_bytes().to_vec(); + raw.extend_from_slice(&addr.to_le_bytes()[..offset_size as usize]); + raw.extend_from_slice(&index.to_le_bytes()); + raw + } + + #[test] + fn strings_end_at_the_first_nul() { + // libhdf5 hands each VL string over as a C string, so h5py sees + // "a\0b" as "a"; we used to return the NUL and what followed. + let mut file_data = vec![0u8; 512]; + build_gcol_at(&mut file_data, 64, &[(1, b"a\0b"), (2, b"cd")]); + let mut raw = element(3, 64, 1, 8); + raw.extend(element(2, 64, 2, 8)); + let mut r = VlResolver::new(&file_data, 8, 8); + assert_eq!( + r.string_bytes(&raw).unwrap(), + vec![b"a".to_vec(), b"cd".to_vec()] + ); + assert_eq!( + read_vl_strings(&file_data, &raw, 2, 8, 8).unwrap(), + ["a", "cd"] + ); + } + + #[test] + fn a_heap_object_of_the_wrong_size_is_an_error() { + // libhdf5: "Expected global heap object size does not match". We + // used to return the object cut to the element's length. + let mut file_data = vec![0u8; 512]; + build_gcol_at(&mut file_data, 64, &[(1, b"cdefgh"), (2, &[1, 0, 0, 0])]); + let mut r = VlResolver::new(&file_data, 8, 8); + assert!(r.string_bytes(&element(3, 64, 1, 8)).is_err()); + assert!(r.string_bytes(&element(9, 64, 1, 8)).is_err()); + assert!(read_vl_strings(&file_data, &element(3, 64, 1, 8), 1, 8, 8).is_err()); + // A sequence of one i32 is 4 bytes; of two, 8. + assert_eq!( + r.sequences(&element(1, 64, 2, 8), 4).unwrap(), + vec![vec![1, 0, 0, 0]] + ); + assert!(r.sequences(&element(2, 64, 2, 8), 4).is_err()); + assert!(r.sequences(&element(1, 64, 2, 8), 0).is_err()); + } + + #[test] + fn address_zero_is_null_whatever_the_length() { + // libhdf5 treats a heap address of 0 as a null element. + let file_data = vec![0u8; 64]; + let mut r = VlResolver::new(&file_data, 8, 8); + assert_eq!( + r.string_bytes(&element(5, 0, 1, 8)).unwrap(), + vec![Vec::::new()] + ); + assert_eq!( + r.sequences(&element(5, 0, 1, 8), 4).unwrap(), + vec![Vec::::new()] + ); + } + + #[test] + fn four_byte_offsets_use_twelve_byte_elements() { + let mut file_data = vec![0u8; 512]; + build_gcol_at(&mut file_data, 64, &[(1, b"one"), (2, b""), (3, b"three")]); + let mut raw = element(3, 64, 1, 4); + raw.extend(element(0, 64, 2, 4)); + raw.extend(element(5, 64, 3, 4)); + assert_eq!(raw.len(), 36); + let mut r = VlResolver::new(&file_data, 4, 8); + assert_eq!(r.element_size(), 12); + assert_eq!(r.strings(&raw).unwrap(), ["one", "", "three"]); + // Not a whole number of elements. + assert!(r.strings(&raw[..30]).is_err()); + } + + #[test] + fn element_size_is_checked_against_the_offset_size() { + assert!(check_element_size(16, 8).is_ok()); + assert!(check_element_size(12, 4).is_ok()); + assert!(check_element_size(16, 4).is_err()); + assert!(check_element_size(524_304, 8).is_err()); + } + #[test] fn parse_vl_references_truncated_error() { let raw = vec![0u8; 10]; // too short for 1 element with offset_size=8 -- 2.54.0 From f7d88bb4fb082909c535ec83e48ab456a1f6b9f4 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:21:09 -0500 Subject: [PATCH 07/36] ci: build the Python package with maturin and run its tests against h5py ci-test.sh gains a step that lints clawhdf5-py, builds its wheel with maturin, unpacks it under target/ (the interpreter's environment is not touched) and runs the pytest suite, which compares reads with h5py. It skips without maturin/pytest, and fails instead under CLAWHDF5_REQUIRE_INTEROP=1. The CI interop venv installs maturin and pytest, so CI runs it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .gitea/workflows/ci.yml | 4 +++- CHANGELOG.md | 5 +++++ scripts/ci-test.sh | 37 +++++++++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index cfd9624..06b42e3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -38,7 +38,9 @@ jobs: # (clawhdf5-tools) interop tests compare against. apt-get install -y --no-install-recommends python3 python3-venv cmake hdf5-tools python3 -m venv /opt/interop - /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin + # maturin + pytest: ci-test.sh builds the Python package + # (crates/clawhdf5-py) and runs its tests against h5py. + /opt/interop/bin/pip install --no-cache-dir h5py numpy netCDF4 xarray hdf5plugin maturin pytest echo "/opt/interop/bin" >> "$GITHUB_PATH" - name: Show interop library versions # h5dump's version too: the h5rs dump test requires its exact output diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a80f4..b873d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,11 @@ h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's: it returns variable-length sequences of big-endian floats unswapped; this package returns the stored values. +- **CI builds and tests the Python package.** It was excluded from CI. + `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with + maturin, unpacks it under `target/` and runs the pytest suite; skipped + without maturin/pytest in `$CLAWHDF5_PYTHON`, a failure then under + `CLAWHDF5_REQUIRE_INTEROP=1`. The CI interop venv installs both. ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files diff --git a/scripts/ci-test.sh b/scripts/ci-test.sh index e99cbf1..be25825 100755 --- a/scripts/ci-test.sh +++ b/scripts/ci-test.sh @@ -7,7 +7,9 @@ # # Environment: # CLAWHDF5_REQUIRE_INTEROP=1 Fail (instead of skip) when python3 with -# h5py/netCDF4/xarray is missing. CI sets this. +# h5py/netCDF4/xarray is missing, or without +# maturin/pytest for the Python package step. +# CI sets this. # Unset locally, the interop steps are skipped # if python3+h5py is not importable. # CLAWHDF5_FUZZ_SECONDS=N Run each cargo-fuzz target for N seconds @@ -55,7 +57,8 @@ run_step "cargo fmt --check" cargo fmt --check # 2. Clippy over every target (lib, bins, tests, benches, examples). Without # --all-targets, test and bench code is never linted. clawhdf5-py is -# excluded because it needs PyO3/Python headers. +# excluded here: PyO3 needs a Python interpreter to build, so it is +# linted in the Python package step (5b) instead. run_step "cargo clippy --all-targets" cargo clippy \ --workspace \ --exclude clawhdf5-py \ @@ -216,6 +219,36 @@ else STEPS+=("SKIP: h5py interop (format, ignored tests)") fi +# 5b. The Python package (crates/clawhdf5-py): lint it, build the wheel with +# maturin and run its pytest suite, which compares every read with h5py. +# The wheel is unpacked under target/ and put on PYTHONPATH, so the +# interpreter's environment is left as it was. Needs maturin and pytest +# in $PYTHON (CI installs both); skipped without them, and a failure +# instead when CLAWHDF5_REQUIRE_INTEROP=1. +python_package() { + local root="$SCRIPT_DIR/.." out + out="${CARGO_TARGET_DIR:-$root/target}/py-package" + rm -rf "$out" && mkdir -p "$out/wheel" "$out/site" || return 1 + cargo clippy -p clawhdf5-py --all-targets -- -D warnings || return 1 + "$PYTHON" -m maturin build \ + -m "$root/crates/clawhdf5-py/Cargo.toml" \ + -i "$PYTHON" \ + --out "$out/wheel" || return 1 + "$PYTHON" -m pip install --quiet --no-deps --target "$out/site" "$out"/wheel/*.whl || return 1 + PYTHONPATH="$out/site" "$PYTHON" -m pytest -q -p no:cacheprovider \ + "$root/crates/clawhdf5-py/tests" +} +if "$PYTHON" -m maturin --version >/dev/null 2>&1 && "$PYTHON" -c "import pytest" >/dev/null 2>&1; then + run_step "Python package (maturin build + pytest vs h5py)" python_package +elif [ "${CLAWHDF5_REQUIRE_INTEROP:-0}" = "1" ]; then + run_step "Python package (maturin build + pytest vs h5py)" \ + bash -c "echo \"maturin and pytest are required in $PYTHON (CLAWHDF5_REQUIRE_INTEROP=1)\"; exit 1" +else + echo "" + echo "==> [Python package] SKIPPED: needs maturin and pytest in $PYTHON" + STEPS+=("SKIP: Python package (maturin build + pytest)") +fi + # 6. Benches must keep compiling (they are not run). run_step "cargo bench --no-run" cargo bench \ --workspace \ -- 2.54.0 From 2bc4cb46a67f0abfb56b2e6e14982799efc09118 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:21:47 -0500 Subject: [PATCH 08/36] perf: copy contiguous hyperslab and point reads run by run A 256 x 256 hyperslab of a contiguous f32 dataset read at an eighth of h5py's speed: partial_read copied the bounding box out of the file, the extractor then walked it element by element (a recursive call and two bounds checks per element) into a second buffer, and read_f32_selection converted that into a third. Selections of contiguous data are now copied straight from the file, one memcpy per run of elements contiguous in the file (gather.rs: a block along the last dimension, touching blocks as one range, whole rows merged), with no zero-filled intermediate and no full copy for large selections. The typed selection readers copy into their Vec directly when the dataset stores T natively (new data_read::read_selection_native and sealed NativeElement trait, which the read_as_* fast paths now share; read_as_u64 gains one) and convert as before otherwise. The general extractor used by the chunked paths runs on the same run walker, keeping its old handling of unvalidated selections. Checked against h5py (contiguous_read_interop.rs) for strided, blocked, adjacent-block and whole-row hyperslabs, points and empty selections of every 1-8-byte type in both byte orders, ranks 1-4. Also keeps the huge-page threshold constant out of no_std builds, where it was unused. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 21 ++ crates/clawhdf5-format/src/bulk_alloc.rs | 1 + crates/clawhdf5-format/src/data_read.rs | 279 +++++++++-------- crates/clawhdf5-format/src/gather.rs | 343 +++++++++++++++++++++ crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/partial_read.rs | 61 ++-- crates/clawhdf5/src/reader.rs | 41 ++- docs/known-issues.md | 9 +- 8 files changed, 586 insertions(+), 170 deletions(-) create mode 100644 crates/clawhdf5-format/src/gather.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f86f2..bb70d01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,27 @@ `crates/clawhdf5/tests/contiguous_read_interop.rs` covers every 1-8-byte integer and float type in both byte orders, ranks 1-4, and datasets past the 4 MiB threshold. +- **Hyperslab and point reads of contiguous data copy runs, not elements.** + A 256 x 256 hyperslab of a contiguous `f32` dataset read at an eighth of + h5py's speed: the selection's bounding box was copied out of the file, + then walked element by element (a recursive call and two bounds checks per + element) into a second buffer, which `read_f32_selection` converted into + a third. Selections of contiguous data are now copied straight from the + file, one `memcpy` per run of elements that is contiguous in the file + (a block along the last dimension, blocks that touch, and whole rows when + the inner dimensions are selected in full, merged), with no zero-filled + intermediate; a selection covering most of the dataset no longer makes a + full copy first. The typed selection readers (`read_f32_selection`, + `read_f64_selection`, `read_i32_selection`, `read_i64_selection`) copy + directly into their output when the dataset stores that type natively, + and convert as before otherwise (big-endian, other widths). The chunked + paths use the same run-based extraction. New public + `clawhdf5_format::data_read::read_selection_native` and the sealed + `NativeElement` trait (also used by the `read_as_*` fast paths, which + gained one for native `u64`). Values are unchanged: checked against h5py + by `contiguous_read_interop.rs` (strided, blocked, adjacent-block and + whole-row hyperslabs, points, empty selections; every type, both byte + orders, ranks 1-4). ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files diff --git a/crates/clawhdf5-format/src/bulk_alloc.rs b/crates/clawhdf5-format/src/bulk_alloc.rs index 22a8bf2..085f125 100644 --- a/crates/clawhdf5-format/src/bulk_alloc.rs +++ b/crates/clawhdf5-format/src/bulk_alloc.rs @@ -19,6 +19,7 @@ use alloc::vec::Vec; /// Buffers smaller than this are left alone (numpy uses the same threshold). +#[cfg(any(target_os = "linux", test))] pub(crate) const HUGE_PAGE_THRESHOLD: usize = 4 << 20; /// Advise the kernel to back `[ptr, ptr + len)` with transparent huge pages, diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 1c53418..633ecd7 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -531,6 +531,11 @@ pub fn extract_selection_from_buffer( block, } => { let rank = dims.len(); + if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { + return Err(FormatError::SelectionOutOfBounds(format!( + "hyperslab rank does not match dataset rank {rank}" + ))); + } let output_elements = count .iter() .zip(block.iter()) @@ -540,96 +545,40 @@ pub fn extract_selection_from_buffer( crate::chunked_read::checked_byte_len(output_elements, elem_size)?, )?; - // Compute dataset strides (row-major) - let mut ds_strides = vec![1usize; rank]; - for i in (0..rank.saturating_sub(1)).rev() { - ds_strides[i] = ds_strides[i + 1] * dims[i + 1] as usize; - } - - // Compute output shape and strides - let output_dims: Vec = count - .iter() - .zip(block.iter()) - .map(|(&c, &b)| (c * b) as usize) - .collect(); - let mut out_strides = vec![1usize; rank]; - for i in (0..rank.saturating_sub(1)).rev() { - out_strides[i] = out_strides[i + 1] * output_dims[i + 1]; - } - - // Iterate over all selected elements - // For each block in the hyperslab, copy the elements - let mut out_linear = 0usize; - let _block_coords = vec![0u64; rank]; - - #[allow(clippy::too_many_arguments)] - fn iterate_hyperslab( - d: usize, - rank: usize, - start: &[u64], - stride: &[u64], - count: &[u64], - block: &[u64], - dims: &[u64], - ds_strides: &[usize], - elem_size: usize, - full_data: &[u8], - output: &mut [u8], - out_linear: &mut usize, - current_ds_offset: usize, - ) { - if d == rank { - // Copy one element - let src = current_ds_offset * elem_size; - let dst = *out_linear * elem_size; - if src + elem_size <= full_data.len() && dst + elem_size <= output.len() { - output[dst..dst + elem_size] - .copy_from_slice(&full_data[src..src + elem_size]); - } - *out_linear += 1; - return; - } - - for bi in 0..count[d] { - let block_start = start[d] + bi * stride[d]; - for bj in 0..block[d] { - let coord = block_start + bj; - if coord < dims[d] { - iterate_hyperslab( - d + 1, - rank, - start, - stride, - count, - block, - dims, - ds_strides, - elem_size, - full_data, - output, - out_linear, - current_ds_offset + coord as usize * ds_strides[d], - ); + // One copy per run of elements contiguous in `full_data` + // (`gather`'s runs). Coordinates past the extent are skipped and + // runs past the end of `full_data` left as zeros, element by + // element, as this extractor always did; validated selections + // never hit either. + let mut out_at = 0usize; + crate::gather::hyperslab_runs(dims, start, stride, count, block, |first, n| { + let big = |v: u64| usize::try_from(v).unwrap_or(usize::MAX); + let (first, n) = (big(first), big(n)); + let len = n.saturating_mul(elem_size); + let src = first.saturating_mul(elem_size); + let out_end = out_at.saturating_add(len); + if let (Some(from), Some(to)) = ( + full_data.get(src..src.saturating_add(len)), + output.get_mut(out_at..out_end), + ) { + to.copy_from_slice(from); + } else { + for k in 0..n { + let s = first.saturating_add(k).saturating_mul(elem_size); + let o = out_at.saturating_add(k.saturating_mul(elem_size)); + if o >= output.len() { + break; + } + if let (Some(from), Some(to)) = ( + full_data.get(s..s.saturating_add(elem_size)), + output.get_mut(o..o.saturating_add(elem_size)), + ) { + to.copy_from_slice(from); } } } - } - - iterate_hyperslab( - 0, - rank, - start, - stride, - count, - block, - dims, - &ds_strides, - elem_size, - full_data, - &mut output, - &mut out_linear, - 0, - ); + out_at = out_end; + }); Ok(output) } @@ -757,22 +706,76 @@ fn get_size(dt: &Datatype) -> usize { dt.type_size() as usize } -/// Reinterpret little-endian bytes as `count` native values of `T` on a -/// little-endian target, in one copy. +mod sealed { + pub trait Sealed {} +} + +/// A numeric type whose values can be copied straight out of a dataset's +/// bytes when the dataset stores exactly that type in the target's byte +/// order: `u8`, `i32`, `i64`, `u64`, `f32` and `f64`. +/// +/// # Safety +/// +/// Implementors have no padding and no invalid bit patterns, so a buffer of +/// them may be filled by copying bytes. The trait is sealed. +pub unsafe trait NativeElement: sealed::Sealed + Copy + 'static { + /// Whether `datatype`'s stored bytes are this type's native in-memory + /// representation (same size, byte order, signedness, full precision, + /// IEEE layout), so reading needs a copy and no conversion. + fn is_native(datatype: &Datatype) -> bool; +} + +/// A full-width fixed-point type of `size` bytes and the given signedness in +/// the target's byte order. +fn is_native_int(datatype: &Datatype, size: u32, want_signed: bool) -> bool { + let order = if cfg!(target_endian = "little") { + DatatypeByteOrder::LittleEndian + } else { + DatatypeByteOrder::BigEndian + }; + matches!( + datatype, + Datatype::FixedPoint { size: s, signed, byte_order, .. } + if *s == size && *signed == want_signed && (size == 1 || *byte_order == order) + ) && is_full_width(datatype) +} + +macro_rules! native_element { + ($($t:ty => |$dt:ident| $check:expr;)*) => {$( + impl sealed::Sealed for $t {} + // SAFETY: a primitive integer or float: no padding, and every bit + // pattern is a valid value. + unsafe impl NativeElement for $t { + fn is_native($dt: &Datatype) -> bool { + $check + } + } + )*}; +} + +native_element! { + u8 => |dt| is_native_int(dt, 1, false); + i32 => |dt| is_native_int(dt, 4, true); + i64 => |dt| is_native_int(dt, 8, true); + u64 => |dt| is_native_int(dt, 8, false); + f32 => |dt| cfg!(target_endian = "little") && is_native_le_float(dt, FloatFormat::Single); + f64 => |dt| cfg!(target_endian = "little") && is_native_le_float(dt, FloatFormat::Double); +} + +/// Copy `count` values of `T` out of `raw`, which holds them in `T`'s native +/// representation (see [`NativeElement::is_native`]), in one copy. /// /// The buffer is allocated uninitialised and filled by the copy. It used to be /// `vec![0; count]` first, which for a large dataset meant writing every page /// twice (zero it, then overwrite it) — about as expensive as the copy itself. -#[cfg(target_endian = "little")] -fn native_le_to_vec(raw: &[u8], count: usize) -> Vec { +fn native_to_vec(raw: &[u8], count: usize) -> Vec { let bytes = count * core::mem::size_of::(); - debug_assert!(bytes <= raw.len()); + assert!(bytes <= raw.len(), "native_to_vec: source too short"); let mut result: Vec = crate::bulk_alloc::vec_for_bulk(count); // SAFETY: `result` has capacity for `count` values of `T`, i.e. `bytes` - // bytes; `raw` holds at least `bytes` bytes (callers derive `count` from - // `raw.len() / size_of::()`); the regions cannot overlap because - // `result` was just allocated. Every `T` used here (f32/f64/i32/i64) is - // valid for any bit pattern, so after the copy all `count` values are + // bytes; `raw` holds at least `bytes` bytes (asserted); the regions + // cannot overlap because `result` was just allocated. `T: NativeElement` + // is valid for any bit pattern, so after the copy all `count` values are // initialised and `set_len` is sound. unsafe { core::ptr::copy_nonoverlapping(raw.as_ptr(), result.as_mut_ptr().cast::(), bytes); @@ -781,6 +784,44 @@ fn native_le_to_vec(raw: &[u8], count: usize) -> Vec { result } +/// Read `selection` of a dataset whose raw bytes (all of them, row-major, of +/// shape `dims`) are `raw` — typically a contiguous dataset's bytes borrowed +/// from the file — straight into a `Vec`, copying each contiguous run of +/// selected elements once. +/// +/// Returns `Ok(None)` when `datatype` is not `T`'s native representation +/// ([`NativeElement::is_native`]); the caller then converts through +/// [`read_raw_data_selection`] and the `read_as_*` functions. The selection is +/// validated like every selection read: out-of-range coordinates are +/// [`FormatError::SelectionOutOfBounds`]. +pub fn read_selection_native( + raw: &[u8], + dims: &[u64], + datatype: &Datatype, + selection: &crate::selection::Selection, +) -> Result>, FormatError> { + if !T::is_native(datatype) { + return Ok(None); + } + let elem_size = core::mem::size_of::(); + let total = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or_else(|| FormatError::Overflow("dataset shape overflows".into()))?; + let expected = crate::chunked_read::checked_byte_len(total, elem_size)?; + if raw.len() != expected { + return Err(FormatError::DataSizeMismatch { + expected, + actual: raw.len(), + }); + } + if let crate::selection::Selection::All = selection { + return Ok(Some(native_to_vec(raw, expected / elem_size))); + } + crate::partial_read::validate(selection, dims)?; + crate::gather::gather::(raw, dims, elem_size, selection).map(Some) +} + /// Convert raw bytes to `f64` values. pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatError> { // Array datatypes read as a flat sequence of their base elements, and @@ -799,9 +840,8 @@ pub fn read_as_f64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr let count = raw.len() / elem_size; // Fast path: native-endian f64 — single bulk memcpy - #[cfg(target_endian = "little")] - if is_native_le_float(datatype, FloatFormat::Double) { - return Ok(native_le_to_vec::(raw, count)); + if f64::is_native(datatype) { + return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); @@ -941,19 +981,8 @@ pub fn read_as_i64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr let count = raw.len() / elem_size; // Fast path: native LE i64 — single bulk memcpy - #[cfg(target_endian = "little")] - if elem_size == 8 - && is_full_width(datatype) - && matches!( - datatype, - Datatype::FixedPoint { - byte_order: DatatypeByteOrder::LittleEndian, - signed: true, - .. - } - ) - { - return Ok(native_le_to_vec::(raw, count)); + if i64::is_native(datatype) { + return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); @@ -986,6 +1015,12 @@ pub fn read_as_u64(raw: &[u8], datatype: &Datatype) -> Result, FormatEr }); } let count = raw.len() / elem_size; + + // Fast path: native u64 — single bulk memcpy + if u64::is_native(datatype) { + return Ok(native_to_vec::(raw, count)); + } + let order = get_byte_order(datatype); let mut result = crate::bulk_alloc::vec_for_bulk(count); for i in 0..count { @@ -1013,9 +1048,8 @@ pub fn read_as_f32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr let count = raw.len() / elem_size; // Fast path: native-endian f32 — single bulk memcpy - #[cfg(target_endian = "little")] - if is_native_le_float(datatype, FloatFormat::Single) { - return Ok(native_le_to_vec::(raw, count)); + if f32::is_native(datatype) { + return Ok(native_to_vec::(raw, count)); } // Little-endian IEEE half precision (numpy float16): widen directly. if is_native_le_float(datatype, FloatFormat::Half) { @@ -1103,19 +1137,8 @@ pub fn read_as_i32(raw: &[u8], datatype: &Datatype) -> Result, FormatEr let count = raw.len() / elem_size; // Fast path: native LE i32 — single bulk memcpy - #[cfg(target_endian = "little")] - if elem_size == 4 - && is_full_width(datatype) - && matches!( - datatype, - Datatype::FixedPoint { - byte_order: DatatypeByteOrder::LittleEndian, - signed: true, - .. - } - ) - { - return Ok(native_le_to_vec::(raw, count)); + if i32::is_native(datatype) { + return Ok(native_to_vec::(raw, count)); } let order = get_byte_order(datatype); diff --git a/crates/clawhdf5-format/src/gather.rs b/crates/clawhdf5-format/src/gather.rs new file mode 100644 index 0000000..22d2ae7 --- /dev/null +++ b/crates/clawhdf5-format/src/gather.rs @@ -0,0 +1,343 @@ +//! Copying a selection out of a row-major buffer one contiguous run at a time. +//! +//! A selection's elements, in output order, fall into runs that are adjacent +//! in the source: a whole block along the last dimension, blocks that touch +//! (`stride == block`), and whole rows when the inner dimensions are selected +//! in full. Copying run by run turns a 256 x 256 hyperslab of a 1024-wide +//! dataset into 256 `memcpy`s of 1 KiB, where the old extractor recursed and +//! bounds-checked once per element. + +#[cfg(not(feature = "std"))] +use alloc::{vec, vec::Vec}; + +use crate::data_read::NativeElement; +use crate::error::FormatError; +use crate::selection::Selection; + +/// Row-major element strides of `dims` (the last dimension has stride 1). +fn strides(dims: &[u64]) -> Vec { + let mut s = vec![1u64; dims.len()]; + for d in (0..dims.len().saturating_sub(1)).rev() { + s[d] = s[d + 1].wrapping_mul(dims[d + 1]); + } + s +} + +/// Merges adjacent runs before handing them on. +struct Coalesce { + start: u64, + len: u64, + emit: F, +} + +impl Coalesce { + #[inline] + fn push(&mut self, start: u64, len: u64) { + if len == 0 { + return; + } + if self.len > 0 && self.start.wrapping_add(self.len) == start { + self.len += len; + return; + } + self.flush(); + self.start = start; + self.len = len; + } + + fn flush(&mut self) { + if self.len > 0 { + (self.emit)(self.start, self.len); + self.len = 0; + } + } +} + +/// Call `emit(first_element, element_count)` for each run of a hyperslab's +/// elements that is contiguous in a row-major dataset of shape `dims`, in +/// the order the selection returns them. Adjacent runs are merged. +/// +/// Coordinates at or past a dimension's extent are skipped, as the +/// element-wise extractor always did; callers that want them to be an error +/// validate the selection first. The four vectors must have `dims.len()` +/// entries. +pub(crate) fn hyperslab_runs( + dims: &[u64], + start: &[u64], + stride: &[u64], + count: &[u64], + block: &[u64], + emit: impl FnMut(u64, u64), +) { + let rank = dims.len(); + let mut out = Coalesce { + start: 0, + len: 0, + emit, + }; + if rank == 0 { + out.push(0, 1); + out.flush(); + return; + } + if (0..rank).any(|d| count[d] == 0 || block[d] == 0) { + return; + } + let strides = strides(dims); + let last = rank - 1; + // Odometer over the outer dimensions: (block index, offset in block). + let mut ci = vec![0u64; last]; + let mut bi = vec![0u64; last]; + 'outer: loop { + // Base offset of this row, or skip it if a coordinate is out of range. + let mut base = 0u64; + let mut in_range = true; + for d in 0..last { + let coord = start[d] + .saturating_add(ci[d].saturating_mul(stride[d])) + .saturating_add(bi[d]); + if coord >= dims[d] { + in_range = false; + break; + } + base = base.wrapping_add(coord.wrapping_mul(strides[d])); + } + if in_range && (stride[last] == block[last] || count[last] == 1) { + // Blocks that touch (the common unit-stride case: block 1, + // stride 1) are one range; don't split it into per-element runs. + let s = start[last]; + let e = s + .saturating_add(count[last].saturating_mul(block[last])) + .min(dims[last]); + if s < e { + out.push(base.wrapping_add(s), e - s); + } + } else if in_range { + for c in 0..count[last] { + let s = start[last].saturating_add(c.saturating_mul(stride[last])); + if s >= dims[last] { + continue; + } + let e = s.saturating_add(block[last]).min(dims[last]); + out.push(base.wrapping_add(s), e - s); + } + } + // Advance the odometer, last outer dimension fastest. + let mut d = last; + loop { + if d == 0 { + break 'outer; + } + d -= 1; + bi[d] += 1; + if bi[d] < block[d] { + break; + } + bi[d] = 0; + ci[d] += 1; + if ci[d] < count[d] { + break; + } + ci[d] = 0; + } + } + out.flush(); +} + +/// The selected elements of `src` — a row-major dataset of shape `dims` and +/// `elem_size`-byte elements — copied into a fresh `Vec`, one `memcpy` per +/// contiguous run, with no zero-filling of the output first. +/// +/// For `T` other than `u8`, `elem_size` must equal `size_of::()`. The +/// selection must be a validated hyperslab, point list or `None` (`All` is the +/// caller's to handle); `src` must hold exactly the dataset. Anything that +/// would read outside `src` is an error, never a partial result. +pub(crate) fn gather( + src: &[u8], + dims: &[u64], + elem_size: usize, + selection: &Selection, +) -> Result, FormatError> { + let t_size = core::mem::size_of::(); + if elem_size == 0 || (t_size != 1 && t_size != elem_size) { + return Err(FormatError::DataSizeMismatch { + expected: t_size, + actual: elem_size, + }); + } + let n_elements = match selection { + Selection::None => 0, + Selection::Hyperslab { count, block, .. } => count + .iter() + .zip(block) + .try_fold(1u64, |acc, (&c, &b)| acc.checked_mul(c.checked_mul(b)?)) + .ok_or_else(|| FormatError::Overflow("hyperslab count x block overflows".into()))?, + Selection::Points(points) => points.len() as u64, + Selection::All => { + return Err(FormatError::SelectionOutOfBounds( + "gather does not take Selection::All".into(), + )); + } + }; + let out_bytes = crate::chunked_read::checked_byte_len(n_elements, elem_size)?; + let out_len = out_bytes / t_size; + let mut out: Vec = crate::bulk_alloc::vec_for_bulk(out_len); + let dst = out.as_mut_ptr().cast::(); + let mut written = 0usize; + let mut failed = false; + let mut copy_run = |first: u64, n: u64| { + if failed { + return; + } + let range = usize::try_from(first) + .ok() + .and_then(|f| f.checked_mul(elem_size)) + .zip( + usize::try_from(n) + .ok() + .and_then(|n| n.checked_mul(elem_size)), + ) + .and_then(|(at, len)| Some((at, len, at.checked_add(len)?))); + match range { + Some((at, len, end)) if end <= src.len() && written + len <= out_bytes => { + // SAFETY: `src[at..end]` is in bounds (checked above), and + // `dst + written .. + len` lies within `out`'s capacity of + // `out_bytes` bytes (checked above); `out` is a fresh + // allocation, so the regions do not overlap. + unsafe { + core::ptr::copy_nonoverlapping(src.as_ptr().add(at), dst.add(written), len) + }; + written += len; + } + _ => failed = true, + } + }; + let mut bad_point = false; + match selection { + Selection::Hyperslab { + start, + stride, + count, + block, + } => { + let rank = dims.len(); + if [start.len(), stride.len(), count.len(), block.len()] != [rank; 4] { + return Err(FormatError::SelectionOutOfBounds( + "hyperslab rank does not match dataset rank".into(), + )); + } + hyperslab_runs(dims, start, stride, count, block, &mut copy_run); + } + Selection::Points(points) => { + let strides = strides(dims); + let mut runs = Coalesce { + start: 0, + len: 0, + emit: &mut copy_run, + }; + for p in points { + if p.len() != dims.len() || p.iter().zip(dims).any(|(c, n)| c >= n) { + bad_point = true; + break; + } + let at = p + .iter() + .zip(&strides) + .fold(0u64, |acc, (c, s)| acc.wrapping_add(c.wrapping_mul(*s))); + runs.push(at, 1); + } + runs.flush(); + } + Selection::None | Selection::All => {} + } + if failed || bad_point || written != out_bytes { + return Err(FormatError::SelectionOutOfBounds( + "selection addresses elements outside the dataset".into(), + )); + } + // SAFETY: all `out_bytes` bytes, i.e. `out_len` values of `T`, were + // written above, and every bit pattern is a valid `T` (`NativeElement`). + unsafe { out.set_len(out_len) }; + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn runs(dims: &[u64], sel: [&[u64]; 4]) -> Vec<(u64, u64)> { + let mut v = Vec::new(); + hyperslab_runs(dims, sel[0], sel[1], sel[2], sel[3], |s, n| v.push((s, n))); + v + } + + #[test] + fn runs_merge_blocks_and_whole_rows() { + // A box: one run per row. + assert_eq!( + runs(&[4, 10], [&[1, 2], &[1, 1], &[2, 3], &[1, 1]]), + vec![(12, 3), (22, 3)] + ); + // Whole rows: one run. + assert_eq!( + runs(&[4, 10], [&[1, 0], &[1, 1], &[3, 10], &[1, 1]]), + vec![(10, 30)] + ); + // stride == block: blocks merge. + assert_eq!( + runs(&[1, 10], [&[0, 1], &[1, 2], &[1, 4], &[1, 2]]), + vec![(1, 8)] + ); + // Strided with blocks along both dimensions. + assert_eq!( + runs(&[6, 10], [&[0, 1], &[3, 4], &[2, 2], &[2, 2]]), + vec![ + (1, 2), + (5, 2), + (11, 2), + (15, 2), + (31, 2), + (35, 2), + (41, 2), + (45, 2) + ] + ); + // Empty. + assert!(runs(&[4, 10], [&[0, 0], &[1, 1], &[0, 3], &[1, 1]]).is_empty()); + // Scalar. + assert_eq!(runs(&[], [&[], &[], &[], &[]]), vec![(0, 1)]); + } + + #[test] + fn gather_matches_element_order_and_rejects_out_of_range() { + let dims = [3u64, 4]; + let src: Vec = (0..12u16).flat_map(|v| v.to_le_bytes()).collect(); + let sel = Selection::Hyperslab { + start: vec![0, 1], + stride: vec![2, 2], + count: vec![2, 2], + block: vec![1, 1], + }; + let got: Vec = gather(&src, &dims, 2, &sel).unwrap(); + let want: Vec = [1u16, 3, 9, 11] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(); + assert_eq!(got, want); + let pts = Selection::Points(vec![vec![2, 3], vec![0, 0], vec![0, 1]]); + let got: Vec = gather(&src, &dims, 2, &pts).unwrap(); + let want: Vec = [11u16, 0, 1].iter().flat_map(|v| v.to_le_bytes()).collect(); + assert_eq!(got, want); + // Past the extent, or a source shorter than the dataset: an error. + let bad = Selection::Points(vec![vec![3, 0]]); + assert!(gather::(&src, &dims, 2, &bad).is_err()); + let past = Selection::Hyperslab { + start: vec![2, 0], + stride: vec![1, 1], + count: vec![2, 4], + block: vec![1, 1], + }; + assert!(gather::(&src, &dims, 2, &past).is_err()); + assert!(gather::(&src[..20], &dims, 2, &pts).is_err()); + } +} diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 05ebd46..23840b8 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -94,6 +94,7 @@ mod filters_szip; pub mod fixed_array; pub mod float16; pub mod fractal_heap; +mod gather; pub mod global_heap; pub mod group_info; pub mod group_v1; diff --git a/crates/clawhdf5-format/src/partial_read.rs b/crates/clawhdf5-format/src/partial_read.rs index 7d73599..9865c46 100644 --- a/crates/clawhdf5-format/src/partial_read.rs +++ b/crates/clawhdf5-format/src/partial_read.rs @@ -3,11 +3,13 @@ //! //! [`crate::data_read::read_raw_data_selection`] used to decode the *entire* //! dataset and then pick elements out of it, so reading a 64x64 window of a -//! large dataset took about as long as reading all of it. Here the selection's -//! bounding box is materialised instead — only the rows of a contiguous -//! dataset, or only the chunks, that overlap it — and the existing extractor -//! runs over that small buffer with the selection translated to the box's -//! origin. Extraction semantics are therefore exactly the full-read ones. +//! large dataset took about as long as reading all of it. A contiguous +//! dataset's selection is now copied straight out of the file, one `memcpy` +//! per contiguous run of selected elements (`crate::gather`). For chunked +//! data the selection's bounding box is materialised — only the chunks that +//! overlap it — and the extractor runs over that small buffer with the +//! selection translated to the box's origin. Extraction semantics are +//! therefore exactly the full-read ones. #[cfg(not(feature = "std"))] use alloc::string as alloc_or_std; @@ -250,10 +252,33 @@ pub fn read_selection( if dims.is_empty() || elem_size == 0 { return Ok(None); } + let total = dataspace.checked_num_elements()?; + // Contiguous data is addressable in place: copy the selection's runs + // straight out of it, whatever fraction of the dataset it covers, with no + // intermediate box (and no full copy for a large selection). + if let ( + DataLayout::Contiguous { + address: Some(address), + .. + }, + Selection::Hyperslab { .. } | Selection::Points(_), + ) = (layout, selection) + { + validate(selection, dims)?; + let base = usize::try_from(*address) + .map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?; + let data = file_data + .get(base..) + .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?)) + .ok_or(FormatError::UnexpectedEof { + expected: base, + available: file_data.len(), + })?; + return crate::gather::gather::(data, dims, elem_size, selection).map(Some); + } let Some((box_start, box_extent)) = bounding_box(selection, dims) else { return Ok(None); }; - let total = dataspace.checked_num_elements()?; let box_elements = box_extent .iter() .try_fold(1u64, |acc, &e| acc.checked_mul(e)) @@ -265,30 +290,6 @@ pub fn read_selection( let mut boxed = alloc_output(checked_byte_len(box_elements, elem_size)?)?; match layout { - DataLayout::Contiguous { - address: Some(address), - .. - } => { - let base = usize::try_from(*address) - .map_err(|_| FormatError::Overflow("data address exceeds usize".into()))?; - let data = file_data - .get(base..) - .and_then(|d| d.get(..checked_byte_len(total, elem_size).ok()?)) - .ok_or(FormatError::UnexpectedEof { - expected: base, - available: file_data.len(), - })?; - let origin = vec![0u64; dims.len()]; - copy_overlap( - data, - &origin, - dims, - &mut boxed, - &box_start, - &box_extent, - elem_size, - ); - } DataLayout::Chunked { btree_address: Some(_), .. diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 83cbfa8..74c268d 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -569,9 +569,7 @@ impl<'f> Dataset<'f> { &self, selection: &clawhdf5_format::selection::Selection, ) -> Result, Error> { - let raw = self.read_selection(selection)?; - let dt = self.datatype()?; - Ok(data_read::read_as_f64(&raw, &dt)?) + self.read_typed_selection(selection, data_read::read_as_f64, || self.read_f64()) } /// Read selected elements as `f32` values. @@ -579,9 +577,7 @@ impl<'f> Dataset<'f> { &self, selection: &clawhdf5_format::selection::Selection, ) -> Result, Error> { - let raw = self.read_selection(selection)?; - let dt = self.datatype()?; - Ok(data_read::read_as_f32(&raw, &dt)?) + self.read_typed_selection(selection, data_read::read_as_f32, || self.read_f32()) } /// Read selected elements as `i32` values. @@ -589,9 +585,7 @@ impl<'f> Dataset<'f> { &self, selection: &clawhdf5_format::selection::Selection, ) -> Result, Error> { - let raw = self.read_selection(selection)?; - let dt = self.datatype()?; - Ok(data_read::read_as_i32(&raw, &dt)?) + self.read_typed_selection(selection, data_read::read_as_i32, || self.read_i32()) } /// Read selected elements as `i64` values. @@ -599,9 +593,34 @@ impl<'f> Dataset<'f> { &self, selection: &clawhdf5_format::selection::Selection, ) -> Result, Error> { - let raw = self.read_selection(selection)?; + self.read_typed_selection(selection, data_read::read_as_i64, || self.read_i64()) + } + + /// The typed selection readers. `All` is a full read. A contiguous dataset + /// that stores `T` natively is copied from the file straight into the + /// `Vec`, one copy per contiguous run of selected elements; anything + /// else reads the selection's bytes and converts them with `convert`. + fn read_typed_selection( + &self, + selection: &clawhdf5_format::selection::Selection, + convert: fn(&[u8], &Datatype) -> Result, FormatError>, + full: impl FnOnce() -> Result, Error>, + ) -> Result, Error> { + if matches!(selection, clawhdf5_format::selection::Selection::All) { + return full(); + } let dt = self.datatype()?; - Ok(data_read::read_as_i64(&raw, &dt)?) + if T::is_native(&dt) + && let Ok(Some(raw)) = self.read_raw_ref() + { + let dims = self.dataspace()?.dimensions; + if let Some(values) = data_read::read_selection_native::(raw, &dims, &dt, selection)? + { + return Ok(values); + } + } + let raw = self.read_selection(selection)?; + Ok(convert(&raw, &dt)?) } /// Zero-copy read of contiguous raw data. diff --git a/docs/known-issues.md b/docs/known-issues.md index bf8f77a..60fc9d8 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -9,7 +9,8 @@ deleting it. ## Concurrent and contiguous read performance (measured 2026-09-26) -**Status:** open. Measured on tank with `concurrent_read` against h5py +**Status:** open for chunked full reads; the contiguous item is fixed +(2026-09-26). Measured on tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, "Concurrent reads"): - Full reads of chunked datasets from several threads through one `File` stop scaling at about 4 threads (880 MB/s on deflate data vs 4424 MB/s @@ -17,6 +18,12 @@ deleting it. scale to 1244 MB/s, so the `File`'s shared chunk cache is the suspect. - Contiguous datasets read 4x slower than h5py on one thread (2.5 vs 9.8 GB/s full, 0.12x for 256 x 256 hyperslabs). + **Fixed 2026-09-26** (not yet re-measured for `BENCHMARKS.md`): full + reads were dominated by 4 KiB page faults on the fresh output buffer, + which is now backed by transparent huge pages as numpy's is; hyperslab + reads copied the selection three times, element by element, and now copy + each contiguous run once, straight from the file into the output (see + `CHANGELOG.md`). The chunked-read scaling item above is still open. Values are correct; this is speed only. ## Silent wrong data found by the 2026-09-25 HDF5 audit -- 2.54.0 From c3850a0b66748dbbf1291820682261302818c2b3 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:21:48 -0500 Subject: [PATCH 09/36] =?UTF-8?q?docs:=20the=20Python=20package=20?= =?UTF-8?q?=E2=80=94=20install=20with=20maturin,=20h5py-style=20reading?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README gains a Python section (maturin develop into a venv, a reading example that was run against an h5py-written file, the supported types and keys, what writing covers). The crate README says the same in more detail. QUICKSTART showed clawhdf5.open()/read_f64(), which never existed; it now shows File(...)[...]. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 42 +++++++++++++++++++++++ crates/clawhdf5-py/README.md | 64 +++++++++++++++++++++++++++++++----- docs/QUICKSTART.md | 15 +++++---- 3 files changed, 107 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b9e3aa1..c266f3b 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,48 @@ let values = ds.read_f64()?; assert_eq!(values, vec![22.5, 23.1, 21.8]); ``` +### Python + +`crates/clawhdf5-py` is a Python package (PyO3 + numpy) that reads HDF5 with +an h5py-shaped API and no libhdf5. It is not on PyPI; build it with +[maturin](https://www.maturin.rs) into a virtualenv: + +```bash +python -m venv .venv && . .venv/bin/activate +pip install maturin numpy +maturin develop --release -m crates/clawhdf5-py/Cargo.toml +python -c "import clawhdf5; print(clawhdf5.__version__)" +``` + +```python +import numpy as np +import clawhdf5 + +with clawhdf5.File("data.h5", "r") as f: + print(list(f.keys())) # sorted member names, like h5py + ds = f["group/temperatures"] # relative or absolute ("/group/...") paths + print(ds.shape, ds.dtype) # dtype is the numpy dtype h5py reports + block = ds[100:200, ::4] # reads only the selected elements + row = ds[-1] # integers drop the axis + picked = ds[[1, 5, 9], :] # one increasing index list per key + units = ds.attrs["units"] # attributes come back as h5py returns them + everything = np.asarray(ds) + + records = f["table"] # compound -> numpy structured array + ids = records["id"] # one field +``` + +Reads cover integers and IEEE floats of every width in either byte order, +`bool`, enums, complex, fixed and variable-length strings, variable-length +sequences, opaque, HDF5 array types and compounds; other types (references, +bitfields, ...) raise `TypeError` instead of returning guessed data. Keys +follow h5py (negative steps, `None` and boolean masks are refused). The +read itself runs with the GIL released, so Python threads read in parallel. +Writing (`File(path, "w")`, `create_dataset`, `create_group`, `attrs[...] =`) +covers `float64`, `float32`, `int64`, `int32` and `uint8` arrays. The tests +in `crates/clawhdf5-py/tests` compare every read with h5py; run them with +`pip install pytest h5py && pytest crates/clawhdf5-py/tests`. + ### Agent Memory ```rust diff --git a/crates/clawhdf5-py/README.md b/crates/clawhdf5-py/README.md index 6554f32..9cff9ab 100644 --- a/crates/clawhdf5-py/README.md +++ b/crates/clawhdf5-py/README.md @@ -3,23 +3,71 @@ [![crates.io](https://img.shields.io/crates/v/clawhdf5-py.svg)](https://crates.io/crates/clawhdf5-py) [![docs.rs](https://docs.rs/clawhdf5-py/badge.svg)](https://docs.rs/clawhdf5-py) -Python bindings for clawhdf5 — a pure-Rust HDF5 library. +Python bindings for clawhdf5 — a pure-Rust HDF5 library. The package is +`clawhdf5` (`import clawhdf5`); it needs numpy and no libhdf5. -## Features +## Install -- h5py-compatible API (`File`, `Group`, `Dataset`) -- NumPy array integration -- Read and write HDF5 files from Python with no C dependencies +Not on PyPI yet. Build it into a virtualenv with [maturin](https://www.maturin.rs): -## Usage +```bash +pip install maturin numpy +cd crates/clawhdf5-py +maturin develop --release +python -c "import clawhdf5; print(clawhdf5.__version__)" +``` + +## Reading + +The read API follows h5py: ```python +import numpy as np import clawhdf5 -with clawhdf5.File('data.h5', 'r') as f: - data = f['/dataset'][:] +with clawhdf5.File("data.h5", "r") as f: + f.keys(), f["group"].items(), "group/data" in f + ds = f["group/data"] # or f["/group/data"], f["group"]["data"] + ds.shape, ds.dtype, ds.attrs["units"] + ds[10:20, ::2] # only the selected elements are read + ds[-1], ds[..., 0], ds[[1, 4, 7]] + np.asarray(ds) + f["table"]["id"] # a compound field ``` +- `Dataset.dtype` is the numpy dtype h5py reports: integers and IEEE floats + of every width in either byte order, `bool`, enums (with + `dtype.metadata['enum']`), complex, `S` fixed strings, `object` for + variable-length strings (`bytes` values) and sequences (array values), + `V` opaque, array types, and compounds as structured dtypes. + Other types raise `TypeError`. +- Keys are h5py's: integers, slices with a positive step, `...`, one + increasing list of integers, compound field names. Each maps onto a + hyperslab selection. `None`, negative steps and boolean masks are refused + with h5py's errors. +- The bytes the library reads become the numpy array's buffer without a + copy, and the read runs with the GIL released, so threads read in + parallel. +- Attributes return what h5py returns; `clawhdf5.Empty` stands for a null + dataspace (h5py's `Empty`). + +## Writing + +`clawhdf5.File(path, "w")` with `create_dataset(name, data=array, +chunks=..., compression="gzip")`, `create_group` and `attrs[...] = ...` +writes `float64`, `float32`, `int64`, `int32` and `uint8` arrays; the file is +written on `close()`. + +## Tests + +```bash +pip install pytest h5py +pytest crates/clawhdf5-py/tests +``` + +`tests/test_read_vs_h5py.py` compares every read with h5py on a file h5py +writes. `scripts/ci-test.sh` builds the wheel and runs these in CI. + ## License MIT diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 2ede177..34b5325 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -395,19 +395,22 @@ clawhdf5 --path agent.h5 snapshot backup_2026-03-19.h5 Read HDF5 files from Python without libhdf5: ```bash -pip install clawhdf5 # coming soon — build from source for now -cd crates/clawhdf5-py && maturin develop +# Not on PyPI yet: build from source into a virtualenv +pip install maturin numpy +cd crates/clawhdf5-py && maturin develop --release ``` ```python import clawhdf5 -# Read -f = clawhdf5.open("data.h5") -temps = f.read_f64("temperatures") -print(temps) # [22.5, 23.1, 21.8] +# Read (h5py-style) +with clawhdf5.File("data.h5", "r") as f: + temps = f["temperatures"][:] + print(temps) # [22.5 23.1 21.8] ``` +See `crates/clawhdf5-py/README.md` for the supported types and indexing. + --- ## Common Patterns -- 2.54.0 From 8ce6eca34da93fa28407fad7f00595d60ba03d31 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:25:05 -0500 Subject: [PATCH 10/36] feat(facade): read VL strings and VL sequences through File VL-string datasets (h5py's default str dtype) failed read_string with "type mismatch: expected String, got VariableLength". read_string now reads fixed- and variable-length strings, with h5py's values (a string ends at a NUL, a null element is ""). New: - Dataset::read_string_bytes: each VL string's exact bytes; - Dataset::read_string_selection: hyperslabs/points of either kind; - Dataset::read_vlen::() and read_vlen_selection::(): VL sequences of numbers as Vec>, T in f64/f32/i64/i32/u64, converted like the other typed readers; - File::decode_strings / decode_string_bytes / decode_vlen: VL values in compound fields and AttrValue::Raw attributes; - MmapDataset and LazyDataset: read_string for VL strings, read_string_bytes and read_vlen. tests/vl_data_interop.rs checks every path against h5py with 8- and 4-byte offsets: scalar, 1-D and 2-D, ASCII and UTF-8, empty strings, contiguous, compact, chunked with gzip and shuffle, unwritten and partly written chunks, hyperslabs, compound members, attributes, a big-endian base type, and a patched file with an embedded NUL and mis-sized heap objects. NetCDF-4 string variables read too (netCDF4-python test). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 18 + .../clawhdf5-netcdf4/tests/interop_tests.rs | 27 ++ crates/clawhdf5/src/lazy.rs | 40 +- crates/clawhdf5/src/lib.rs | 2 + crates/clawhdf5/src/mmap_file.rs | 40 +- crates/clawhdf5/src/reader.rs | 101 +++- crates/clawhdf5/src/vlen.rs | 143 ++++++ crates/clawhdf5/tests/vl_data_interop.rs | 444 ++++++++++++++++++ docs/known-issues.md | 11 +- 9 files changed, 818 insertions(+), 8 deletions(-) create mode 100644 crates/clawhdf5/src/vlen.rs create mode 100644 crates/clawhdf5/tests/vl_data_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 40781cf..0a16e0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,24 @@ a VL datatype whose stored size is not 4 + offset size + 4 (libhdf5 ignores the stored size). The conformance probe resolves VL elements with `VlResolver` too; conformance unchanged at 575 of 697. +- **VL data through the facade.** VL-string datasets (h5py's default `str` + dtype) failed `read_string` with "type mismatch: expected String, got + VariableLength". `Dataset::read_string` now reads fixed- and + variable-length strings; new `read_string_bytes` (a VL string's exact + bytes, as h5py's `Dataset[()]` returns them), `read_string_selection`, + `read_vlen::()` / `read_vlen_selection::()` for VL sequences of + numbers (`T` = `f64`, `f32`, `i64`, `i32`, `u64`; converted like the + other typed readers), and `File::decode_strings` / `decode_string_bytes` + / `decode_vlen` for VL values in compound fields and `AttrValue::Raw` + attributes. `MmapDataset` and `LazyDataset` gain `read_string` for VL + strings, `read_string_bytes` and `read_vlen`. Checked against h5py with + 8- and 4-byte offsets: scalar and 1-/2-D, ASCII and UTF-8, empty strings, + contiguous, compact, chunked with gzip/shuffle, never-written and + partly written chunks, hyperslab selections, VL members of compound + datasets and attributes (`crates/clawhdf5/tests/vl_data_interop.rs`). + NetCDF-4 `string` variables now read through + `clawhdf5_netcdf4::Variable::read_string` (checked against netCDF4-python + in `crates/clawhdf5-netcdf4/tests/interop_tests.rs`). ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files diff --git a/crates/clawhdf5-netcdf4/tests/interop_tests.rs b/crates/clawhdf5-netcdf4/tests/interop_tests.rs index af75136..eba85ac 100644 --- a/crates/clawhdf5-netcdf4/tests/interop_tests.rs +++ b/crates/clawhdf5-netcdf4/tests/interop_tests.rs @@ -350,3 +350,30 @@ ds.close() let press_vals = press_var.read_raw_f32().unwrap(); assert_eq!(press_vals, vec![1000.0f32, 850.0, 500.0, 200.0]); } + +#[test] +fn netcdf4_python_string_variable_clawhdf5_reads() { + // NC_STRING variables are HDF5 variable-length strings, which + // `read_string` refused ("expected String, got VariableLength") until + // 2026-09-26. + skip_if_no_netcdf4!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("strings.nc"); + let path_str = path.display().to_string(); + let script = format!( + r#" +import netCDF4 as nc +import numpy as np +ds = nc.Dataset("{path_str}", "w", format="NETCDF4") +ds.createDimension("station", 4) +v = ds.createVariable("name", str, ("station",)) +v[:] = np.array(["Oslo", "", "São Paulo", "x"], dtype=object) +ds.close() +"# + ); + run_python(&script); + + let file = NetCDF4File::open(&path).unwrap(); + let names = file.variable("name").unwrap().read_string().unwrap(); + assert_eq!(names, vec!["Oslo", "", "São Paulo", "x"]); +} diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index d893485..a33b9bb 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -422,11 +422,47 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { Ok(data_read::read_as_u64(&raw, &dt)?) } - /// Read all data as `String` values. + /// Read all data as `String` values: fixed- or variable-length strings + /// (see [`Dataset::read_string`](crate::Dataset::read_string)). pub fn read_string(&self) -> Result, Error> { let raw = self.read_raw()?; let dt = self.datatype()?; - Ok(data_read::read_as_strings(&raw, &dt)?) + crate::vlen::decode_strings( + self.file.hdf5_bytes(), + &dt, + &raw, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// Read a variable-length string dataset as the exact bytes of each + /// string (see + /// [`Dataset::read_string_bytes`](crate::Dataset::read_string_bytes)). + pub fn read_string_bytes(&self) -> Result>, Error> { + let raw = self.read_raw()?; + let dt = self.datatype()?; + crate::vlen::decode_string_bytes( + self.file.hdf5_bytes(), + &dt, + &raw, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// Read a variable-length sequence dataset as one `Vec` per element + /// (see [`Dataset::read_vlen`](crate::Dataset::read_vlen)). + pub fn read_vlen(&self) -> Result>, Error> { + let raw = self.read_raw()?; + let dt = self.datatype()?; + crate::vlen::decode_vlen( + self.file.hdf5_bytes(), + &dt, + &raw, + self.file.offset_size(), + self.file.length_size(), + ) } /// Read all attributes of this dataset. diff --git a/crates/clawhdf5/src/lib.rs b/crates/clawhdf5/src/lib.rs index 12e2f69..f8098fa 100644 --- a/crates/clawhdf5/src/lib.rs +++ b/crates/clawhdf5/src/lib.rs @@ -30,6 +30,7 @@ pub mod lazy; pub mod mmap_file; pub mod reader; pub mod types; +pub mod vlen; pub mod writer; pub use error::Error; @@ -38,6 +39,7 @@ pub use lazy::{LazyDataset, LazyFile, LazyGroup}; pub use mmap_file::{MmapDataset, MmapFile, MmapGroup}; pub use reader::{Dataset, File, Group}; pub use types::{AttrValue, DType}; +pub use vlen::VlenValue; pub use writer::FileBuilder; #[cfg(feature = "parallel")] pub use writer::{DatasetSpec, create_datasets_parallel}; diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 7f544ca..76d9ea1 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -336,11 +336,47 @@ impl<'f> MmapDataset<'f> { Ok(data_read::read_as_u64(&raw, &dt)?) } - /// Read all data as `String` values. + /// Read all data as `String` values: fixed- or variable-length strings + /// (see [`Dataset::read_string`](crate::Dataset::read_string)). pub fn read_string(&self) -> Result, Error> { let raw = self.read_raw()?; let dt = self.datatype()?; - Ok(data_read::read_as_strings(&raw, &dt)?) + crate::vlen::decode_strings( + self.file.hdf5_bytes(), + &dt, + &raw, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// Read a variable-length string dataset as the exact bytes of each + /// string (see + /// [`Dataset::read_string_bytes`](crate::Dataset::read_string_bytes)). + pub fn read_string_bytes(&self) -> Result>, Error> { + let raw = self.read_raw()?; + let dt = self.datatype()?; + crate::vlen::decode_string_bytes( + self.file.hdf5_bytes(), + &dt, + &raw, + self.file.offset_size(), + self.file.length_size(), + ) + } + + /// Read a variable-length sequence dataset as one `Vec` per element + /// (see [`Dataset::read_vlen`](crate::Dataset::read_vlen)). + pub fn read_vlen(&self) -> Result>, Error> { + let raw = self.read_raw()?; + let dt = self.datatype()?; + crate::vlen::decode_vlen( + self.file.hdf5_bytes(), + &dt, + &raw, + self.file.offset_size(), + self.file.length_size(), + ) } /// For contiguous datasets, return a zero-copy slice into the mmap. diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 83cbfa8..f0afa6d 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -262,6 +262,56 @@ impl File { } } + /// Decode the strings in `raw`, a buffer of elements of `datatype` read + /// from this file — for instance a variable-length string field of a + /// compound ([`clawhdf5_format::data_read::read_compound_fields`]) or an + /// [`AttrValue::Raw`] attribute. Variable-length strings are resolved in + /// this file's global heap; see [`Dataset::read_string`] for the values. + pub fn decode_strings(&self, datatype: &Datatype, raw: &[u8]) -> Result, Error> { + crate::vlen::decode_strings( + self.as_bytes(), + datatype, + raw, + self.offset_size(), + self.length_size(), + ) + } + + /// Like [`decode_strings`](Self::decode_strings) for variable-length + /// strings, returning each string's exact bytes (see + /// [`Dataset::read_string_bytes`]). + pub fn decode_string_bytes( + &self, + datatype: &Datatype, + raw: &[u8], + ) -> Result>, Error> { + crate::vlen::decode_string_bytes( + self.as_bytes(), + datatype, + raw, + self.offset_size(), + self.length_size(), + ) + } + + /// Decode the variable-length sequences in `raw`, a buffer of elements + /// of the sequence type `datatype` read from this file (a compound + /// field, an [`AttrValue::Raw`] attribute, ...). See + /// [`Dataset::read_vlen`]. + pub fn decode_vlen( + &self, + datatype: &Datatype, + raw: &[u8], + ) -> Result>, Error> { + crate::vlen::decode_vlen( + self.as_bytes(), + datatype, + raw, + self.offset_size(), + self.length_size(), + ) + } + fn parse_header(&self, address: u64) -> Result { ObjectHeader::parse( self.data.as_bytes(), @@ -498,11 +548,58 @@ impl<'f> Dataset<'f> { Ok(data_read::read_as_u64(&raw, &dt)?) } - /// Read all data as `String` values. + /// Read all data as `String` values, in row-major order. + /// + /// Works for fixed-length and variable-length string datasets (h5py's + /// default `str` dtype). A variable-length string ends at its first NUL + /// and a null element (e.g. never written) is `""`, as h5py returns + /// them; bytes that are not valid UTF-8 are replaced with U+FFFD — use + /// [`read_string_bytes`](Self::read_string_bytes) for the exact bytes. pub fn read_string(&self) -> Result, Error> { let raw = self.read_raw()?; let dt = self.datatype()?; - Ok(data_read::read_as_strings(&raw, &dt)?) + self.file.decode_strings(&dt, &raw) + } + + /// Read a variable-length string dataset as the exact bytes of each + /// string (what h5py's `Dataset[()]` returns), in row-major order. + pub fn read_string_bytes(&self) -> Result>, Error> { + let raw = self.read_raw()?; + let dt = self.datatype()?; + self.file.decode_string_bytes(&dt, &raw) + } + + /// Read the selected elements of a fixed- or variable-length string + /// dataset (see [`read_string`](Self::read_string)). + pub fn read_string_selection( + &self, + selection: &clawhdf5_format::selection::Selection, + ) -> Result, Error> { + let raw = self.read_selection(selection)?; + let dt = self.datatype()?; + self.file.decode_strings(&dt, &raw) + } + + /// Read a variable-length sequence dataset (h5py + /// `vlen_dtype(np.int32)`, ...) as one `Vec` per element, in + /// row-major order. The base type must be an integer or float type; it + /// is converted to `T` as [`read_f64`](Self::read_f64) and the other + /// typed readers convert. A null element is an empty sequence. + pub fn read_vlen(&self) -> Result>, Error> { + let raw = self.read_raw()?; + let dt = self.datatype()?; + self.file.decode_vlen(&dt, &raw) + } + + /// Read the selected elements of a variable-length sequence dataset + /// (see [`read_vlen`](Self::read_vlen)). + pub fn read_vlen_selection( + &self, + selection: &clawhdf5_format::selection::Selection, + ) -> Result>, Error> { + let raw = self.read_selection(selection)?; + let dt = self.datatype()?; + self.file.decode_vlen(&dt, &raw) } // ----- Selection-based read methods ----- diff --git a/crates/clawhdf5/src/vlen.rs b/crates/clawhdf5/src/vlen.rs new file mode 100644 index 0000000..01ba02d --- /dev/null +++ b/crates/clawhdf5/src/vlen.rs @@ -0,0 +1,143 @@ +//! Variable-length data: VL strings and VL sequences of numbers. +//! +//! A variable-length element stores a reference into the file's global heap; +//! these helpers resolve the references in a buffer of raw elements (from a +//! dataset read, a selection, a compound field or an [`AttrValue::Raw`] +//! attribute) against the file they came from. +//! +//! Values match libhdf5 (and h5py): a string ends at its first NUL, a null +//! element is an empty string or sequence, and a heap object whose size +//! disagrees with its element is an error rather than a truncated value. +//! +//! [`AttrValue::Raw`]: crate::AttrValue::Raw + +use clawhdf5_format::data_read; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::vl_data::{VlResolver, check_element_size}; + +use crate::error::Error; + +mod sealed { + pub trait Sealed {} +} + +/// A number type that [`Dataset::read_vlen`](crate::Dataset::read_vlen) can +/// return: the sequence's base type is converted to it as libhdf5 converts +/// numbers (the same rules as `read_f64`, `read_i64`, ...). +pub trait VlenValue: sealed::Sealed + Sized { + #[doc(hidden)] + fn decode(raw: &[u8], base: &Datatype) -> Result, FormatError>; +} + +macro_rules! vlen_value { + ($t:ty, $f:path) => { + impl sealed::Sealed for $t {} + impl VlenValue for $t { + fn decode(raw: &[u8], base: &Datatype) -> Result, FormatError> { + $f(raw, base) + } + } + }; +} + +vlen_value!(f64, data_read::read_as_f64); +vlen_value!(f32, data_read::read_as_f32); +vlen_value!(i64, data_read::read_as_i64); +vlen_value!(i32, data_read::read_as_i32); +vlen_value!(u64, data_read::read_as_u64); + +fn class_name(dt: &Datatype) -> &'static str { + match dt { + Datatype::FixedPoint { .. } => "integer", + Datatype::FloatingPoint { .. } => "float", + Datatype::Time { .. } => "time", + Datatype::String { .. } => "fixed-length string", + Datatype::BitField { .. } => "bitfield", + Datatype::Opaque { .. } => "opaque", + Datatype::Compound { .. } => "compound", + Datatype::Reference { .. } => "reference", + Datatype::Enumeration { .. } => "enum", + Datatype::VariableLength { + is_string: true, .. + } => "variable-length string", + Datatype::VariableLength { .. } => "variable-length sequence", + Datatype::Array { .. } => "array", + } +} + +/// The strings in `raw`, elements of `dt`: fixed-length strings decoded as +/// `read_string` always has, variable-length strings resolved in the heap. +pub(crate) fn decode_strings( + file_data: &[u8], + dt: &Datatype, + raw: &[u8], + offset_size: u8, + length_size: u8, +) -> Result, Error> { + match dt { + Datatype::VariableLength { + size, + is_string: true, + .. + } => { + check_element_size(*size, offset_size)?; + Ok(VlResolver::new(file_data, offset_size, length_size).strings(raw)?) + } + _ => Ok(data_read::read_as_strings(raw, dt)?), + } +} + +/// The exact bytes of the variable-length strings in `raw`. +pub(crate) fn decode_string_bytes( + file_data: &[u8], + dt: &Datatype, + raw: &[u8], + offset_size: u8, + length_size: u8, +) -> Result>, Error> { + match dt { + Datatype::VariableLength { + size, + is_string: true, + .. + } => { + check_element_size(*size, offset_size)?; + Ok(VlResolver::new(file_data, offset_size, length_size).string_bytes(raw)?) + } + other => Err(Error::Format(FormatError::TypeMismatch { + expected: "variable-length string", + actual: class_name(other), + })), + } +} + +/// The sequences in `raw`, elements of the variable-length sequence type +/// `dt`, converted to `T`. +pub(crate) fn decode_vlen( + file_data: &[u8], + dt: &Datatype, + raw: &[u8], + offset_size: u8, + length_size: u8, +) -> Result>, Error> { + let Datatype::VariableLength { + size, + is_string: false, + base_type, + .. + } = dt + else { + return Err(Error::Format(FormatError::TypeMismatch { + expected: "variable-length sequence", + actual: class_name(dt), + })); + }; + check_element_size(*size, offset_size)?; + let base_size = base_type.type_size() as usize; + VlResolver::new(file_data, offset_size, length_size) + .sequences(raw, base_size)? + .iter() + .map(|bytes| Ok(T::decode(bytes, base_type)?)) + .collect() +} diff --git a/crates/clawhdf5/tests/vl_data_interop.rs b/crates/clawhdf5/tests/vl_data_interop.rs new file mode 100644 index 0000000..63f97f9 --- /dev/null +++ b/crates/clawhdf5/tests/vl_data_interop.rs @@ -0,0 +1,444 @@ +//! Variable-length data (VL strings and VL sequences) read through the +//! facade, checked against h5py/libhdf5. +//! +//! h5py writes each file — once with the default 8-byte offsets and once +//! with 4-byte offsets and lengths (`sizeof_addr = 4`) — and prints what +//! libhdf5 reads back; `File`, `MmapFile` and `LazyFile` must return the same +//! values. Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +// `Selection::slice(&[0..1])` is one range per dimension, not a Vec of a range. +#![allow(clippy::single_range_in_vec_init)] + +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use clawhdf5::{AttrValue, File, LazyFile, MmapFile, Selection}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Run `script` and return its stdout as `key -> value`, one +/// `keyvalue` line per key. +fn run_python(script: &str) -> HashMap { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| { + let (k, v) = line.split_once('\t')?; + Some((k.to_string(), v.to_string())) + }) + .collect() +} + +/// `hex,hex,...` -> the strings' bytes. +fn parse_strings(v: &str) -> Vec> { + v.split(',') + .map(|h| { + (0..h.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&h[i..i + 2], 16).unwrap()) + .collect() + }) + .collect() +} + +/// `1 2 3|| -5` -> sequences. +fn parse_seqs(v: &str) -> Vec> { + v.split('|') + .map(|s| s.split_whitespace().map(|x| x.parse().unwrap()).collect()) + .collect() +} + +fn utf8(bytes: &[Vec]) -> Vec { + bytes + .iter() + .map(|b| String::from_utf8(b.clone()).unwrap()) + .collect() +} + +/// Writes `vl8.h5` (8-byte offsets) and `vl4.h5` (4-byte offsets and +/// lengths) into `dir` and prints h5py's reading of both. +const SCRIPT: &str = r#" +import sys, h5py, numpy as np +d = sys.argv[1] +S = h5py.string_dtype('utf-8'); A = h5py.string_dtype('ascii') +def make(path, sizes): + if sizes: + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(*sizes) + f = h5py.File(h5py.h5f.create(path.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl)) + else: + f = h5py.File(path, 'w') + f.create_dataset('scalar_utf8', data='héllo', dtype=S) + f.create_dataset('scalar_ascii', data=b'hello', dtype=A) + f.create_dataset('d1', data=np.array(['a', '', 'ccc', 'δδ'], dtype=object), dtype=S) + f.create_dataset('d2', data=np.array([['x', 'yy', 'zzz'], ['', 'w', 'vv']], dtype=object), dtype=S) + f.create_dataset('chunked', data=np.array(['s%d' % i * (i % 5) for i in range(100)], dtype=object), + dtype=S, chunks=(7,), compression='gzip') + f.create_dataset('chunked2d', data=np.array([['r%dc%d' % (r, c) for c in range(9)] for r in range(11)], dtype=object), + dtype=S, chunks=(4, 4), compression='gzip', shuffle=True) + f.create_dataset('unwritten', shape=(5,), dtype=S, chunks=(2,)) + p = f.create_dataset('partial', shape=(6,), dtype=S, chunks=(2,)); p[0] = 'first'; p[5] = 'last' + f.create_dataset('contig_empty', shape=(3,), dtype=S) + dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE); dcpl.set_layout(h5py.h5d.COMPACT) + f.create_dataset('compact', data=np.array(['c1', '', 'c3'], dtype=object), dtype=S, dcpl=dcpl) + assert f['compact'].id.get_create_plist().get_layout() == h5py.h5d.COMPACT + f.attrs['vlattr'] = 'attr-value' + f.attrs.create('vlattr_arr', np.array(['p', 'qq', ''], dtype=object), dtype=S) + ct = np.dtype([('id', 'u2'))) + v[0] = [1, 65535]; v[1] = [300] + f.attrs.create('vlen_attr', np.array([np.array([1, 2], dtype=' HashMap { + let script = format!( + "import sys; sys.argv = ['x', {:?}]\n{SCRIPT}", + dir.display().to_string() + ); + run_python(&script) +} + +const STRING_DATASETS: [&str; 10] = [ + "compact", + "scalar_utf8", + "scalar_ascii", + "d1", + "d2", + "chunked", + "chunked2d", + "unwritten", + "partial", + "contig_empty", +]; + +#[test] +fn vl_string_datasets_read_like_h5py() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let expected = make_files(dir.path()); + for tag in ["8", "4"] { + let path = dir.path().join(format!("vl{tag}.h5")); + let file = File::open(&path).unwrap(); + let mmap = MmapFile::open(&path).unwrap(); + let lazy = LazyFile::open_mmap(&path).unwrap(); + for name in STRING_DATASETS { + let want = parse_strings(&expected[&format!("{tag}:{name}")]); + let ctx = format!("vl{tag}.h5 {name}"); + let ds = file.dataset(name).unwrap(); + assert_eq!(ds.read_string_bytes().unwrap(), want, "{ctx}"); + assert_eq!(ds.read_string().unwrap(), utf8(&want), "{ctx}"); + let m = mmap.dataset(name).unwrap(); + assert_eq!(m.read_string_bytes().unwrap(), want, "{ctx} (mmap)"); + assert_eq!(m.read_string().unwrap(), utf8(&want), "{ctx} (mmap)"); + let l = lazy.dataset(name).unwrap(); + assert_eq!(l.read_string_bytes().unwrap(), want, "{ctx} (lazy)"); + assert_eq!(l.read_string().unwrap(), utf8(&want), "{ctx} (lazy)"); + } + } +} + +#[test] +fn vl_string_selections_read_like_h5py() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let expected = make_files(dir.path()); + let hyperslab = |start: &[u64], stride: &[u64], count: &[u64]| Selection::Hyperslab { + start: start.to_vec(), + stride: stride.to_vec(), + count: count.to_vec(), + block: vec![1; start.len()], + }; + let cases = [ + ("d2", "d2[1,1:3]", Selection::slice(&[1..2, 1..3])), + ("chunked", "chunked[5:60:3]", hyperslab(&[5], &[3], &[19])), + ( + "chunked2d", + "chunked2d[2:9:2,3:8]", + hyperslab(&[2, 3], &[2, 1], &[4, 5]), + ), + ]; + for tag in ["8", "4"] { + let file = File::open(dir.path().join(format!("vl{tag}.h5"))).unwrap(); + for (name, key, sel) in &cases { + let want = utf8(&parse_strings(&expected[&format!("{tag}:{key}")])); + let got = file + .dataset(name) + .unwrap() + .read_string_selection(sel) + .unwrap(); + assert_eq!(got, want, "vl{tag}.h5 {key}"); + } + // A selection of VL integers is not strings. + assert!( + file.dataset("vlen_i4") + .unwrap() + .read_string_selection(&Selection::slice(&[0..1])) + .is_err() + ); + } +} + +#[test] +fn vl_values_in_compounds_and_attributes_read_like_h5py() { + // With 4-byte offsets these failed with GlobalHeapObjectNotFound or came + // back as `AttrValue::Raw`: the VL type claimed 16-byte elements and the + // global heap was read without the padding libhdf5 puts after its + // headers. + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let expected = make_files(dir.path()); + for tag in ["8", "4"] { + let file = File::open(dir.path().join(format!("vl{tag}.h5"))).unwrap(); + let want = |key: &str| utf8(&parse_strings(&expected[&format!("{tag}:{key}")])); + + let attrs = file.root().attrs().unwrap(); + match &attrs["vlattr"] { + AttrValue::String(s) => assert_eq!(*s, want("vlattr")[0], "vl{tag}.h5"), + other => panic!("vl{tag}.h5 vlattr: {other:?}"), + } + match &attrs["vlattr_arr"] { + AttrValue::StringArray(s) => assert_eq!(*s, want("vlattr_arr"), "vl{tag}.h5"), + other => panic!("vl{tag}.h5 vlattr_arr: {other:?}"), + } + + // Compound with a VL string member: dataset and attribute. + let ds = file.dataset("compound").unwrap(); + let dt = ds.raw_datatype().unwrap(); + let raw = ds.read_selection(&Selection::All).unwrap(); + let fields = clawhdf5_format::data_read::read_compound_fields(&raw, &dt).unwrap(); + let name = fields.iter().find(|f| f.name == "name").unwrap(); + assert_eq!( + file.decode_strings(&name.datatype, &name.raw_data).unwrap(), + want("compound.name"), + "vl{tag}.h5 compound" + ); + let id = fields.iter().find(|f| f.name == "id").unwrap(); + assert_eq!( + clawhdf5_format::data_read::read_as_i64(&id.raw_data, &id.datatype).unwrap(), + vec![1, 2, 3] + ); + let v = fields.iter().find(|f| f.name == "v").unwrap(); + assert_eq!( + clawhdf5_format::data_read::read_as_f64(&v.raw_data, &v.datatype).unwrap(), + vec![0.5, 1.5, 2.5] + ); + + let AttrValue::Raw { datatype, data, .. } = &attrs["compound_attr"] else { + panic!("compound attribute is Raw"); + }; + let fields = clawhdf5_format::data_read::read_compound_fields(data, datatype).unwrap(); + let name = fields.iter().find(|f| f.name == "name").unwrap(); + assert_eq!( + file.decode_strings(&name.datatype, &name.raw_data).unwrap(), + want("compound_attr.name"), + "vl{tag}.h5 compound attribute" + ); + + // A VL sequence attribute. + let AttrValue::Raw { datatype, data, .. } = &attrs["vlen_attr"] else { + panic!("vlen attribute is Raw"); + }; + let got: Vec> = file.decode_vlen(datatype, data).unwrap(); + let want_seqs = parse_seqs(&expected[&format!("{tag}:vlen_attr")]); + assert_eq!( + got, + want_seqs + .iter() + .map(|s| s.iter().map(|&x| x as i64).collect::>()) + .collect::>(), + "vl{tag}.h5 vlen_attr" + ); + } +} + +#[test] +fn vl_sequence_datasets_read_like_h5py() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let expected = make_files(dir.path()); + for tag in ["8", "4"] { + let path = dir.path().join(format!("vl{tag}.h5")); + let file = File::open(&path).unwrap(); + let seqs = |key: &str| parse_seqs(&expected[&format!("{tag}:{key}")]); + + let i4 = file.dataset("vlen_i4").unwrap(); + let want: Vec> = seqs("vlen_i4") + .iter() + .map(|s| s.iter().map(|&x| x as i32).collect()) + .collect(); + assert_eq!(i4.read_vlen::().unwrap(), want, "vl{tag}.h5 vlen_i4"); + let as_f64: Vec> = i4.read_vlen().unwrap(); + assert_eq!(as_f64, seqs("vlen_i4")); + + let f8 = file.dataset("vlen_f8").unwrap(); + assert_eq!( + f8.read_vlen::().unwrap(), + seqs("vlen_f8"), + "vl{tag}.h5" + ); + assert_eq!( + f8.read_vlen_selection::(&Selection::slice(&[1..2, 0..2])) + .unwrap(), + seqs("vlen_f8[1,:]"), + "vl{tag}.h5 vlen_f8[1,:]" + ); + + // h5py returns big-endian VL elements byte-swapped (an h5py bug, see + // CONFORMANCE.md); the values written are [1, 65535] and [300]. + assert_eq!( + file.dataset("vlen_u2_be") + .unwrap() + .read_vlen::() + .unwrap(), + vec![vec![1, 65535], vec![300]] + ); + + let mmap = MmapFile::open(&path).unwrap(); + assert_eq!( + mmap.dataset("vlen_f8").unwrap().read_vlen::().unwrap(), + seqs("vlen_f8") + ); + let lazy = LazyFile::open_mmap(&path).unwrap(); + assert_eq!( + lazy.dataset("vlen_f8").unwrap().read_vlen::().unwrap(), + seqs("vlen_f8") + ); + + // Wrong kind of data is an error, not a value. + assert!(i4.read_string().is_err()); + assert!(i4.read_string_bytes().is_err()); + assert!(file.dataset("d1").unwrap().read_vlen::().is_err()); + } +} + +#[test] +fn vl_strings_end_at_nul_and_mis_sized_elements_fail_like_h5py() { + // h5py cannot write a VL string with a NUL in it, so the file is patched: + // one string gets an embedded NUL, and two elements get a length that + // disagrees with their heap object. libhdf5 returns the string up to the + // NUL and refuses the others ("Expected global heap object size does + // not match"); we used to return the NUL and a truncated string. + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("patched.h5"); + let script = format!( + r#" +import struct, h5py, numpy as np +path = {path:?} +with h5py.File(path, 'w') as f: + f.create_dataset('d', data=np.array(['aXb', 'cdefgh', 'ij', 'ok'], dtype=object), + dtype=h5py.string_dtype()) + s = f.create_dataset('seq', shape=(2,), dtype=h5py.vlen_dtype(np.dtype(' 3 +struct.pack_into(' 9 +struct.pack_into(' 2 +open(path, 'wb').write(bytes(b)) +with h5py.File(path, 'r') as f: + for i in range(4): + try: + print('d%d\t%s' % (i, f['d'][i].hex())) + except OSError as e: + print('d%d\terror' % i) + for i in range(2): + try: + print('seq%d\t%s' % (i, ' '.join(str(x) for x in f['seq'][i]))) + except OSError as e: + print('seq%d\terror' % i) +"#, + path = path.display().to_string() + ); + let expected = run_python(&script); + assert_eq!(expected["d0"], "61", "h5py cuts 'a\\0b' at the NUL"); + assert_eq!(expected["d1"], "error"); + assert_eq!(expected["d2"], "error"); + assert_eq!(expected["d3"], "6f6b"); + assert_eq!(expected["seq0"], "error"); + assert_eq!(expected["seq1"], "4"); + + let file = File::open(&path).unwrap(); + let d = file.dataset("d").unwrap(); + let one = |i: u64| d.read_string_selection(&Selection::slice(&[i..i + 1])); + assert_eq!(one(0).unwrap(), vec!["a"]); + assert!(one(1).is_err()); + assert!(one(2).is_err()); + assert_eq!(one(3).unwrap(), vec!["ok"]); + assert!(d.read_string().is_err()); + let seq = file.dataset("seq").unwrap(); + let one = |i: u64| seq.read_vlen_selection::(&Selection::slice(&[i..i + 1])); + assert!(one(0).is_err()); + assert_eq!(one(1).unwrap(), vec![vec![4]]); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index b9a19b6..4ea8307 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -140,7 +140,13 @@ fill-value item that did is fixed). is left out of `attrs()` (reported by `attrs_with_errors()`) instead of failing the others. - **Other readers:** - - VL-string datasets are not readable through `File`. + - VL-string datasets are not readable through `File`. **Fixed + 2026-09-26:** `read_string` reads them (also `read_string_bytes`, + `read_string_selection`, and on `MmapFile`/`LazyFile`), with h5py's + values: strings end at a NUL, null elements are `""`; VL sequences of + numbers read with `read_vlen::()`, and VL values inside compounds or + `AttrValue::Raw` attributes decode with `File::decode_strings` / + `File::decode_vlen` (`crates/clawhdf5/tests/vl_data_interop.rs`). - Variable-length values inside a compound (and VL-string attributes) in a file with 4-byte offsets (`sizeof_addr = 4`) fail with `GlobalHeapObjectNotFound` or come back as `Raw`: these paths assume @@ -505,7 +511,8 @@ which is what libhdf5 itself writes. followed (no file system). - Variable-length string datasets are read by decoding `read_selection`'s bytes with `clawhdf5_format::vl_data` in the wasm crate; `File` itself still - cannot (see the audit gaps above). + cannot (see the audit gaps above). (`File` can since 2026-09-26; the wasm + crate still decodes them itself.) ## The Node.js package (`packages/clawhdf5-node`) does not work -- 2.54.0 From 6e8421a81e71bacddbe0a36e75857508f56f654f Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:27:09 -0500 Subject: [PATCH 11/36] build: record libc in the conformance probe's lockfile clawhdf5-format now depends on libc on Linux (huge-page advice for read buffers); the probe's committed lockfile picks that up. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/conformance/probe/Cargo.lock b/conformance/probe/Cargo.lock index 964e0ab..36cfe85 100644 --- a/conformance/probe/Cargo.lock +++ b/conformance/probe/Cargo.lock @@ -64,6 +64,7 @@ dependencies = [ "bzip2", "flate2", "libaec-sys", + "libc", "lz4_flex", "pco", "portable-atomic", -- 2.54.0 From d102c063064d369c4764f1e92d349a25b7dafc15 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:33:46 -0500 Subject: [PATCH 12/36] feat(format): nested groups, soft/hard/external links and creation order in the writer FileWriter wrote the root group plus one level of groups, and refused path-like names. The writer now flattens its builders into a group tree (writer_tree.rs) before layout: - A name may be a path ("a/b/x", "/a/b/x" at the root); missing intermediate groups are created as h5py does, and GroupBuilder gains create_group/add_group so builders nest to any depth. A group added at a path that already holds a group is merged into it (require_group); any other repeated name, an empty or "." component, or an absolute path below the root is an error. - add_soft_link, add_hard_link and add_external_link on FileWriter, FileBuilder and GroupBuilder. Hard-link targets are resolved to objects at finish (through other hard links; a missing target, a soft link on the way or a cycle of paths is an error). Objects with several hard links get an Object Reference Count message so libhdf5 can delete one link without freeing the object. - track_order(true) per group, or as the file default, tracks and indexes link creation order: Link Info flags and max order, the order in each Link message, and a type-6 creation-order B-tree for dense groups. - A group's link index is one B-tree leaf; more than 65535 links is an error. Groups are laid out depth-first from the root, datasets group by group, and untracked groups keep writing datasets, then groups, then other links: files with one level of groups are byte-identical to before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 38 + README.md | 24 + crates/clawhdf5-format/src/file_writer.rs | 848 ++++++++++-------- crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/type_builders.rs | 122 ++- crates/clawhdf5-format/src/writer_tree.rs | 446 +++++++++ .../tests/writer_meta_tests.rs | 47 +- crates/clawhdf5-tools/tests/h5rs_interop.rs | 85 ++ crates/clawhdf5/src/writer.rs | 43 +- .../clawhdf5/tests/writer_groups_interop.rs | 460 +++++++++- docs/known-issues.md | 13 +- 11 files changed, 1694 insertions(+), 433 deletions(-) create mode 100644 crates/clawhdf5-format/src/writer_tree.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index cc1010f..bb939de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,44 @@ ## Unreleased ### Writer: groups and links (2026-09-26) +- **Nested groups, to any depth.** `FileWriter`/`FileBuilder` wrote the root + group plus one level, and refused path-like names. Now a name may be a path + (`create_dataset("a/b/x")`, `create_group("a/b")`, a leading `/` at the + root) and missing intermediate groups are created, as h5py does; groups + also nest through the new `GroupBuilder::create_group`/`add_group`. A group + added at a path that already holds a group is merged into it (h5py's + `require_group`); a name used twice otherwise, an empty or `"."` + component (`"a//b"`, `"a/"`) or an absolute path below the root is an + error. Datasets, attributes, dense attribute storage and dense link + storage work at every level. +- **Soft, hard and external links at any depth:** `add_soft_link(name, + target)` (h5py's `SoftLink`; the target may dangle), + `add_hard_link(name, target)` (h5py's `f[name] = f[target]`; the target + path is resolved when the file is written, may go through other hard + links, and a missing target, a soft link on the way or a cycle of + hard-link paths is an error) and `add_external_link`, on `FileWriter`, + `FileBuilder` and `GroupBuilder`. An object with several hard links gets + an Object Reference Count message, so libhdf5 can delete one of the links + without freeing the object. +- **Link creation order:** `track_order(true)` on a `GroupBuilder`, or on + `FileWriter`/`FileBuilder` for every group that does not set its own, + tracks and indexes link creation order (h5py's `track_order=True`): the + Link Info message carries the flags, each link its order, and a dense + group a creation-order B-tree (type 6). h5py then lists members in + insertion order. Attribute creation order is not tracked. +- A group holds at most 65 535 links (its link index is one B-tree leaf); + more is an error. `GroupBuilder`'s fields changed (they were + crate-private); `FinishedGroup` is unchanged for callers. +- Files that use one level of groups and no new link kinds are laid out as + before: byte-identical to the writer with the Group Info fix below + (compared on simple, mixed dense/chunked/compact/external-link and paged + files). Tests: h5py and clawhdf5 read the same + tree (every path, attribute and value) from a 5-level file; soft, hard, + external and cyclic hard links; 10 000 links in one group, with and + without creation order; libhdf5 adding and deleting links in our groups; + `h5rs check` passes and `h5rs dump` equals h5dump + (`crates/clawhdf5/tests/writer_groups_interop.rs`, + `crates/clawhdf5-tools/tests/h5rs_interop.rs`). - **libhdf5 could not add links to groups we wrote.** h5py in `"r+"` mode failed with "Unable to create link (message type not found)" on every group `FileWriter` wrote: libhdf5 reads a group's Group Info message before diff --git a/README.md b/README.md index b9e3aa1..3b3fd22 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,30 @@ let values = ds.read_f64()?; assert_eq!(values, vec![22.5, 23.1, 21.8]); ``` +### Groups and links + +```rust +use clawhdf5::{AttrValue, FileBuilder}; + +let mut b = FileBuilder::new(); +// A path creates its missing intermediate groups, as in h5py. +b.create_dataset("run/2026/temps").with_f64_data(&[22.5, 23.1]); +// Builders nest; a group added at an existing path is merged into it. +let mut run = b.create_group("run"); +run.set_attr("operator", AttrValue::String("ana".into())); +let mut cal = run.create_group("calibration"); +cal.track_order(true); // h5py lists members in insertion order +cal.create_dataset("offset").with_f64_data(&[0.1]); +run.add_group(cal.finish()); +b.add_group(run.finish()); +b.add_soft_link("latest", "/run/2026"); // h5py.SoftLink +b.add_hard_link("temps", "/run/2026/temps"); // f["temps"] = f["run/2026/temps"] +b.add_external_link("raw", "raw.h5", "/data"); +b.write("groups.h5")?; +``` + +A group holds at most 65 535 links; more is an error. + ### Agent Memory ```rust diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 36c3fbb..b4a90aa 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -4,7 +4,7 @@ //! link messages, contiguous datasets, inline and dense attributes. #[cfg(not(feature = "std"))] -use alloc::{format, string::String, string::ToString, vec, vec::Vec}; +use alloc::{format, vec, vec::Vec}; use crate::attribute::AttributeMessage; use crate::chunked_write::{ @@ -21,6 +21,7 @@ use crate::superblock::Superblock; use crate::type_builders::{ DatasetBuilder, FinishedGroup, GroupBuilder, build_attr_message, fill_value_message, }; +use crate::writer_tree::{self, LinkTo}; // Re-export public types that moved to type_builders for API compatibility. #[cfg(feature = "provenance")] @@ -63,19 +64,6 @@ fn build_paged_superblock_extension(page_size: u32) -> Result, FormatErr w.serialize() } -/// A group or dataset name must be one path component: not empty, not ".", -/// and without '/'. `FileWriter` writes a root group plus one level of -/// groups, and cannot create intermediate groups for a path. -fn check_link_name(name: &str) -> Result<(), FormatError> { - if name.is_empty() || name == "." || name.contains('/') { - return Err(FormatError::SerializationError(format!( - "invalid object name {name:?}: names must be a single path component \ - (FileWriter does not create nested groups)" - ))); - } - Ok(()) -} - /// Threshold for switching from compact (inline) to dense attribute storage. const DENSE_ATTR_THRESHOLD: usize = 8; @@ -86,6 +74,7 @@ const DENSE_LINK_THRESHOLD: usize = 8; // ---- OH builders ---- +#[allow(clippy::too_many_arguments)] pub(crate) fn build_chunked_dataset_oh( dt: &Datatype, ds: &Dataspace, @@ -94,6 +83,7 @@ pub(crate) fn build_chunked_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_message: &[u8], + refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); @@ -110,9 +100,11 @@ pub(crate) fn build_chunked_dataset_oh( w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); } } + add_refcount(&mut w, refcount); w.serialize() } +#[allow(clippy::too_many_arguments)] pub(crate) fn build_dataset_oh( dt: &Datatype, ds: &Dataspace, @@ -121,6 +113,7 @@ pub(crate) fn build_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_message: &[u8], + refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); @@ -145,6 +138,7 @@ pub(crate) fn build_dataset_oh( w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); } } + add_refcount(&mut w, refcount); w.serialize() } @@ -156,6 +150,7 @@ pub(crate) fn build_compact_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_message: &[u8], + refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); @@ -175,34 +170,29 @@ pub(crate) fn build_compact_dataset_oh( w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); } } + add_refcount(&mut w, refcount); w.serialize() } +/// Build a group's object header. `link_info` is its Link Info message; +/// with `dense_links` the links live in the fractal heap it points at and are +/// not written inline. pub(crate) fn build_group_oh( links: &[LinkMessage], - dense_link_info: Option<&[u8]>, + link_info: &[u8], + dense_links: bool, attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, + refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); - if let Some(li) = dense_link_info { - // Dense link storage: a LinkInfo pointing at the fractal heap + name - // B-tree, and no inline Link messages. - w.add_message(MessageType::LinkInfo, li.to_vec()); - } else { - let mut li = Vec::new(); - li.push(0); // version - li.push(0); // flags - li.extend_from_slice(&u64::MAX.to_le_bytes()); // fractal heap addr = UNDEF - li.extend_from_slice(&u64::MAX.to_le_bytes()); // btree name index addr = UNDEF - w.add_message(MessageType::LinkInfo, li); - } + w.add_message(MessageType::LinkInfo, link_info.to_vec()); // Group Info (version 0, default link-phase thresholds, no estimates). // Readers don't need it, but libhdf5 reads it before inserting a link: // without one, adding a link to a group we wrote (h5py in "r+" mode) // failed with "message type not found". w.add_message(MessageType::GroupInfo, vec![0, 0]); - if dense_link_info.is_none() { + if !dense_links { for link in links { w.add_message(MessageType::Link, link.serialize(OFFSET_SIZE)); } @@ -214,30 +204,58 @@ pub(crate) fn build_group_oh( w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); } } + add_refcount(&mut w, refcount); w.serialize() } -pub(crate) fn make_link(name: &str, addr: u64) -> LinkMessage { - LinkMessage { - name: name.to_string(), - link_target: LinkTarget::Hard { - object_header_address: addr, +/// An object with more than one hard link records the count in an Object +/// Reference Count message (libhdf5 omits it for a count of one). Without +/// it, libhdf5 deleting one of the links would free an object that is still +/// linked. +fn add_refcount(w: &mut ObjectHeaderWriter, refcount: u32) { + if refcount > 1 { + let mut msg = vec![0u8]; // version + msg.extend_from_slice(&refcount.to_le_bytes()); + w.add_message(MessageType::ObjectReferenceCount, msg); + } +} + +/// The Link message for `link`, whose group and dataset targets are at the +/// given addresses (indexed as in the writer tree). +fn link_message(link: &writer_tree::Link, group_addrs: &[u64], ds_addrs: &[u64]) -> LinkMessage { + let link_target = match &link.to { + LinkTo::Group(g) => LinkTarget::Hard { + object_header_address: group_addrs.get(*g).copied().unwrap_or(0), }, - creation_order: None, + LinkTo::Dataset(d) => LinkTarget::Hard { + object_header_address: ds_addrs.get(*d).copied().unwrap_or(0), + }, + LinkTo::Soft(target_path) => LinkTarget::Soft { + target_path: target_path.clone(), + }, + LinkTo::External { file, path } => LinkTarget::External { + filename: file.clone(), + object_path: path.clone(), + }, + }; + LinkMessage { + name: link.name.clone(), + link_target, + creation_order: link.creation_order, charset: CharacterSet::Ascii, } } -pub(crate) fn make_external_link(name: &str, filename: &str, object_path: &str) -> LinkMessage { - LinkMessage { - name: name.to_string(), - link_target: LinkTarget::External { - filename: filename.to_string(), - object_path: object_path.to_string(), - }, - creation_order: None, - charset: CharacterSet::Ascii, - } +/// Link Info message for a group with compact (inline) links: no heap, no +/// B-trees. A group tracking creation order records the next order to use. +fn compact_link_info(track_order: bool, nlinks: usize) -> Vec { + let max_corder = track_order.then_some(nlinks as u64); + serialize_link_info( + max_corder, + u64::MAX, + u64::MAX, + track_order.then_some(u64::MAX), + ) } // ---- Dense attribute blob ---- @@ -765,97 +783,181 @@ pub(crate) fn build_dense_attrs(attrs: &[AttributeMessage], base_address: u64) - pub(crate) struct DenseLinkBlob { /// Serialized LinkInfo message (to embed in the group's object header). pub(crate) link_info_message: Vec, - /// The combined fractal heap header + direct block + B-tree v2 bytes. + /// The combined fractal heap, name-index B-tree and (when creation order + /// is tracked) creation-order-index B-tree bytes. pub(crate) blob: Vec, } +/// A v2 B-tree of `btree_type` holding `records` (already in key order) in a +/// single leaf, laid out at `addr`: the header, then the leaf. +fn single_leaf_v2_btree( + btree_type: u8, + record_size: u16, + records: &[Vec], + addr: u64, +) -> Result, FormatError> { + let os = OFFSET_SIZE as usize; + let ls = LENGTH_SIZE as usize; + // The root node's record count is a 2-byte field; more records need + // internal nodes, which the writer does not build. + let num_records = u16::try_from(records.len()).map_err(|_| { + FormatError::SerializationError(format!( + "{} links in one group: a group holds at most {} links \ + (a deeper link index is not implemented)", + records.len(), + u16::MAX + )) + })?; + let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4; + let btlf_size = 4 + 1 + 1 + (records.len() * record_size as usize) + 4; + let node_size = btlf_size.next_power_of_two().max(512) as u32; + let btlf_addr = addr + bthd_size as u64; + + let mut out = Vec::with_capacity(bthd_size + node_size as usize); + out.extend_from_slice(b"BTHD"); + out.push(0); // version + out.push(btree_type); + out.extend_from_slice(&node_size.to_le_bytes()); + out.extend_from_slice(&record_size.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); // depth = 0 (single leaf) + out.push(100); // split_percent + out.push(40); // merge_percent + write_offset(&mut out, btlf_addr, OFFSET_SIZE); + out.extend_from_slice(&num_records.to_le_bytes()); + write_length(&mut out, records.len() as u64, LENGTH_SIZE); + let checksum = crate::checksum::jenkins_lookup3(&out); + out.extend_from_slice(&checksum.to_le_bytes()); + debug_assert_eq!(out.len(), bthd_size); + + let mut btlf = Vec::with_capacity(node_size as usize); + btlf.extend_from_slice(b"BTLF"); + btlf.push(0); // version + btlf.push(btree_type); + for rec in records { + debug_assert_eq!(rec.len(), record_size as usize); + btlf.extend_from_slice(rec); + } + // The checksum follows the records, not the end of the node. + let checksum = crate::checksum::jenkins_lookup3(&btlf); + btlf.extend_from_slice(&checksum.to_le_bytes()); + btlf.resize(node_size as usize, 0); + out.extend_from_slice(&btlf); + Ok(out) +} + /// Build dense link storage for a group's links, laid out at `base_address`. /// /// Mirrors [`build_dense_attrs`]: each link is stored as a serialized Link -/// message in a single-direct-block fractal heap, indexed by a v2 B-tree of -/// **type 5** (link-name index, record = name hash + heap ID). The returned -/// LinkInfo message points at the heap and the name B-tree. -pub(crate) fn build_dense_links(links: &[LinkMessage], base_address: u64) -> DenseLinkBlob { +/// message in a fractal heap, indexed by a v2 B-tree of **type 5** (link-name +/// index, record = name hash + heap ID). With `track_order` (every link then +/// carries its creation order) a **type 6** B-tree (creation-order index, +/// record = creation order + heap ID) follows, as libhdf5 writes for a group +/// created with an indexed creation order. The returned LinkInfo message +/// points at the heap and the B-trees. +pub(crate) fn build_dense_links( + links: &[LinkMessage], + base_address: u64, + track_order: bool, +) -> Result { let serialized: Vec> = links.iter().map(|l| l.serialize(OFFSET_SIZE)).collect(); - let name_hashes: Vec = links - .iter() - .map(|l| crate::checksum::jenkins_lookup3(l.name.as_bytes())) - .collect(); - - let os = OFFSET_SIZE as usize; - let ls = LENGTH_SIZE as usize; // libhdf5's link heap uses max_heap_size 32 / heap ID length 7 (vs 40/8 for // attributes), giving a 7-byte heap ID and an 11-byte type-5 record. let heap = build_single_block_fractal_heap(&serialized, base_address, 32, 7); let heap_id_length = heap.heap_id_length; - // B-tree v2 type 5 records: hash(4) + heap_id(heap_id_length). The B-tree - // search key is the name hash, so records are sorted by (hash, order). - let record_size: u16 = 4 + heap_id_length; - let mut records: Vec<(u32, u32, Vec)> = Vec::with_capacity(links.len()); - for (i, heap_id) in heap.heap_ids.iter().enumerate() { - let mut rec = Vec::with_capacity(record_size as usize); - rec.extend_from_slice(&name_hashes[i].to_le_bytes()); // hash - rec.extend_from_slice(heap_id); // heap ID - records.push((name_hashes[i], i as u32, rec)); - } - records.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); + // Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash, + // so records are sorted by (hash, order). + let mut by_name: Vec<(u32, usize)> = links + .iter() + .enumerate() + .map(|(i, l)| (crate::checksum::jenkins_lookup3(l.name.as_bytes()), i)) + .collect(); + by_name.sort_unstable(); + let name_records: Vec> = by_name + .iter() + .map(|&(hash, i)| { + let mut rec = hash.to_le_bytes().to_vec(); + rec.extend_from_slice(&heap.heap_ids[i]); + rec + }) + .collect(); + let name_bt_addr = heap.btree_addr; + let mut blob = heap.blob; + blob.extend_from_slice(&single_leaf_v2_btree( + 5, + 4 + heap_id_length, + &name_records, + name_bt_addr, + )?); - let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4; - let num_records = links.len(); - let btlf_size = 4 + 1 + 1 + (num_records * record_size as usize) + 4; - let node_size = btlf_size.next_power_of_two().max(512) as u32; + let link_info_message = if track_order { + // Type 6 records: creation order(8) + heap_id, sorted by order. + let mut by_order: Vec<(u64, usize)> = links + .iter() + .enumerate() + .map(|(i, l)| (l.creation_order.unwrap_or(i as u64), i)) + .collect(); + by_order.sort_unstable(); + let order_records: Vec> = by_order + .iter() + .map(|&(order, i)| { + let mut rec = order.to_le_bytes().to_vec(); + rec.extend_from_slice(&heap.heap_ids[i]); + rec + }) + .collect(); + let order_bt_addr = base_address + blob.len() as u64; + blob.extend_from_slice(&single_leaf_v2_btree( + 6, + 8 + heap_id_length, + &order_records, + order_bt_addr, + )?); + let next_order = by_order.last().map_or(0, |&(o, _)| o + 1); + serialize_link_info( + Some(next_order), + heap.frhp_addr, + name_bt_addr, + Some(order_bt_addr), + ) + } else { + serialize_link_info(None, heap.frhp_addr, name_bt_addr, None) + }; - let bthd_addr = heap.btree_addr; - let btlf_addr = bthd_addr + bthd_size as u64; - - let mut bthd = Vec::with_capacity(bthd_size); - bthd.extend_from_slice(b"BTHD"); - bthd.push(0); // version - bthd.push(5); // type = link name index - bthd.extend_from_slice(&node_size.to_le_bytes()); - bthd.extend_from_slice(&record_size.to_le_bytes()); - bthd.extend_from_slice(&0u16.to_le_bytes()); // depth = 0 (single leaf) - bthd.push(100); // split_percent - bthd.push(40); // merge_percent - write_offset(&mut bthd, btlf_addr, OFFSET_SIZE); - bthd.extend_from_slice(&(num_records as u16).to_le_bytes()); - write_length(&mut bthd, num_records as u64, LENGTH_SIZE); - let bthd_checksum = crate::checksum::jenkins_lookup3(&bthd); - bthd.extend_from_slice(&bthd_checksum.to_le_bytes()); - debug_assert_eq!(bthd.len(), bthd_size); - - let mut btlf = Vec::with_capacity(node_size as usize); - btlf.extend_from_slice(b"BTLF"); - btlf.push(0); // version - btlf.push(5); // type - for (_, _, rec) in &records { - btlf.extend_from_slice(rec); - } - let btlf_checksum = crate::checksum::jenkins_lookup3(&btlf); - btlf.extend_from_slice(&btlf_checksum.to_le_bytes()); - btlf.resize(node_size as usize, 0); - - let mut blob = Vec::with_capacity(heap.blob.len() + bthd.len() + btlf.len()); - blob.extend_from_slice(&heap.blob); - blob.extend_from_slice(&bthd); - blob.extend_from_slice(&btlf); - - DenseLinkBlob { - link_info_message: serialize_link_info(heap.frhp_addr, bthd_addr), + Ok(DenseLinkBlob { + link_info_message, blob, - } + }) } -/// Serialize a LinkInfo message (version 0, no creation-order index) pointing -/// at a fractal heap and a v2 B-tree name index. -fn serialize_link_info(fh_addr: u64, btree_name_addr: u64) -> Vec { +/// Serialize a LinkInfo message (version 0). `max_creation_order` (the next +/// creation order to assign) is present when creation order is tracked, and +/// `btree_corder_addr` when it is indexed; both set flag bits. +fn serialize_link_info( + max_creation_order: Option, + fh_addr: u64, + btree_name_addr: u64, + btree_corder_addr: Option, +) -> Vec { let mut data = Vec::new(); data.push(0); // version - data.push(0x00); // flags: no creation-order tracking + let mut flags = 0u8; + if max_creation_order.is_some() { + flags |= 0x01; // creation order tracked + } + if btree_corder_addr.is_some() { + flags |= 0x02; // creation order indexed + } + data.push(flags); + if let Some(m) = max_creation_order { + data.extend_from_slice(&m.to_le_bytes()); + } write_offset(&mut data, fh_addr, OFFSET_SIZE); write_offset(&mut data, btree_name_addr, OFFSET_SIZE); + if let Some(a) = btree_corder_addr { + write_offset(&mut data, a, OFFSET_SIZE); + } data } @@ -953,6 +1055,7 @@ pub(crate) fn build_vds_dataset_oh( attrs: &[AttributeMessage], dense_blob: Option<&DenseAttrBlob>, fill_message: &[u8], + refcount: u32, ) -> Result, FormatError> { let mut w = ObjectHeaderWriter::new(); w.add_message_with_flags(MessageType::Datatype, dt.serialize(), 0x01); @@ -972,6 +1075,7 @@ pub(crate) fn build_vds_dataset_oh( w.add_message(MessageType::Attribute, attr.serialize(LENGTH_SIZE)); } } + add_refcount(&mut w, refcount); w.serialize() } @@ -997,10 +1101,16 @@ fn write_undef_offset(buf: &mut Vec, offset_size: u8) { // ---- FileWriter ---- /// The main file creation API. +/// +/// Groups nest to any depth: a name may be a path (`"a/b/x"`), and missing +/// intermediate groups are created, as h5py does; groups also nest through +/// [`GroupBuilder::add_group`]. See [`GroupBuilder`] for how names and +/// repeated groups are handled, and for soft, hard and external links. pub struct FileWriter { - root_datasets: Vec, - root_attrs: Vec<(String, AttrValue)>, - groups: Vec, + /// The root group's contents (its name is unused). + root: GroupBuilder, + /// Default for groups that do not call [`GroupBuilder::track_order`]. + track_order: bool, /// Global alignment threshold: datasets with raw data >= this many bytes /// will have their data aligned to `alignment_bytes`. alignment_threshold: usize, @@ -1018,12 +1128,98 @@ impl Default for FileWriter { } } +/// A dataset ready for layout. +struct DsFlat { + dt: Datatype, + ds: Dataspace, + raw: Vec, + attrs: Vec, + chunk_options: ChunkOptions, + maxshape: Option>, + /// Serialized Fill Value message. + fill_message: Vec, + compact: bool, + alignment: usize, + /// VDS source mappings (set for Virtual datasets). + virtual_sources: Option>, + /// Number of hard links to the dataset. + refcount: u32, +} + +/// Convert a DatasetBuilder into a DsFlat, handling VDS (which does not +/// require a `data` field). +fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result { + let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?; + let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?; + let is_vds = db.virtual_sources.is_some(); + let raw = if is_vds { + // VDS datasets have no raw data stored in this file. + db.data.unwrap_or_default() + } else { + db.data.ok_or(FormatError::DatasetMissingData)? + }; + let max_dimensions = db.maxshape.clone(); + let dspace = Dataspace { + space_type: if shape.is_empty() { + DataspaceType::Scalar + } else { + DataspaceType::Simple + }, + rank: shape.len() as u8, + dimensions: shape, + max_dimensions, + }; + let mut attrs = Vec::new(); + for (n, v) in &db.attrs { + attrs.push(build_attr_message(n, v)); + } + #[cfg(feature = "provenance")] + if let Some(ref prov) = db.provenance { + let p = crate::provenance::Provenance { + creator: prov.creator.clone(), + timestamp: prov.timestamp.clone(), + source: prov.source.clone(), + }; + attrs.extend(p.build_attrs(&raw)); + } + let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?; + Ok(DsFlat { + dt, + ds: dspace, + raw, + attrs, + chunk_options: db.chunk_options, + maxshape: db.maxshape, + fill_message, + compact: db.compact, + alignment: db.alignment, + virtual_sources: db.virtual_sources, + refcount, + }) +} + +/// A group ready for layout. +struct GrpFlat { + attrs: Vec, + links: Vec, + track_order: bool, + refcount: u32, +} + +impl GrpFlat { + fn link_messages(&self, group_addrs: &[u64], ds_addrs: &[u64]) -> Vec { + self.links + .iter() + .map(|l| link_message(l, group_addrs, ds_addrs)) + .collect() + } +} + impl FileWriter { pub fn new() -> Self { Self { - root_datasets: Vec::new(), - root_attrs: Vec::new(), - groups: Vec::new(), + root: GroupBuilder::new("/"), + track_order: false, alignment_threshold: 0, alignment_bytes: 0, page_size: None, @@ -1055,21 +1251,61 @@ impl FileWriter { self } + /// Track (and index) link creation order in every group that does not + /// set its own [`GroupBuilder::track_order`], the root included — as + /// h5py's `track_order=True`: libhdf5 then lists members in the order + /// they were added. Off by default (members are listed by name). + pub fn track_order(&mut self, track: bool) -> &mut Self { + self.track_order = track; + self + } + + /// Start a group. The builder is detached: fill it, then pass + /// `finish()`'s result to [`Self::add_group`]. `name` may be a path + /// (`"a/b"`); missing intermediate groups are created. pub fn create_group(&mut self, name: &str) -> GroupBuilder { GroupBuilder::new(name) } + /// Add a finished group to the root group. pub fn add_group(&mut self, group: FinishedGroup) { - self.groups.push(group); + self.root.add_group(group); } + /// Create a dataset. `name` may be a path (`"a/b/x"`, or `"/a/b/x"`); + /// missing intermediate groups are created. pub fn create_dataset(&mut self, name: &str) -> &mut DatasetBuilder { - self.root_datasets.push(DatasetBuilder::new(name)); - self.root_datasets.last_mut().unwrap() + self.root.create_dataset(name) } pub fn set_root_attr(&mut self, name: &str, value: AttrValue) { - self.root_attrs.push((name.to_string(), value)); + self.root.set_attr(name, value); + } + + /// Add a soft link `name` (a path from the root) to `target`. See + /// [`GroupBuilder::add_soft_link`]. + pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self { + self.root.add_soft_link(name, target); + self + } + + /// Add another hard link `name` (a path from the root) to the object at + /// `target`. See [`GroupBuilder::add_hard_link`]. + pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self { + self.root.add_hard_link(name, target); + self + } + + /// Add an external link `name` (a path from the root) to `target_path` + /// in the file `target_file`. + pub fn add_external_link( + &mut self, + name: &str, + target_file: &str, + target_path: &str, + ) -> &mut Self { + self.root.add_external_link(name, target_file, target_path); + self } pub fn finish(self) -> Result, FormatError> { @@ -1082,131 +1318,35 @@ impl FileWriter { {MIN_FILE_SPACE_PAGE_SIZE}..={MAX_FILE_SPACE_PAGE_SIZE} bytes" ))); } - struct DsFlat { - name: String, - dt: Datatype, - ds: Dataspace, - raw: Vec, - attrs: Vec, - chunk_options: ChunkOptions, - maxshape: Option>, - /// Serialized Fill Value message. - fill_message: Vec, - compact: bool, - alignment: usize, - /// VDS source mappings (set for Virtual datasets). - virtual_sources: Option>, - } - struct GrpFlat { - name: String, - attrs: Vec, - ds_indices: Vec, - /// (link_name, target_file, target_path) - external_links: Vec<(String, String, String)>, - } - // Helper: convert a DatasetBuilder into DsFlat, handling VDS (which - // does not require a `data` field). - let flatten_ds = |db: DatasetBuilder| -> Result { - let dt = db.datatype.ok_or(FormatError::DatasetMissingData)?; - let shape = db.shape.ok_or(FormatError::DatasetMissingShape)?; - let is_vds = db.virtual_sources.is_some(); - let raw = if is_vds { - // VDS datasets have no raw data stored in this file. - db.data.unwrap_or_default() - } else { - db.data.ok_or(FormatError::DatasetMissingData)? - }; - let max_dimensions = db.maxshape.clone(); - let dspace = Dataspace { - space_type: if shape.is_empty() { - DataspaceType::Scalar - } else { - DataspaceType::Simple - }, - rank: shape.len() as u8, - dimensions: shape, - max_dimensions, - }; - let mut attrs = Vec::new(); - for (n, v) in &db.attrs { - attrs.push(build_attr_message(n, v)); - } - #[cfg(feature = "provenance")] - if let Some(ref prov) = db.provenance { - let p = crate::provenance::Provenance { - creator: prov.creator.clone(), - timestamp: prov.timestamp.clone(), - source: prov.source.clone(), - }; - attrs.extend(p.build_attrs(&raw)); - } - let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?; - Ok(DsFlat { - name: db.name, - dt, - ds: dspace, - raw, - attrs, - chunk_options: db.chunk_options, - maxshape: db.maxshape, - fill_message, - compact: db.compact, - alignment: db.alignment, - virtual_sources: db.virtual_sources, + // The group tree, in layout order: groups depth-first from the root, + // then every group's datasets in the same order. + let tree = writer_tree::build(self.root, self.track_order)?; + let all_ds: Vec = tree + .datasets + .into_iter() + .map(|(db, refcount)| flatten_ds(db, refcount)) + .collect::>()?; + let groups: Vec = tree + .groups + .into_iter() + .map(|g| GrpFlat { + attrs: g + .attrs + .iter() + .map(|(n, v)| build_attr_message(n, v)) + .collect(), + links: g.links, + track_order: g.track_order, + refcount: g.refcount, }) - }; - - // Every name becomes a single link in its parent group. The writer - // has no nested groups, so a path like "a/b" would be stored as one - // link literally named "a/b" — which no HDF5 reader can resolve. - let root_names = self.root_datasets.iter().map(|d| d.name.as_str()); - let group_names = self.groups.iter().flat_map(|g| { - core::iter::once(g.name.as_str()) - .chain(g.datasets.iter().map(|d| d.name.as_str())) - .chain(g.external_links.iter().map(|l| l.0.as_str())) - }); - for name in root_names.chain(group_names) { - check_link_name(name)?; - } - - let mut all_ds: Vec = Vec::new(); - let mut groups: Vec = Vec::new(); - let mut root_ds_indices: Vec = Vec::new(); - - for db in self.root_datasets { - root_ds_indices.push(all_ds.len()); - all_ds.push(flatten_ds(db)?); - } - - for g in self.groups.into_iter() { - let mut gattrs = Vec::new(); - for (n, v) in &g.attrs { - gattrs.push(build_attr_message(n, v)); - } - let mut ds_idx = Vec::new(); - for db in g.datasets { - ds_idx.push(all_ds.len()); - all_ds.push(flatten_ds(db)?); - } - groups.push(GrpFlat { - name: g.name, - attrs: gattrs, - ds_indices: ds_idx, - external_links: g.external_links, - }); - } - - let mut root_attrs: Vec = Vec::new(); - for (n, v) in &self.root_attrs { - root_attrs.push(build_attr_message(n, v)); - } + .collect(); // Every datatype must have an on-disk encoding before anything is laid // out: `Datatype::serialize` itself cannot report a failure. let group_attrs = groups.iter().flat_map(|g| &g.attrs); let ds_attrs = all_ds.iter().flat_map(|d| &d.attrs); - for a in root_attrs.iter().chain(group_attrs).chain(ds_attrs) { + for a in group_attrs.chain(ds_attrs) { a.datatype.check_encodable()?; } for d in &all_ds { @@ -1232,7 +1372,6 @@ impl FileWriter { !is_vds[i] && !is_chunked[i] && d.compact && d.raw.len() <= MAX_COMPACT_DATA_SIZE }) .collect(); - let root_dense = root_attrs.len() > DENSE_ATTR_THRESHOLD; let group_dense: Vec = groups .iter() .map(|g| g.attrs.len() > DENSE_ATTR_THRESHOLD) @@ -1244,51 +1383,41 @@ impl FileWriter { // Dense link decision: a group with more than the compact threshold of // links stores them in a fractal heap + v2 B-tree instead of inline. - let root_link_count = root_ds_indices.len() + groups.len(); - let root_links_dense = root_link_count > DENSE_LINK_THRESHOLD; let group_links_dense: Vec = groups .iter() - .map(|g| g.ds_indices.len() + g.external_links.len() > DENSE_LINK_THRESHOLD) + .map(|g| g.links.len() > DENSE_LINK_THRESHOLD) .collect(); - // The dense LinkInfo message is a fixed size regardless of address, so a - // dummy is sufficient for OH size computation. - let dummy_link_info = serialize_link_info(0, 0); - // Pass 1: compute OH sizes with dummy addresses + // Pass 1: compute OH sizes with dummy addresses. Link messages and + // the Link Info message are the same size whatever the addresses. let group_oh_sizes: Vec = groups .iter() .enumerate() .map(|(gi, g)| { - let mut dummy_links: Vec = g - .ds_indices - .iter() - .map(|&i| make_link(&all_ds[i].name, 0)) - .collect(); - for (lname, fname, opath) in &g.external_links { - dummy_links.push(make_external_link(lname, fname, opath)); - } + let dummy_links = g.link_messages(&[], &[]); let attr_blob = group_dense[gi].then(|| build_dense_attrs(&g.attrs, 0)); - let dl = group_links_dense[gi].then_some(dummy_link_info.as_slice()); - build_group_oh(&dummy_links, dl, &g.attrs, attr_blob.as_ref()).map(|oh| oh.len()) + let li = if group_links_dense[gi] { + serialize_link_info( + g.track_order.then_some(0), + 0, + 0, + g.track_order.then_some(0), + ) + } else { + compact_link_info(g.track_order, g.links.len()) + }; + build_group_oh( + &dummy_links, + &li, + group_links_dense[gi], + &g.attrs, + attr_blob.as_ref(), + g.refcount, + ) + .map(|oh| oh.len()) }) .collect::>()?; - let root_dummy_links: Vec = { - let mut links = Vec::new(); - for &i in &root_ds_indices { - links.push(make_link(&all_ds[i].name, 0)); - } - for g in &groups { - links.push(make_link(&g.name, 0)); - } - links - }; - let root_oh_size = { - let attr_blob = root_dense.then(|| build_dense_attrs(&root_attrs, 0)); - let dl = root_links_dense.then_some(dummy_link_info.as_slice()); - build_group_oh(&root_dummy_links, dl, &root_attrs, attr_blob.as_ref())?.len() - }; - struct DataBlob { data: Vec, oh_bytes: Vec, @@ -1300,14 +1429,10 @@ impl FileWriter { let mut dummy_blobs: Vec = Vec::new(); let mut dummy_cursor = 0u64; for (i, d) in all_ds.iter().enumerate() { + let dense_blob = ds_dense[i].then(|| build_dense_attrs(&d.attrs, 0)); if is_vds[i] { // VDS: dummy OH with address 0 to get the OH size. The global // heap blob will be placed after the OHs in pass 2. - let dense_blob = if ds_dense[i] { - Some(build_dense_attrs(&d.attrs, 0)) - } else { - None - }; let oh = build_vds_dataset_oh( &d.dt, &d.ds, @@ -1315,6 +1440,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), &d.fill_message, + d.refcount, )?; // Global heap blob size is address-independent; compute it now // so pass 2 can place it correctly. @@ -1346,11 +1472,6 @@ impl FileWriter { d.maxshape.as_deref(), )?; dummy_cursor += result.data_bytes.len() as u64; - let dense_blob = if ds_dense[i] { - Some(build_dense_attrs(&d.attrs, 0)) - } else { - None - }; let oh = build_chunked_dataset_oh( &d.dt, &d.ds, @@ -1359,6 +1480,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), &d.fill_message, + d.refcount, )?; dummy_blobs.push(DataBlob { data: result.data_bytes, @@ -1366,11 +1488,6 @@ impl FileWriter { precompressed: Some(pre), }); } else if is_compact[i] { - let dense_blob = if ds_dense[i] { - Some(build_dense_attrs(&d.attrs, 0)) - } else { - None - }; let oh = build_compact_dataset_oh( &d.dt, &d.ds, @@ -1378,6 +1495,7 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), &d.fill_message, + d.refcount, )?; dummy_blobs.push(DataBlob { data: vec![], @@ -1385,11 +1503,6 @@ impl FileWriter { precompressed: None, }); } else { - let dense_blob = if ds_dense[i] { - Some(build_dense_attrs(&d.attrs, 0)) - } else { - None - }; let oh = build_dataset_oh( &d.dt, &d.ds, @@ -1398,9 +1511,10 @@ impl FileWriter { &d.attrs, dense_blob.as_ref(), &d.fill_message, + d.refcount, )?; dummy_blobs.push(DataBlob { - data: d.raw.clone(), + data: vec![], oh_bytes: oh, precompressed: None, }); @@ -1416,61 +1530,37 @@ impl FileWriter { .map(build_paged_superblock_extension) .transpose()?; let superblock_size = SUPERBLOCK_SIZE + sb_ext.as_ref().map_or(0, Vec::len); - let root_group_addr = superblock_size as u64; - let mut cursor2 = superblock_size + root_oh_size; - - // Each group is laid out as: object header, then (if dense) its link - // blob, then (if dense) its attribute blob. Link blobs are sized with - // dummy target addresses here — link message size is address-independent - // — and rebuilt with real addresses in the final pass. - let root_link_blob_addr = if root_links_dense { - let addr = cursor2 as u64; - cursor2 += build_dense_links(&root_dummy_links, addr).blob.len(); - Some(addr) - } else { - None - }; - let root_dense_blob = if root_dense { - let blob = build_dense_attrs(&root_attrs, cursor2 as u64); - cursor2 += blob.blob.len(); - Some(blob) - } else { - None - }; + let mut cursor2 = superblock_size; + // Each group (the root first) is laid out as: object header, then (if + // dense) its link blob, then (if dense) its attribute blob. Link blobs + // are sized with dummy target addresses here — link message size is + // address-independent — and rebuilt with real addresses when written. let mut group_link_blob_addrs: Vec> = Vec::new(); let mut group_dense_blobs: Vec> = Vec::new(); - let group_addrs2: Vec = group_oh_sizes - .iter() - .enumerate() - .map(|(gi, &sz)| { - let addr = cursor2 as u64; - cursor2 += sz; - if group_links_dense[gi] { - let mut dummy_links: Vec = groups[gi] - .ds_indices - .iter() - .map(|&i| make_link(&all_ds[i].name, 0)) - .collect(); - for (lname, fname, opath) in &groups[gi].external_links { - dummy_links.push(make_external_link(lname, fname, opath)); - } - let blob_addr = cursor2 as u64; - cursor2 += build_dense_links(&dummy_links, blob_addr).blob.len(); - group_link_blob_addrs.push(Some(blob_addr)); - } else { - group_link_blob_addrs.push(None); - } - if group_dense[gi] { - let blob = build_dense_attrs(&groups[gi].attrs, cursor2 as u64); - cursor2 += blob.blob.len(); - group_dense_blobs.push(Some(blob)); - } else { - group_dense_blobs.push(None); - } - addr - }) - .collect(); + let mut group_addrs2: Vec = Vec::with_capacity(groups.len()); + for (gi, g) in groups.iter().enumerate() { + group_addrs2.push(cursor2 as u64); + cursor2 += group_oh_sizes[gi]; + if group_links_dense[gi] { + let blob_addr = cursor2 as u64; + let dummy = g.link_messages(&[], &[]); + cursor2 += build_dense_links(&dummy, blob_addr, g.track_order)? + .blob + .len(); + group_link_blob_addrs.push(Some(blob_addr)); + } else { + group_link_blob_addrs.push(None); + } + if group_dense[gi] { + let blob = build_dense_attrs(&g.attrs, cursor2 as u64); + cursor2 += blob.blob.len(); + group_dense_blobs.push(Some(blob)); + } else { + group_dense_blobs.push(None); + } + } + let root_group_addr = group_addrs2[0]; let mut ds_dense_blobs: Vec> = Vec::new(); let ds_oh_addrs2: Vec = actual_ds_oh_sizes @@ -1507,6 +1597,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), &d.fill_message, + d.refcount, )?; ds_blobs2.push(DataBlob { data: gcol_bytes.clone(), @@ -1534,6 +1625,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), &d.fill_message, + d.refcount, )?; ds_blobs2.push(DataBlob { data: result.data_bytes, @@ -1549,6 +1641,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), &d.fill_message, + d.refcount, )?; ds_blobs2.push(DataBlob { data: vec![], @@ -1574,6 +1667,7 @@ impl FileWriter { &d.attrs, ds_dense_blobs[i].as_ref(), &d.fill_message, + d.refcount, )?; let mut data = vec![0u8; padding]; data.extend_from_slice(&d.raw); @@ -1623,51 +1717,29 @@ impl FileWriter { buf.extend_from_slice(ext); } - // Root group OH - let mut root_links: Vec = Vec::new(); - for &i in &root_ds_indices { - root_links.push(make_link(&all_ds[i].name, ds_oh_addrs2[i])); - } - for (gi, g) in groups.iter().enumerate() { - root_links.push(make_link(&g.name, group_addrs2[gi])); - } - // Rebuild the root link blob with real target addresses (same size as - // the dummy used for layout); its LinkInfo goes in the OH. - let root_link_blob = root_link_blob_addr.map(|addr| build_dense_links(&root_links, addr)); - let root_dl = root_link_blob - .as_ref() - .map(|b| b.link_info_message.as_slice()); - buf.extend_from_slice(&build_group_oh( - &root_links, - root_dl, - &root_attrs, - root_dense_blob.as_ref(), - )?); - if let Some(ref b) = root_link_blob { - buf.extend_from_slice(&b.blob); - } - if let Some(ref blob) = root_dense_blob { - buf.extend_from_slice(&blob.blob); - } - // Group OHs + dense blobs (link blob, then attr blob, matching pass 2) for (gi, g) in groups.iter().enumerate() { - let mut links: Vec = g - .ds_indices - .iter() - .map(|&i| make_link(&all_ds[i].name, ds_oh_addrs2[i])) - .collect(); - for (lname, fname, opath) in &g.external_links { - links.push(make_external_link(lname, fname, opath)); - } - let link_blob = group_link_blob_addrs[gi].map(|addr| build_dense_links(&links, addr)); - let dl = link_blob.as_ref().map(|b| b.link_info_message.as_slice()); - buf.extend_from_slice(&build_group_oh( + let links = g.link_messages(&group_addrs2, &ds_oh_addrs2); + // Rebuild the link blob with real target addresses (same size as + // the dummy used for layout); its LinkInfo goes in the OH. + let link_blob = group_link_blob_addrs[gi] + .map(|addr| build_dense_links(&links, addr, g.track_order)) + .transpose()?; + let li = match &link_blob { + Some(b) => b.link_info_message.clone(), + None => compact_link_info(g.track_order, links.len()), + }; + let oh = build_group_oh( &links, - dl, + &li, + link_blob.is_some(), &g.attrs, group_dense_blobs[gi].as_ref(), - )?); + g.refcount, + )?; + debug_assert_eq!(oh.len(), group_oh_sizes[gi]); + debug_assert_eq!(buf.len() as u64, group_addrs2[gi]); + buf.extend_from_slice(&oh); if let Some(ref b) = link_blob { buf.extend_from_slice(&b.blob); } diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 905ce84..180cf07 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -130,6 +130,7 @@ mod test_fuzz; pub mod type_builders; pub mod vds; pub mod vl_data; +mod writer_tree; #[cfg(feature = "provenance")] pub mod provenance; diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index f055a88..fa64465 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -903,34 +903,117 @@ impl DatasetBuilder { // ---- Group builder ---- -/// Builder for groups. +/// One entry of a [`GroupBuilder`], kept in the order it was added (the +/// order a group that tracks creation order lists its links in). +pub(crate) enum GroupItem { + Dataset(Box), + Group(GroupBuilder), + /// A soft link: `name` resolves to whatever `target` names when read. + Soft { + name: String, + target: String, + }, + /// An extra hard link to the object at `target` (a path in this file). + Hard { + name: String, + target: String, + }, + /// An external link to `path` in the file `file`. + External { + name: String, + file: String, + path: String, + }, +} + +/// Builder for a group: its datasets, subgroups, links and attributes. +/// +/// Names are paths relative to the group: `create_dataset("a/b/x")` creates +/// the groups `a` and `a/b` as needed, as h5py does. A group added where a +/// group of the same path already exists (added by another builder, or +/// created as an intermediate group) is merged into it, like h5py's +/// `require_group`; any other name used twice in a group is an error when the +/// file is written. A path component must not be empty or `"."`. pub struct GroupBuilder { pub(crate) name: String, - pub(crate) datasets: Vec, + pub(crate) items: Vec, pub(crate) attrs: Vec<(String, AttrValue)>, - /// (link_name, target_file, target_path) - pub(crate) external_links: Vec<(String, String, String)>, + /// Track (and index) link creation order; `None` follows the file's + /// default (`FileWriter::track_order`). + pub(crate) track_order: Option, } impl GroupBuilder { pub(crate) fn new(name: &str) -> Self { Self { name: name.to_string(), - datasets: Vec::new(), + items: Vec::new(), attrs: Vec::new(), - external_links: Vec::new(), + track_order: None, } } + /// Create a dataset in this group. `name` may be a relative path + /// (`"a/b/x"`); missing intermediate groups are created. pub fn create_dataset(&mut self, name: &str) -> &mut DatasetBuilder { - self.datasets.push(DatasetBuilder::new(name)); - self.datasets.last_mut().unwrap() + self.items + .push(GroupItem::Dataset(Box::new(DatasetBuilder::new(name)))); + match self.items.last_mut() { + Some(GroupItem::Dataset(d)) => d, + _ => unreachable!("just pushed a dataset"), + } + } + + /// Start a subgroup of this group. Like `FileWriter::create_group`, the + /// builder is detached: fill it, then pass `finish()`'s result to + /// [`Self::add_group`]. `name` may be a relative path. + pub fn create_group(&self, name: &str) -> GroupBuilder { + GroupBuilder::new(name) + } + + /// Add a finished subgroup to this group. + pub fn add_group(&mut self, group: FinishedGroup) -> &mut Self { + self.items.push(GroupItem::Group(group.group)); + self } pub fn set_attr(&mut self, name: &str, value: AttrValue) { self.attrs.push((name.to_string(), value)); } + /// Track the creation order of this group's links, and index it, as + /// h5py's `track_order=True` does: libhdf5 (and h5py) then list the + /// group's members in the order they were added rather than by name. + /// Applies to links only, not to attributes. + pub fn track_order(&mut self, track: bool) -> &mut Self { + self.track_order = Some(track); + self + } + + /// Add a soft link `name` to the path `target` (absolute, or relative to + /// this group), like h5py's `grp[name] = h5py.SoftLink(target)`. The + /// target need not exist. + pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self { + self.items.push(GroupItem::Soft { + name: name.to_string(), + target: target.to_string(), + }); + self + } + + /// Add another hard link `name` to the group or dataset at `target` + /// (absolute, or relative to this group), like h5py's + /// `grp[name] = f[target]`. The target must be written in the same file; + /// its path may go through other hard links, but not through soft or + /// external links. + pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self { + self.items.push(GroupItem::Hard { + name: name.to_string(), + target: target.to_string(), + }); + self + } + /// Add an external link: a named pointer to an object in another HDF5 file. pub fn add_external_link( &mut self, @@ -938,30 +1021,21 @@ impl GroupBuilder { target_file: &str, target_path: &str, ) -> &mut Self { - self.external_links.push(( - name.to_string(), - target_file.to_string(), - target_path.to_string(), - )); + self.items.push(GroupItem::External { + name: name.to_string(), + file: target_file.to_string(), + path: target_path.to_string(), + }); self } /// Consume the builder, returning a FinishedGroup to add to FileWriter. pub fn finish(self) -> FinishedGroup { - FinishedGroup { - name: self.name, - datasets: self.datasets, - attrs: self.attrs, - external_links: self.external_links, - } + FinishedGroup { group: self } } } /// A finished group ready for the file writer. pub struct FinishedGroup { - pub(crate) name: String, - pub(crate) datasets: Vec, - pub(crate) attrs: Vec<(String, AttrValue)>, - /// (link_name, target_file, target_path) - pub(crate) external_links: Vec<(String, String, String)>, + pub(crate) group: GroupBuilder, } diff --git a/crates/clawhdf5-format/src/writer_tree.rs b/crates/clawhdf5-format/src/writer_tree.rs new file mode 100644 index 0000000..284867e --- /dev/null +++ b/crates/clawhdf5-format/src/writer_tree.rs @@ -0,0 +1,446 @@ +//! The group hierarchy `FileWriter` writes: builders flattened into a tree +//! of groups, datasets and links, with path names expanded into +//! intermediate groups, hard links resolved to objects, reference counts +//! counted, and everything put in layout order. + +#[cfg(not(feature = "std"))] +use alloc::{ + collections::BTreeMap, + format, + string::{String, ToString}, + vec, + vec::Vec, +}; +#[cfg(feature = "std")] +use std::collections::BTreeMap; + +use crate::error::FormatError; +use crate::type_builders::{AttrValue, DatasetBuilder, GroupBuilder, GroupItem}; + +/// Hard links followed while resolving one hard-link target path. Guards +/// against hard links whose targets name each other. +const MAX_LINK_DEPTH: usize = 64; + +fn err(msg: String) -> FormatError { + FormatError::SerializationError(msg) +} + +/// A link name must be one path component: not empty, not ".", and without +/// '/' (a '/' separates components, so it cannot be part of a name). +fn check_link_name(name: &str, path: &str) -> Result<(), FormatError> { + if name.is_empty() || name == "." || name.contains('/') { + return Err(err(format!( + "invalid object name {path:?}: every path component must be a \ + non-empty name other than \".\"" + ))); + } + Ok(()) +} + +/// What a link in the final tree points at. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum LinkTo { + /// A group, by index into [`Tree::groups`] (layout order). + Group(usize), + /// A dataset, by index into [`Tree::datasets`] (layout order). + Dataset(usize), + Soft(String), + External { + file: String, + path: String, + }, +} + +pub(crate) struct Link { + pub(crate) name: String, + pub(crate) to: LinkTo, + /// Set when the group tracks creation order. + pub(crate) creation_order: Option, +} + +pub(crate) struct Group { + pub(crate) attrs: Vec<(String, AttrValue)>, + /// Links in the order they are written. + pub(crate) links: Vec, + pub(crate) track_order: bool, + /// Number of hard links to this group (the root counts one for the + /// superblock's reference). + pub(crate) refcount: u32, +} + +/// The flattened file: groups (root first) and datasets, both in the order +/// they are laid out in the file. +pub(crate) struct Tree { + pub(crate) groups: Vec, + pub(crate) datasets: Vec<(DatasetBuilder, u32)>, +} + +// ---- construction ---- + +enum Target { + Group(usize), + Dataset(usize), + Soft(String), + Hard(String), + External { file: String, path: String }, +} + +struct BuildGroup { + /// Full path, for messages. + path: String, + attrs: Vec<(String, AttrValue)>, + links: Vec<(String, Target)>, + by_name: BTreeMap, + track_order: Option, +} + +struct Builder { + groups: Vec, + datasets: Vec, +} + +fn join(parent: &str, name: &str) -> String { + if parent == "/" { + format!("/{name}") + } else { + format!("{parent}/{name}") + } +} + +impl Builder { + fn new_group(&mut self, path: String) -> usize { + self.groups.push(BuildGroup { + path, + attrs: Vec::new(), + links: Vec::new(), + by_name: BTreeMap::new(), + track_order: None, + }); + self.groups.len() - 1 + } + + /// Split `path` (relative to group `g`) into the group holding its last + /// component, creating missing intermediate groups, and that component. + fn parent_of<'p>(&mut self, g: usize, path: &'p str) -> Result<(usize, &'p str), FormatError> { + // An absolute path is accepted at the root only. + let rel = match path.strip_prefix('/') { + Some(rest) if g == 0 => rest, + Some(_) => { + return Err(err(format!( + "invalid object name {path:?} in {}: absolute paths are accepted \ + only at the root", + self.groups[g].path + ))); + } + None => path, + }; + let mut comps: Vec<&str> = rel.split('/').collect(); + let last = comps.pop().unwrap_or(""); + check_link_name(last, path)?; + let mut cur = g; + for c in comps { + check_link_name(c, path)?; + cur = match self.groups[cur].by_name.get(c).copied() { + Some(i) => match self.groups[cur].links[i].1 { + Target::Group(child) => child, + _ => { + return Err(err(format!( + "cannot create {path:?} in {}: {c:?} exists and is not a group", + self.groups[g].path + ))); + } + }, + None => { + let child = self.new_group(join(&self.groups[cur].path, c)); + self.push_link(cur, c, Target::Group(child))?; + child + } + }; + } + Ok((cur, last)) + } + + fn push_link(&mut self, g: usize, name: &str, to: Target) -> Result<(), FormatError> { + let grp = &mut self.groups[g]; + if grp.by_name.contains_key(name) { + return Err(err(format!("{:?} already exists", join(&grp.path, name)))); + } + grp.by_name.insert(name.to_string(), grp.links.len()); + grp.links.push((name.to_string(), to)); + Ok(()) + } + + /// Add `item` to group `g`. + fn add_item(&mut self, g: usize, item: GroupItem) -> Result<(), FormatError> { + match item { + GroupItem::Dataset(db) => { + let (parent, name) = self.parent_of(g, &db.name)?; + let name = name.to_string(); + self.push_link(parent, &name, Target::Dataset(self.datasets.len()))?; + self.datasets.push(*db); + } + GroupItem::Group(gb) => self.add_group(g, gb)?, + GroupItem::Soft { name, target } => { + if target.is_empty() { + return Err(err(format!("soft link {name:?} has an empty target"))); + } + let (parent, last) = self.parent_of(g, &name)?; + self.push_link(parent, last, Target::Soft(target))?; + } + GroupItem::Hard { name, target } => { + let (parent, last) = self.parent_of(g, &name)?; + self.push_link(parent, last, Target::Hard(target))?; + } + GroupItem::External { name, file, path } => { + if file.is_empty() || path.is_empty() { + return Err(err(format!( + "external link {name:?} needs a file name and an object path" + ))); + } + let (parent, last) = self.parent_of(g, &name)?; + self.push_link(parent, last, Target::External { file, path })?; + } + } + Ok(()) + } + + /// Add the group `gb` (named by a path relative to group `g`), merging it + /// into a group already at that path. + fn add_group(&mut self, g: usize, gb: GroupBuilder) -> Result<(), FormatError> { + let (parent, last) = self.parent_of(g, &gb.name)?; + let idx = match self.groups[parent].by_name.get(last).copied() { + Some(i) => match self.groups[parent].links[i].1 { + Target::Group(child) => child, + _ => { + return Err(err(format!( + "{:?} already exists and is not a group", + join(&self.groups[parent].path, last) + ))); + } + }, + None => { + let child = self.new_group(join(&self.groups[parent].path, last)); + self.push_link(parent, last, Target::Group(child))?; + child + } + }; + self.merge_into(idx, gb) + } + + /// Merge a builder's attributes, setting and items into group `idx`. + fn merge_into(&mut self, idx: usize, gb: GroupBuilder) -> Result<(), FormatError> { + for (name, value) in gb.attrs { + if self.groups[idx].attrs.iter().any(|(n, _)| *n == name) { + return Err(err(format!( + "attribute {name:?} set twice on {}", + self.groups[idx].path + ))); + } + self.groups[idx].attrs.push((name, value)); + } + if let Some(t) = gb.track_order { + match self.groups[idx].track_order { + Some(old) if old != t => { + return Err(err(format!( + "conflicting track_order settings for {}", + self.groups[idx].path + ))); + } + _ => self.groups[idx].track_order = Some(t), + } + } + for item in gb.items { + self.add_item(idx, item)?; + } + Ok(()) + } + + /// The object a hard link's `target` path names, from group `from`. + fn resolve(&self, from: usize, target: &str, depth: usize) -> Result { + if depth > MAX_LINK_DEPTH { + return Err(err(format!( + "hard link target {target:?}: too many hard links to follow (a cycle?)" + ))); + } + let (mut cur, rest) = match target.strip_prefix('/') { + Some(rest) => (0, rest), + None => (from, target), + }; + if target.is_empty() { + return Err(err("a hard link needs a target path".to_string())); + } + let comps: Vec<&str> = rest + .split('/') + .filter(|c| !c.is_empty() && *c != ".") + .collect(); + let mut obj = Obj::Group(cur); + for (i, c) in comps.iter().enumerate() { + let Obj::Group(g) = obj else { + return Err(err(format!( + "hard link target {target:?}: {:?} is not a group", + comps[..i].join("/") + ))); + }; + cur = g; + let grp = &self.groups[cur]; + let Some(&li) = grp.by_name.get(*c) else { + return Err(err(format!( + "hard link target {target:?} does not exist in the file" + ))); + }; + obj = match &grp.links[li].1 { + Target::Group(child) => Obj::Group(*child), + Target::Dataset(d) => Obj::Dataset(*d), + Target::Hard(p) => self.resolve(cur, p, depth + 1)?, + Target::Soft(_) | Target::External { .. } => { + return Err(err(format!( + "hard link target {target:?} goes through a soft or external \ + link ({:?}); name the object by its hard-link path", + join(&grp.path, c) + ))); + } + }; + } + Ok(obj) + } +} + +#[derive(Clone, Copy)] +enum Obj { + Group(usize), + Dataset(usize), +} + +/// Flatten the root group builder into a [`Tree`]. `default_track_order` +/// applies to every group that does not set its own. +pub(crate) fn build(root: GroupBuilder, default_track_order: bool) -> Result { + let mut b = Builder { + groups: Vec::new(), + datasets: Vec::new(), + }; + b.new_group("/".to_string()); + b.merge_into(0, root)?; + + // Resolve hard links and count references. + let mut group_refs = vec![0u32; b.groups.len()]; + let mut ds_refs = vec![0u32; b.datasets.len()]; + group_refs[0] = 1; // the superblock's reference to the root + let mut resolved: Vec>> = Vec::with_capacity(b.groups.len()); + for (gi, g) in b.groups.iter().enumerate() { + let mut row = Vec::with_capacity(g.links.len()); + for (_, t) in &g.links { + let obj = match t { + Target::Group(i) => Some(Obj::Group(*i)), + Target::Dataset(d) => Some(Obj::Dataset(*d)), + Target::Hard(p) => Some(b.resolve(gi, p, 0)?), + Target::Soft(_) | Target::External { .. } => None, + }; + match obj { + Some(Obj::Group(i)) => group_refs[i] += 1, + Some(Obj::Dataset(d)) => ds_refs[d] += 1, + None => {} + } + row.push(obj); + } + resolved.push(row); + } + + // The order each group's links are written in: creation order when + // tracked; otherwise datasets, then groups, then other links (the order + // earlier versions wrote, so one-level files keep their layout). + let tracked: Vec = b + .groups + .iter() + .map(|g| g.track_order.unwrap_or(default_track_order)) + .collect(); + let link_order: Vec> = b + .groups + .iter() + .enumerate() + .map(|(gi, g)| { + let mut idx: Vec = (0..g.links.len()).collect(); + if !tracked[gi] { + idx.sort_by_key(|&i| match g.links[i].1 { + Target::Dataset(_) => 0, + Target::Group(_) => 1, + _ => 2, + }); + } + idx + }) + .collect(); + + // Layout order: groups depth-first from the root, following the links + // that created them; datasets group by group in that order. + let mut group_order = Vec::with_capacity(b.groups.len()); + let mut stack = vec![0usize]; + while let Some(g) = stack.pop() { + group_order.push(g); + let children: Vec = link_order[g] + .iter() + .filter_map(|&i| match b.groups[g].links[i].1 { + Target::Group(c) => Some(c), + _ => None, + }) + .collect(); + stack.extend(children.into_iter().rev()); + } + let mut ds_order = Vec::with_capacity(b.datasets.len()); + for &g in &group_order { + for &i in &link_order[g] { + if let Target::Dataset(d) = b.groups[g].links[i].1 { + ds_order.push(d); + } + } + } + let mut group_pos = vec![0usize; b.groups.len()]; + for (pos, &g) in group_order.iter().enumerate() { + group_pos[g] = pos; + } + let mut ds_pos = vec![0usize; b.datasets.len()]; + for (pos, &d) in ds_order.iter().enumerate() { + ds_pos[d] = pos; + } + + let mut groups_by_id: Vec> = b.groups.into_iter().map(Some).collect(); + let mut groups = Vec::with_capacity(group_order.len()); + for &g in &group_order { + let bg = groups_by_id[g].take().expect("each group is laid out once"); + let mut targets: Vec> = bg.links.into_iter().map(Some).collect(); + let links = link_order[g] + .iter() + .map(|&i| { + let (name, t) = targets[i].take().expect("each link is written once"); + let to = match (resolved[g][i], t) { + (Some(Obj::Group(c)), _) => LinkTo::Group(group_pos[c]), + (Some(Obj::Dataset(d)), _) => LinkTo::Dataset(ds_pos[d]), + (None, Target::Soft(s)) => LinkTo::Soft(s), + (None, Target::External { file, path }) => LinkTo::External { file, path }, + (None, _) => unreachable!("hard links are resolved"), + }; + Link { + name, + to, + creation_order: tracked[g].then_some(i as u64), + } + }) + .collect(); + groups.push(Group { + attrs: bg.attrs, + links, + track_order: tracked[g], + refcount: group_refs[g], + }); + } + let mut ds_by_id: Vec> = b.datasets.into_iter().map(Some).collect(); + let datasets = ds_order + .iter() + .map(|&d| { + ( + ds_by_id[d].take().expect("each dataset is laid out once"), + ds_refs[d], + ) + }) + .collect(); + Ok(Tree { groups, datasets }) +} diff --git a/crates/clawhdf5-format/tests/writer_meta_tests.rs b/crates/clawhdf5-format/tests/writer_meta_tests.rs index a79e52d..3e907ab 100644 --- a/crates/clawhdf5-format/tests/writer_meta_tests.rs +++ b/crates/clawhdf5-format/tests/writer_meta_tests.rs @@ -528,33 +528,50 @@ fn h5py_reads_all_attributes_next_to_an_empty_string() { // ---- 6. path-like names ---- #[test] -fn slash_in_a_group_or_dataset_name_is_an_error() { - // Measured: create_group("a/b") wrote one link literally named "a/b", - // which h5py cannot reach ("component not found"). The writer has no - // nested groups, so such names are refused. +fn path_names_create_nested_groups() { + // create_group("a/b") used to write one link literally named "a/b", + // which h5py cannot reach ("component not found"); then such names were + // refused. Now a path creates its missing intermediate groups, as h5py + // does. let mut fw = FileWriter::new(); let mut g = fw.create_group("a/b"); g.create_dataset("c").with_f64_data(&[1.0]); fw.add_group(g.finish()); - assert!(fw.finish().is_err()); - - let mut fw = FileWriter::new(); - fw.create_dataset("x/y").with_f64_data(&[1.0]); - assert!(fw.finish().is_err()); - - let mut fw = FileWriter::new(); + fw.create_dataset("x/y").with_f64_data(&[2.0]); + fw.create_dataset("/a/b/z").with_f64_data(&[3.0]); let mut g = fw.create_group("g"); - g.create_dataset("x/y").with_f64_data(&[1.0]); + g.create_dataset("x/y").with_f64_data(&[4.0]); fw.add_group(g.finish()); - assert!(fw.finish().is_err()); + let bytes = fw.finish().unwrap(); + for path in ["a", "a/b", "a/b/c", "a/b/z", "x", "x/y", "g/x", "g/x/y"] { + header_at(&bytes, path); + } +} - for bad in ["", "."] { +#[test] +fn names_that_are_not_valid_link_names_are_errors() { + for bad in ["", ".", "a//b", "a/", "a/./b", "/"] { let mut fw = FileWriter::new(); fw.create_dataset(bad).with_f64_data(&[1.0]); assert!(fw.finish().is_err(), "{bad:?}"); } + // An absolute path inside a group, and a name used twice. + let mut fw = FileWriter::new(); + let mut g = fw.create_group("g"); + g.create_dataset("/x").with_f64_data(&[1.0]); + fw.add_group(g.finish()); + assert!(fw.finish().is_err()); + let mut fw = FileWriter::new(); + fw.create_dataset("x").with_f64_data(&[1.0]); + fw.create_dataset("x").with_f64_data(&[1.0]); + assert!(fw.finish().is_err()); + // A dataset in the way of a path. + let mut fw = FileWriter::new(); + fw.create_dataset("x").with_f64_data(&[1.0]); + fw.create_dataset("x/y").with_f64_data(&[1.0]); + assert!(fw.finish().is_err()); - // One level of groups still works, and '/' stays legal in attribute names. + // '/' stays legal in attribute names. let mut fw = FileWriter::new(); let mut g = fw.create_group("g"); g.create_dataset("c").with_f64_data(&[1.0]); diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 7d97cc5..519bc6d 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -773,3 +773,88 @@ fn every_subcommand_rejects_a_non_hdf5_file_cleanly() { assert_eq!(code(&h5rs(&["ls"])), 2); assert_eq!(code(&h5rs(&["--help"])), 0); } + +// --------------------------------------------------------------------------- +// files clawhdf5 writes: nested groups and links +// --------------------------------------------------------------------------- + +/// Nested groups (4 levels, by builders and by path names), soft, hard and +/// external links, creation-order tracking, and dense link and attribute +/// storage, as `FileBuilder` writes them. +fn write_nested_links(dir: &Path) -> Vec { + use clawhdf5::{AttrValue, FileBuilder}; + let mut b = FileBuilder::new(); + b.set_attr("title", AttrValue::String("links".into())); + b.create_dataset("x/y").with_f64_data(&[1.0, 2.0]); + b.create_dataset("a/b/c/d/leaf") + .with_i32_data(&[4, 5]) + .set_attr("depth", AttrValue::I64(5)); + b.add_soft_link("soft", "/x/y"); + b.add_soft_link("dangling", "/nowhere"); + b.add_hard_link("alias", "/x/y"); + b.add_external_link("ext", "other.h5", "/data"); + let mut g = b.create_group("a/b"); + g.set_attr("merged", AttrValue::I64(1)); + for i in 0..10 { + g.set_attr(&format!("attr{i}"), AttrValue::F64(i as f64)); + } + b.add_group(g.finish()); + let mut g = b.create_group("ordered"); + g.track_order(true); + for i in (0..40).rev() { + g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]); + } + g.add_hard_link("back", "/a/b/c"); + b.add_group(g.finish()); + let mut g = b.create_group("compact_ordered"); + g.track_order(true); + g.create_dataset("z").with_i32_data(&[1]); + g.create_dataset("a").with_i32_data(&[2]); + b.add_group(g.finish()); + let nested = dir.join("nested.h5"); + b.write(&nested).unwrap(); + + let mut b = FileBuilder::new(); + let mut g = b.create_group("many"); + for i in 0..10_000 { + g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]); + } + b.add_group(g.finish()); + let many = dir.join("many.h5"); + b.write(&many).unwrap(); + [nested, many] + .iter() + .map(|p| p.to_string_lossy().into_owned()) + .collect() +} + +#[test] +fn check_and_dump_files_with_nested_groups_and_links() { + let dir = tempfile::tempdir().unwrap(); + let files = write_nested_links(dir.path()); + // `--data` reads every dataset by path, and a lookup in a dense group + // scans all of its links: 10 000 datasets take minutes in a debug + // build, so the big file is checked structurally only (and not dumped). + for (p, data) in [(&files[0], true), (&files[1], false)] { + let args: &[&str] = if data { + &["check", "--data", p] + } else { + &["check", p] + }; + let o = h5rs(args); + assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o)); + assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o)); + } + if missing(tool_available("h5dump"), "h5dump") { + return; + } + for p in &files[..1] { + let name = Path::new(p).file_name().unwrap().to_string_lossy(); + let ours = h5rs(&["dump", p]); + assert!(ours.status.success(), "{p}: {ours:?}"); + let reference = run("h5dump", &[p]); + assert!(reference.status.success(), "h5dump {p}: {reference:?}"); + let r = stdout(&reference).replacen(p.as_str(), &name, 1); + assert_eq!(stdout(&ours), r, "{name}"); + } +} diff --git a/crates/clawhdf5/src/writer.rs b/crates/clawhdf5/src/writer.rs index 59904ba..bfa9393 100644 --- a/crates/clawhdf5/src/writer.rs +++ b/crates/clawhdf5/src/writer.rs @@ -42,14 +42,17 @@ impl FileBuilder { } } - /// Create a dataset at the root level. Returns a mutable reference to - /// a `DatasetBuilder` for configuring data, shape, and attributes. + /// Create a dataset. Returns a mutable reference to a `DatasetBuilder` + /// for configuring data, shape, and attributes. `name` may be a path + /// (`"a/b/x"`): missing intermediate groups are created, as in h5py. pub fn create_dataset(&mut self, name: &str) -> &mut FormatDatasetBuilder { self.writer.create_dataset(name) } /// Create a group builder. Call `.finish()` on the returned builder - /// to complete it, then pass to `add_group()`. + /// to complete it, then pass to `add_group()`. `name` may be a path; + /// groups nest to any depth (see `GroupBuilder::add_group`), and a group + /// added at a path that already holds a group is merged into it. pub fn create_group(&mut self, name: &str) -> FormatGroupBuilder { self.writer.create_group(name) } @@ -59,6 +62,40 @@ impl FileBuilder { self.writer.add_group(group); } + /// Add a soft link `name` to the path `target`, like h5py's + /// `f[name] = h5py.SoftLink(target)`. The target need not exist. + pub fn add_soft_link(&mut self, name: &str, target: &str) -> &mut Self { + self.writer.add_soft_link(name, target); + self + } + + /// Add another hard link `name` to the object at `target`, like h5py's + /// `f[name] = f[target]`. The target must be written in this file. + pub fn add_hard_link(&mut self, name: &str, target: &str) -> &mut Self { + self.writer.add_hard_link(name, target); + self + } + + /// Add an external link `name` to `target_path` in `target_file`. + pub fn add_external_link( + &mut self, + name: &str, + target_file: &str, + target_path: &str, + ) -> &mut Self { + self.writer + .add_external_link(name, target_file, target_path); + self + } + + /// Track link creation order in every group that does not set its own + /// (`GroupBuilder::track_order`), as h5py's `track_order=True`: libhdf5 + /// then lists members in the order they were added. + pub fn track_order(&mut self, track: bool) -> &mut Self { + self.writer.track_order(track); + self + } + /// Set an attribute on the root group. pub fn set_attr(&mut self, name: &str, value: AttrValue) { self.writer.set_root_attr(name, value); diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 9ea3053..1e8f879 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -6,7 +6,7 @@ use std::process::Command; -use clawhdf5::{File, FileBuilder}; +use clawhdf5::{AttrValue, File, FileBuilder, Group}; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) @@ -134,3 +134,461 @@ fn h5py_can_add_links_to_groups_we_wrote() { ); assert_eq!(f.dataset("alias").unwrap().read_f64().unwrap(), [1.0, 2.0]); } + +// ---- the whole tree, as h5py and as clawhdf5 read it ---- + +fn fmt_num(x: f64) -> String { + format!("{x:.6}") +} + +fn fmt_attr(v: &AttrValue) -> String { + let join = |v: Vec| v.join(","); + match v { + AttrValue::F64(x) => fmt_num(*x), + AttrValue::I64(x) => fmt_num(*x as f64), + AttrValue::U64(x) => fmt_num(*x as f64), + AttrValue::F64Array(a) => join(a.iter().map(|x| fmt_num(*x)).collect()), + AttrValue::I64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()), + AttrValue::U64Array(a) => join(a.iter().map(|x| fmt_num(*x as f64)).collect()), + AttrValue::String(s) => s.clone(), + AttrValue::StringArray(a) => a.join(","), + AttrValue::Raw { .. } => "raw".to_string(), + } +} + +fn fmt_attrs(attrs: std::collections::HashMap) -> String { + let mut v: Vec<_> = attrs.into_iter().collect(); + v.sort_by(|a, b| a.0.cmp(&b.0)); + v.iter() + .map(|(k, a)| format!("{k}={}", fmt_attr(a))) + .collect::>() + .join(";") +} + +fn child_path(path: &str, name: &str) -> String { + if path == "/" { + format!("/{name}") + } else { + format!("{path}/{name}") + } +} + +/// Every group and dataset reachable from `g` (following hard and soft +/// links; the tree must be acyclic), one line each: path, kind, attributes +/// and (datasets) values. +fn walk(g: &Group<'_>, path: &str, out: &mut Vec) { + out.push(format!("{path}|group|{}", fmt_attrs(g.attrs().unwrap()))); + let mut names: Vec<(String, bool)> = g + .datasets() + .unwrap() + .into_iter() + .map(|n| (n, false)) + .chain(g.groups().unwrap().into_iter().map(|n| (n, true))) + .collect(); + names.sort(); + for (name, is_group) in names { + let p = child_path(path, &name); + if is_group { + walk(&g.group(&name).unwrap(), &p, out); + } else { + let ds = g.dataset(&name).unwrap(); + let values: Vec = ds.read_f64().unwrap().into_iter().map(fmt_num).collect(); + out.push(format!( + "{p}|dataset|{}|{}", + fmt_attrs(ds.attrs().unwrap()), + values.join(",") + )); + } + } +} + +fn clawhdf5_tree(path: &str) -> String { + let f = File::open(path).unwrap(); + let mut out = Vec::new(); + walk(&f.root(), "/", &mut out); + out.join("\n") +} + +/// The same listing as [`walk`], from h5py. External and dangling soft +/// links are skipped, as clawhdf5's group listings skip them. +const H5PY_WALK: &str = r#" +def fmt(v): + if isinstance(v, bytes): return v.decode() + if isinstance(v, str): return v + a = np.asarray(v) + if a.dtype.kind in 'SUO': + return ','.join(x.decode() if isinstance(x, bytes) else str(x) for x in a.ravel()) + if a.ndim == 0: return '%.6f' % float(a) + return ','.join('%.6f' % float(x) for x in a.ravel()) +def attrs(o): return ';'.join(f'{k}={fmt(o.attrs[k])}' for k in sorted(o.attrs)) +out = [] +def walk(g, path): + out.append(f'{path}|group|{attrs(g)}') + for k in sorted(g.keys()): + if isinstance(g.get(k, getlink=True), h5py.ExternalLink): continue + o = g.get(k) + if o is None: continue + p = '/' + k if path == '/' else path + '/' + k + if isinstance(o, h5py.Group): walk(o, p) + else: + vals = ','.join('%.6f' % float(x) for x in np.asarray(o[()]).ravel()) + out.append(f'{p}|dataset|{attrs(o)}|{vals}') +with h5py.File(path, 'r') as f: + walk(f, '/') +print('\n'.join(out)) +"#; + +fn h5py_tree(path: &str) -> String { + h5py(path, H5PY_WALK) +} + +/// A four-level tree with attributes on every object: nested builders, +/// path names (with intermediate groups made on the way) and a group added +/// twice (merged), with dense attribute storage at one level and dense link +/// storage at another. +fn nested_builder() -> FileBuilder { + let mut b = FileBuilder::new(); + b.set_attr("title", AttrValue::String("nested".into())); + let mut l1 = b.create_group("l1"); + l1.set_attr("depth", AttrValue::I64(1)); + l1.create_dataset("d1") + .with_f64_data(&[1.0, 1.5]) + .set_attr("unit", AttrValue::String("m".into())); + let mut l2 = l1.create_group("l2"); + l2.set_attr("depth", AttrValue::I64(2)); + l2.create_dataset("d2").with_i32_data(&[2, 3, 4]); + let mut l3 = l2.create_group("l3"); + for i in 0..10 { + l3.set_attr(&format!("a{i}"), AttrValue::F64(i as f64 / 4.0)); // dense + } + for i in 0..12 { + l3.create_dataset(&format!("x{i:02}")) // dense links + .with_i64_data(&[i, -i]) + .set_attr("i", AttrValue::I64(i)); + } + let mut l4 = l3.create_group("l4"); + l4.set_attr("depth", AttrValue::I64(4)); + l4.create_dataset("leaf") + .with_f64_data(&[4.0, 4.25, 4.5]) + .set_attr( + "tags", + AttrValue::StringArray(vec!["a".into(), "bc".into()]), + ); + l3.add_group(l4.finish()); + l2.add_group(l3.finish()); + l1.add_group(l2.finish()); + b.add_group(l1.finish()); + // Path names: /p, /p/q and /p/q/r are made on the way to the dataset. + b.create_dataset("p/q/r/s") + .with_f64_data(&[7.0]) + .set_attr("deep", AttrValue::I64(4)); + // A group at an existing path is merged into it. + let mut pq = b.create_group("p/q"); + pq.set_attr("merged", AttrValue::I64(1)); + pq.create_dataset("t").with_i32_data(&[8]); + b.add_group(pq.finish()); + let mut l1b = b.create_group("l1/l2/l3/l4/l5"); + l1b.set_attr("depth", AttrValue::I64(5)); + b.add_group(l1b.finish()); + b +} + +const NESTED_TREE: &str = "\ +/|group|title=nested +/l1|group|depth=1.000000 +/l1/d1|dataset|unit=m|1.000000,1.500000 +/l1/l2|group|depth=2.000000 +/l1/l2/d2|dataset||2.000000,3.000000,4.000000"; + +#[test] +fn nested_groups_read_the_same_in_h5py_and_clawhdf5() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = write(&dir, "nested.h5", nested_builder()); + let ours = clawhdf5_tree(&path); + let theirs = h5py_tree(&path); + assert_eq!(ours, theirs); + assert!(ours.starts_with(NESTED_TREE), "{ours}"); + for line in [ + "/l1/l2/l3|group|a0=0.000000;a1=0.250000;a2=0.500000;a3=0.750000;a4=1.000000;\ + a5=1.250000;a6=1.500000;a7=1.750000;a8=2.000000;a9=2.250000", + "/l1/l2/l3/l4|group|depth=4.000000", + "/l1/l2/l3/l4/l5|group|depth=5.000000", + "/l1/l2/l3/l4/leaf|dataset|tags=a,bc|4.000000,4.250000,4.500000", + "/l1/l2/l3/x11|dataset|i=11.000000|11.000000,-11.000000", + "/p|group|", + "/p/q|group|merged=1.000000", + "/p/q/r/s|dataset|deep=4.000000|7.000000", + "/p/q/t|dataset||8.000000", + ] { + assert!( + ours.lines().any(|l| l == line), + "missing {line:?} in\n{ours}" + ); + } + assert_eq!(ours.lines().count(), 26, "{ours}"); + let dump = h5dump_ok(&path); + if !dump.is_empty() { + assert!(dump.contains("GROUP \"l5\""), "{dump}"); + assert!(dump.contains("DATASET \"leaf\""), "{dump}"); + } +} + +#[test] +fn soft_hard_and_external_links() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mut other = FileBuilder::new(); + other.create_dataset("data").with_i32_data(&[42, 43]); + write(&dir, "other.h5", other); + + let mut b = FileBuilder::new(); + b.create_dataset("x/y").with_f64_data(&[1.0, 2.0, 3.0]); + b.create_dataset("x/z").with_i32_data(&[9]); + b.add_soft_link("soft_abs", "/x/y"); + b.add_soft_link("dangling", "/nowhere"); + b.add_hard_link("alias", "/x/y"); + b.add_hard_link("x_again", "x"); + b.add_external_link("ext", "other.h5", "/data"); + let mut g = b.create_group("a/b/c"); + g.add_soft_link("rel", "sib"); // relative to /a/b/c + g.create_dataset("sib").with_i32_data(&[5]); + g.add_hard_link("deep_alias", "/x_again/z"); // through a hard link + g.add_soft_link("to_group", "/x"); + b.add_group(g.finish()); + let path = write(&dir, "links.h5", b); + + let out = h5py( + &path, + "import os\nos.chdir(os.path.dirname(path))\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 def kind(g, k):\n\ + \x20 l = g.get(k, getlink=True)\n\ + \x20 if isinstance(l, h5py.SoftLink): return 'soft:' + l.path\n\ + \x20 if isinstance(l, h5py.ExternalLink): return 'ext:' + l.filename + ':' + l.path\n\ + \x20 return 'hard'\n\ + \x20 print(json.dumps({\n\ + \x20 'root': {k: kind(f, k) for k in f},\n\ + \x20 'abc': {k: kind(f['a/b/c'], k) for k in f['a/b/c']},\n\ + \x20 'same': [f['alias'].id == f['x/y'].id, f['x_again'].id == f['x'].id,\n\ + \x20 f['a/b/c/deep_alias'].id == f['x/z'].id],\n\ + \x20 'rc': [h5py.h5o.get_info(f['x/y'].id).rc, h5py.h5o.get_info(f['x'].id).rc,\n\ + \x20 h5py.h5o.get_info(f['x/z'].id).rc, h5py.h5o.get_info(f['a'].id).rc],\n\ + \x20 'vals': [f['soft_abs'][()].tolist(), f['ext'][()].tolist(),\n\ + \x20 f['a/b/c/rel'][()].tolist(), sorted(f['a/b/c/to_group'])],\n\ + \x20 'dangling': f.get('dangling') is None,\n\ + \x20 }, sort_keys=True))", + ); + assert_eq!( + out, + r#"{"abc": {"deep_alias": "hard", "rel": "soft:sib", "sib": "hard", "to_group": "soft:/x"}, "dangling": true, "rc": [2, 2, 2, 1], "root": {"a": "hard", "alias": "hard", "dangling": "soft:/nowhere", "ext": "ext:other.h5:/data", "soft_abs": "soft:/x/y", "x": "hard", "x_again": "hard"}, "same": [true, true, true], "vals": [[1.0, 2.0, 3.0], [42, 43], [5], ["y", "z"]]}"# + ); + assert_eq!(clawhdf5_tree(&path), h5py_tree(&path)); + h5dump_ok(&path); + + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("alias").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + assert_eq!( + f.dataset("soft_abs").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + assert_eq!(f.dataset("a/b/c/rel").unwrap().read_i32().unwrap(), [5]); + assert_eq!( + f.dataset("a/b/c/deep_alias").unwrap().read_i32().unwrap(), + [9] + ); + assert_eq!( + f.dataset("x_again/y").unwrap().read_f64().unwrap(), + [1.0, 2.0, 3.0] + ); + drop(f); + + // The reference counts let libhdf5 delete one of two hard links and + // keep the object; with a count of 1 it would free an object still + // linked from elsewhere. + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 del f['alias']\n\ + \x20 del f['x_again']\n\ + \x20 f.create_dataset('filler', data=np.arange(1000))\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([f['x/y'][()].tolist(), sorted(f['x']), h5py.h5o.get_info(f['x/y'].id).rc]))", + ); + assert_eq!(out, r#"[[1.0, 2.0, 3.0], ["y", "z"], 1]"#); + h5dump_ok(&path); +} + +#[test] +fn a_hard_link_can_make_a_cycle() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + let mut g = b.create_group("g"); + g.create_dataset("v").with_i32_data(&[1]); + g.add_hard_link("up", "/"); + g.add_hard_link("me", "."); + b.add_group(g.finish()); + let path = write(&dir, "cycle.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([sorted(f['g/up/g']), f['g/up/g/me/me/v'][()].tolist(),\n\ + \x20 h5py.h5o.get_info(f.id).rc, h5py.h5o.get_info(f['g'].id).rc]))", + ); + assert_eq!(out, r#"[["me", "up", "v"], [1], 2, 2]"#); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("g/up/g/me/v").unwrap().read_i32().unwrap(), [1]); +} + +#[test] +fn bad_links_are_errors() { + for setup in [ + |b: &mut FileBuilder| { + b.add_hard_link("h", "/missing"); + }, + |b: &mut FileBuilder| { + b.create_dataset("x").with_i32_data(&[1]); + b.add_soft_link("s", "/x"); + b.add_hard_link("h", "/s"); // through a soft link + }, + |b: &mut FileBuilder| { + b.add_hard_link("h1", "/h2"); + b.add_hard_link("h2", "/h1"); + }, + |b: &mut FileBuilder| { + b.create_dataset("x").with_i32_data(&[1]); + b.add_hard_link("h", "/x/y"); // a dataset is not a group + }, + |b: &mut FileBuilder| { + b.add_soft_link("s", ""); + }, + |b: &mut FileBuilder| { + b.add_external_link("e", "", "/x"); + }, + |b: &mut FileBuilder| { + b.create_dataset("x").with_i32_data(&[1]); + b.add_soft_link("x", "/y"); // name taken + }, + ] { + let mut b = FileBuilder::new(); + setup(&mut b); + assert!(b.finish().is_err()); + } +} + +#[test] +fn ten_thousand_links_in_one_group() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + let mut g = b.create_group("many"); + for i in 0..10_000 { + g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]); + } + g.set_attr("n", AttrValue::I64(10_000)); + b.add_group(g.finish()); + // The same in creation order, added in reverse name order, with soft + // links among them. + let mut g = b.create_group("ordered"); + g.track_order(true); + for i in (0..10_000).rev() { + if i % 1000 == 0 { + g.add_soft_link(&format!("s{i:05}"), &format!("/many/d{i:05}")); + } + g.create_dataset(&format!("d{i:05}")).with_i32_data(&[i]); + } + b.add_group(g.finish()); + let path = write(&dir, "many.h5", b); + + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 m, o = f['many'], f['ordered']\n\ + \x20 names = list(m)\n\ + \x20 onames = list(o)\n\ + \x20 print(json.dumps([len(names), names == sorted(names), names[:2], int(m.attrs['n']),\n\ + \x20 [int(m['d%05d' % i][0]) for i in (0, 1, 4096, 9999)],\n\ + \x20 len(onames), onames[:3], onames[-2:], int(o['s05000'][0]),\n\ + \x20 o.id.get_create_plist().get_link_creation_order()]))", + ); + assert_eq!( + out, + r#"[10000, true, ["d00000", "d00001"], 10000, [0, 1, 4096, 9999], 10010, ["d09999", "d09998", "d09997"], ["s00000", "d00000"], 5000, 3]"# + ); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + let g = f.group("many").unwrap(); + assert_eq!(g.datasets().unwrap().len(), 10_000); + assert_eq!(g.dataset("d09999").unwrap().read_i32().unwrap(), [9999]); + assert_eq!( + f.dataset("ordered/s05000").unwrap().read_i32().unwrap(), + [5000] + ); +} + +#[test] +fn more_links_than_one_index_leaf_holds_is_an_error() { + let mut b = FileBuilder::new(); + for i in 0..70_000 { + b.add_soft_link(&format!("s{i}"), "/x"); + } + let err = b.finish().unwrap_err().to_string(); + assert!(err.contains("at most 65535 links"), "{err}"); +} + +#[test] +fn track_order_lists_members_in_creation_order() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let names = ["zeta", "alpha", "mid", "beta"]; + let mut b = FileBuilder::new(); + b.track_order(true); // the root and every group without its own setting + for n in names { + b.create_dataset(n).with_i32_data(&[1]); + } + let mut g = b.create_group("by_name"); + g.track_order(false); + for n in names { + g.create_dataset(n).with_i32_data(&[2]); + } + b.add_group(g.finish()); + let mut g = b.create_group("dense"); + for i in (0..20).rev() { + g.create_dataset(&format!("n{i:02}")).with_i32_data(&[i]); + } + g.add_soft_link("soft", "/zeta"); + b.add_group(g.finish()); + b.create_dataset("made/on/the/way").with_i32_data(&[3]); + let path = write(&dir, "order.h5", b); + + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([list(f), list(f['by_name']), list(f['dense'])[:3],\n\ + \x20 list(f['dense'])[-2:], list(f['made/on'])]))", + ); + assert_eq!( + out, + r#"[["zeta", "alpha", "mid", "beta", "by_name", "dense", "made"], ["alpha", "beta", "mid", "zeta"], ["n19", "n18", "n17"], ["n00", "soft"], ["the"]]"# + ); + h5dump_ok(&path); + assert_eq!(clawhdf5_tree(&path), h5py_tree(&path)); + + // libhdf5 keeps the order when it adds to (and converts) these groups. + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 f['aaa'] = np.arange(2)\n\ + \x20 f['dense']['aaa'] = np.arange(2)\n\ + \x20 del f['dense/n10']\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([list(f)[-1], list(f['dense'])[-2:], len(f['dense'])]))", + ); + assert_eq!(out, r#"["aaa", ["soft", "aaa"], 21]"#); + h5dump_ok(&path); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index bf8f77a..76fc86e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -202,8 +202,17 @@ fill-value item that did is fixed). and h5dump 1.14.6 rejects 9 of those (tank, 2026-09-26; 28 and 21 before these checks). - **Writer:** - - Nested groups beyond one level: path-like names are now refused, not - created. + - ~~Nested groups beyond one level: path-like names are now refused, not + created.~~ **Fixed 2026-09-26:** groups nest to any depth (path names + create intermediate groups, as h5py does), with soft, extra hard and + external links at any depth and optional creation-order tracking; + h5py, h5dump and `h5rs check --data` read them + (`crates/clawhdf5/tests/writer_groups_interop.rs`, + `crates/clawhdf5-tools/tests/h5rs_interop.rs`). Still missing: a group + with more than 65 535 links (its link index is one B-tree leaf) is an + error, and attribute creation order is not tracked. + - ~~libhdf5 could not add a link to a group we wrote (no Group Info + message).~~ **Fixed 2026-09-26.** - Dense attribute storage for attributes over 64 KiB. - Output that HDF5 1.8 can read. - A B-tree v2 chunk index larger than one leaf, so datasets with several -- 2.54.0 From bd36fe883bfa31478a872c27bbcd55c61f87a536 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:34:04 -0500 Subject: [PATCH 13/36] fix(format): flag non-ASCII link names as UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The writer marked every link name ASCII, so a name such as "größe" was stored as UTF-8 bytes under the ASCII character set (h5py reports cset 0 for it). Names that are not plain ASCII now carry the UTF-8 flag, as h5py writes them; ASCII names are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 +++ crates/clawhdf5-format/src/file_writer.rs | 12 +++++++++++- crates/clawhdf5/tests/writer_groups_interop.rs | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb939de..3092ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ `h5rs check` passes and `h5rs dump` equals h5dump (`crates/clawhdf5/tests/writer_groups_interop.rs`, `crates/clawhdf5-tools/tests/h5rs_interop.rs`). +- **Non-ASCII link names were marked ASCII.** A group or dataset name such as + `größe` was written with the ASCII character set flag (h5py reported + `cset` 0 for it); it is now flagged UTF-8, as h5py writes it. - **libhdf5 could not add links to groups we wrote.** h5py in `"r+"` mode failed with "Unable to create link (message type not found)" on every group `FileWriter` wrote: libhdf5 reads a group's Group Info message before diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index b4a90aa..2d318f5 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -220,6 +220,16 @@ fn add_refcount(w: &mut ObjectHeaderWriter, refcount: u32) { } } +/// The character set a link name is written with: UTF-8 when it is not +/// plain ASCII, as h5py writes it. +fn name_charset(name: &str) -> CharacterSet { + if name.is_ascii() { + CharacterSet::Ascii + } else { + CharacterSet::Utf8 + } +} + /// The Link message for `link`, whose group and dataset targets are at the /// given addresses (indexed as in the writer tree). fn link_message(link: &writer_tree::Link, group_addrs: &[u64], ds_addrs: &[u64]) -> LinkMessage { @@ -242,7 +252,7 @@ fn link_message(link: &writer_tree::Link, group_addrs: &[u64], ds_addrs: &[u64]) name: link.name.clone(), link_target, creation_order: link.creation_order, - charset: CharacterSet::Ascii, + charset: name_charset(&link.name), } } diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 1e8f879..fb7aa23 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -592,3 +592,21 @@ fn track_order_lists_members_in_creation_order() { assert_eq!(out, r#"["aaa", ["soft", "aaa"], 21]"#); h5dump_ok(&path); } + +#[test] +fn non_ascii_names_are_utf8() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.create_dataset("größe/wert").with_i32_data(&[1]); + let path = write(&dir, "utf8.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 l = f.id.links.get_info('größe'.encode())\n\ + \x20 print(json.dumps([list(f), list(f['größe']), l.cset], ensure_ascii=False))", + ); + assert_eq!(out, r#"[["größe"], ["wert"], 1]"#); + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("größe/wert").unwrap().read_i32().unwrap(), [1]); +} -- 2.54.0 From b0a1e4f9a61f3b0e1a925f76a31a55f9173a7278 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:35:16 -0500 Subject: [PATCH 14/36] fix(format): a group attribute set again replaces the earlier value Setting a group or root attribute twice wrote two attribute messages with the same name, and h5py read back the first value: set_attr("w", 1) then set_attr("w", "two") read as 1. The later value now replaces the earlier one, as `attrs[name] = v` does in h5py, including when a group is merged from two builders. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 ++++ crates/clawhdf5-format/src/writer_tree.rs | 12 ++++---- .../clawhdf5/tests/writer_groups_interop.rs | 28 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3092ae5..080045d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,11 @@ `h5rs check` passes and `h5rs dump` equals h5dump (`crates/clawhdf5/tests/writer_groups_interop.rs`, `crates/clawhdf5-tools/tests/h5rs_interop.rs`). +- **A group attribute set twice read back as its first value.** Setting a + group (or root) attribute again wrote a second attribute message with the + same name, and h5py returned the first value. The later value now replaces + the earlier one, as `attrs[name] = v` does in h5py — also when a group is + merged from two builders. - **Non-ASCII link names were marked ASCII.** A group or dataset name such as `größe` was written with the ASCII character set flag (h5py reported `cset` 0 for it); it is now flagged UTF-8, as h5py writes it. diff --git a/crates/clawhdf5-format/src/writer_tree.rs b/crates/clawhdf5-format/src/writer_tree.rs index 284867e..724ba70 100644 --- a/crates/clawhdf5-format/src/writer_tree.rs +++ b/crates/clawhdf5-format/src/writer_tree.rs @@ -229,14 +229,14 @@ impl Builder { /// Merge a builder's attributes, setting and items into group `idx`. fn merge_into(&mut self, idx: usize, gb: GroupBuilder) -> Result<(), FormatError> { + // An attribute set again (by this builder or a merged one) takes the + // new value, as assigning `attrs[name]` in h5py does. for (name, value) in gb.attrs { - if self.groups[idx].attrs.iter().any(|(n, _)| *n == name) { - return Err(err(format!( - "attribute {name:?} set twice on {}", - self.groups[idx].path - ))); + let attrs = &mut self.groups[idx].attrs; + match attrs.iter_mut().find(|(n, _)| *n == name) { + Some(slot) => slot.1 = value, + None => attrs.push((name, value)), } - self.groups[idx].attrs.push((name, value)); } if let Some(t) = gb.track_order { match self.groups[idx].track_order { diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index fb7aa23..ba2d121 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -610,3 +610,31 @@ fn non_ascii_names_are_utf8() { let f = File::open(&path).unwrap(); assert_eq!(f.dataset("größe/wert").unwrap().read_i32().unwrap(), [1]); } + +#[test] +fn a_group_attribute_set_again_takes_the_new_value() { + skip_if_no_python!(); + // Setting a group attribute twice wrote two attribute messages with one + // name. Now the later value replaces the earlier, as `attrs[name] = v` + // does in h5py — also across a group merged from two builders. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.set_attr("v", AttrValue::I64(1)); + b.set_attr("v", AttrValue::I64(2)); + let mut g = b.create_group("g"); + g.set_attr("w", AttrValue::I64(1)); + b.add_group(g.finish()); + let mut g = b.create_group("g"); + g.set_attr("w", AttrValue::String("two".into())); + b.add_group(g.finish()); + let path = write(&dir, "attrs.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([list(f.attrs), int(f.attrs['v']), list(f['g'].attrs),\n\ + \x20 f['g'].attrs['w'].decode()]))", + ); + assert_eq!(out, r#"[["v"], 2, ["w"], "two"]"#); + let f = File::open(&path).unwrap(); + assert!(matches!(f.root().attrs().unwrap()["v"], AttrValue::I64(2))); +} -- 2.54.0 From a5e41c1a535481c195e9c991a0a93925de2c9768 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:40:10 -0500 Subject: [PATCH 15/36] fix(read): decode on the calling thread when rayon's pool has one thread Full reads of chunked datasets handed their chunks to rayon. With a one-thread pool (concurrent_read --decode-threads 1, RAYON_NUM_THREADS=1) every thread reading through a File queued behind that single worker, so 16 readers decoded on one core: per-thread CPU time showed one thread doing all the decoding and the readers almost none, and full reads stopped at about 2x one thread. The cached full-read path and the uncached reader behind verify_provenance now decode inline when the pool cannot parallelise (parallel_read::pool_can_parallelise). The File's chunk cache was the suspect but not the cause: datasets over its budget were already read without inserting, and skipping its lookups gained only a few percent at 16 threads. The regression test keeps a one-thread global pool's worker busy and requires a full read and verify_provenance to finish anyway; before the fix both waited for the worker (timed out). Co-Authored-By: Claude Opus 5.5 (1M context) --- BENCHMARKS.md | 6 +- CHANGELOG.md | 13 +++ crates/clawhdf5-format/src/chunked_read.rs | 18 ++-- crates/clawhdf5-format/src/parallel_read.rs | 14 +++ .../tests/single_thread_decode_pool.rs | 90 +++++++++++++++++++ docs/known-issues.md | 25 ++++-- 6 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 crates/clawhdf5/tests/single_thread_decode_pool.rs diff --git a/BENCHMARKS.md b/BENCHMARKS.md index c105698..5d55961 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -528,7 +528,11 @@ What this shows: (about 880 MB/s) while h5py processes reach 4424 MB/s. Hyperslab reads, which bypass the `File`'s chunk cache, keep scaling, so the cache (one mutex and one 16 MiB budget per `File`, thrashed by 64 MiB - datasets) is the suspect. + datasets) is the suspect. **Fixed after these measurements + (2026-09-26); the table above predates the fix and has not been + re-measured.** The cause was not the cache: with `--decode-threads 1` + every full read queued its chunks for the pool's single rayon worker; + see `docs/known-issues.md`. - *Contiguous reads are slow*: 2.5 GB/s for a single-threaded full read against h5py's 9.8 GB/s (0.25x), and 0.12x for 256 x 256 hyperslabs. Threads close the gap (about 1.0x h5py at 16), but single-thread diff --git a/CHANGELOG.md b/CHANGELOG.md index ef4bd2f..91d56f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +### Concurrent reads (2026-09-26) +- **Full reads of chunked datasets scale with threads again when rayon's + pool has one thread.** Each full read handed its chunks to rayon to + decode; with a one-thread pool (`RAYON_NUM_THREADS=1`, or + `concurrent_read --decode-threads 1`) every thread reading through a + `File` queued behind that single worker, so N readers decoded on one core + and throughput stopped at about 2x one thread. Such reads, and + `verify_provenance`'s uncached reader, now decode on the calling thread + (`clawhdf5_format::parallel_read::pool_can_parallelise`). The `File`'s + chunk cache, the suspect in `docs/known-issues.md`, was not the cause: + reads of datasets larger than its budget already skipped inserting, and + its lookups cost a few percent at 16 threads. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index cbbaa69..9169ab7 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -40,6 +40,7 @@ fn decompress_all_chunks( { if let Some(pl) = pipeline && parallel_read::should_use_parallel(chunks.len()) + && parallel_read::pool_can_parallelise() { // Seed from the first chunk's address and count for determinism. let seed = chunks.first().map(|c| c.address).unwrap_or(0) ^ (chunks.len() as u64); @@ -1056,7 +1057,9 @@ pub fn read_chunked_data_cached( // Decompress what the cache didn't have, a bounded batch at a time — in // parallel with the `parallel` feature (this path, the one the facade - // uses, was sequential; only the uncached reader was parallel). Chunks are + // uses, was sequential; only the uncached reader was parallel), unless the + // pool has one thread: then every reading thread would queue behind that + // one worker, so each decodes its own chunks instead. Chunks are // cached only when the whole dataset fits: pushing a larger dataset // through the cache just evicts each chunk moments after inserting it. let cache_them = total_bytes <= cache.max_bytes(); @@ -1073,12 +1076,13 @@ pub fn read_chunked_data_cached( }; for batch in misses.chunks(DECODE_BATCH) { #[cfg(feature = "parallel")] - let decoded: Vec, FormatError>> = if batch.len() >= 4 { - use rayon::prelude::*; - batch.par_iter().map(decode).collect() - } else { - batch.iter().map(decode).collect() - }; + let decoded: Vec, FormatError>> = + if batch.len() >= 4 && parallel_read::pool_can_parallelise() { + use rayon::prelude::*; + batch.par_iter().map(decode).collect() + } else { + batch.iter().map(decode).collect() + }; #[cfg(not(feature = "parallel"))] let decoded: Vec, FormatError>> = batch.iter().map(decode).collect(); diff --git a/crates/clawhdf5-format/src/parallel_read.rs b/crates/clawhdf5-format/src/parallel_read.rs index 6593132..1b940c6 100644 --- a/crates/clawhdf5-format/src/parallel_read.rs +++ b/crates/clawhdf5-format/src/parallel_read.rs @@ -27,6 +27,20 @@ pub fn should_use_parallel(chunk_count: usize) -> bool { chunk_count > PARALLEL_THRESHOLD } +/// Whether handing a read's chunks to rayon can decode them faster than the +/// calling thread would alone. +/// +/// `false` when the pool the work would go to (the current pool inside a +/// rayon worker, else the global one) has a single thread. Handing work to +/// that pool is then worse than useless: the caller blocks while the one +/// worker decodes, and every other thread reading at the same time queues +/// behind the same worker, so N reader threads decode on one core. (That is +/// how full reads with `--decode-threads 1` stopped scaling at about 2x in +/// the `concurrent_read` benchmark.) +pub fn pool_can_parallelise() -> bool { + rayon::current_num_threads() > 1 +} + /// Decompress chunks in parallel using lane-partitioned assignment. /// /// Instead of naive `par_iter`, chunks are deterministically assigned to lanes diff --git a/crates/clawhdf5/tests/single_thread_decode_pool.rs b/crates/clawhdf5/tests/single_thread_decode_pool.rs new file mode 100644 index 0000000..455ed90 --- /dev/null +++ b/crates/clawhdf5/tests/single_thread_decode_pool.rs @@ -0,0 +1,90 @@ +//! With a one-thread rayon pool, full reads of chunked datasets must decode +//! on the calling thread. +//! +//! Handing a read's chunks to a one-worker pool made every reading thread +//! queue behind that single worker: N threads reading through one `File` +//! decoded on one core, and full reads stopped scaling at about 2x in the +//! `concurrent_read` benchmark with `--decode-threads 1` (see +//! `docs/known-issues.md`). The test makes that queueing observable: it keeps +//! the pool's only worker busy and requires reads to finish anyway. +//! +//! One test in its own binary: it configures the process-wide rayon pool. + +#![cfg(feature = "parallel")] + +use std::sync::mpsc; +use std::time::Duration; + +use clawhdf5::{File, FileBuilder}; + +const N: usize = 4096; // 64 chunks of 64 elements + +fn values() -> Vec { + (0..N).map(|i| i as f64 * 0.5).collect() +} + +fn build() -> File { + let mut b = FileBuilder::new(); + b.create_dataset("data") + .with_f64_data(&values()) + .with_shape(&[N as u64]) + .with_chunks(&[64]) + .with_deflate(1) + .with_provenance("test-suite", "2026-09-26T00:00:00Z", None); + File::from_bytes(b.finish().unwrap()).unwrap() +} + +/// Run `f` on a fresh thread; `None` if it has not finished within `limit`. +fn finishes_within( + limit: Duration, + f: impl FnOnce() -> T + Send + 'static, +) -> Option { + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(limit).ok() +} + +#[test] +fn full_reads_do_not_wait_for_a_busy_one_thread_pool() { + rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build_global() + .expect("this test binary configures the global pool first"); + + // Built first: the writer compresses on the pool too. + let file = std::sync::Arc::new(build()); + let file2 = std::sync::Arc::clone(&file); + + // Occupy the pool's only worker until the reads are done. + let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + rayon::spawn(move || { + started_tx.send(()).unwrap(); + let _ = release_rx.recv(); + }); + started_rx.recv().unwrap(); + + let limit = Duration::from_secs(20); + // Cached full read (the path `read_*` uses), then the uncached reader + // behind `verify_provenance`. + let read = finishes_within(limit, move || { + file.dataset("data").unwrap().read_f64().unwrap() + }); + let verified = finishes_within(limit, move || { + file2.dataset("data").unwrap().verify_provenance().unwrap() + }); + // Free the worker before asserting, so a failure does not hang the + // blocked reader threads forever. + release_tx.send(()).unwrap(); + + assert_eq!( + read.expect("a full read waited for the busy one-thread rayon pool"), + values() + ); + assert_eq!( + verified.expect("verify_provenance waited for the busy one-thread rayon pool"), + clawhdf5::provenance::VerifyResult::Ok + ); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index bf8f77a..a08e914 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -9,12 +9,25 @@ deleting it. ## Concurrent and contiguous read performance (measured 2026-09-26) -**Status:** open. Measured on tank with `concurrent_read` against h5py -3.16 / HDF5 2.0 (`BENCHMARKS.md`, "Concurrent reads"): -- Full reads of chunked datasets from several threads through one `File` - stop scaling at about 4 threads (880 MB/s on deflate data vs 4424 MB/s - for 16 h5py processes). Hyperslab reads, which skip the chunk cache, - scale to 1244 MB/s, so the `File`'s shared chunk cache is the suspect. +**Status:** first bullet fixed (2026-09-26), second open. Measured on +tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, +"Concurrent reads"): +- **Fixed 2026-09-26.** Full reads of chunked datasets from several threads + through one `File` stop scaling at about 4 threads (880 MB/s on deflate + data vs 4424 MB/s for 16 h5py processes). Hyperslab reads, which skip the + chunk cache, scale to 1244 MB/s, so the `File`'s shared chunk cache is the + suspect. *Cause:* not the cache. Those numbers were taken with + `--decode-threads 1`, a one-thread rayon pool, and every full read handed + its chunks to that pool, so all reader threads queued behind its single + worker (per-thread CPU time: one thread did all the decoding, the 16 + readers almost none). Hyperslab reads touch one chunk each and never used + the pool. Reads now decode on the calling thread when the pool has one + thread (`tests/single_thread_decode_pool.rs`). Datasets larger than the + cache's budget were already read without inserting into it, and skipping + its lookups entirely gained only a few percent at 16 threads. Remaining + per-read overhead, not yet addressed: each full `read_f32` of a chunked + dataset faults in about three times its size in fresh pages (the output, + the `f32` copy of it, and a new buffer per decoded chunk). - Contiguous datasets read 4x slower than h5py on one thread (2.5 vs 9.8 GB/s full, 0.12x for 256 x 256 hyperslabs). Values are correct; this is speed only. -- 2.54.0 From 37770f594a2943338df7a7ad0070b28b7e100e21 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:48:05 -0500 Subject: [PATCH 16/36] docs: the rayon fix covers a one-thread pool, not the h5py-process gap The review measured the default pool unchanged (about 2900 MB/s at 16 threads before and after) and still short of 16 h5py processes; small pools still make outside readers wait. Say so instead of marking the scaling issue fixed. Co-Authored-By: Claude Opus 5.5 (1M context) --- BENCHMARKS.md | 10 +++++----- CHANGELOG.md | 3 ++- docs/known-issues.md | 10 +++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 5d55961..c4e1705 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -528,11 +528,11 @@ What this shows: (about 880 MB/s) while h5py processes reach 4424 MB/s. Hyperslab reads, which bypass the `File`'s chunk cache, keep scaling, so the cache (one mutex and one 16 MiB budget per `File`, thrashed by 64 MiB - datasets) is the suspect. **Fixed after these measurements - (2026-09-26); the table above predates the fix and has not been - re-measured.** The cause was not the cache: with `--decode-threads 1` - every full read queued its chunks for the pool's single rayon worker; - see `docs/known-issues.md`. + datasets) is the suspect. The cause of the `--decode-threads 1` + ceiling was not the cache: every full read queued its chunks for the + pool's single rayon worker. That case was fixed after these + measurements (2026-09-26, not yet re-measured here). With the default + pool the gap to h5py processes remains (see `docs/known-issues.md`). - *Contiguous reads are slow*: 2.5 GB/s for a single-threaded full read against h5py's 9.8 GB/s (0.25x), and 0.12x for 256 x 256 hyperslabs. Threads close the gap (about 1.0x h5py at 16), but single-thread diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d56f1..9030c3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ (`clawhdf5_format::parallel_read::pool_can_parallelise`). The `File`'s chunk cache, the suspect in `docs/known-issues.md`, was not the cause: reads of datasets larger than its budget already skipped inserting, and - its lookups cost a few percent at 16 threads. + its lookups cost a few percent at 16 threads. Throughput with the default + pool is unchanged, and still short of an h5py process pool. ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files diff --git a/docs/known-issues.md b/docs/known-issues.md index a08e914..f8d7798 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -9,10 +9,10 @@ deleting it. ## Concurrent and contiguous read performance (measured 2026-09-26) -**Status:** first bullet fixed (2026-09-26), second open. Measured on +**Status:** open; one cause of the first bullet fixed (2026-09-26). Measured on tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, "Concurrent reads"): -- **Fixed 2026-09-26.** Full reads of chunked datasets from several threads +- **Partly fixed 2026-09-26.** Full reads of chunked datasets from several threads through one `File` stop scaling at about 4 threads (880 MB/s on deflate data vs 4424 MB/s for 16 h5py processes). Hyperslab reads, which skip the chunk cache, scale to 1244 MB/s, so the `File`'s shared chunk cache is the @@ -22,7 +22,11 @@ tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, worker (per-thread CPU time: one thread did all the decoding, the 16 readers almost none). Hyperslab reads touch one chunk each and never used the pool. Reads now decode on the calling thread when the pool has one - thread (`tests/single_thread_decode_pool.rs`). Datasets larger than the + thread (`tests/single_thread_decode_pool.rs`). **Still open:** this + fixes only a one-thread pool. With the default pool, 16 reader threads + ran at about 2900 MB/s before and after the change, still short of 16 + h5py processes (4424 MB/s); with a small pool (2-4 threads) readers + outside it still wait on its workers. Datasets larger than the cache's budget were already read without inserting into it, and skipping its lookups entirely gained only a few percent at 16 threads. Remaining per-read overhead, not yet addressed: each full `read_f32` of a chunked -- 2.54.0 From 3bcd443e63362a4be91cd3eb8576bf88842c530b Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:49:54 -0500 Subject: [PATCH 17/36] fix(format): selections of v4 implicit-index chunked data no longer panic read_raw_data_selection's chunked fallback (taken when partial_read declines, e.g. a bounding box over half the dataset) handed the layout's chunk dimensions, element-size dimension included, to generate_implicit_chunks, which indexed past the dataset rank. It then decoded the whole dataset regardless, so the enumeration is gone: the arm decodes and extracts for every chunk index. The new test reads small and large hyperslabs of all five v4 indexes written by h5py and compares with h5py's values; it panicked before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 + crates/clawhdf5-format/src/chunked_read.rs | 4 + crates/clawhdf5-format/src/data_read.rs | 82 +----- .../tests/v4_chunk_index_selection.rs | 236 ++++++++++++++++++ 4 files changed, 260 insertions(+), 75 deletions(-) create mode 100644 crates/clawhdf5/tests/v4_chunk_index_selection.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b873d19..d01a200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,19 @@ ## Unreleased ### Python bindings (2026-09-26) +- **Panic: selections of v4 implicit-index chunked datasets** (pre-existing, + facade `Dataset::read_selection`, Rust callers too). A hyperslab whose + bounding box covered more than half of a chunked dataset with the implicit + index (`libver='latest'`, early allocation, no filters) panicked with + "index out of bounds" in `generate_implicit_chunks`: the fallback in + `data_read::read_raw_data_selection` passed the layout's chunk dimensions, + element-size dimension included, and then decoded the whole dataset + anyway. That arm now decodes and extracts directly, for every chunk index. + `crates/clawhdf5/tests/v4_chunk_index_selection.rs` reads small and large + hyperslabs of all five v4 indexes (single chunk, implicit, fixed array, + extensible array, B-tree v2) and compares them with h5py; it panicked + before. The Python bindings made this easy to reach (`ds[0:3]` on + libhdf5's `h5fc_ext*.h5` test files). - **`pip install` / `maturin develop` now gives `import clawhdf5`.** The distribution in `crates/clawhdf5-py/pyproject.toml` was still called `rustyhdf5` while the extension module was `clawhdf5`, and the package's diff --git a/crates/clawhdf5-format/src/chunked_read.rs b/crates/clawhdf5-format/src/chunked_read.rs index cbbaa69..4af42aa 100644 --- a/crates/clawhdf5-format/src/chunked_read.rs +++ b/crates/clawhdf5-format/src/chunked_read.rs @@ -510,6 +510,10 @@ fn collect_chunk_info_inner( /// /// Chunks are stored contiguously starting at `base_address`. No stored index; /// addresses are computed from the chunk position. +/// +/// `chunk_dimensions` are the spatial chunk dimensions, one per entry of +/// `dataset_dims` — not the layout message's list, which carries the element +/// size as an extra last dimension. pub fn generate_implicit_chunks( base_address: u64, dataset_dims: &[u64], diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index b8f2ffe..d10f381 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -356,85 +356,17 @@ pub fn read_raw_data_selection( } DataLayout::Chunked { chunk_dimensions, - btree_address, version, - chunk_index_type, .. } => { + // `partial_read` declined (a bounding box covering most of the + // dataset, or a selection it doesn't box), so decode every chunk + // and pick the selection out, whatever the chunk index. This arm + // used to enumerate the chunks first — passing the layout's + // chunk dimensions, element-size dimension included, to the + // implicit-index generator, which then indexed past the rank and + // panicked — only to decode the full dataset anyway. crate::chunked_read::chunk_geometry(chunk_dimensions, *version, dataspace, elem_size)?; - // For chunked data, only read chunks that intersect the selection - let chunk_dims: Vec = chunk_dimensions.iter().map(|&d| d as u64).collect(); - let rank = dims.len(); - - // Collect chunk info from B-tree - let chunks = if *version == 4 { - match chunk_index_type { - Some(2) => { - // Implicit index - crate::chunked_read::generate_implicit_chunks( - btree_address.unwrap_or(0), - dims, - chunk_dimensions, - elem_size as u32, - ) - } - _ => { - if let Some(_addr) = btree_address { - // Use extensible array or fixed array - // Fall back to full read for complex v4 index types - let full_data = read_raw_data_full( - file_data, - layout, - dataspace, - datatype, - pipeline, - offset_size, - length_size, - )?; - return extract_selection_from_buffer( - &full_data, dims, elem_size, selection, - ); - } else { - return Ok(Vec::new()); - } - } - } - } else { - // v3: B-tree v1 - if let Some(addr) = btree_address { - crate::chunked_read::collect_chunk_info_checked( - file_data, - *addr, - chunk_dimensions, - offset_size, - length_size, - )? - } else { - return Ok(Vec::new()); - } - }; - - // Filter chunks to only those that intersect the selection - let intersecting: Vec<_> = chunks - .iter() - .filter(|ci| { - let offsets: Vec = ci.offsets.iter().take(rank).copied().collect(); - selection.intersects_chunk(&offsets, &chunk_dims[..rank]) - }) - .collect(); - - if intersecting.is_empty() { - return Ok(Vec::new()); - } - - // Decompress only the intersecting chunks - let _chunk_total_bytes: usize = - chunk_dims.iter().map(|&d| d as usize).product::() * elem_size; - let _element_size_u32 = elem_size as u32; - - // First, assemble only the intersecting chunks into a partial buffer, - // then extract the selection. For simplicity, we assemble into a full - // dataset buffer and extract (same as contiguous path). let full_data = read_raw_data_full( file_data, layout, diff --git a/crates/clawhdf5/tests/v4_chunk_index_selection.rs b/crates/clawhdf5/tests/v4_chunk_index_selection.rs new file mode 100644 index 0000000..97cc7c9 --- /dev/null +++ b/crates/clawhdf5/tests/v4_chunk_index_selection.rs @@ -0,0 +1,236 @@ +//! Partial hyperslab reads of every v4 (`libver='latest'`) chunk index type, +//! compared element for element with h5py. +//! +//! `Dataset::read_selection` takes two routes: `partial_read` materialises +//! the selection's bounding box when it covers at most half the dataset, and +//! `data_read::read_raw_data_selection` handles the rest. The second route +//! once passed the layout's full chunk dimensions (which carry the element +//! size as an extra, last dimension) to the implicit-index chunk generator, +//! which then indexed past the dataset's rank and panicked. So every case +//! below reads both a small window and one covering most of the dataset. +//! +//! Each case asserts which chunk index the file actually uses (parsed from +//! the layout message), so a change in how h5py lays the file out can't turn +//! this into a test of the wrong index. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::selection::Selection; +use clawhdf5_format::superblock::Superblock; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// The v4 chunk index type recorded in `name`'s layout message +/// (1 single chunk, 2 implicit, 3 fixed array, 4 extensible array, 5 B-tree v2). +fn chunk_index_type(path: &std::path::Path, name: &str) -> u8 { + let data = std::fs::read(path).unwrap(); + let sb = Superblock::parse(&data, 0).unwrap(); + let addr = clawhdf5_format::group_v2::resolve_path_any(&data, &sb, name).unwrap(); + let hdr = ObjectHeader::parse(&data, addr as usize, sb.offset_size, sb.length_size).unwrap(); + let msg = hdr + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + .expect("layout message"); + match DataLayout::parse(&msg.data, sb.offset_size, sb.length_size).unwrap() { + DataLayout::Chunked { + version: 4, + chunk_index_type: Some(t), + .. + } => t, + other => panic!("{name}: expected a v4 chunked layout, got {other:?}"), + } +} + +struct Case { + name: &'static str, + /// Python keyword arguments to `create_dataset` besides `data`. + kwargs: &'static str, + index_type: u8, +} + +const SHAPE: [u64; 2] = [37, 23]; + +const CASES: &[Case] = &[ + Case { + name: "implicit", + // Early allocation, no filters, fixed maximum: the implicit index. + kwargs: "chunks=(5, 4), dcpl=early()", + index_type: 2, + }, + Case { + name: "fixed_array", + kwargs: "chunks=(5, 4), compression='gzip'", + index_type: 3, + }, + Case { + name: "extensible_array", + kwargs: "chunks=(5, 4), maxshape=(None, 23), compression='gzip'", + index_type: 4, + }, + Case { + name: "btree2", + kwargs: "chunks=(5, 4), maxshape=(None, None), compression='gzip'", + index_type: 5, + }, + Case { + name: "single_chunk", + kwargs: "chunks=(37, 23), compression='gzip'", + index_type: 1, + }, + Case { + name: "single_chunk_unfiltered", + kwargs: "chunks=(37, 23)", + index_type: 1, + }, +]; + +/// `(start, stride, count, block)` per dimension; the first few stay below +/// half the dataset (bounding-box path), the rest exceed it (full path). +fn selections() -> Vec<([u64; 2], [u64; 2], [u64; 2], [u64; 2])> { + vec![ + ([0, 0], [1, 1], [3, 23], [1, 1]), // ds[0:3] + ([7, 3], [1, 1], [9, 6], [1, 1]), // interior window across chunks + ([36, 22], [1, 1], [1, 1], [1, 1]), // last element (edge chunk) + ([2, 1], [3, 4], [4, 3], [1, 1]), // strided, small + ([0, 0], [1, 1], [30, 23], [1, 1]), // most rows + ([1, 0], [2, 1], [18, 23], [1, 1]), // every other row, spanning all + ([0, 2], [1, 1], [37, 20], [1, 1]), // columns 2..22 of every row + ([3, 1], [5, 3], [7, 7], [2, 2]), // strided blocks over everything + ] +} + +fn py_slice(start: u64, stride: u64, count: u64, block: u64) -> String { + // Each case is expressible as a numpy index when block == 1; with a block + // the selected indices are listed explicitly. + let idx: Vec = (0..count) + .flat_map(|c| (0..block).map(move |b| start + c * stride + b)) + .map(|i| i.to_string()) + .collect(); + format!("[{}]", idx.join(",")) +} + +#[test] +fn partial_hyperslabs_of_every_v4_chunk_index_match_h5py() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("v4_index_selection.h5"); + let path_str = path.display().to_string(); + + // h5py writes the file, then reads every selection back and prints the + // values, one line per (case, selection). + let mut script = format!( + "import h5py, numpy as np\n\ + def early():\n\ + \x20 p = h5py.h5p.create(h5py.h5p.DATASET_CREATE)\n\ + \x20 p.set_alloc_time(h5py.h5d.ALLOC_TIME_EARLY)\n\ + \x20 return p\n\ + data = (np.arange({n}, dtype=' = expected + .next() + .expect("h5py printed too few lines") + .split_whitespace() + .map(|v| v.parse().unwrap()) + .collect(); + let raw = ds + .read_selection(&sel) + .unwrap_or_else(|e| panic!("{}: read_selection {sel:?} failed: {e}", case.name)); + let got: Vec = raw + .chunks_exact(4) + .map(|b| i32::from_le_bytes(b.try_into().unwrap())) + .collect(); + assert_eq!(got, want, "{}: selection {sel:?}", case.name); + assert_eq!( + ds.read_i32_selection(&sel).unwrap(), + want, + "{}: read_i32_selection {sel:?}", + case.name + ); + } + } + assert!(expected.next().is_none(), "h5py printed extra lines"); +} -- 2.54.0 From 24412a0e5995e7124f905ccc091fad073c4f4996 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:52:55 -0500 Subject: [PATCH 18/36] fix(py): a panic in the library raises clawhdf5.InternalError, not PanicException MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PanicException derives from BaseException, so `except Exception` let a library bug through. Every call from the bindings into the library now runs under catch_unwind and a panic becomes InternalError (RuntimeError) naming the object. Tests: a hidden hook panics inside the guard; and the v4 chunk indexes are compared with h5py from Python — with the library fix reverted, ds[0:30] of the implicit-index dataset now raises InternalError instead of aborting the test run. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 + crates/clawhdf5-py/src/attrs.rs | 80 +++++----- crates/clawhdf5-py/src/dataset.rs | 141 ++++++++++-------- crates/clawhdf5-py/src/file.rs | 46 +++--- crates/clawhdf5-py/src/group.rs | 22 +-- crates/clawhdf5-py/src/lib.rs | 38 +++++ crates/clawhdf5-py/src/node.rs | 88 ++++++----- crates/clawhdf5-py/tests/test_read_vs_h5py.py | 44 ++++++ 8 files changed, 289 insertions(+), 176 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01a200..d679fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,12 @@ h5py 3.16 (HDF5 2.0) on a file h5py writes. One difference is h5py's: it returns variable-length sequences of big-endian floats unswapped; this package returns the stored values. +- **A panic in the library is an ordinary Python exception.** PyO3 turns a + Rust panic into `PanicException`, a `BaseException` that `except + Exception` does not catch. Every call from the bindings into the library + is now guarded and a panic becomes `clawhdf5.InternalError` (a + `RuntimeError`) naming the object; with the implicit-index panic above + restored, `ds[0:30]` raises it. - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/crates/clawhdf5-py/src/attrs.rs b/crates/clawhdf5-py/src/attrs.rs index ff608ed..d1b06cb 100644 --- a/crates/clawhdf5-py/src/attrs.rs +++ b/crates/clawhdf5-py/src/attrs.rs @@ -191,46 +191,48 @@ fn attr_to_py<'py>( file: &clawhdf5_rs::File, attr: &AttributeMessage, ) -> PyResult> { - let sb = file.superblock(); - let conv = Converter::new(py, &attr.datatype, sb.offset_size) - .map_err(|e| prefix_err(py, &attr.name, e))?; - if node::is_null(&attr.dataspace) { - return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any()); - } - let shape: Vec = attr - .dataspace - .dimensions - .iter() - .map(|&d| d as usize) - .collect(); - let n: usize = shape.iter().product(); - let data = if conv.is_vl() { - let want = n * conv.elem_size; - if attr.raw_data.len() < want { - return Err(PyValueError::new_err(format!( - "attribute {}: {} bytes of variable-length references, expected {want}", - attr.name, - attr.raw_data.len(), - ))); + crate::no_panic(|| { + let sb = file.superblock(); + let conv = Converter::new(py, &attr.datatype, sb.offset_size) + .map_err(|e| prefix_err(py, &attr.name, e))?; + if node::is_null(&attr.dataspace) { + return Ok(PyEmpty::new(conv.dtype).into_pyobject(py)?.into_any()); } - let raw = &attr.raw_data[..want]; - let file_data = file.as_bytes(); - let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit); - Elements::Vl( - py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit)) - .map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?, - ) - } else { - Elements::Bytes(attr.raw_data.clone()) - }; - let arr = conv - .to_array(py, data, &shape, true) - .map_err(|e| prefix_err(py, &attr.name, e))?; - if shape.is_empty() { - // A scalar dataspace: h5py returns the element itself. - return arr.get_item(PyTuple::empty(py)); - } - Ok(arr) + let shape: Vec = attr + .dataspace + .dimensions + .iter() + .map(|&d| d as usize) + .collect(); + let n: usize = shape.iter().product(); + let data = if conv.is_vl() { + let want = n * conv.elem_size; + if attr.raw_data.len() < want { + return Err(PyValueError::new_err(format!( + "attribute {}: {} bytes of variable-length references, expected {want}", + attr.name, + attr.raw_data.len(), + ))); + } + let raw = &attr.raw_data[..want]; + let file_data = file.as_bytes(); + let (osz, lsz, unit) = (sb.offset_size, sb.length_size, conv.vl_unit); + Elements::Vl( + py.detach(|| resolve_vl(file_data, raw, n, osz, lsz, unit)) + .map_err(|e| PyValueError::new_err(format!("attribute {}: {e}", attr.name)))?, + ) + } else { + Elements::Bytes(attr.raw_data.clone()) + }; + let arr = conv + .to_array(py, data, &shape, true) + .map_err(|e| prefix_err(py, &attr.name, e))?; + if shape.is_empty() { + // A scalar dataspace: h5py returns the element itself. + return arr.get_item(PyTuple::empty(py)); + } + Ok(arr) + }) } fn prefix_err(py: Python<'_>, name: &str, e: PyErr) -> PyErr { diff --git a/crates/clawhdf5-py/src/dataset.rs b/crates/clawhdf5-py/src/dataset.rs index cbfc68d..9109a32 100644 --- a/crates/clawhdf5-py/src/dataset.rs +++ b/crates/clawhdf5-py/src/dataset.rs @@ -42,25 +42,27 @@ impl PyDataset { file: Arc, path: String, ) -> PyResult { - let hdr = node::header(&file, &path)?; - let null = node::is_null(&node::dataspace(&file, &hdr)?); - let (shape, datatype) = { - let ds = file.dataset(&path).map_err(to_py_err)?; - let shape = if null { - None - } else { - Some(ds.shape().map_err(to_py_err)?) + crate::no_panic(|| { + let hdr = node::header(&file, &path)?; + let null = node::is_null(&node::dataspace(&file, &hdr)?); + let (shape, datatype) = { + let ds = file.dataset(&path).map_err(to_py_err)?; + let shape = if null { + None + } else { + Some(ds.shape().map_err(to_py_err)?) + }; + (shape, ds.raw_datatype().map_err(to_py_err)?) }; - (shape, ds.raw_datatype().map_err(to_py_err)?) - }; - let conv = Converter::new(py, &datatype, file.superblock().offset_size) - .map_err(|e| e.value(py).to_string()); - Ok(Self { - file, - path, - shape, - datatype, - conv, + let conv = Converter::new(py, &datatype, file.superblock().offset_size) + .map_err(|e| e.value(py).to_string()); + Ok(Self { + file, + path, + shape, + datatype, + conv, + }) }) } @@ -84,39 +86,43 @@ impl PyDataset { let path = self.path.as_str(); let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit); // Everything below touches only Rust data: release the GIL. - let blocks: Vec<(Elements, Vec)> = py - .detach(|| -> Result<_, ReadError> { - let ds = file.dataset(path)?; - let sb = file.superblock(); - let mut blocks = Vec::with_capacity(reads.len()); - for (sel, shape) in reads { - let raw = ds.read_selection(&sel)?; - let n: usize = shape.iter().product(); - let data = if vl { - if raw.len() != n * elem_size { - return Err(ReadError::Other(format!( - "read {} bytes of variable-length references, expected {}", - raw.len(), - n * elem_size - ))); - } - Elements::Vl( - resolve_vl( - file.as_bytes(), - &raw, - n, - sb.offset_size, - sb.length_size, - unit, - ) - .map_err(ReadError::Other)?, + let read = || -> Result<_, ReadError> { + let ds = file.dataset(path)?; + let sb = file.superblock(); + let mut blocks = Vec::with_capacity(reads.len()); + for (sel, shape) in reads { + let raw = ds.read_selection(&sel)?; + let n: usize = shape.iter().product(); + let data = if vl { + if raw.len() != n * elem_size { + return Err(ReadError::Other(format!( + "read {} bytes of variable-length references, expected {}", + raw.len(), + n * elem_size + ))); + } + Elements::Vl( + resolve_vl( + file.as_bytes(), + &raw, + n, + sb.offset_size, + sb.length_size, + unit, ) - } else { - Elements::Bytes(raw) - }; - blocks.push((data, shape)); - } - Ok(blocks) + .map_err(ReadError::Other)?, + ) + } else { + Elements::Bytes(raw) + }; + blocks.push((data, shape)); + } + Ok(blocks) + }; + let blocks: Vec<(Elements, Vec)> = py + .detach(|| { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(read)) + .unwrap_or_else(|p| Err(ReadError::Panic(crate::panic_text(&*p)))) }) .map_err(|e| e.into_py(&self.path))?; @@ -160,6 +166,7 @@ impl PyDataset { enum ReadError { Lib(clawhdf5_rs::Error), Other(String), + Panic(String), } impl From for ReadError { @@ -173,6 +180,10 @@ impl ReadError { match self { ReadError::Lib(e) => to_py_err(e), ReadError::Other(msg) => PyValueError::new_err(format!("{}: {msg}", node::name(path))), + ReadError::Panic(msg) => crate::InternalError::new_err(format!( + "{}: clawhdf5 internal error (please report it): {msg}", + node::name(path) + )), } } } @@ -223,20 +234,22 @@ impl PyDataset { /// The maximum shape (`None` per unlimited dimension), like h5py. #[getter] fn maxshape<'py>(&self, py: Python<'py>) -> PyResult> { - let Some(shape) = &self.shape else { - return Ok(py.None().into_bound(py)); - }; - let max = self - .file - .dataset(&self.path) - .and_then(|ds| ds.max_dimensions()) - .map_err(to_py_err)? - .unwrap_or_else(|| shape.clone()); - let items: Vec> = max - .into_iter() - .map(|d| (d != u64::MAX).then_some(d)) - .collect(); - Ok(PyTuple::new(py, items)?.into_any()) + crate::no_panic(|| { + let Some(shape) = &self.shape else { + return Ok(py.None().into_bound(py)); + }; + let max = self + .file + .dataset(&self.path) + .and_then(|ds| ds.max_dimensions()) + .map_err(to_py_err)? + .unwrap_or_else(|| shape.clone()); + let items: Vec> = max + .into_iter() + .map(|d| (d != u64::MAX).then_some(d)) + .collect(); + Ok(PyTuple::new(py, items)?.into_any()) + }) } /// The dataset's numpy dtype, as h5py reports it. diff --git a/crates/clawhdf5-py/src/file.rs b/crates/clawhdf5-py/src/file.rs index 4f6bb7a..a2c48ea 100644 --- a/crates/clawhdf5-py/src/file.rs +++ b/crates/clawhdf5-py/src/file.rs @@ -57,9 +57,9 @@ impl PyFile { let filename = path.to_string(); match mode { "r" => { - let file = py - .detach(|| clawhdf5_rs::File::open(path)) - .map_err(to_py_err)?; + let file = py.detach(|| { + crate::no_panic(|| clawhdf5_rs::File::open(path).map_err(to_py_err)) + })?; Ok(Self { inner: Some(FileInner::Read(Arc::new(file))), filename, @@ -277,29 +277,31 @@ fn parse_compression( /// Build and write the HDF5 file from accumulated write state. fn finalize_write(state: WriteState) -> PyResult<()> { - let mut builder = clawhdf5_rs::FileBuilder::new(); + crate::no_panic(|| { + let mut builder = clawhdf5_rs::FileBuilder::new(); - // Root attributes - let root_attrs = state.root_attrs.lock().unwrap_or_else(|e| e.into_inner()); - for (name, val) in root_attrs.iter() { - builder.set_attr(name, val.clone().into()); - } - drop(root_attrs); + // Root attributes + let root_attrs = state.root_attrs.lock().unwrap_or_else(|e| e.into_inner()); + for (name, val) in root_attrs.iter() { + builder.set_attr(name, val.clone().into()); + } + drop(root_attrs); - // Root datasets - for spec in &state.root_datasets { - let db = builder.create_dataset(&spec.name); - apply_dataset_spec(db, spec); - } + // Root datasets + for spec in &state.root_datasets { + let db = builder.create_dataset(&spec.name); + apply_dataset_spec(db, spec); + } - // Groups - for group_arc in &state.groups { - let guard = group_arc.lock().unwrap(); - finalize_write_group(&mut builder, &guard); - } + // Groups + for group_arc in &state.groups { + let guard = group_arc.lock().unwrap(); + finalize_write_group(&mut builder, &guard); + } - builder.write(&state.path).map_err(to_py_err)?; - Ok(()) + builder.write(&state.path).map_err(to_py_err)?; + Ok(()) + }) } #[cfg(test)] diff --git a/crates/clawhdf5-py/src/group.rs b/crates/clawhdf5-py/src/group.rs index c97cc17..365846b 100644 --- a/crates/clawhdf5-py/src/group.rs +++ b/crates/clawhdf5-py/src/group.rs @@ -73,16 +73,18 @@ pub(crate) fn get_item( /// Names of the group's datasets and subgroups, sorted (h5py's order). pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult> { - let group = if path.is_empty() { - file.root() - } else { - file.group(path).map_err(to_py_err)? - }; - let mut names = group.datasets().map_err(to_py_err)?; - names.extend(group.groups().map_err(to_py_err)?); - names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); - names.dedup(); - Ok(names) + crate::no_panic(|| { + let group = if path.is_empty() { + file.root() + } else { + file.group(path).map_err(to_py_err)? + }; + let mut names = group.datasets().map_err(to_py_err)?; + names.extend(group.groups().map_err(to_py_err)?); + names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); + names.dedup(); + Ok(names) + }) } pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool { diff --git a/crates/clawhdf5-py/src/lib.rs b/crates/clawhdf5-py/src/lib.rs index 4c58cd6..e3b5619 100644 --- a/crates/clawhdf5-py/src/lib.rs +++ b/crates/clawhdf5-py/src/lib.rs @@ -24,6 +24,42 @@ pub(crate) use dataset::PyDataset; pub(crate) use file::PyFile; pub(crate) use group::PyGroup; +pyo3::create_exception!( + clawhdf5, + InternalError, + pyo3::exceptions::PyRuntimeError, + "A bug in clawhdf5 met while reading or writing a file (a Rust panic, \ + caught). Derived from RuntimeError, so `except Exception` handles it." +); + +/// The text of a caught panic. +pub(crate) fn panic_text(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()) +} + +/// Run `f`, turning a panic in the library into [`InternalError`] instead of +/// PyO3's `PanicException` (a `BaseException`, which `except Exception` +/// does not catch). Wraps every call into the library. +pub(crate) fn no_panic(f: impl FnOnce() -> PyResult) -> PyResult { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).unwrap_or_else(|p| { + Err(InternalError::new_err(format!( + "clawhdf5 internal error (please report it): {}", + panic_text(&*p) + ))) + }) +} + +/// A test hook: panics inside [`no_panic`], so the tests can check that a +/// library panic reaches Python as an ordinary exception. +#[pyfunction] +fn _panic_for_test() -> PyResult<()> { + no_panic(|| panic!("deliberate panic for the test suite")) +} + /// Convert a `clawhdf5_rs::Error` into a `PyErr`. /// /// Maps different error variants to more specific Python exception types: @@ -276,6 +312,8 @@ fn clawhdf5(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add("InternalError", m.py().get_type::())?; + m.add_function(wrap_pyfunction!(_panic_for_test, m)?)?; Ok(()) } diff --git a/crates/clawhdf5-py/src/node.rs b/crates/clawhdf5-py/src/node.rs index 1c68101..0482603 100644 --- a/crates/clawhdf5-py/src/node.rs +++ b/crates/clawhdf5-py/src/node.rs @@ -35,22 +35,24 @@ pub(crate) fn name(path: &str) -> String { /// The object header of the object at `path`. pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult { - let sb = file.superblock(); - let data = file.as_bytes(); - let addr = if path.is_empty() { - sb.root_group_address - } else { - clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| { - PyKeyError::new_err(format!( - "Unable to open object (object '{}' doesn't exist): {e}", - name(path) - )) - })? - }; - let addr = usize::try_from(addr) - .map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?; - ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size) - .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path)))) + crate::no_panic(|| { + let sb = file.superblock(); + let data = file.as_bytes(); + let addr = if path.is_empty() { + sb.root_group_address + } else { + clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| { + PyKeyError::new_err(format!( + "Unable to open object (object '{}' doesn't exist): {e}", + name(path) + )) + })? + }; + let addr = usize::try_from(addr) + .map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?; + ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size) + .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path)))) + }) } /// What an object header describes. @@ -115,20 +117,22 @@ pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool { /// The dataspace message of an object header. pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult { - let sb = file.superblock(); - let msg = hdr - .messages - .iter() - .find(|m| m.msg_type == MessageType::Dataspace) - .ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?; - let data = clawhdf5_format::shared_message::message_data( - file.as_bytes(), - msg, - sb.offset_size, - sb.length_size, - ) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string())) + crate::no_panic(|| { + let sb = file.superblock(); + let msg = hdr + .messages + .iter() + .find(|m| m.msg_type == MessageType::Dataspace) + .ok_or_else(|| PyValueError::new_err("object has no dataspace message"))?; + let data = clawhdf5_format::shared_message::message_data( + file.as_bytes(), + msg, + sb.offset_size, + sb.length_size, + ) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + Dataspace::parse(&data, sb.length_size).map_err(|e| PyValueError::new_err(e.to_string())) + }) } pub(crate) fn is_null(space: &Dataspace) -> bool { @@ -139,17 +143,19 @@ pub(crate) fn is_null(space: &Dataspace) -> bool { /// Attributes whose messages cannot be parsed are left out, as the facade's /// `attrs()` does. pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult> { - let hdr = header(file, path)?; - let sb = file.superblock(); - let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant( - file.as_bytes(), - &hdr, - sb.offset_size, - sb.length_size, - ) - .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?; - attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes())); - Ok(attrs) + crate::no_panic(|| { + let hdr = header(file, path)?; + let sb = file.superblock(); + let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant( + file.as_bytes(), + &hdr, + sb.offset_size, + sb.length_size, + ) + .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path))))?; + attrs.sort_by(|a, b| a.name.as_bytes().cmp(b.name.as_bytes())); + Ok(attrs) + }) } #[cfg(test)] diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py index 9a9178d..66fc643 100644 --- a/crates/clawhdf5-py/tests/test_read_vs_h5py.py +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -467,3 +467,47 @@ def test_threads_read_the_same_file(pair): with ThreadPoolExecutor(8) as pool: list(pool.map(work, range(8))) assert not errors, errors[:3] + + +def _v4_index_fixture(h5py, path): + """One 2-D dataset per v4 chunk index (HDF5 1.10+ layout, libver='latest').""" + data = (np.arange(37 * 23, dtype=" Date: Sat, 26 Sep 2026 08:54:04 -0500 Subject: [PATCH 19/36] fix(py): index lists of padded compounds no longer return uninitialised padding np.concatenate copies structured dtypes field by field into np.empty, so the padding of ds[[0, 3, 6]] held process memory. The runs' bytes are joined in Rust, whole elements at a time, before anything becomes numpy: the padding is the file's bytes (h5py's) and the result is still a view of the Rust buffer. The h5py comparisons now compare every byte of structured values; the new test failed on the padding before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++ crates/clawhdf5-py/src/dataset.rs | 88 ++++++++----------- crates/clawhdf5-py/src/select.rs | 47 +++++++++- crates/clawhdf5-py/tests/test_read_vs_h5py.py | 24 ++++- 4 files changed, 116 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d679fbf..f19acc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,15 @@ is now guarded and a panic becomes `clawhdf5.InternalError` (a `RuntimeError`) naming the object; with the implicit-index panic above restored, `ds[0:30]` raises it. +- **Wrong data: uninitialised padding in compound results of index lists.** + `ds[[0, 3, 6]]` joined one read per run with `np.concatenate`, which + copies structured dtypes field by field into an `np.empty` result, so the + padding bytes held whatever was in memory (pointers were seen) and leaked + through `tobytes()`, hashes and write-backs. The runs' bytes are now joined + in Rust, whole elements at a time, so the result carries the bytes read + from the file (h5py's, zero for files it wrote) and stays zero-copy. + The h5py comparisons now also compare every byte of structured values + (`test_compound_padding_bytes_match_h5py` and `assert_same`). - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/crates/clawhdf5-py/src/dataset.rs b/crates/clawhdf5-py/src/dataset.rs index 9109a32..36467d8 100644 --- a/crates/clawhdf5-py/src/dataset.rs +++ b/crates/clawhdf5-py/src/dataset.rs @@ -82,70 +82,60 @@ impl PyDataset { conv.empty(py, &out_shape)? } else { let (reads, list_axis) = plan.reads(dims); + let read_shape = plan.read_shape(); let file = &*self.file; let path = self.path.as_str(); let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit); // Everything below touches only Rust data: release the GIL. - let read = || -> Result<_, ReadError> { + let read = || -> Result { let ds = file.dataset(path)?; - let sb = file.superblock(); let mut blocks = Vec::with_capacity(reads.len()); for (sel, shape) in reads { let raw = ds.read_selection(&sel)?; - let n: usize = shape.iter().product(); - let data = if vl { - if raw.len() != n * elem_size { - return Err(ReadError::Other(format!( - "read {} bytes of variable-length references, expected {}", - raw.len(), - n * elem_size - ))); - } - Elements::Vl( - resolve_vl( - file.as_bytes(), - &raw, - n, - sb.offset_size, - sb.length_size, - unit, - ) - .map_err(ReadError::Other)?, - ) - } else { - Elements::Bytes(raw) - }; - blocks.push((data, shape)); + let want = shape.iter().product::() * elem_size; + if raw.len() != want { + return Err(ReadError::Other(format!( + "read {} bytes, expected {want}", + raw.len() + ))); + } + blocks.push((raw, shape)); } - Ok(blocks) + // Several blocks only for a list index: join their bytes + // (every byte of every element, padding included) along + // that axis before anything becomes numpy. + let raw = match (blocks.len(), list_axis) { + (1, _) => blocks.pop().expect("one block").0, + (_, Some(axis)) => select::join_along(&blocks, axis, elem_size), + _ => { + return Err(ReadError::Other( + "several reads without an index list".into(), + )); + } + }; + if !vl { + return Ok(Elements::Bytes(raw)); + } + let sb = file.superblock(); + let n = read_shape.iter().product(); + resolve_vl( + file.as_bytes(), + &raw, + n, + sb.offset_size, + sb.length_size, + unit, + ) + .map(Elements::Vl) + .map_err(ReadError::Other) }; - let blocks: Vec<(Elements, Vec)> = py + let data = py .detach(|| { std::panic::catch_unwind(std::panic::AssertUnwindSafe(read)) .unwrap_or_else(|p| Err(ReadError::Panic(crate::panic_text(&*p)))) }) .map_err(|e| e.into_py(&self.path))?; - - let mut arrays = Vec::with_capacity(blocks.len()); - for (data, shape) in blocks { - arrays.push(conv.to_array(py, data, &shape, false)?); - } - let joined = if arrays.len() == 1 { - arrays.pop().expect("one block") - } else { - let axis = list_axis.expect("several reads only for a list index"); - // Name the dtype: left to itself numpy canonicalises a - // structured dtype here (drops padding, native byte order). - let kwargs = pyo3::types::PyDict::new(py); - kwargs.set_item("axis", axis)?; - kwargs.set_item("dtype", arrays[0].getattr("dtype")?)?; - kwargs.set_item("casting", "no")?; - py.import("numpy")?.call_method( - "concatenate", - (PyList::new(py, arrays)?,), - Some(&kwargs), - )? - }; + let joined = conv.to_array(py, data, &read_shape, false)?; // Drop the axes indexed by an integer (length 1 in the blocks). let mut shape = out_shape.clone(); if let crate::convert::Layout::Subarray(sub) = &conv.layout { diff --git a/crates/clawhdf5-py/src/select.rs b/crates/clawhdf5-py/src/select.rs index 1b1c796..dac62ab 100644 --- a/crates/clawhdf5-py/src/select.rs +++ b/crates/clawhdf5-py/src/select.rs @@ -56,6 +56,12 @@ impl Plan { .collect() } + /// The shape of the result before the integer-indexed axes are dropped + /// (they have length 1 here): the shape of the joined reads. + pub fn read_shape(&self) -> Vec { + self.axes.iter().map(|a| a.len() as usize).collect() + } + /// Whether the selection is empty. pub fn is_empty(&self) -> bool { self.axes.iter().any(|a| a.len() == 0) @@ -64,7 +70,7 @@ impl Plan { /// The hyperslab reads that make up this selection, each with the shape /// of its block (index axes kept at length 1). More than one only when an /// axis is indexed by a list: one read per run of consecutive indices, - /// concatenated along `list_axis` afterwards. + /// joined along `list_axis` afterwards (`join_along`). pub fn reads(&self, dims: &[u64]) -> (Vec<(Selection, Vec)>, Option) { let list_axis = self.axes.iter().position(|a| matches!(a, Axis::List(_))); let runs: Vec<(u64, u64)> = match list_axis.map(|i| &self.axes[i]) { @@ -119,6 +125,33 @@ fn consecutive_runs(idx: &[u64]) -> Vec<(u64, u64)> { runs } +/// Join row-major blocks of `elem_size`-byte elements whose shapes differ +/// only along `axis` into one buffer, in order along that axis. Whole +/// elements are copied, so compound padding keeps the bytes that were read. +pub(crate) fn join_along( + blocks: &[(Vec, Vec)], + axis: usize, + elem_size: usize, +) -> Vec { + let Some((_, first)) = blocks.first() else { + return Vec::new(); + }; + let outer: usize = first[..axis].iter().product(); + let inner: usize = first[axis + 1..].iter().product::() * elem_size; + let total: usize = blocks.iter().map(|(_, s)| s[axis]).sum(); + let mut out = vec![0u8; outer * total * inner]; + let mut at = 0; + for (bytes, shape) in blocks { + let len = shape[axis] * inner; + for o in 0..outer { + let dst = (o * total) * inner + at; + out[dst..dst + len].copy_from_slice(&bytes[o * len..(o + 1) * len]); + } + at += len; + } + out +} + /// Parse `key` for a dataset of shape `dims`. pub(crate) fn parse(key: &Bound<'_, PyAny>, dims: &[u64]) -> PyResult { let items: Vec> = match key.cast::() { @@ -311,6 +344,18 @@ mod tests { assert_eq!(consecutive_runs(&[]), vec![]); } + #[test] + fn blocks_join_along_the_list_axis() { + // Two 2x1 and 2x2 blocks of 1-byte elements, joined along axis 1. + let a = (vec![1, 2], vec![2, 1]); + let b = (vec![3, 4, 5, 6], vec![2, 2]); + assert_eq!(join_along(&[a, b], 1, 1), vec![1, 3, 4, 2, 5, 6]); + // Along axis 0 it is concatenation; 2-byte elements stay whole. + let a = (vec![1, 2, 3, 4], vec![1, 2]); + let b = (vec![5, 6, 7, 8], vec![1, 2]); + assert_eq!(join_along(&[a, b], 0, 2), vec![1, 2, 3, 4, 5, 6, 7, 8]); + } + #[test] fn full_selection_reads_everything() { let plan = Plan { diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py index 66fc643..c5876d2 100644 --- a/crates/clawhdf5-py/tests/test_read_vs_h5py.py +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -212,14 +212,19 @@ def assert_same(ours, theirs, what=""): if theirs.dtype == object: for a, b in zip(ours.ravel(), theirs.ravel()): assert_same(a, b, what) - elif theirs.dtype.names is None and theirs.dtype.kind == "V": - assert ours.tobytes() == theirs.tobytes(), what + elif theirs.dtype.kind == "V": + # Structured and opaque: every byte, padding included (h5py's + # padding is zero; uninitialised memory there would leak). + if theirs.dtype.names is not None: + np.testing.assert_array_equal(ours, theirs, err_msg=what) + assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ" else: np.testing.assert_array_equal(ours, theirs, err_msg=what) elif isinstance(theirs, np.generic): assert ours.dtype == theirs.dtype, what if theirs.dtype.names is not None: np.testing.assert_array_equal(np.asarray(ours), np.asarray(theirs), err_msg=what) + assert ours.tobytes() == theirs.tobytes(), f"{what}: bytes differ" else: assert ours == theirs or (ours != ours and theirs != theirs), what else: @@ -511,3 +516,18 @@ def test_a_library_panic_is_an_ordinary_exception(): clawhdf5._panic_for_test() except Exception: # noqa: BLE001 - the point of the test pass + + +def test_compound_padding_bytes_match_h5py(pair): + """Every byte of a padded compound, padding included, is h5py's, for + index lists with many runs as well as slices. Joining the runs with + np.concatenate left the padding uninitialised: process memory ended up + in tobytes().""" + ours, theirs, _ = pair + keys = [[0, 3, 6], [1, 2, 5, 9], [0, 2, 4, 6, 8], slice(None), slice(1, 9, 3), 4, [9]] + for name in ["cmp/padded", "cmp/padded_chunked"]: + for _ in range(20): # garbage varies between runs; zeros do not + for key in keys: + assert ours[name][key].tobytes() == theirs[name][key].tobytes(), f"{name}[{key!r}]" + for key in [(slice(None), [0, 2]), ([0, 2, 3], slice(None)), ([1, 3], 1)]: + assert ours["cmp/nested_2d"][key].tobytes() == theirs["cmp/nested_2d"][key].tobytes(), key -- 2.54.0 From 41b7837d0acac060b8fe1f484c9474eae2551901 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:56:10 -0500 Subject: [PATCH 20/36] fix(format): bound what a VL read retains on a crafted global heap VlResolver kept an owned copy of every object of every heap collection it parsed, for the whole read. Collections nested inside each other's object data, 32 bytes apart with each element pointing at a different one, made retained memory O(elements x file size): 1.58 GB for a 744 KB file (read_vl_strings did the same before VlResolver). Chaining every collection's objects into one shared run of tiny objects made parse time O(elements x objects) as well. libhdf5 refuses these files. - The cache records where each object lies (GlobalHeapCollection:: parse_index, new) instead of copying it, and is dropped past a 32 MiB budget. - A collection overlapping one already read is an error: libhdf5 gives every collection its own block, so only a crafted file has them. - parse and parse_index refuse a collection that runs past the end of the file and an object that runs past the end of its collection. tests/vl_heap_bounds.rs measures peak heap use with a counting allocator: 129 MB and 350 MB live before on its two crafted files (64 KB and 176 KB), 97 KB and 0.9 MB now. Conformance unchanged at 575 of 697. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 + crates/clawhdf5-format/src/global_heap.rs | 108 +++++++-- crates/clawhdf5-format/src/vl_data.rs | 207 ++++++++++++++---- .../clawhdf5-format/tests/vl_heap_bounds.rs | 161 ++++++++++++++ docs/known-issues.md | 26 +++ 5 files changed, 453 insertions(+), 61 deletions(-) create mode 100644 crates/clawhdf5-format/tests/vl_heap_bounds.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a16e0a..847f3c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,18 @@ `clawhdf5_netcdf4::Variable::read_string` (checked against netCDF4-python in `crates/clawhdf5-netcdf4/tests/interop_tests.rs`). +- **Crafted global heaps could exhaust memory.** `VlResolver` kept an owned + copy of every object of every heap collection it read, so collections + nested inside each other's object data made a 744 KB file take 1.58 GB + (and `read_vl_strings` before it did the same). The cache now records + where objects lie instead of copying them, is dropped past a 32 MiB + budget, and a collection overlapping one already read is an error + (libhdf5 never writes one). New `GlobalHeapCollection::parse_index` + locates a collection's objects without copying them; `parse` and + `parse_index` refuse a collection running past the end of the file or an + object running past its collection. Conformance unchanged at 575 of 697 + (`crates/clawhdf5-format/tests/vl_heap_bounds.rs`). + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/src/global_heap.rs b/crates/clawhdf5-format/src/global_heap.rs index 5eab713..474c7fd 100644 --- a/crates/clawhdf5-format/src/global_heap.rs +++ b/crates/clawhdf5-format/src/global_heap.rs @@ -1,7 +1,7 @@ //! HDF5 Global Heap collection parsing. #[cfg(not(feature = "std"))] -use alloc::vec::Vec; +use alloc::{format, string::String, vec::Vec}; use crate::error::FormatError; @@ -52,11 +52,42 @@ fn read_length(data: &[u8], offset: usize, length_size: u8) -> Result String { + format!( + "global heap object {index} ({size} bytes) runs past the end of its \ + {collection_size}-byte collection" + ) +} + /// Round up to next multiple of 8. fn pad8(x: usize) -> usize { (x + 7) & !7 } +/// Where one object of a global heap collection lies in the file, without +/// its data: see [`GlobalHeapCollection::parse_index`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GlobalHeapObjectRef { + /// Object index (1-based; 0 is the free space marker). + pub index: u16, + /// Reference count. + pub reference_count: u16, + /// Offset of the object's data in the file data the collection was + /// parsed from. + pub offset: usize, + /// Size of the object's data in bytes. + pub size: usize, +} + +/// A global heap collection's objects, located but not copied. +#[derive(Debug, Clone)] +pub struct GlobalHeapIndex { + /// Total size of this collection including header. + pub collection_size: u64, + /// The objects, in file order. + pub objects: Vec, +} + impl GlobalHeapCollection { /// Parse a global heap collection at the given offset in the file data. pub fn parse( @@ -64,6 +95,33 @@ impl GlobalHeapCollection { offset: usize, length_size: u8, ) -> Result { + let index = Self::parse_index(file_data, offset, length_size)?; + Ok(GlobalHeapCollection { + collection_size: index.collection_size, + objects: index + .objects + .iter() + .map(|o| GlobalHeapObject { + index: o.index, + reference_count: o.reference_count, + data: file_data[o.offset..o.offset + o.size].to_vec(), + }) + .collect(), + }) + } + + /// Locate the objects of the global heap collection at `offset` without + /// copying their data, so a caller can keep many collections indexed + /// for the cost of their object headers. + /// + /// The collection must lie inside `file_data`, and every object inside + /// the collection, as libhdf5 lays them out; an object that runs past + /// its collection is an error. + pub fn parse_index( + file_data: &[u8], + offset: usize, + length_size: u8, + ) -> Result { // signature(4) + version(1) + reserved(3) + collection_size(length_size), // padded to a multiple of 8 as libhdf5 lays it out (`H5HG_SIZEOF_HDR`). // With 8-byte lengths the padding is 0; with 4-byte lengths it is 4, @@ -81,25 +139,25 @@ impl GlobalHeapCollection { } let collection_size = read_length(file_data, offset + 8, length_size)?; - let collection_size_usize = - usize::try_from(collection_size).map_err(|_| FormatError::UnexpectedEof { - expected: u64::MAX as usize, + let collection_end = usize::try_from(collection_size) + .ok() + .and_then(|size| offset.checked_add(size)) + .ok_or(FormatError::UnexpectedEof { + expected: usize::MAX, available: file_data.len(), })?; - let collection_end = - offset - .checked_add(collection_size_usize) - .ok_or(FormatError::UnexpectedEof { - expected: usize::MAX, - available: file_data.len(), - })?; + if collection_end > file_data.len() { + return Err(FormatError::UnexpectedEof { + expected: collection_end, + available: file_data.len(), + }); + } let mut pos = offset + header_size; let mut objects = Vec::new(); // Parse objects until we hit index 0 (free space) or run out of space while pos + 2 <= collection_end { - ensure_len(file_data, pos, 2)?; let object_index = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); if object_index == 0 { @@ -110,26 +168,36 @@ impl GlobalHeapCollection { // object_index(2) + reference_count(2) + reserved(4) + // object_size(length_size), padded to 8 (`H5HG_SIZEOF_OBJHDR`). let obj_header_size = pad8(8 + length_size as usize); - ensure_len(file_data, pos, obj_header_size)?; + ensure_len(&file_data[..collection_end], pos, obj_header_size)?; let reference_count = u16::from_le_bytes([file_data[pos + 2], file_data[pos + 3]]); - let object_size = read_length(file_data, pos + 8, length_size)? as usize; + let object_size = usize::try_from(read_length(file_data, pos + 8, length_size)?) + .map_err(|_| FormatError::Overflow("global heap object size".into()))?; pos += obj_header_size; - ensure_len(file_data, pos, object_size)?; - let data = file_data[pos..pos + object_size].to_vec(); + if pos + .checked_add(object_size) + .is_none_or(|end| end > collection_end) + { + return Err(FormatError::VlDataError(object_overrun_msg( + object_index, + object_size, + collection_size, + ))); + } - objects.push(GlobalHeapObject { + objects.push(GlobalHeapObjectRef { index: object_index, reference_count, - data, + offset: pos, + size: object_size, }); // Advance past data + padding to 8-byte boundary - pos += pad8(object_size); + pos = pos.saturating_add(pad8(object_size)); } - Ok(GlobalHeapCollection { + Ok(GlobalHeapIndex { collection_size, objects, }) diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index 58015a4..f183ef0 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -10,7 +10,7 @@ use alloc::{collections::BTreeMap, format, string::String, vec, vec::Vec}; use std::collections::BTreeMap; use crate::error::FormatError; -use crate::global_heap::GlobalHeapCollection; +use crate::global_heap::{GlobalHeapCollection, GlobalHeapIndex}; /// A parsed variable-length element reference (global heap ID). #[derive(Debug, Clone)] @@ -134,38 +134,43 @@ pub fn check_element_size(stored_size: u32, offset_size: u8) -> Result<(), Forma Ok(()) } -/// A parsed collection, with its objects indexed for lookup. +/// A collection's objects, located in the file data but not copied: +/// `(index, offset, size)` of the first object with each index, sorted by +/// index. struct CachedCollection { - collection: GlobalHeapCollection, - /// `slots[index]` is the position in `collection.objects` of the first - /// object with that index. - slots: Vec>, + objects: Vec<(u16, usize, usize)>, } impl CachedCollection { - fn new(collection: GlobalHeapCollection) -> Self { - let max = collection + fn new(index: GlobalHeapIndex) -> Self { + let mut objects: Vec<(u16, usize, usize)> = index .objects .iter() - .map(|o| o.index as usize) - .max() - .unwrap_or(0); - let mut slots = vec![None; max + 1]; - for (pos, obj) in collection.objects.iter().enumerate() { - let slot = &mut slots[obj.index as usize]; - if slot.is_none() { - *slot = Some(pos); - } - } - Self { collection, slots } + .map(|o| (o.index, o.offset, o.size)) + .collect(); + // Stable, so the first object with a repeated index is kept. + objects.sort_by_key(|o| o.0); + objects.dedup_by_key(|o| o.0); + Self { objects } } - fn get(&self, index: u32) -> Option<&[u8]> { - let pos = (*self.slots.get(usize::try_from(index).ok()?)?)?; - Some(&self.collection.objects[pos].data) + /// What this entry costs to keep, in bytes (roughly). + fn cost(&self) -> usize { + 64 + self.objects.len() * core::mem::size_of::<(u16, usize, usize)>() + } + + fn get(&self, index: u32) -> Option<(usize, usize)> { + let index = u16::try_from(index).ok()?; + let i = self.objects.binary_search_by_key(&index, |o| o.0).ok()?; + Some((self.objects[i].1, self.objects[i].2)) } } +/// How many bytes of collection indexes a [`VlResolver`] keeps before it +/// drops them and starts again. Values are never copied into the cache, so +/// this bounds what a read retains however many collections it visits. +const CACHE_BUDGET: usize = 32 << 20; + /// Resolves variable-length elements against a file's global heap, parsing /// each heap collection once however many elements point into it. /// @@ -173,11 +178,22 @@ impl CachedCollection { /// empty string or sequence), and an element whose heap object is not /// exactly `length × base size` bytes is an error ("Expected global heap /// object size does not match"), not a truncated or padded value. +/// +/// Memory stays bounded on hostile files: the cache holds where each +/// object lies, not a copy of it, up to a fixed budget; and collections +/// that overlap one another are refused (libhdf5 never writes them), so a +/// file cannot make the resolver parse the same bytes as the objects of +/// many collections. pub struct VlResolver<'a> { file_data: &'a [u8], offset_size: u8, length_size: u8, cache: BTreeMap, + cached_bytes: usize, + budget: usize, + /// Start → end of every collection parsed so far (kept when the cache + /// is dropped, to check overlaps). + extents: BTreeMap, } impl<'a> VlResolver<'a> { @@ -189,6 +205,9 @@ impl<'a> VlResolver<'a> { offset_size, length_size, cache: BTreeMap::new(), + cached_bytes: 0, + budget: CACHE_BUDGET, + extents: BTreeMap::new(), } } @@ -210,11 +229,18 @@ impl<'a> VlResolver<'a> { } /// The bytes of one element: `length × base_size` bytes from the heap, - /// or empty for a null or zero-length element. - fn resolve(&mut self, vl: &VlElement, base_size: usize) -> Result<&[u8], FormatError> { + /// or `None` for a null element. + fn resolve( + &mut self, + vl: &VlElement, + base_size: usize, + ) -> Result, FormatError> { let addr = vl.collection_address; - if addr == 0 || (vl.length == 0 && is_undefined_address(addr, self.offset_size)) { - return Ok(&[]); + if addr == 0 { + return Ok(None); + } + if vl.length == 0 && is_undefined_address(addr, self.offset_size) { + return Ok(Some(&[])); } let data = self.object(vl)?; let expected = (vl.length as usize) @@ -229,7 +255,27 @@ impl<'a> VlResolver<'a> { vl.length ))); } - Ok(data) + Ok(Some(data)) + } + + /// One element (the first [`element_size`](Self::element_size) bytes of + /// `elem`) of a variable-length sequence whose base type is `base_size` + /// bytes: its `length × base_size` bytes, or `None` for a null element + /// (heap address 0). + pub fn element( + &mut self, + elem: &[u8], + base_size: usize, + ) -> Result, FormatError> { + let vl = parse_vl_references(elem, 1, self.offset_size)?; + self.resolve(&vl[0], base_size) + } + + /// One variable-length string element: its bytes up to the first NUL, + /// or `None` for a null element (h5dump prints it as `NULL`, h5py + /// returns it as empty). + pub fn string_element(&mut self, elem: &[u8]) -> Result, FormatError> { + Ok(self.element(elem, 1)?.map(cut_at_nul)) } /// The strings of the variable-length string elements in `raw`, as @@ -238,11 +284,7 @@ impl<'a> VlResolver<'a> { pub fn string_bytes(&mut self, raw: &[u8]) -> Result>, FormatError> { self.elements(raw)? .iter() - .map(|vl| { - let s = self.resolve(vl, 1)?; - let end = s.iter().position(|&b| b == 0).unwrap_or(s.len()); - Ok(s[..end].to_vec()) - }) + .map(|vl| Ok(self.resolve(vl, 1)?.map(cut_at_nul).unwrap_or(&[]).to_vec())) .collect() } @@ -270,11 +312,16 @@ impl<'a> VlResolver<'a> { } self.elements(raw)? .iter() - .map(|vl| self.resolve(vl, base_size).map(<[u8]>::to_vec)) + .map(|vl| Ok(self.resolve(vl, base_size)?.unwrap_or(&[]).to_vec())) .collect() } } +/// A string's bytes up to its first NUL. +fn cut_at_nul(s: &[u8]) -> &[u8] { + &s[..s.iter().position(|&b| b == 0).unwrap_or(s.len())] +} + /// Resolve VL strings from raw data by looking up each element in the global heap. /// /// Reads the first `num_elements` elements of `raw`. Strings end at their @@ -342,25 +389,66 @@ pub fn read_vl_bytes( Ok(result) } -impl VlResolver<'_> { +impl<'a> VlResolver<'a> { /// The heap object `vl` points to, whatever its size; its collection is /// parsed on first use. - fn object(&mut self, vl: &VlElement) -> Result<&[u8], FormatError> { + fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> { let addr = vl.collection_address; if !self.cache.contains_key(&addr) { let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof { expected: usize::MAX, available: self.file_data.len(), })?; - let coll = GlobalHeapCollection::parse(self.file_data, offset, self.length_size)?; - self.cache.insert(addr, CachedCollection::new(coll)); + let index = + GlobalHeapCollection::parse_index(self.file_data, offset, self.length_size)?; + // parse_index checked that the collection lies in the file. + let end = offset + index.collection_size as usize; + self.check_overlap(offset, end)?; + let coll = CachedCollection::new(index); + if self.cached_bytes.saturating_add(coll.cost()) > self.budget { + self.cache.clear(); + self.cached_bytes = 0; + } + self.cached_bytes += coll.cost(); + self.cache.insert(addr, coll); } - self.cache[&addr] - .get(vl.object_index) - .ok_or(FormatError::GlobalHeapObjectNotFound { + let (start, size) = self.cache[&addr].get(vl.object_index).ok_or( + FormatError::GlobalHeapObjectNotFound { collection_address: addr, index: vl.object_index as u16, - }) + }, + )?; + Ok(&self.file_data[start..start + size]) + } + + /// Record the collection at `start..end`, refusing one that overlaps a + /// collection already read. libhdf5 allocates each collection its own + /// block; overlapping ones only come from a crafted file, where they let + /// every byte be parsed again as the objects of each collection. + fn check_overlap(&mut self, start: usize, end: usize) -> Result<(), FormatError> { + if let Some(&known) = self.extents.get(&start) { + return if known == end { + Ok(()) + } else { + Err(FormatError::VlDataError(format!( + "global heap collection at {start} changed size" + ))) + }; + } + let before = self.extents.range(..start).next_back(); + let after = self.extents.range(start..).next(); + let clash = match (before, after) { + (Some((&s, &e)), _) if e > start => Some(s), + (_, Some((&s, _))) if s < end => Some(s), + _ => None, + }; + if let Some(other) = clash { + return Err(FormatError::VlDataError(format!( + "global heap collection at {start} overlaps the one at {other}" + ))); + } + self.extents.insert(start, end); + Ok(()) } } @@ -581,6 +669,43 @@ mod tests { assert!(r.strings(&raw[..30]).is_err()); } + #[test] + fn the_cache_stays_within_its_budget_and_rereads_what_it_dropped() { + // Twenty collections of three objects each; a budget that holds + // about two of them. Reading every element twice must still return + // the right strings after the cache is dropped. + let mut file_data = vec![0u8; 64]; + let mut raw = Vec::new(); + for c in 0..20u64 { + let at = file_data.len(); + let names: Vec = (0..3).map(|i| format!("c{c}o{i}")).collect(); + let objs: Vec<(u16, &[u8])> = names + .iter() + .enumerate() + .map(|(i, n)| (i as u16 + 1, n.as_bytes())) + .collect(); + build_gcol_at(&mut file_data, at, &objs); + for (i, n) in names.iter().enumerate() { + raw.extend(element(n.len() as u32, at as u64, i as u32 + 1, 8)); + } + } + raw.extend(raw.clone()); + let mut r = VlResolver::new(&file_data, 8, 8); + let one = CachedCollection { + objects: vec![(0, 0, 0); 3], + } + .cost(); + r.budget = 2 * one + 1; + let want: Vec = (0..2) + .flat_map(|_| (0..20).flat_map(|c| (0..3).map(move |i| format!("c{c}o{i}")))) + .collect(); + for (k, chunk) in raw.chunks(16).enumerate() { + assert_eq!(r.strings(chunk).unwrap(), [want[k].clone()]); + assert!(r.cached_bytes <= r.budget); + assert!(r.cache.len() <= 2); + } + } + #[test] fn element_size_is_checked_against_the_offset_size() { assert!(check_element_size(16, 8).is_ok()); diff --git a/crates/clawhdf5-format/tests/vl_heap_bounds.rs b/crates/clawhdf5-format/tests/vl_heap_bounds.rs new file mode 100644 index 0000000..2993087 --- /dev/null +++ b/crates/clawhdf5-format/tests/vl_heap_bounds.rs @@ -0,0 +1,161 @@ +//! Crafted files cannot make variable-length reads retain memory, or take +//! time, out of proportion to the file. +//! +//! `VlResolver` used to keep an owned copy of every object of every +//! collection it parsed, for the whole read. A file whose global heap +//! collections nest inside each other's object data — each element +//! pointing at a different one — then made retained memory O(K × file +//! size): a 744 KB file took 1.58 GB. The same nesting, with every +//! collection's object chain jumping to one shared run of tiny objects, +//! made the parse time O(K × M) as well. libhdf5 never writes overlapping +//! collections; they are now refused, and the cache holds only where +//! objects lie. +//! +//! Peak heap use is measured with a counting global allocator, so the +//! cases run one after another in a single test. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use clawhdf5_format::vl_data::VlResolver; + +struct Counting; + +static CURRENT: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + let now = CURRENT.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(now, Ordering::Relaxed); + } + p + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } +} + +#[global_allocator] +static ALLOC: Counting = Counting; + +/// Bytes allocated at the peak of `f`, above what was live when it started. +fn peak_during(f: impl FnOnce() -> T) -> (T, usize) { + let base = CURRENT.load(Ordering::Relaxed); + PEAK.store(base, Ordering::Relaxed); + let out = f(); + (out, PEAK.load(Ordering::Relaxed) - base) +} + +fn put_header(file: &mut [u8], at: usize, size: u64) { + file[at..at + 4].copy_from_slice(b"GCOL"); + file[at + 4] = 1; + file[at + 8..at + 16].copy_from_slice(&size.to_le_bytes()); +} + +fn put_object(file: &mut [u8], at: usize, index: u16, size: u64) { + file[at..at + 2].copy_from_slice(&index.to_le_bytes()); + file[at + 2..at + 4].copy_from_slice(&1u16.to_le_bytes()); + file[at + 8..at + 16].copy_from_slice(&size.to_le_bytes()); +} + +fn element(length: u32, addr: u64, index: u32) -> Vec { + let mut e = length.to_le_bytes().to_vec(); + e.extend_from_slice(&addr.to_le_bytes()); + e.extend_from_slice(&index.to_le_bytes()); + e +} + +/// K collections 32 bytes apart, each running to the end of the file with +/// one object covering the rest of it (and so every later collection). +/// Element i is that object of collection i. +fn nested(k: usize) -> (Vec, Vec) { + let base = 64; + let end = base + 32 * k + 64; + let mut file = vec![0u8; end]; + let mut raw = Vec::new(); + for i in 0..k { + let at = base + 32 * i; + put_header(&mut file, at, (end - at) as u64); + let obj = (end - at - 32) as u64; + put_object(&mut file, at + 16, 1, obj); + raw.extend(element(obj as u32, at as u64, 1)); + } + (file, raw) +} + +/// K collections 32 bytes apart, each with a first object that jumps over +/// the later collections to one shared run of M empty objects, so parsing +/// every collection walks all M. +fn shared_tail(k: usize, m: usize) -> (Vec, Vec) { + let base = 64; + let tail = base + 32 * k + 32; + let end = tail + 16 * m + 16; + let mut file = vec![0u8; end]; + let mut raw = Vec::new(); + for i in 0..k { + let at = base + 32 * i; + put_header(&mut file, at, (end - at) as u64); + let jump = (tail - at - 32) as u64; + put_object(&mut file, at + 16, 1, jump); + raw.extend(element(jump as u32, at as u64, 1)); + } + for j in 0..m { + put_object(&mut file, tail + 16 * j, (j % 65_000 + 2) as u16, 0); + } + (file, raw) +} + +#[test] +fn overlapping_collections_are_refused_in_bounded_memory_and_time() { + for (name, (file, raw)) in [ + ("nested", nested(2000)), + ("shared tail", shared_tail(500, 10_000)), + ] { + let start = Instant::now(); + let (result, peak) = peak_during(|| { + let mut r = VlResolver::new(&file, 8, 8); + (r.string_bytes(&raw), r.sequences(&raw, 1).map(|s| s.len())) + }); + let took = start.elapsed(); + // libhdf5 never writes overlapping collections, and refuses these + // files; so do we, rather than returning what they claim. + let (strings, sequences) = result; + let e = strings.expect_err(name).to_string(); + assert!(e.contains("overlaps"), "{name}: {e}"); + assert!(sequences.is_err(), "{name}"); + // Measured before the fix: 129 MB ("nested", 64 KB file) and 350 MB + // ("shared tail", 176 KB file) live at the peak; after, 97 KB and + // 0.9 MB. + assert!( + peak < 4 * file.len() + (1 << 20), + "{name}: peak {peak} bytes for a {}-byte file", + file.len() + ); + assert!(took < Duration::from_secs(5), "{name}: took {took:?}"); + } +} + +/// Collections that do not overlap still read, however many elements point +/// into them, and the first object of a collection is returned for its +/// index (as before). +#[test] +fn separate_collections_still_read() { + let mut file = vec![0u8; 64 + 3 * 64]; + let mut raw = Vec::new(); + for i in 0..3usize { + let at = 64 + 64 * i; + put_header(&mut file, at, 64); + put_object(&mut file, at + 16, 1, 3); + file[at + 32..at + 35].copy_from_slice(format!("s{i}!").as_bytes()); + raw.extend(element(3, at as u64, 1)); + } + raw.extend(element(3, 64, 1)); + let mut r = VlResolver::new(&file, 8, 8); + assert_eq!(r.strings(&raw).unwrap(), ["s0!", "s1!", "s2!", "s0!"]); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 4ea8307..ceb6ab6 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -409,6 +409,32 @@ has produced more records than the file could physically hold. --- +## Crafted global heaps exhaust the variable-length reader's memory + +**Status:** fixed on `feat/p2-vl-strings` (2026-09-26). Not a regression of +that branch: every earlier release is affected through `read_vl_strings`. + +Reading variable-length values kept an owned copy of every object of every +global heap collection visited, for the whole read. A file whose collections +nest inside one another's object data (32 bytes apart, each element pointing +at a different one) made retained memory O(elements × file size): a 744 KB +file reached 1.58 GB. Letting every collection's object chain jump to one +shared run of tiny objects made the parse time O(elements × objects) too. +libhdf5 refuses such files. + +Now `VlResolver` caches where each object lies instead of a copy, drops its +cache past a 32 MiB budget, and refuses a collection that overlaps one it +has already read (libhdf5 gives each collection its own block, so only a +crafted file has them). `GlobalHeapCollection::parse` (and the new +`parse_index`) also refuse a collection that runs past the end of the file, +or an object that runs past the end of its collection. Guarded by +`crates/clawhdf5-format/tests/vl_heap_bounds.rs`, which measures peak heap +use with a counting allocator. Still open: a file may point many elements +at one large heap object, and a VL-*sequence* read then returns that +object once per element, as h5py would. + +--- + ## Extensible Array chunk indexes read back wrong data past the inline elements **Status:** fixed on `main` (2026-09-20), after v2.6.0. **Every release up to -- 2.54.0 From 81a0e8685db41080442dc4c4e4c4a73b92783940 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:57:01 -0500 Subject: [PATCH 21/36] fix(format): write child indirect blocks in big fractal heaps Dense link and attribute storage keeps its messages in a fractal heap. Its root indirect block holds direct blocks up to 64 KiB, 512 KiB in all; rows past that are child indirect blocks. The writer kept adding rows of direct blocks instead, and libhdf5 and h5rs read them as indirect blocks: a group with 20 000 links of 20-byte names was written without error and could not be listed ("incorrect metadata checksum"), and 150 dense attributes of up to 56 KB could not be opened. The heap writer now follows the doubling table: rows past the direct ones hold child indirect blocks, each with its own rows, nested as deep as the heap needs. Two more heap bugs are fixed on the way. An object bigger than the next block's free space was written into it anyway and cut off; the block is now left unallocated and the object goes in the first block big enough, as libhdf5 skips blocks. And the header's next-block offset was 0, so libhdf5 adding a link to such a group overwrote the heap's first block ("bad version number for message"); it is now the offset after the last block. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 446 ++++++++++++------ crates/clawhdf5-tools/tests/h5rs_interop.rs | 31 ++ .../clawhdf5/tests/writer_groups_interop.rs | 127 +++++ 3 files changed, 467 insertions(+), 137 deletions(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 2d318f5..1574dc1 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -307,7 +307,7 @@ pub(crate) fn build_single_block_fractal_heap( base_address: u64, max_heap_size: u16, heap_id_length: u16, -) -> FractalHeapBlock { +) -> Result { let os = OFFSET_SIZE as usize; let ls = LENGTH_SIZE as usize; let block_offset_bytes = (max_heap_size as usize).div_ceil(8); @@ -435,183 +435,346 @@ pub(crate) fn build_single_block_fractal_heap( blob.extend_from_slice(&frhp); blob.extend_from_slice(&dblock); - FractalHeapBlock { + Ok(FractalHeapBlock { blob, frhp_addr, btree_addr, heap_ids, heap_id_length, - } + }) } -/// Build a multi-block fractal heap: a root indirect block (FHIB) over multiple -/// direct blocks sized by the doubling table. Used when the objects don't fit -/// in a single direct block. Objects do not span blocks (no huge-object path). +/// Build a multi-block fractal heap: a root indirect block (FHIB) over direct +/// blocks sized by the doubling table. Used when the objects don't fit in a +/// single direct block. +/// +/// Rows of the doubling table whose block size exceeds the maximum direct +/// block size hold child indirect blocks, as the HDF5 spec (and libhdf5) +/// reads them: a child in row `r` spans that row's block size of heap space +/// and has `log2(size) - log2(start * width) + 1` rows of its own, which may +/// in turn hold indirect blocks. Objects are packed into direct blocks in +/// heap-offset order and never span blocks; a block too small for the next +/// object is left unallocated (an undefined address), as libhdf5 skips rows +/// when it needs a bigger block. There is no huge-object path. fn build_multiblock_fractal_heap( serialized: &[Vec], base_address: u64, max_heap_size: u16, heap_id_length: u16, -) -> FractalHeapBlock { +) -> Result { let os = OFFSET_SIZE as usize; let block_offset_bytes = (max_heap_size as usize).div_ceil(8); - let max_direct_block_size: u64 = 65536; - let table_width: u16 = 4; - let starting_block_size: u64 = 512; - let dblock_header_size = 4 + 1 + os + block_offset_bytes + 4; - let block_capacity = - |row: usize| block_size_for_row(starting_block_size, row) - dblock_header_size as u64; + let geom = HeapGeometry { + width: 4, + starting_block_size: 512, + max_direct_block_size: 65536, + dblock_header_size: 4 + 1 + os + block_offset_bytes + 4, + iblock_fixed_size: 5 + os + block_offset_bytes + 4, + max_heap_size, + }; - // ---- Pack objects into direct blocks (row-major over the doubling table) ---- - struct Blk { - row: usize, - size: u64, - heap_offset: u64, - data: Vec, - } - let mut blocks: Vec = Vec::new(); - // Each object's (heap_offset, length) for the heap ID. - let mut obj_loc: Vec<(u64, u64)> = vec![(0, 0); serialized.len()]; - - let mut row = 0usize; - let mut col = 0u16; - let mut heap_off = 0u64; - let mut cur: Option = None; - - for (idx, s) in serialized.iter().enumerate() { - loop { - if cur.is_none() { - let size = block_size_for_row(starting_block_size, row); - cur = Some(Blk { - row, - size, - heap_offset: heap_off, - data: Vec::new(), - }); - } - let blk = cur.as_mut().unwrap(); - let cap = block_capacity(blk.row) as usize; - if !blk.data.is_empty() && blk.data.len() + s.len() > cap { - // Doesn't fit; finalize this block and advance to the next slot. - let finished = cur.take().unwrap(); - heap_off += finished.size; - blocks.push(finished); - col += 1; - if col >= table_width { - col = 0; - row += 1; - } - continue; - } - // Place the object (a fresh block always accepts at least one object - // up to its capacity; objects larger than a max block are unsupported). - let pos_in_block = dblock_header_size + blk.data.len(); - obj_loc[idx] = (blk.heap_offset + pos_in_block as u64, s.len() as u64); - blk.data.extend_from_slice(s); - break; - } - } - if let Some(b) = cur.take() { - blocks.push(b); - } - - let cur_rows = (blocks.last().map(|b| b.row).unwrap_or(0) + 1) as u16; + // ---- Pack objects into the doubling table ---- + let mut packer = HeapPacker { + geom: &geom, + objects: serialized, + next: 0, + blocks: Vec::new(), + obj_loc: vec![(0, 0); serialized.len()], + }; + let root = packer.fill(0, None)?; + let HeapPacker { + blocks, obj_loc, .. + } = packer; // ---- Addresses ---- let frhp_size = frhp_header_size(os, LENGTH_SIZE as usize); let frhp_addr = base_address; let fhib_addr = frhp_addr + frhp_size as u64; - let fhib_entries = cur_rows as usize * table_width as usize; - let fhib_size = 5 + os + block_offset_bytes + fhib_entries * os + 4; - let first_dblock_addr = fhib_addr + fhib_size as u64; + let heap_len = root.subtree_size(&geom, &blocks); + let btree_addr = fhib_addr + heap_len; - // Assign each used block an address (laid out consecutively after the FHIB). - let mut blk_addrs: Vec = Vec::with_capacity(blocks.len()); - let mut a = first_dblock_addr; - for b in &blocks { - blk_addrs.push(a); - a += b.size; - } - let heap_end = a; - let btree_addr = heap_end; - - // Bookkeeping totals. - let managed_space: u64 = (0..cur_rows as usize) - .map(|r| block_size_for_row(starting_block_size, r) * table_width as u64) - .sum(); + // Bookkeeping totals, as libhdf5 keeps them: the managed space is what + // the root's rows span, the allocated space the direct blocks written, + // and the allocation iterator the heap offset after the last of them. + let cur_rows = root.nrows as u16; + let managed_space: u64 = (0..root.nrows).map(|r| geom.row_size(r) * geom.width).sum(); let alloc_space: u64 = blocks.iter().map(|b| b.size).sum(); let used: u64 = blocks .iter() - .map(|b| dblock_header_size as u64 + b.data.len() as u64) + .map(|b| geom.dblock_header_size as u64 + b.data.len() as u64) .sum(); let free_space = alloc_space.saturating_sub(used); + let alloc_iter = blocks.last().map_or(0, |b| b.heap_offset + b.size); // ---- FRHP header ---- - let max_managed = max_direct_block_size as u32 - dblock_header_size as u32; + let max_managed = geom.max_managed(); let frhp = write_frhp(WriteFrhp { heap_id_length, max_managed, free_space, managed_space, alloc_space, + alloc_iter, nobjects: serialized.len() as u64, - table_width, - starting_block_size, - max_direct_block_size, + table_width: geom.width as u16, + starting_block_size: geom.starting_block_size, + max_direct_block_size: geom.max_direct_block_size, max_heap_size, root_addr: fhib_addr, cur_rows, }); debug_assert_eq!(frhp.len(), frhp_size); - // ---- Root indirect block (FHIB) ---- - let mut fhib = Vec::with_capacity(fhib_size); - fhib.extend_from_slice(b"FHIB"); - fhib.push(0); // version - write_offset(&mut fhib, frhp_addr, OFFSET_SIZE); - fhib.extend_from_slice(&vec![0u8; block_offset_bytes]); // block offset = 0 (root) - for &addr in &blk_addrs { - write_offset(&mut fhib, addr, OFFSET_SIZE); - } - // Remaining slots within the current rows are unallocated. - for _ in blk_addrs.len()..fhib_entries { - write_undef_offset(&mut fhib, OFFSET_SIZE); - } - let fhib_checksum = crate::checksum::jenkins_lookup3(&fhib); - fhib.extend_from_slice(&fhib_checksum.to_le_bytes()); - debug_assert_eq!(fhib.len(), fhib_size); - - // ---- Direct blocks ---- + // ---- Indirect and direct blocks, depth first after the root ---- let mut blob = frhp; - blob.extend_from_slice(&fhib); - for b in &blocks { - let mut dblock = Vec::with_capacity(b.size as usize); - dblock.extend_from_slice(b"FHDB"); - dblock.push(0); // version - write_offset(&mut dblock, frhp_addr, OFFSET_SIZE); - let mut bo = b.heap_offset.to_le_bytes().to_vec(); - bo.truncate(block_offset_bytes); - dblock.extend_from_slice(&bo); - let cksum_pos = dblock.len(); - dblock.extend_from_slice(&[0u8; 4]); // checksum placeholder - dblock.extend_from_slice(&b.data); - dblock.resize(b.size as usize, 0); - let cksum = crate::checksum::jenkins_lookup3(&dblock); - dblock[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes()); - blob.extend_from_slice(&dblock); - } + root.emit(&geom, &blocks, frhp_addr, fhib_addr, &mut blob); + debug_assert_eq!(blob.len() as u64, frhp_size as u64 + heap_len); let heap_ids: Vec> = obj_loc .iter() .map(|(off, len)| encode_managed_id(*off, *len, max_heap_size, heap_id_length)) .collect(); - FractalHeapBlock { + Ok(FractalHeapBlock { blob, frhp_addr, btree_addr, heap_ids, heap_id_length, + }) +} + +/// The doubling table of a heap the writer builds. +struct HeapGeometry { + width: u64, + starting_block_size: u64, + max_direct_block_size: u64, + dblock_header_size: usize, + /// An indirect block's size without its child entries. + iblock_fixed_size: usize, + max_heap_size: u16, +} + +impl HeapGeometry { + fn row_size(&self, row: usize) -> u64 { + block_size_for_row(self.starting_block_size, row) + } + + /// Rows holding direct blocks: `log2(max_direct / start) + 2`. + fn max_direct_rows(&self) -> usize { + (self.max_direct_block_size / self.starting_block_size).ilog2() as usize + 2 + } + + /// `log2(start * width)`, libhdf5's `first_row_bits`. + fn first_row_bits(&self) -> u32 { + (self.starting_block_size * self.width).ilog2() + } + + /// Rows of an indirect block spanning `size` bytes of heap space + /// (libhdf5's `H5HF__dtable_size_to_rows`). + fn rows_for_size(&self, size: u64) -> usize { + (size.ilog2() - self.first_row_bits() + 1) as usize + } + + /// Rows the root indirect block can have: enough to span the heap's + /// whole `2^max_heap_size` address space. + fn max_root_rows(&self) -> usize { + (u32::from(self.max_heap_size) - self.first_row_bits() + 1) as usize + } + + /// The largest object a direct block holds. + fn max_managed(&self) -> u32 { + (self.max_direct_block_size - self.dblock_header_size as u64) as u32 + } +} + +/// A direct block the packer filled. +struct HeapDirectBlock { + size: u64, + heap_offset: u64, + data: Vec, +} + +/// One entry of an indirect block. +enum HeapSlot { + /// Not allocated (undefined address). + Empty, + /// Index into the packer's direct blocks. + Direct(usize), + Indirect(HeapIndirectBlock), +} + +struct HeapIndirectBlock { + heap_offset: u64, + nrows: usize, + /// `nrows * width` entries, row-major. + slots: Vec, +} + +impl HeapIndirectBlock { + fn own_size(&self, geom: &HeapGeometry) -> u64 { + (geom.iblock_fixed_size + self.slots.len() * OFFSET_SIZE as usize) as u64 + } + + /// Bytes of this block and everything below it. + fn subtree_size(&self, geom: &HeapGeometry, blocks: &[HeapDirectBlock]) -> u64 { + self.own_size(geom) + + self + .slots + .iter() + .map(|s| match s { + HeapSlot::Empty => 0, + HeapSlot::Direct(i) => blocks[*i].size, + HeapSlot::Indirect(ib) => ib.subtree_size(geom, blocks), + }) + .sum::() + } + + /// Append this block at `addr` (= `out`'s current end, relative to the + /// same base as `frhp_addr`), then its children in entry order. + fn emit( + &self, + geom: &HeapGeometry, + blocks: &[HeapDirectBlock], + frhp_addr: u64, + addr: u64, + out: &mut Vec, + ) { + let block_offset_bytes = (geom.max_heap_size as usize).div_ceil(8); + let start = out.len(); + out.extend_from_slice(b"FHIB"); + out.push(0); // version + write_offset(out, frhp_addr, OFFSET_SIZE); + out.extend_from_slice(&self.heap_offset.to_le_bytes()[..block_offset_bytes]); + let mut child = addr + self.own_size(geom); + for s in &self.slots { + match s { + HeapSlot::Empty => write_undef_offset(out, OFFSET_SIZE), + HeapSlot::Direct(i) => { + write_offset(out, child, OFFSET_SIZE); + child += blocks[*i].size; + } + HeapSlot::Indirect(ib) => { + write_offset(out, child, OFFSET_SIZE); + child += ib.subtree_size(geom, blocks); + } + } + } + let checksum = crate::checksum::jenkins_lookup3(&out[start..]); + out.extend_from_slice(&checksum.to_le_bytes()); + + let mut child = addr + self.own_size(geom); + for s in &self.slots { + match s { + HeapSlot::Empty => {} + HeapSlot::Direct(i) => { + let b = &blocks[*i]; + let d = out.len(); + out.extend_from_slice(b"FHDB"); + out.push(0); // version + write_offset(out, frhp_addr, OFFSET_SIZE); + out.extend_from_slice(&b.heap_offset.to_le_bytes()[..block_offset_bytes]); + let cksum_pos = out.len(); + out.extend_from_slice(&[0u8; 4]); // checksum placeholder + out.extend_from_slice(&b.data); + out.resize(d + b.size as usize, 0); + let cksum = crate::checksum::jenkins_lookup3(&out[d..]); + out[cksum_pos..cksum_pos + 4].copy_from_slice(&cksum.to_le_bytes()); + child += b.size; + } + HeapSlot::Indirect(ib) => { + ib.emit(geom, blocks, frhp_addr, child, out); + child += ib.subtree_size(geom, blocks); + } + } + } + } +} + +/// Packs objects into a heap's doubling table in heap-offset order. +struct HeapPacker<'a> { + geom: &'a HeapGeometry, + objects: &'a [Vec], + /// The next object to place. + next: usize, + blocks: Vec, + /// Each object's (heap offset, length). + obj_loc: Vec<(u64, u64)>, +} + +impl HeapPacker<'_> { + /// Fill an indirect block at `heap_offset` with `nrows` rows, or, for the + /// root (`None`), with as many rows as the objects need. + fn fill( + &mut self, + heap_offset: u64, + nrows: Option, + ) -> Result { + let geom = self.geom; + let width = geom.width as usize; + let mut slots = Vec::new(); + let mut off = heap_offset; + let mut row = 0usize; + while self.next < self.objects.len() && nrows.is_none_or(|n| row < n) { + if nrows.is_none() && row >= geom.max_root_rows() { + return Err(FormatError::SerializationError(format!( + "fractal heap: {} objects do not fit its {}-bit address space", + self.objects.len(), + geom.max_heap_size + ))); + } + let size = geom.row_size(row); + for _ in 0..width { + if self.next == self.objects.len() { + slots.push(HeapSlot::Empty); + } else if row < geom.max_direct_rows() { + slots.push(self.fill_direct(off, size)); + } else { + let child = self.fill(off, Some(geom.rows_for_size(size)))?; + let used = child.slots.iter().any(|s| !matches!(s, HeapSlot::Empty)); + slots.push(if used { + HeapSlot::Indirect(child) + } else { + HeapSlot::Empty + }); + } + off += size; + } + row += 1; + } + let nrows = nrows.unwrap_or(row); + slots.resize_with(nrows * width, || HeapSlot::Empty); + Ok(HeapIndirectBlock { + heap_offset, + nrows, + slots, + }) + } + + /// Fill the direct block at `heap_offset` with as many of the next + /// objects as fit; leave it unallocated if not even the next one does. + fn fill_direct(&mut self, heap_offset: u64, size: u64) -> HeapSlot { + let header = self.geom.dblock_header_size; + let capacity = size as usize - header; + let mut data = Vec::new(); + while let Some(obj) = self.objects.get(self.next) { + if data.len() + obj.len() > capacity { + break; + } + self.obj_loc[self.next] = + (heap_offset + (header + data.len()) as u64, obj.len() as u64); + data.extend_from_slice(obj); + self.next += 1; + } + if data.is_empty() && self.objects.get(self.next).is_some_and(|o| !o.is_empty()) { + return HeapSlot::Empty; + } + self.blocks.push(HeapDirectBlock { + size, + heap_offset, + data, + }); + HeapSlot::Direct(self.blocks.len() - 1) } } @@ -661,6 +824,8 @@ struct WriteFrhp { free_space: u64, managed_space: u64, alloc_space: u64, + /// Heap offset of the next direct block to allocate. + alloc_iter: u64, nobjects: u64, table_width: u16, starting_block_size: u64, @@ -685,7 +850,7 @@ fn write_frhp(p: WriteFrhp) -> Vec { write_undef_offset(&mut frhp, OFFSET_SIZE); // free_space_mgr_addr write_length(&mut frhp, p.managed_space, LENGTH_SIZE); // managed_space_in_heap write_length(&mut frhp, p.alloc_space, LENGTH_SIZE); // allocated_managed_space - write_length(&mut frhp, 0, LENGTH_SIZE); // dblock_alloc_iter + write_length(&mut frhp, p.alloc_iter, LENGTH_SIZE); // dblock_alloc_iter write_length(&mut frhp, p.nobjects, LENGTH_SIZE); // managed_objects_count write_length(&mut frhp, 0, LENGTH_SIZE); // huge_objects_size write_length(&mut frhp, 0, LENGTH_SIZE); // huge_objects_count @@ -704,7 +869,10 @@ fn write_frhp(p: WriteFrhp) -> Vec { } /// Build dense attribute storage for a set of attributes. -pub(crate) fn build_dense_attrs(attrs: &[AttributeMessage], base_address: u64) -> DenseAttrBlob { +pub(crate) fn build_dense_attrs( + attrs: &[AttributeMessage], + base_address: u64, +) -> Result { // Dense attrs use v3 attribute messages (adds character set encoding byte). let serialized: Vec> = attrs.iter().map(|a| a.serialize_v3(LENGTH_SIZE)).collect(); @@ -717,7 +885,7 @@ pub(crate) fn build_dense_attrs(attrs: &[AttributeMessage], base_address: u64) - let ls = LENGTH_SIZE as usize; // Attribute heaps use max_heap_size 40 / heap ID length 8 (matching libhdf5). - let heap = build_single_block_fractal_heap(&serialized, base_address, 40, 8); + let heap = build_single_block_fractal_heap(&serialized, base_address, 40, 8)?; let frhp_addr = heap.frhp_addr; let btree_addr = heap.btree_addr; let heap_id_length = heap.heap_id_length; @@ -781,10 +949,10 @@ pub(crate) fn build_dense_attrs(attrs: &[AttributeMessage], base_address: u64) - let attr_info = serialize_attribute_info(frhp_addr, bthd_addr); - DenseAttrBlob { + Ok(DenseAttrBlob { attr_info_message: attr_info, blob, - } + }) } // ---- Dense link blob ---- @@ -873,7 +1041,7 @@ pub(crate) fn build_dense_links( // libhdf5's link heap uses max_heap_size 32 / heap ID length 7 (vs 40/8 for // attributes), giving a 7-byte heap ID and an 11-byte type-5 record. - let heap = build_single_block_fractal_heap(&serialized, base_address, 32, 7); + let heap = build_single_block_fractal_heap(&serialized, base_address, 32, 7)?; let heap_id_length = heap.heap_id_length; // Type 5 records: hash(4) + heap_id. The B-tree's key is the name hash, @@ -1405,7 +1573,9 @@ impl FileWriter { .enumerate() .map(|(gi, g)| { let dummy_links = g.link_messages(&[], &[]); - let attr_blob = group_dense[gi].then(|| build_dense_attrs(&g.attrs, 0)); + let attr_blob = group_dense[gi] + .then(|| build_dense_attrs(&g.attrs, 0)) + .transpose()?; let li = if group_links_dense[gi] { serialize_link_info( g.track_order.then_some(0), @@ -1439,7 +1609,9 @@ impl FileWriter { let mut dummy_blobs: Vec = Vec::new(); let mut dummy_cursor = 0u64; for (i, d) in all_ds.iter().enumerate() { - let dense_blob = ds_dense[i].then(|| build_dense_attrs(&d.attrs, 0)); + let dense_blob = ds_dense[i] + .then(|| build_dense_attrs(&d.attrs, 0)) + .transpose()?; if is_vds[i] { // VDS: dummy OH with address 0 to get the OH size. The global // heap blob will be placed after the OHs in pass 2. @@ -1563,7 +1735,7 @@ impl FileWriter { group_link_blob_addrs.push(None); } if group_dense[gi] { - let blob = build_dense_attrs(&g.attrs, cursor2 as u64); + let blob = build_dense_attrs(&g.attrs, cursor2 as u64)?; cursor2 += blob.blob.len(); group_dense_blobs.push(Some(blob)); } else { @@ -1580,15 +1752,15 @@ impl FileWriter { let addr = cursor2 as u64; cursor2 += sz; if ds_dense[i] { - let blob = build_dense_attrs(&all_ds[i].attrs, cursor2 as u64); + let blob = build_dense_attrs(&all_ds[i].attrs, cursor2 as u64)?; cursor2 += blob.blob.len(); ds_dense_blobs.push(Some(blob)); } else { ds_dense_blobs.push(None); } - addr + Ok(addr) }) - .collect(); + .collect::>()?; let mut ds_blobs2: Vec = Vec::new(); let global_align_threshold = self.alignment_threshold; diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 519bc6d..8d5bd44 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -858,3 +858,34 @@ fn check_and_dump_files_with_nested_groups_and_links() { assert_eq!(stdout(&ours), r, "{name}"); } } + +#[test] +fn check_files_with_big_dense_storage() { + // Dense links and attributes past the 512 KiB the root indirect block's + // direct blocks hold: the heap then needs child indirect blocks, which + // the writer used to write as direct blocks ("fractal heap indirect + // block: bad signature"). + use clawhdf5::{AttrValue, FileBuilder}; + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + let x = b.create_dataset("x"); + x.with_i32_data(&[7]); + for i in 0..150usize { + let len = if i % 3 == 0 { 7_000 } else { 1 + i }; + x.set_attr( + &format!("a{i:03}"), + AttrValue::F64Array(vec![i as f64; len]), + ); + } + let mut g = b.create_group("g"); + for i in 0..40_000 { + g.add_hard_link(&format!("link_{i:06}_{}", "x".repeat(88)), "/x"); + } + b.add_group(g.finish()); + let p = dir.path().join("big.h5").to_string_lossy().into_owned(); + b.write(&p).unwrap(); + // Structure only: `--data` looks every link up by a linear scan. + let o = h5rs(&["check", &p]); + assert_eq!(code(&o), 0, "{p}:\n{}", stdout(&o)); + assert!(stdout(&o).contains("no problems found"), "{}", stdout(&o)); +} diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index ba2d121..3b8727d 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -638,3 +638,130 @@ fn a_group_attribute_set_again_takes_the_new_value() { let f = File::open(&path).unwrap(); assert!(matches!(f.root().attrs().unwrap()["v"], AttrValue::I64(2))); } + +// ---- big dense storage: child indirect blocks in the fractal heap ---- + +/// A name `len` bytes long, unique per `i`. +fn long_name(i: usize, len: usize) -> String { + let n = format!("link_{i:06}_"); + format!("{n}{}", "x".repeat(len - n.len())) +} + +#[test] +fn dense_links_past_the_direct_blocks_of_the_root() { + skip_if_no_python!(); + // A dense group's links live in a fractal heap whose root indirect + // block holds direct blocks up to 64 KiB: 512 KiB of link messages. + // Rows past that are child indirect blocks. The writer used to write + // them as direct blocks, which libhdf5 cannot read ("incorrect metadata + // checksum"), from about 17 000 links with 20-byte names. + // `g` crosses the first boundary (0.6 MB of links); `deep` has 65 535 + // links of about 110 bytes (7 MB), so its heap reaches the child indirect + // blocks that hold indirect blocks themselves. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.create_dataset("x").with_i32_data(&[7]); + let mut g = b.create_group("g"); + for i in 0..20_000 { + g.create_dataset(&format!("dataset_number_{i:06}")) + .with_i32_data(&[i]); + } + b.add_group(g.finish()); + let mut g = b.create_group("deep"); + g.track_order(true); + for i in 0..usize::from(u16::MAX) { + g.add_hard_link(&long_name(i, 100), "/x"); + } + b.add_group(g.finish()); + let path = write(&dir, "big_links.h5", b); + + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 g, d = f['g'], f['deep']\n\ + \x20 names = list(g)\n\ + \x20 dn = list(d)\n\ + \x20 print(json.dumps([len(names), names[-1], int(g[names[-1]][0]),\n\ + \x20 sum(int(g[n][0]) for n in names), len(dn), dn[0][:12], dn[-1][:12],\n\ + \x20 int(d[dn[-1]][0]), h5py.h5o.get_info(f['x'].id).rc]))", + ); + assert_eq!( + out, + r#"[20000, "dataset_number_019999", 19999, 199990000, 65535, "link_000000_", "link_065534_", 7, 65536]"# + ); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + let g = f.group("g").unwrap(); + assert_eq!(g.datasets().unwrap().len(), 20_000); + assert_eq!( + g.dataset("dataset_number_019999") + .unwrap() + .read_i32() + .unwrap(), + [19999] + ); + let d = f.group("deep").unwrap(); + assert_eq!(d.datasets().unwrap().len(), usize::from(u16::MAX)); + assert_eq!( + d.dataset(&long_name(65_534, 100)) + .unwrap() + .read_i32() + .unwrap(), + [7] + ); + + // libhdf5 can add to and delete from the heap. It could not when the + // header's block allocation offset was 0: its next block overwrote the + // first ("bad version number for message"). + let out = h5py( + &path, + "with h5py.File(path, 'r+') as f:\n\ + \x20 f['g']['zz_new'] = np.arange(3)\n\ + \x20 del f['g/dataset_number_000005']\n\ + with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]),\n\ + \x20 int(f['g/dataset_number_019998'][0])]))", + ); + assert_eq!(out, r#"[20000, 2, 19998]"#); + h5dump_ok(&path); +} + +#[test] +fn dense_attributes_past_the_direct_blocks_of_the_root() { + skip_if_no_python!(); + // Dense attributes share the heap writer. 150 attributes of up to 56 KB + // (8 MB) need child indirect blocks, and a big attribute after small + // ones must skip the small blocks rather than overrun one. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + let ds = b.create_dataset("x"); + ds.with_i32_data(&[1]); + for i in 0..150usize { + let len = if i % 3 == 0 { 7_000 } else { 1 + i }; + let v: Vec = (0..len).map(|k| (i * 100_000 + k) as f64).collect(); + ds.set_attr(&format!("a{i:03}"), AttrValue::F64Array(v)); + } + let path = write(&dir, "big_attrs.h5", b); + + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 a = f['x'].attrs\n\ + \x20 ok = all(np.array_equal(a['a%03d' % i],\n\ + \x20 np.arange(7000 if i % 3 == 0 else 1 + i) + i * 100000) for i in range(150))\n\ + \x20 print(json.dumps([len(a), ok]))", + ); + assert_eq!(out, "[150, true]"); + h5dump_ok(&path); + let f = File::open(&path).unwrap(); + let attrs = f.dataset("x").unwrap().attrs().unwrap(); + assert_eq!(attrs.len(), 150); + for i in [0usize, 1, 147, 149] { + let len = if i % 3 == 0 { 7_000 } else { 1 + i }; + let want: Vec = (0..len).map(|k| (i * 100_000 + k) as f64).collect(); + match &attrs[&format!("a{i:03}")] { + AttrValue::F64Array(v) => assert_eq!(*v, want, "a{i:03}"), + other => panic!("a{i:03}: {other:?}"), + } + } +} -- 2.54.0 From b43bd2e67f6a07fefda1db378d08571c781fa983 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:57:16 -0500 Subject: [PATCH 22/36] perf(py): read an index list one group of chunks at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each run of consecutive indices was its own uncached hyperslab read, so a list over a compressed chunked dataset decoded the same chunk once per run (d[range(0, 200000, 40)] over 20 gzip chunks: 8 s, h5py 0.014 s). Plan::reads now groups the indices — a group ends only where a whole chunk holds no selected index, or, unchunked, at a gap over 64 KiB — and the selected rows are gathered from each group's block in Rust. Now 3.8 ms (h5py 4.1 ms, release, tank). The new test (1-D, 2-D and contiguous, compared with h5py, 2 s bound) took 5.8 s before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +- crates/clawhdf5-py/src/dataset.rs | 28 ++- crates/clawhdf5-py/src/node.rs | 27 +++ crates/clawhdf5-py/src/select.rs | 193 +++++++++++++++--- crates/clawhdf5-py/tests/test_read_vs_h5py.py | 32 +++ 5 files changed, 259 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f19acc4..a29e25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,8 @@ read the whole dataset and slice it in numpy, and knew six dtypes. Now integers (negative from the end), slices with positive steps, `...`, one increasing list of integers per key and compound field names map onto the - facade's hyperslab selection (a list becomes one hyperslab per run of - consecutive indices), with h5py's results (numpy scalar for an all-integer + facade's hyperslab selection (a list is read one group of neighbouring + chunks at a time and picked from in memory), with h5py's results (numpy scalar for an all-integer key, 0-d array for `scalar[...]`) and h5py's errors for everything else (negative steps, `None`, boolean masks, out-of-range indices). `Dataset.dtype` is the numpy dtype h5py reports, for every integer and @@ -69,6 +69,17 @@ from the file (h5py's, zero for files it wrote) and stays zero-copy. The h5py comparisons now also compare every byte of structured values (`test_compound_padding_bytes_match_h5py` and `assert_same`). +- **Index lists no longer decode the same chunks once per run.** A list + index was one uncached hyperslab read per run of consecutive indices, so + on a chunked, compressed dataset every run decoded its chunk again: + `d[list(range(0, 200000, 40))]` over 20 gzip chunks took 8 s (h5py: + 0.014 s). The list is now read in groups — for a chunked dataset a group + ends only where a whole chunk holds no selected index, so each chunk is + decoded once; otherwise at a gap of more than 64 KiB — and the selected + rows are picked from each group in Rust. The same read now takes 3.8 ms + (h5py 4.1 ms; release build on tank, best of 5). + `test_a_long_index_list_decodes_each_chunk_once` compares 1-D, 2-D and + contiguous cases with h5py under a 2 s bound (5.8 s before, debug build). - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/crates/clawhdf5-py/src/dataset.rs b/crates/clawhdf5-py/src/dataset.rs index 36467d8..aa75e7d 100644 --- a/crates/clawhdf5-py/src/dataset.rs +++ b/crates/clawhdf5-py/src/dataset.rs @@ -31,6 +31,8 @@ pub struct PyDataset { path: String, /// `None` for a dataset with a null dataspace (h5py's `Empty`). shape: Option>, + /// The chunk shape, for a chunked dataset. + chunks: Option>, datatype: Datatype, /// Why the datatype cannot be read into numpy, if it cannot. conv: Result, @@ -56,10 +58,14 @@ impl PyDataset { }; let conv = Converter::new(py, &datatype, file.superblock().offset_size) .map_err(|e| e.value(py).to_string()); + let chunks = shape + .as_ref() + .and_then(|s| node::chunk_shape(&file, &hdr, s.len())); Ok(Self { file, path, shape, + chunks, datatype, conv, }) @@ -81,17 +87,23 @@ impl PyDataset { let arr = if plan.is_empty() { conv.empty(py, &out_shape)? } else { - let (reads, list_axis) = plan.reads(dims); + let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit); + let list_axis = plan.list_axis(); + let chunk_len = match (&self.chunks, list_axis) { + (Some(c), Some(a)) => c.get(a).copied(), + _ => None, + }; + let (reads, list_axis) = plan.reads(dims, chunk_len, elem_size); let read_shape = plan.read_shape(); let file = &*self.file; let path = self.path.as_str(); - let (vl, elem_size, unit) = (conv.is_vl(), conv.elem_size, conv.vl_unit); // Everything below touches only Rust data: release the GIL. let read = || -> Result { let ds = file.dataset(path)?; let mut blocks = Vec::with_capacity(reads.len()); - for (sel, shape) in reads { - let raw = ds.read_selection(&sel)?; + for read in reads { + let raw = ds.read_selection(&read.sel)?; + let mut shape = read.shape; let want = shape.iter().product::() * elem_size; if raw.len() != want { return Err(ReadError::Other(format!( @@ -99,6 +111,14 @@ impl PyDataset { raw.len() ))); } + let raw = match (&read.pick, list_axis) { + (Some(pick), Some(axis)) => { + let kept = select::gather_along(&raw, &shape, axis, pick, elem_size); + shape[axis] = pick.len(); + kept + } + _ => raw, + }; blocks.push((raw, shape)); } // Several blocks only for a list index: join their bytes diff --git a/crates/clawhdf5-py/src/node.rs b/crates/clawhdf5-py/src/node.rs index 0482603..bb6c2d2 100644 --- a/crates/clawhdf5-py/src/node.rs +++ b/crates/clawhdf5-py/src/node.rs @@ -135,6 +135,33 @@ pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResul }) } +/// The chunk shape of a chunked dataset (one entry per dataset dimension), +/// or `None` for other layouts or a layout message that does not parse. +pub(crate) fn chunk_shape( + file: &clawhdf5_rs::File, + hdr: &ObjectHeader, + rank: usize, +) -> Option> { + let sb = file.superblock(); + let msg = hdr + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout)?; + match clawhdf5_format::data_layout::DataLayout::parse(&msg.data, sb.offset_size, sb.length_size) + .ok()? + { + clawhdf5_format::data_layout::DataLayout::Chunked { + chunk_dimensions, .. + } if chunk_dimensions.len() >= rank => Some( + chunk_dimensions[..rank] + .iter() + .map(|&d| u64::from(d)) + .collect(), + ), + _ => None, + } +} + pub(crate) fn is_null(space: &Dataspace) -> bool { space.space_type == DataspaceType::Null } diff --git a/crates/clawhdf5-py/src/select.rs b/crates/clawhdf5-py/src/select.rs index dac62ab..6b358b9 100644 --- a/crates/clawhdf5-py/src/select.rs +++ b/crates/clawhdf5-py/src/select.rs @@ -62,6 +62,11 @@ impl Plan { self.axes.iter().map(|a| a.len() as usize).collect() } + /// The axis indexed by a list, if any. + pub fn list_axis(&self) -> Option { + self.axes.iter().position(|a| matches!(a, Axis::List(_))) + } + /// Whether the selection is empty. pub fn is_empty(&self) -> bool { self.axes.iter().any(|a| a.len() == 0) @@ -69,16 +74,42 @@ impl Plan { /// The hyperslab reads that make up this selection, each with the shape /// of its block (index axes kept at length 1). More than one only when an - /// axis is indexed by a list: one read per run of consecutive indices, - /// joined along `list_axis` afterwards (`join_along`). - pub fn reads(&self, dims: &[u64]) -> (Vec<(Selection, Vec)>, Option) { - let list_axis = self.axes.iter().position(|a| matches!(a, Axis::List(_))); - let runs: Vec<(u64, u64)> = match list_axis.map(|i| &self.axes[i]) { - Some(Axis::List(idx)) => consecutive_runs(idx), - _ => vec![(0, 0)], + /// axis is indexed by a list; those are joined along `list_axis` + /// afterwards (`join_along`), after keeping each read's `pick` rows. + /// + /// A list is read in groups, each one hyperslab over a stretch of the + /// axis, not once per index: every read decodes the chunks it touches + /// (and lists the dataset's chunks), so a read per run of indices decoded + /// the same chunk again and again. For a chunked dataset (`chunk_len` is + /// the chunk's length along the list axis) a group ends only where a + /// whole chunk holds no selected index, so no chunk is decoded twice or + /// without need. Otherwise a group ends at a gap of more than + /// [`MAX_GAP_BYTES`] of unselected data. + pub fn reads( + &self, + dims: &[u64], + chunk_len: Option, + elem_size: usize, + ) -> (Vec, Option) { + let list_axis = self.list_axis(); + let groups: Vec<&[u64]> = match list_axis.map(|i| &self.axes[i]) { + Some(Axis::List(idx)) => { + let row_bytes = self.row_bytes(elem_size); + group_indices(idx, |last, next| match chunk_len { + Some(c) if c > 0 => next / c <= last / c + 1, + _ => (next - last - 1).saturating_mul(row_bytes) <= MAX_GAP_BYTES, + }) + } + _ => vec![&[]], }; - let mut out = Vec::with_capacity(runs.len()); - for (run_start, run_len) in runs { + let mut out = Vec::with_capacity(groups.len()); + for group in groups { + let (first, span) = match (group.first(), group.last()) { + (Some(&f), Some(&l)) => (f, l - f + 1), + _ => (0, 0), + }; + let pick = (span != group.len() as u64) + .then(|| group.iter().map(|&i| (i - first) as usize).collect()); let mut start = Vec::with_capacity(dims.len()); let mut stride = Vec::with_capacity(dims.len()); let mut count = Vec::with_capacity(dims.len()); @@ -86,14 +117,14 @@ impl Plan { let (s, st, c) = match axis { Axis::Index(i) => (*i, 1, 1), Axis::Slice { start, step, count } => (*start, *step, *count), - Axis::List(_) => (run_start, 1, run_len), + Axis::List(_) => (first, 1, span), }; start.push(s); // A stride only matters between blocks; keep it >= 1. stride.push(if c <= 1 { 1 } else { st }); count.push(c); } - let block_shape: Vec = count.iter().map(|&c| c as usize).collect(); + let shape: Vec = count.iter().map(|&c| c as usize).collect(); let whole = start.iter().all(|&s| s == 0) && stride.iter().all(|&s| s == 1) && count.as_slice() == dims; @@ -108,21 +139,74 @@ impl Plan { block, } }; - out.push((sel, block_shape)); + out.push(Read { sel, shape, pick }); } (out, list_axis) } + + /// Bytes of one step along the list axis within a read's bounding box. + fn row_bytes(&self, elem_size: usize) -> u64 { + self.axes + .iter() + .map(|a| match a { + Axis::Slice { step, count, .. } if *count > 0 => (count - 1) * step + 1, + _ => 1, + }) + .fold(elem_size as u64, u64::saturating_mul) + } } -fn consecutive_runs(idx: &[u64]) -> Vec<(u64, u64)> { - let mut runs: Vec<(u64, u64)> = Vec::new(); - for &i in idx { - match runs.last_mut() { - Some((s, n)) if *s + *n == i => *n += 1, - _ => runs.push((i, 1)), +/// Unselected data a read of a non-chunked dataset copies through rather +/// than start another read. +pub(crate) const MAX_GAP_BYTES: u64 = 64 * 1024; + +/// One hyperslab read of a selection. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Read { + pub sel: Selection, + /// The block's shape (index axes at length 1). + pub shape: Vec, + /// For a list: the positions along the list axis, within the block, to + /// keep (`None`: all of them). + pub pick: Option>, +} + +/// Split increasing indices into groups; `joins(last, next)` says whether +/// `next` extends the group whose last index is `last`. +fn group_indices(idx: &[u64], joins: impl Fn(u64, u64) -> bool) -> Vec<&[u64]> { + let mut groups = Vec::new(); + let mut from = 0; + for k in 1..idx.len() { + if !joins(idx[k - 1], idx[k]) { + groups.push(&idx[from..k]); + from = k; } } - runs + if from < idx.len() { + groups.push(&idx[from..]); + } + groups +} + +/// Keep the elements at positions `pick` along `axis` of a row-major block. +pub(crate) fn gather_along( + bytes: &[u8], + shape: &[usize], + axis: usize, + pick: &[usize], + elem_size: usize, +) -> Vec { + let outer: usize = shape[..axis].iter().product(); + let inner: usize = shape[axis + 1..].iter().product::() * elem_size; + let len = shape[axis]; + let mut out = Vec::with_capacity(outer * pick.len() * inner); + for o in 0..outer { + for &p in pick { + let at = (o * len + p) * inner; + out.extend_from_slice(&bytes[at..at + inner]); + } + } + out } /// Join row-major blocks of `elem_size`-byte elements whose shapes differ @@ -336,12 +420,53 @@ mod tests { use super::*; #[test] - fn runs_group_consecutive_indices() { + fn indices_group_by_chunk() { + let chunked = |c: u64| move |last: u64, next: u64| next / c <= last / c + 1; + // Chunks of 10: 3, 5 and 15 are in neighbouring chunks; 42 skips two. + let idx = [3, 5, 15, 42, 43, 99]; assert_eq!( - consecutive_runs(&[1, 2, 3, 7, 9, 10]), - vec![(1, 3), (7, 1), (9, 2)] + group_indices(&idx, chunked(10)), + vec![&[3, 5, 15][..], &[42, 43], &[99]] + ); + assert_eq!(group_indices(&[], chunked(10)), Vec::<&[u64]>::new()); + } + + #[test] + fn a_list_reads_once_per_group() { + let plan = Plan { + axes: vec![ + Axis::List(vec![0, 2, 3, 40]), + Axis::Slice { + start: 0, + step: 1, + count: 5, + }, + ], + fields: vec![], + scalar: false, + }; + // Chunks of 8 rows: rows 0-3 are one read, row 40 another. + let (reads, axis) = plan.reads(&[50, 5], Some(8), 4); + assert_eq!(axis, Some(0)); + assert_eq!(reads.len(), 2); + assert_eq!(reads[0].shape, vec![4, 5]); + assert_eq!(reads[0].pick, Some(vec![0, 2, 3])); + assert_eq!(reads[1].shape, vec![1, 5]); + assert_eq!(reads[1].pick, None); + // Not chunked: a gap under MAX_GAP_BYTES is read through. + let (reads, _) = plan.reads(&[50, 5], None, 4); + assert_eq!(reads.len(), 1); + assert_eq!(reads[0].pick, Some(vec![0, 2, 3, 40])); + } + + #[test] + fn gather_keeps_picked_rows() { + // A 2x3 block of 1-byte elements; keep columns 0 and 2. + let block = [1, 2, 3, 4, 5, 6]; + assert_eq!( + gather_along(&block, &[2, 3], 1, &[0, 2], 1), + vec![1, 3, 4, 6] ); - assert_eq!(consecutive_runs(&[]), vec![]); } #[test] @@ -374,9 +499,16 @@ mod tests { fields: vec![], scalar: false, }; - let (reads, list) = plan.reads(&[4, 3]); + let (reads, list) = plan.reads(&[4, 3], None, 8); assert_eq!(list, None); - assert_eq!(reads, vec![(Selection::All, vec![4, 3])]); + assert_eq!( + reads, + vec![Read { + sel: Selection::All, + shape: vec![4, 3], + pick: None + }] + ); } #[test] @@ -393,18 +525,19 @@ mod tests { fields: vec![], scalar: false, }; - let (reads, _) = plan.reads(&[4, 8]); + let (reads, _) = plan.reads(&[4, 8], None, 8); assert_eq!( reads, - vec![( - Selection::Hyperslab { + vec![Read { + sel: Selection::Hyperslab { start: vec![2, 1], stride: vec![1, 3], count: vec![1, 2], block: vec![1, 1], }, - vec![1, 2] - )] + shape: vec![1, 2], + pick: None + }] ); assert_eq!(plan.out_shape(), vec![2]); } diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py index c5876d2..f51c534 100644 --- a/crates/clawhdf5-py/tests/test_read_vs_h5py.py +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -531,3 +531,35 @@ def test_compound_padding_bytes_match_h5py(pair): assert ours[name][key].tobytes() == theirs[name][key].tobytes(), f"{name}[{key!r}]" for key in [(slice(None), [0, 2]), ([0, 2, 3], slice(None)), ([1, 3], 1)]: assert ours["cmp/nested_2d"][key].tobytes() == theirs["cmp/nested_2d"][key].tobytes(), key + + +def test_a_long_index_list_decodes_each_chunk_once(h5py, tmp_path): + """An index list is read one group of chunks at a time, not one + hyperslab per run of indices: 5000 runs over 20 gzip chunks used to + decode the chunks 5000 times (8 s, against h5py's 0.014 s).""" + import time + + path = str(tmp_path / "long_list.h5") + data = np.arange(200000, dtype=" Date: Sat, 26 Sep 2026 08:57:52 -0500 Subject: [PATCH 23/36] fix(format): refuse a dense link or attribute too big for the heap A message in dense storage is a fractal heap object, and an object must fit one direct block: 65 515 bytes here, since the writer has no huge-object path. A bigger one (a soft link with a 80 000-byte target in a group of more than 8 links) was written without error, cut off at the end of its block, and libhdf5 could not list the group ("object overruns end of direct block"). finish() now fails with an error that names the limit, for links and for dense attributes; a 65 001-byte soft link target still works and h5py reads it back. The heap packer also skips a child indirect block whose blocks are all too small for the next object instead of walking it. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 30 +++++++++++-- .../clawhdf5/tests/writer_groups_interop.rs | 42 +++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 1574dc1..65181a0 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -316,6 +316,18 @@ pub(crate) fn build_single_block_fractal_heap( // Direct block layout: sig(4) + ver(1) + heap_addr(os) + block_offset(bo_bytes) // + checksum(4) [when flags bit 1 set] + data... let dblock_header_size = 4 + 1 + os + block_offset_bytes + 4; // +4 for checksum + + // An object must fit one direct block: the writer has no huge-object + // path, and libhdf5 cannot read an object that overruns its block. + let max_managed = max_direct_block_size as usize - dblock_header_size; + if let Some(big) = serialized.iter().find(|s| s.len() > max_managed) { + return Err(FormatError::SerializationError(format!( + "a {}-byte message cannot go in dense storage: a fractal heap \ + object holds at most {max_managed} bytes (huge heap objects are \ + not written)", + big.len() + ))); + } let total_data_size: usize = serialized.iter().map(|s| s.len()).sum(); let dblock_content_size = dblock_header_size + total_data_size; let starting_block_size = dblock_content_size.next_power_of_two().max(512) as u64; @@ -373,8 +385,7 @@ pub(crate) fn build_single_block_fractal_heap( frhp.extend_from_slice(&heap_id_length.to_le_bytes()); frhp.extend_from_slice(&0u16.to_le_bytes()); // io_filter_encoded_length frhp.push(0x02); // flags: bit 1 = checksum direct blocks - let max_managed = max_direct_block_size as u32 - dblock_header_size as u32; - frhp.extend_from_slice(&max_managed.to_le_bytes()); + frhp.extend_from_slice(&(max_managed as u32).to_le_bytes()); write_length(&mut frhp, 0, LENGTH_SIZE); // next_huge_object_id write_undef_offset(&mut frhp, OFFSET_SIZE); // btree_huge_objects_address write_length(&mut frhp, free_space as u64, LENGTH_SIZE); // free_space_managed_blocks @@ -455,7 +466,8 @@ pub(crate) fn build_single_block_fractal_heap( /// in turn hold indirect blocks. Objects are packed into direct blocks in /// heap-offset order and never span blocks; a block too small for the next /// object is left unallocated (an undefined address), as libhdf5 skips rows -/// when it needs a bigger block. There is no huge-object path. +/// when it needs a bigger block. The caller has checked that every object +/// fits a maximum-size direct block (there is no huge-object path). fn build_multiblock_fractal_heap( serialized: &[Vec], base_address: u64, @@ -730,7 +742,17 @@ impl HeapPacker<'_> { } else if row < geom.max_direct_rows() { slots.push(self.fill_direct(off, size)); } else { - let child = self.fill(off, Some(geom.rows_for_size(size)))?; + let child_rows = geom.rows_for_size(size); + // A child whose biggest direct block cannot hold the + // next object is skipped whole, not walked. + let biggest = geom.row_size(child_rows.min(geom.max_direct_rows()) - 1); + if self.objects[self.next].len() > (biggest as usize - geom.dblock_header_size) + { + slots.push(HeapSlot::Empty); + off += size; + continue; + } + let child = self.fill(off, Some(child_rows))?; let used = child.slots.iter().any(|s| !matches!(s, HeapSlot::Empty)); slots.push(if used { HeapSlot::Indirect(child) diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 3b8727d..770b835 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -765,3 +765,45 @@ fn dense_attributes_past_the_direct_blocks_of_the_root() { } } } + +#[test] +fn a_link_too_big_for_dense_storage_is_an_error() { + // A link message must fit one fractal heap direct block (64 KiB less + // its header); the writer has no huge-object path. It used to be + // written anyway, cut off, and libhdf5 could not list the group. + let mut b = FileBuilder::new(); + for i in 0..10 { + b.create_dataset(&format!("d{i}")).with_i32_data(&[i]); + } + b.add_soft_link("s", &"/y".repeat(40_000)); + let err = b.finish().unwrap_err().to_string(); + assert!(err.contains("fractal heap object holds at most"), "{err}"); + // The same for a dense attribute. + let mut b = FileBuilder::new(); + let x = b.create_dataset("x"); + x.with_i32_data(&[1]); + for i in 0..9 { + x.set_attr(&format!("a{i}"), AttrValue::I64(i)); + } + x.set_attr("big", AttrValue::F64Array(vec![0.5; 9_000])); + let err = b.finish().unwrap_err().to_string(); + assert!(err.contains("fractal heap object holds at most"), "{err}"); + + // Just under the limit is fine, and libhdf5 reads it back. + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + for i in 0..10 { + b.create_dataset(&format!("d{i}")).with_i32_data(&[i]); + } + let target = format!("/{}", "y".repeat(65_000)); + b.add_soft_link("s", &target); + let path = write(&dir, "long_soft.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([len(f), len(f.get('s', getlink=True).path)]))", + ); + assert_eq!(out, "[11, 65001]"); + h5dump_ok(&path); +} -- 2.54.0 From bd1d8f1a593d8d6e5842f9e8dffdfb981f45f245 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 08:58:49 -0500 Subject: [PATCH 24/36] fix(format): keep a dense index leaf within 65 535 records The link and attribute name indexes are one v2 B-tree leaf, sized to the next power of two. libhdf5 takes a leaf's capacity from that node size, but a leaf's record count is a 2-byte field. From about 47 700 links the node had room for more than 65 535 records, so adding a link in h5py overflowed the count: a group of 65 535 links crashed h5py, or could no longer be listed ("unknown link class"). The node is now capped at a full leaf of 65 535 records, so libhdf5 splits it instead. Dense attributes now go through the same index builder. Their record count was written modulo 65 536, without error; more than 65 535 attributes on one object are now refused, like links. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 71 ++++++------------- .../clawhdf5/tests/writer_groups_interop.rs | 30 ++++++-- 2 files changed, 48 insertions(+), 53 deletions(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 65181a0..0b318a9 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -903,9 +903,6 @@ pub(crate) fn build_dense_attrs( .map(|a| crate::checksum::jenkins_lookup3(a.name.as_bytes())) .collect(); - let os = OFFSET_SIZE as usize; - let ls = LENGTH_SIZE as usize; - // Attribute heaps use max_heap_size 40 / heap ID length 8 (matching libhdf5). let heap = build_single_block_fractal_heap(&serialized, base_address, 40, 8)?; let frhp_addr = heap.frhp_addr; @@ -926,48 +923,16 @@ pub(crate) fn build_dense_attrs( } records.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); - let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4; - let num_records = attrs.len(); - let btlf_size = 4 + 1 + 1 + (num_records * record_size as usize) + 4; - let node_size = btlf_size.next_power_of_two().max(512) as u32; - + let records: Vec> = records.into_iter().map(|(_, _, rec)| rec).collect(); let bthd_addr = btree_addr; - let btlf_addr = bthd_addr + bthd_size as u64; - - let mut bthd = Vec::with_capacity(bthd_size); - bthd.extend_from_slice(b"BTHD"); - bthd.push(0); // version - bthd.push(8); // type = attribute name index - bthd.extend_from_slice(&node_size.to_le_bytes()); - bthd.extend_from_slice(&record_size.to_le_bytes()); - bthd.extend_from_slice(&0u16.to_le_bytes()); // depth = 0 - bthd.push(100); // split_percent - bthd.push(40); // merge_percent - write_offset(&mut bthd, btlf_addr, OFFSET_SIZE); - bthd.extend_from_slice(&(num_records as u16).to_le_bytes()); - write_length(&mut bthd, num_records as u64, LENGTH_SIZE); - let bthd_checksum = crate::checksum::jenkins_lookup3(&bthd); - bthd.extend_from_slice(&bthd_checksum.to_le_bytes()); - debug_assert_eq!(bthd.len(), bthd_size); - - let mut btlf = Vec::with_capacity(node_size as usize); - btlf.extend_from_slice(b"BTLF"); - btlf.push(0); // version - btlf.push(8); // type - for (_, _, rec) in &records { - btlf.extend_from_slice(rec); - } - // Checksum goes immediately after records (NOT at end of node). - // HDF5 C library computes checksum over sig+ver+type+records only. - let btlf_checksum = crate::checksum::jenkins_lookup3(&btlf); - btlf.extend_from_slice(&btlf_checksum.to_le_bytes()); - // Pad to node_size - btlf.resize(node_size as usize, 0); - - let mut blob = Vec::with_capacity(heap.blob.len() + bthd.len() + btlf.len()); - blob.extend_from_slice(&heap.blob); - blob.extend_from_slice(&bthd); - blob.extend_from_slice(&btlf); + let mut blob = heap.blob; + blob.extend_from_slice(&single_leaf_v2_btree( + 8, + record_size, + &records, + bthd_addr, + "attributes on one object", + )?); let attr_info = serialize_attribute_info(frhp_addr, bthd_addr); @@ -989,12 +954,14 @@ pub(crate) struct DenseLinkBlob { } /// A v2 B-tree of `btree_type` holding `records` (already in key order) in a -/// single leaf, laid out at `addr`: the header, then the leaf. +/// single leaf, laid out at `addr`: the header, then the leaf. `what` names +/// the records in the error for too many ("links in one group"). fn single_leaf_v2_btree( btree_type: u8, record_size: u16, records: &[Vec], addr: u64, + what: &str, ) -> Result, FormatError> { let os = OFFSET_SIZE as usize; let ls = LENGTH_SIZE as usize; @@ -1002,15 +969,21 @@ fn single_leaf_v2_btree( // internal nodes, which the writer does not build. let num_records = u16::try_from(records.len()).map_err(|_| { FormatError::SerializationError(format!( - "{} links in one group: a group holds at most {} links \ - (a deeper link index is not implemented)", + "{} {what}: at most {} can be written \ + (a deeper B-tree index is not implemented)", records.len(), u16::MAX )) })?; let bthd_size = 4 + 1 + 1 + 4 + 2 + 2 + 1 + 1 + os + 2 + ls + 4; let btlf_size = 4 + 1 + 1 + (records.len() * record_size as usize) + 4; - let node_size = btlf_size.next_power_of_two().max(512) as u32; + // libhdf5 sizes a leaf's capacity from the node size, and a leaf's + // record count is a 2-byte field: a node with room for more than + // 65 535 records makes it overflow that count when it adds one (the + // group can then no longer be listed). Cap the node at a full leaf. + let max_node = btlf_size - records.len() * record_size as usize + + usize::from(u16::MAX) * record_size as usize; + let node_size = btlf_size.next_power_of_two().max(512).min(max_node) as u32; let btlf_addr = addr + bthd_size as u64; let mut out = Vec::with_capacity(bthd_size + node_size as usize); @@ -1089,6 +1062,7 @@ pub(crate) fn build_dense_links( 4 + heap_id_length, &name_records, name_bt_addr, + "links in one group", )?); let link_info_message = if track_order { @@ -1113,6 +1087,7 @@ pub(crate) fn build_dense_links( 8 + heap_id_length, &order_records, order_bt_addr, + "links in one group", )?); let next_order = by_order.last().map_or(0, |&(o, _)| o + 1); serialize_link_info( diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 770b835..27379bb 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -538,7 +538,23 @@ fn more_links_than_one_index_leaf_holds_is_an_error() { b.add_soft_link(&format!("s{i}"), "/x"); } let err = b.finish().unwrap_err().to_string(); - assert!(err.contains("at most 65535 links"), "{err}"); + assert!( + err.contains("70000 links in one group: at most 65535"), + "{err}" + ); + // Dense attributes have the same one-leaf index. Their count used to + // be written modulo 65 536. + let mut b = FileBuilder::new(); + let x = b.create_dataset("x"); + x.with_i32_data(&[1]); + for i in 0..70_000 { + x.set_attr(&format!("a{i}"), AttrValue::I64(i)); + } + let err = b.finish().unwrap_err().to_string(); + assert!( + err.contains("70000 attributes on one object: at most 65535"), + "{err}" + ); } #[test] @@ -712,17 +728,21 @@ fn dense_links_past_the_direct_blocks_of_the_root() { // libhdf5 can add to and delete from the heap. It could not when the // header's block allocation offset was 0: its next block overwrote the - // first ("bad version number for message"). + // first ("bad version number for message"). Adding to `deep` also + // needs its index's leaf node to have room for at most 65 535 records: + // a bigger node made libhdf5 overflow the leaf's 2-byte record count + // (a crash, or "unknown link class" when listing). let out = h5py( &path, "with h5py.File(path, 'r+') as f:\n\ \x20 f['g']['zz_new'] = np.arange(3)\n\ + \x20 f['deep']['zz_new'] = np.arange(4)\n\ \x20 del f['g/dataset_number_000005']\n\ with h5py.File(path, 'r') as f:\n\ - \x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]),\n\ - \x20 int(f['g/dataset_number_019998'][0])]))", + \x20 print(json.dumps([len(f['g']), int(f['g/zz_new'][2]), len(f['deep']),\n\ + \x20 list(f['deep'])[-1], int(f['g/dataset_number_019998'][0])]))", ); - assert_eq!(out, r#"[20000, 2, 19998]"#); + assert_eq!(out, r#"[20000, 2, 65536, "zz_new", 19998]"#); h5dump_ok(&path); } -- 2.54.0 From 400e3a9feca335f0877c88a1ada8ff9dd79a2ba1 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:01:13 -0500 Subject: [PATCH 25/36] fix(format): resolve each hard link once A hard link's target may go through other hard links, and each was resolved again every time a path went through it. With each link's target naming the previous link twice (g/s{i} -> /g/s{i-1}/s{i-1}) the work doubled per link: finish() took 46 s for 26 links in a debug build, and 60 would never finish. Resolved links are now remembered, so the work is linear in the links, and a hard link met again while it is being resolved is reported as a cycle by name. The depth limit (64) still bounds the recursion through links not yet resolved. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/writer_tree.rs | 62 ++++++++++++++++--- .../clawhdf5/tests/writer_groups_interop.rs | 49 +++++++++++++++ 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/crates/clawhdf5-format/src/writer_tree.rs b/crates/clawhdf5-format/src/writer_tree.rs index 724ba70..275b73d 100644 --- a/crates/clawhdf5-format/src/writer_tree.rs +++ b/crates/clawhdf5-format/src/writer_tree.rs @@ -17,8 +17,8 @@ use std::collections::BTreeMap; use crate::error::FormatError; use crate::type_builders::{AttrValue, DatasetBuilder, GroupBuilder, GroupItem}; -/// Hard links followed while resolving one hard-link target path. Guards -/// against hard links whose targets name each other. +/// Depth of the chain of unresolved hard links followed while resolving one +/// hard-link target path (a bound on recursion; cycles are found exactly). const MAX_LINK_DEPTH: usize = 64; fn err(msg: String) -> FormatError { @@ -256,10 +256,22 @@ impl Builder { } /// The object a hard link's `target` path names, from group `from`. - fn resolve(&self, from: usize, target: &str, depth: usize) -> Result { + /// + /// Hard links met on the way are resolved once and remembered in + /// `memo` (by group and link index), so a target that goes through + /// other hard links costs time linear in the links, not exponential; a + /// hard link met again while it is being resolved is a cycle. + fn resolve( + &self, + memo: &mut [Vec], + from: usize, + target: &str, + depth: usize, + ) -> Result { if depth > MAX_LINK_DEPTH { return Err(err(format!( - "hard link target {target:?}: too many hard links to follow (a cycle?)" + "hard link target {target:?}: more than {MAX_LINK_DEPTH} hard links \ + to follow" ))); } let (mut cur, rest) = match target.strip_prefix('/') { @@ -291,7 +303,22 @@ impl Builder { obj = match &grp.links[li].1 { Target::Group(child) => Obj::Group(*child), Target::Dataset(d) => Obj::Dataset(*d), - Target::Hard(p) => self.resolve(cur, p, depth + 1)?, + Target::Hard(p) => match memo[cur][li] { + Resolution::Done(o) => o, + Resolution::InProgress => { + return Err(err(format!( + "hard link target {target:?}: the hard link {:?} leads \ + back to itself (a cycle)", + join(&grp.path, c) + ))); + } + Resolution::Todo => { + memo[cur][li] = Resolution::InProgress; + let o = self.resolve(memo, cur, p, depth + 1)?; + memo[cur][li] = Resolution::Done(o); + o + } + }, Target::Soft(_) | Target::External { .. } => { return Err(err(format!( "hard link target {target:?} goes through a soft or external \ @@ -305,6 +332,14 @@ impl Builder { } } +/// Where resolving one hard link has got to. +#[derive(Clone, Copy)] +enum Resolution { + Todo, + InProgress, + Done(Obj), +} + #[derive(Clone, Copy)] enum Obj { Group(usize), @@ -325,14 +360,27 @@ pub(crate) fn build(root: GroupBuilder, default_track_order: bool) -> Result> = b + .groups + .iter() + .map(|g| vec![Resolution::Todo; g.links.len()]) + .collect(); let mut resolved: Vec>> = Vec::with_capacity(b.groups.len()); for (gi, g) in b.groups.iter().enumerate() { let mut row = Vec::with_capacity(g.links.len()); - for (_, t) in &g.links { + for (li, (_, t)) in g.links.iter().enumerate() { let obj = match t { Target::Group(i) => Some(Obj::Group(*i)), Target::Dataset(d) => Some(Obj::Dataset(*d)), - Target::Hard(p) => Some(b.resolve(gi, p, 0)?), + Target::Hard(p) => Some(match memo[gi][li] { + Resolution::Done(o) => o, + _ => { + memo[gi][li] = Resolution::InProgress; + let o = b.resolve(&mut memo, gi, p, 0)?; + memo[gi][li] = Resolution::Done(o); + o + } + }), Target::Soft(_) | Target::External { .. } => None, }; match obj { diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index 27379bb..e8488c9 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -827,3 +827,52 @@ fn a_link_too_big_for_dense_storage_is_an_error() { assert_eq!(out, "[11, 65001]"); h5dump_ok(&path); } + +#[test] +fn chained_hard_links_resolve_in_linear_time() { + skip_if_no_python!(); + // Each link's target goes through the previous link twice. Resolving + // them without remembering resolved links doubled the work per link: + // 26 links took 46 s in a debug build, so 60 would never finish. + fn chain(reverse: bool) -> FileBuilder { + let mut b = FileBuilder::new(); + let mut g = b.create_group("g"); + g.create_dataset("v").with_i32_data(&[5]); + b.add_group(g.finish()); + let mut order: Vec = (0..60).collect(); + if reverse { + order.reverse(); + } + for i in order { + if i == 0 { + b.add_hard_link("g/s0", "/g"); + } else { + b.add_hard_link(&format!("g/s{i}"), &format!("/g/s{}/s{}", i - 1, i - 1)); + } + } + b + } + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let bytes = [false, true].map(|r| chain(r).finish().unwrap()); + tx.send(bytes).unwrap(); + }); + let [forward, reverse] = rx + .recv_timeout(std::time::Duration::from_secs(60)) + .expect("resolving 60 chained hard links took over a minute"); + + let dir = tempfile::tempdir().unwrap(); + for (name, bytes) in [("forward.h5", forward), ("reverse.h5", reverse)] { + let path = dir.path().join(name).display().to_string(); + std::fs::write(&path, bytes).unwrap(); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 print(json.dumps([h5py.h5o.get_info(f['g'].id).rc, len(f['g']),\n\ + \x20 int(f['g/s59/s30/s0/v'][0]), f['g/s59'] == f['g']]))", + ); + assert_eq!(out, "[61, 61, 5, true]", "{name}"); + let f = File::open(&path).unwrap(); + assert_eq!(f.dataset("g/s59/s0/v").unwrap().read_i32().unwrap(), [5]); + } +} -- 2.54.0 From f0ecae38b68d519bc357e1a2601639184d2ea827 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:01:54 -0500 Subject: [PATCH 26/36] perf(py): datasets and groups keep their address; groups their links Every ds[...] and g[k] resolved the path from the root again, two or three times per open, and resolving a name in a large group scans its links: visiting a group was O(n^2). 4000 scalar datasets in one group took 39 s (v1 group) and 131 s (dense) to list, read and re-read; now 0.3 s each. A Dataset keeps its object address, a Group (and the file's root) its address and, after the first lookup, its link table. New facade API File::dataset_at(address), tested in integration_tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 + crates/clawhdf5-py/src/attrs.rs | 7 +- crates/clawhdf5-py/src/dataset.rs | 22 +- crates/clawhdf5-py/src/file.rs | 41 +-- crates/clawhdf5-py/src/group.rs | 276 +++++++++++------- crates/clawhdf5-py/src/node.rs | 74 +++-- crates/clawhdf5-py/tests/test_read_vs_h5py.py | 33 +++ crates/clawhdf5/src/reader.rs | 16 + crates/clawhdf5/tests/integration_tests.rs | 32 ++ 9 files changed, 349 insertions(+), 163 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29e25a..8e61de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,17 @@ (h5py 4.1 ms; release build on tank, best of 5). `test_a_long_index_list_decodes_each_chunk_once` compares 1-D, 2-D and contiguous cases with h5py under a 2 s bound (5.8 s before, debug build). +- **Groups and datasets remember where they are.** Every `ds[...]`, and + every `g[k]`, resolved its path from the root again (two or three times + per open), and in a large group each resolution scans the group's links, + so visiting a group was quadratic: 4000 scalar datasets in one group took + 39 s (`libver='earliest'`) and 131 s (`'latest'`) to list, read and + re-read in `test_big_groups_are_not_quadratic`; now 0.3 s each (debug + build). A `Dataset` keeps its object's address, and a `Group` (and the + file's root) its address and, once listed, its link table. New facade + API: `File::dataset_at(address)` opens a dataset without resolving a + path. libhdf5's `h5stat_newgrat.h5` (35001 members in the root): listing + takes 0.03 s and 2000 opens 1 ms (h5py: 0.022 s). - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/crates/clawhdf5-py/src/attrs.rs b/crates/clawhdf5-py/src/attrs.rs index d1b06cb..fd3f0b3 100644 --- a/crates/clawhdf5-py/src/attrs.rs +++ b/crates/clawhdf5-py/src/attrs.rs @@ -34,9 +34,10 @@ pub struct PyAttrs { } impl PyAttrs { - /// The attributes of the object at `path` in a file opened for reading. - pub(crate) fn read(file: Arc, path: &str) -> PyResult { - let attrs = node::attributes(&file, path)?; + /// The attributes of the object at `addr` (whose path is `path`) in a + /// file opened for reading. + pub(crate) fn read(file: Arc, addr: u64, path: &str) -> PyResult { + let attrs = node::attributes(&file, addr, path)?; Ok(Self { inner: AttrsInner::Read { file, attrs }, }) diff --git a/crates/clawhdf5-py/src/dataset.rs b/crates/clawhdf5-py/src/dataset.rs index aa75e7d..c20f293 100644 --- a/crates/clawhdf5-py/src/dataset.rs +++ b/crates/clawhdf5-py/src/dataset.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::object_header::ObjectHeader; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyList, PyTuple}; @@ -29,6 +30,9 @@ use crate::{PyEmpty, node, to_py_err}; pub struct PyDataset { file: Arc, path: String, + /// Where the dataset's object header is: reads open it from here rather + /// than resolve `path` again. + addr: u64, /// `None` for a dataset with a null dataspace (h5py's `Empty`). shape: Option>, /// The chunk shape, for a chunked dataset. @@ -43,12 +47,13 @@ impl PyDataset { py: Python<'_>, file: Arc, path: String, + addr: u64, + hdr: &ObjectHeader, ) -> PyResult { crate::no_panic(|| { - let hdr = node::header(&file, &path)?; - let null = node::is_null(&node::dataspace(&file, &hdr)?); + let null = node::is_null(&node::dataspace(&file, hdr)?); let (shape, datatype) = { - let ds = file.dataset(&path).map_err(to_py_err)?; + let ds = file.dataset_at(addr).map_err(to_py_err)?; let shape = if null { None } else { @@ -60,10 +65,11 @@ impl PyDataset { .map_err(|e| e.value(py).to_string()); let chunks = shape .as_ref() - .and_then(|s| node::chunk_shape(&file, &hdr, s.len())); + .and_then(|s| node::chunk_shape(&file, hdr, s.len())); Ok(Self { file, path, + addr, shape, chunks, datatype, @@ -96,10 +102,10 @@ impl PyDataset { let (reads, list_axis) = plan.reads(dims, chunk_len, elem_size); let read_shape = plan.read_shape(); let file = &*self.file; - let path = self.path.as_str(); + let addr = self.addr; // Everything below touches only Rust data: release the GIL. let read = || -> Result { - let ds = file.dataset(path)?; + let ds = file.dataset_at(addr)?; let mut blocks = Vec::with_capacity(reads.len()); for read in reads { let raw = ds.read_selection(&read.sel)?; @@ -250,7 +256,7 @@ impl PyDataset { }; let max = self .file - .dataset(&self.path) + .dataset_at(self.addr) .and_then(|ds| ds.max_dimensions()) .map_err(to_py_err)? .unwrap_or_else(|| shape.clone()); @@ -288,7 +294,7 @@ impl PyDataset { /// The dataset's attributes (read-only, dict-like). #[getter] fn attrs(&self) -> PyResult { - PyAttrs::read(Arc::clone(&self.file), &self.path) + PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path) } /// Read with h5py indexing: integers, slices with positive steps, diff --git a/crates/clawhdf5-py/src/file.rs b/crates/clawhdf5-py/src/file.rs index a2c48ea..5f7f51b 100644 --- a/crates/clawhdf5-py/src/file.rs +++ b/crates/clawhdf5-py/src/file.rs @@ -3,12 +3,11 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; -use pyo3::exceptions::PyKeyError; use pyo3::prelude::*; use pyo3::types::PyList; use crate::attrs::PyAttrs; -use crate::group::{self, PyGroup, WriteGroupState, finalize_write_group}; +use crate::group::{PyGroup, ReadGroup, WriteGroupState, finalize_write_group}; use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, to_py_err}; /// Internal state for write mode. @@ -40,7 +39,8 @@ pub struct PyFile { } enum FileInner { - Read(Arc), + /// The root group; it holds the file. + Read(ReadGroup), Write(WriteState), } @@ -61,7 +61,7 @@ impl PyFile { crate::no_panic(|| clawhdf5_rs::File::open(path).map_err(to_py_err)) })?; Ok(Self { - inner: Some(FileInner::Read(Arc::new(file))), + inner: Some(FileInner::Read(root_group(Arc::new(file)))), filename, }) } @@ -110,33 +110,28 @@ impl PyFile { /// Get a child object (dataset or group) by path; `f['/']` is the root. fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { - group::get_item(py, self.read_file()?, "", key) + self.read_file()?.get_item(py, key) } /// `f.get(key, default=None)`. #[pyo3(signature = (key, default=None))] fn get(&self, py: Python<'_>, key: &str, default: Option>) -> PyResult> { - match group::get_item(py, self.read_file()?, "", key) { - Err(e) if e.is_instance_of::(py) => { - Ok(default.unwrap_or_else(|| py.None())) - } - other => other, - } + self.read_file()?.get(py, key, default) } /// List the names of all children in the root group. fn keys(&self, py: Python<'_>) -> PyResult> { - let names = group::member_names(self.read_file()?, "")?; + let names = self.read_file()?.member_names()?; Ok(PyList::new(py, names)?.into_any().unbind()) } fn values(&self, py: Python<'_>) -> PyResult> { - let vals = group::values(py, self.read_file()?, "")?; + let vals = self.read_file()?.values(py)?; Ok(PyList::new(py, vals)?.into_any().unbind()) } fn items(&self, py: Python<'_>) -> PyResult> { - let items = group::items(py, self.read_file()?, "")?; + let items = self.read_file()?.items(py)?; Ok(PyList::new(py, items)?.into_any().unbind()) } @@ -145,7 +140,7 @@ impl PyFile { } fn __len__(&self) -> PyResult { - Ok(group::member_names(self.read_file()?, "")?.len()) + Ok(self.read_file()?.member_names()?.len()) } /// The root group's name, `/`. @@ -211,7 +206,7 @@ impl PyFile { #[getter] fn attrs(&self) -> PyResult { match self.inner.as_ref() { - Some(FileInner::Read(file)) => PyAttrs::read(Arc::clone(file), ""), + Some(FileInner::Read(root)) => root.attrs(), Some(FileInner::Write(state)) => Ok(PyAttrs::from_write(Arc::clone(&state.root_attrs))), None => Err(PyErr::new::( "file is closed", @@ -221,8 +216,8 @@ impl PyFile { fn __repr__(&self) -> String { match &self.inner { - Some(FileInner::Read(f)) => { - format!("", f.as_bytes().len()) + Some(FileInner::Read(root)) => { + format!("", root.file.as_bytes().len()) } Some(FileInner::Write(s)) => { format!("", s.path.display()) @@ -232,12 +227,13 @@ impl PyFile { } fn __contains__(&self, key: &str) -> PyResult { - Ok(group::contains(self.read_file()?, "", key)) + Ok(self.read_file()?.contains(key)) } } impl PyFile { - fn read_file(&self) -> PyResult<&Arc> { + /// The root group of a file opened for reading. + fn read_file(&self) -> PyResult<&ReadGroup> { match &self.inner { Some(FileInner::Read(f)) => Ok(f), Some(FileInner::Write(_)) => Err(PyErr::new::( @@ -275,6 +271,11 @@ fn parse_compression( } } +fn root_group(file: Arc) -> ReadGroup { + let root = file.superblock().root_group_address; + ReadGroup::new(file, String::new(), root) +} + /// Build and write the HDF5 file from accumulated write state. fn finalize_write(state: WriteState) -> PyResult<()> { crate::no_panic(|| { diff --git a/crates/clawhdf5-py/src/group.rs b/crates/clawhdf5-py/src/group.rs index 365846b..7585e57 100644 --- a/crates/clawhdf5-py/src/group.rs +++ b/crates/clawhdf5-py/src/group.rs @@ -1,13 +1,14 @@ //! PyGroup — navigable HDF5 group with read and write support. -use std::sync::{Arc, Mutex}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; -use pyo3::exceptions::{PyIOError, PyKeyError}; +use pyo3::exceptions::{PyIOError, PyKeyError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyList; use crate::attrs::PyAttrs; -use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node, to_py_err}; +use crate::{DatasetSpec, OwnedAttrValue, apply_dataset_spec, extract_numpy_data, node}; /// Shared state for a group being written. pub(crate) struct WriteGroupState { @@ -28,17 +29,14 @@ pub struct PyGroup { } enum GroupInner { - Read { - file: Arc, - path: String, - }, + Read(ReadGroup), Write(Arc>), } impl PyGroup { - pub(crate) fn from_read(file: Arc, path: String) -> Self { + pub(crate) fn from_read(file: Arc, path: String, addr: u64) -> Self { Self { - inner: GroupInner::Read { file, path }, + inner: GroupInner::Read(ReadGroup::new(file, path, addr)), } } @@ -48,9 +46,9 @@ impl PyGroup { } } - fn read_parts(&self, what: &str) -> PyResult<(&Arc, &str)> { + fn read_group(&self, what: &str) -> PyResult<&ReadGroup> { match &self.inner { - GroupInner::Read { file, path } => Ok((file, path)), + GroupInner::Read(g) => Ok(g), GroupInner::Write(_) => Err(PyIOError::new_err(format!( "cannot {what} a group opened for writing" ))), @@ -58,77 +56,90 @@ impl PyGroup { } } -// Read-mode operations shared by `Group` and `File` (a file is its root -// group, as in h5py). - -/// `group[key]`. -pub(crate) fn get_item( - py: Python<'_>, - file: &Arc, - path: &str, - key: &str, -) -> PyResult> { - node::open(py, file, node::join(path, key)) +/// A group in a file opened for reading (a file is its root group, as in +/// h5py). It keeps its own address and, once listed, its links, so looking +/// up a child neither resolves the path from the root nor scans the group's +/// links again: visiting every member of a large group is linear, not +/// quadratic. +pub(crate) struct ReadGroup { + pub file: Arc, + pub path: String, + pub addr: u64, + /// Link name -> object address (soft links resolved), filled on first use. + links: OnceLock>, + /// Names of the datasets and subgroups, sorted (h5py's order). + members: OnceLock>, } -/// Names of the group's datasets and subgroups, sorted (h5py's order). -pub(crate) fn member_names(file: &clawhdf5_rs::File, path: &str) -> PyResult> { - crate::no_panic(|| { - let group = if path.is_empty() { - file.root() - } else { - file.group(path).map_err(to_py_err)? - }; - let mut names = group.datasets().map_err(to_py_err)?; - names.extend(group.groups().map_err(to_py_err)?); - names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); - names.dedup(); - Ok(names) - }) -} - -pub(crate) fn contains(file: &clawhdf5_rs::File, path: &str, key: &str) -> bool { - node::exists(file, &node::join(path, key)) -} - -pub(crate) fn values( - py: Python<'_>, - file: &Arc, - path: &str, -) -> PyResult>> { - member_names(file, path)? - .iter() - .map(|n| get_item(py, file, path, n)) - .collect() -} - -pub(crate) fn items( - py: Python<'_>, - file: &Arc, - path: &str, -) -> PyResult)>> { - member_names(file, path)? - .into_iter() - .map(|n| { - let v = get_item(py, file, path, &n)?; - Ok((n, v)) - }) - .collect() -} - -#[pymethods] -impl PyGroup { - /// Get a child object (dataset or subgroup) by name or path. - fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { - let (file, path) = self.read_parts("read children from")?; - get_item(py, file, path, key) +impl ReadGroup { + pub(crate) fn new(file: Arc, path: String, addr: u64) -> Self { + Self { + file, + path, + addr, + links: OnceLock::new(), + members: OnceLock::new(), + } } - /// `group.get(key, default=None)`. - #[pyo3(signature = (key, default=None))] - fn get(&self, py: Python<'_>, key: &str, default: Option>) -> PyResult> { - let (file, path) = self.read_parts("read children from")?; - match get_item(py, file, path, key) { + fn links(&self) -> PyResult<&HashMap> { + if let Some(links) = self.links.get() { + return Ok(links); + } + let entries = crate::no_panic(|| { + clawhdf5_format::group_v2::resolve_group_children( + self.file.as_bytes(), + self.file.superblock(), + self.addr, + ) + .map_err(|e| PyValueError::new_err(format!("{}: {e}", node::name(&self.path)))) + })?; + let map = entries + .into_iter() + .map(|e| (e.name, e.object_header_address)) + .collect(); + Ok(self.links.get_or_init(|| map)) + } + + /// The path and address of `key` (a name, a relative or an absolute path). + fn locate(&self, key: &str) -> PyResult<(String, u64)> { + let path = node::join(&self.path, key); + let rel = if self.path.is_empty() { + Some(path.as_str()) + } else if path == self.path { + Some("") + } else { + path.strip_prefix(self.path.as_str()) + .and_then(|r| r.strip_prefix('/')) + }; + let addr = match rel { + // A direct child: the link table, when it has the name. + Some(name) if !name.is_empty() && !name.contains('/') => { + match self.links()?.get(name) { + Some(&a) => a, + None => node::resolve_from(&self.file, self.addr, name, &path)?, + } + } + Some(rel) => node::resolve_from(&self.file, self.addr, rel, &path)?, + None => node::address(&self.file, &path)?, + }; + Ok((path, addr)) + } + + /// `group[key]`. + pub(crate) fn get_item(&self, py: Python<'_>, key: &str) -> PyResult> { + let (path, addr) = self.locate(key)?; + node::open(py, &self.file, path, addr) + } + + /// `group.get(key, default)`. + pub(crate) fn get( + &self, + py: Python<'_>, + key: &str, + default: Option>, + ) -> PyResult> { + match self.get_item(py, key) { Err(e) if e.is_instance_of::(py) => { Ok(default.unwrap_or_else(|| py.None())) } @@ -136,11 +147,70 @@ impl PyGroup { } } + /// Names of the group's datasets and subgroups, sorted (h5py's order). + pub(crate) fn member_names(&self) -> PyResult<&[String]> { + if let Some(m) = self.members.get() { + return Ok(m); + } + let mut names = Vec::new(); + for (name, &addr) in self.links()? { + let hdr = node::header_at(&self.file, addr, &node::join(&self.path, name))?; + if matches!( + node::kind(&hdr), + Some(node::Kind::Dataset | node::Kind::Group) + ) { + names.push(name.clone()); + } + } + names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes())); + Ok(self.members.get_or_init(|| names)) + } + + pub(crate) fn contains(&self, key: &str) -> bool { + self.locate(key) + .and_then(|(path, addr)| node::header_at(&self.file, addr, &path)) + .ok() + .and_then(|h| node::kind(&h)) + .is_some_and(|k| k != node::Kind::Datatype) + } + + pub(crate) fn values(&self, py: Python<'_>) -> PyResult>> { + self.member_names()? + .iter() + .map(|n| self.get_item(py, n)) + .collect() + } + + pub(crate) fn items(&self, py: Python<'_>) -> PyResult)>> { + self.member_names()? + .iter() + .map(|n| Ok((n.clone(), self.get_item(py, n)?))) + .collect() + } + + pub(crate) fn attrs(&self) -> PyResult { + PyAttrs::read(Arc::clone(&self.file), self.addr, &self.path) + } +} + +#[pymethods] +impl PyGroup { + /// Get a child object (dataset or subgroup) by name or path. + fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { + self.read_group("read children from")?.get_item(py, key) + } + + /// `group.get(key, default=None)`. + #[pyo3(signature = (key, default=None))] + fn get(&self, py: Python<'_>, key: &str, default: Option>) -> PyResult> { + self.read_group("read children from")?.get(py, key, default) + } + /// List the names of all children (datasets and subgroups). fn keys(&self, py: Python<'_>) -> PyResult> { match &self.inner { - GroupInner::Read { file, path } => { - let list = PyList::new(py, member_names(file, path)?)?; + GroupInner::Read(g) => { + let list = PyList::new(py, g.member_names()?)?; Ok(list.into_any().unbind()) } GroupInner::Write(state) => { @@ -153,15 +223,13 @@ impl PyGroup { } fn values(&self, py: Python<'_>) -> PyResult> { - let (file, path) = self.read_parts("read children from")?; - Ok(PyList::new(py, values(py, file, path)?)? - .into_any() - .unbind()) + let g = self.read_group("read children from")?; + Ok(PyList::new(py, g.values(py)?)?.into_any().unbind()) } fn items(&self, py: Python<'_>) -> PyResult> { - let (file, path) = self.read_parts("read children from")?; - Ok(PyList::new(py, items(py, file, path)?)?.into_any().unbind()) + let g = self.read_group("read children from")?; + Ok(PyList::new(py, g.items(py)?)?.into_any().unbind()) } fn __iter__(&self, py: Python<'_>) -> PyResult> { @@ -170,7 +238,7 @@ impl PyGroup { fn __len__(&self) -> PyResult { match &self.inner { - GroupInner::Read { file, path } => Ok(member_names(file, path)?.len()), + GroupInner::Read(g) => Ok(g.member_names()?.len()), GroupInner::Write(state) => Ok(state.lock().unwrap().datasets.len()), } } @@ -179,7 +247,7 @@ impl PyGroup { #[getter] fn name(&self) -> String { match &self.inner { - GroupInner::Read { path, .. } => node::name(path), + GroupInner::Read(g) => node::name(&g.path), GroupInner::Write(state) => node::name(&state.lock().unwrap().name), } } @@ -235,7 +303,7 @@ impl PyGroup { #[getter] fn attrs(&self) -> PyResult { match &self.inner { - GroupInner::Read { file, path } => PyAttrs::read(Arc::clone(file), path), + GroupInner::Read(g) => g.attrs(), GroupInner::Write(state) => { let store = Arc::clone(&state.lock().unwrap().attrs); Ok(PyAttrs::from_write(store)) @@ -245,9 +313,9 @@ impl PyGroup { fn __repr__(&self) -> String { match &self.inner { - GroupInner::Read { file, path } => { - let n = member_names(file, path).map_or(0, |m| m.len()); - format!("", node::name(path)) + GroupInner::Read(g) => { + let n = g.member_names().map_or(0, |m| m.len()); + format!("", node::name(&g.path)) } GroupInner::Write(state) => { let name = &state.lock().unwrap().name; @@ -258,7 +326,7 @@ impl PyGroup { fn __contains__(&self, key: &str) -> PyResult { match &self.inner { - GroupInner::Read { file, path } => Ok(contains(file, path, key)), + GroupInner::Read(g) => Ok(g.contains(key)), GroupInner::Write(state) => { let guard = state.lock().unwrap(); Ok(guard.datasets.iter().any(|d| d.name == key)) @@ -299,15 +367,19 @@ mod tests { let finished = g.finish(); b.add_group(finished); let bytes = b.finish().unwrap(); - let file = clawhdf5_rs::File::from_bytes(bytes).unwrap(); - assert_eq!( - member_names(&file, "").unwrap(), - vec!["alpha", "mid", "zeta"] - ); - assert_eq!(member_names(&file, "mid").unwrap(), vec!["x"]); - assert!(contains(&file, "", "mid/x")); - assert!(contains(&file, "mid", "/alpha")); - assert!(!contains(&file, "", "nope")); + let file = Arc::new(clawhdf5_rs::File::from_bytes(bytes).unwrap()); + let root = file.superblock().root_group_address; + let top = ReadGroup::new(Arc::clone(&file), String::new(), root); + assert_eq!(top.member_names().unwrap(), ["alpha", "mid", "zeta"]); + let (path, addr) = top.locate("mid").unwrap(); + assert_eq!(path, "mid"); + let mid = ReadGroup::new(Arc::clone(&file), path, addr); + assert_eq!(mid.member_names().unwrap(), ["x"]); + assert!(top.contains("mid/x")); + assert!(mid.contains("/alpha")); + assert!(mid.contains("x") && mid.contains("./x")); + assert!(!top.contains("nope")); + assert!(!mid.contains("alpha")); } #[test] diff --git a/crates/clawhdf5-py/src/node.rs b/crates/clawhdf5-py/src/node.rs index bb6c2d2..fa9ede7 100644 --- a/crates/clawhdf5-py/src/node.rs +++ b/crates/clawhdf5-py/src/node.rs @@ -33,24 +33,40 @@ pub(crate) fn name(path: &str) -> String { format!("/{path}") } -/// The object header of the object at `path`. -pub(crate) fn header(file: &clawhdf5_rs::File, path: &str) -> PyResult { +/// The address of the object at `path`, resolved from the root group. +pub(crate) fn address(file: &clawhdf5_rs::File, path: &str) -> PyResult { + resolve_from(file, file.superblock().root_group_address, path, path) +} + +/// The address of `rel` resolved from the group at `group` (`full` is the +/// resulting path, for the error message). +pub(crate) fn resolve_from( + file: &clawhdf5_rs::File, + group: u64, + rel: &str, + full: &str, +) -> PyResult { + if rel.is_empty() { + return Ok(group); + } crate::no_panic(|| { - let sb = file.superblock(); - let data = file.as_bytes(); - let addr = if path.is_empty() { - sb.root_group_address - } else { - clawhdf5_format::group_v2::resolve_path_any(data, sb, path).map_err(|e| { + clawhdf5_format::group_v2::resolve_path_from(file.as_bytes(), file.superblock(), group, rel) + .map_err(|e| { PyKeyError::new_err(format!( "Unable to open object (object '{}' doesn't exist): {e}", - name(path) + name(full) )) - })? - }; - let addr = usize::try_from(addr) + }) + }) +} + +/// The object header at `addr` (the object at `path`). +pub(crate) fn header_at(file: &clawhdf5_rs::File, addr: u64, path: &str) -> PyResult { + crate::no_panic(|| { + let sb = file.superblock(); + let at = usize::try_from(addr) .map_err(|_| PyValueError::new_err(format!("{}: address out of range", name(path))))?; - ObjectHeader::parse(data, addr, sb.offset_size, sb.length_size) + ObjectHeader::parse(file.as_bytes(), at, sb.offset_size, sb.length_size) .map_err(|e| PyValueError::new_err(format!("{}: {e}", name(path)))) }) } @@ -80,19 +96,21 @@ pub(crate) fn kind(hdr: &ObjectHeader) -> Option { } } -/// Open the object at `path` as a `Dataset` or `Group`. +/// Open the object at `addr` (whose path is `path`) as a `Dataset` or +/// `Group`. Both keep the address, so later reads resolve nothing. pub(crate) fn open( py: Python<'_>, file: &Arc, path: String, + addr: u64, ) -> PyResult> { - let hdr = header(file, &path)?; + let hdr = header_at(file, addr, &path)?; match kind(&hdr) { - Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path)? + Some(Kind::Dataset) => Ok(PyDataset::open(py, Arc::clone(file), path, addr, &hdr)? .into_pyobject(py)? .into_any() .unbind()), - Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path) + Some(Kind::Group) => Ok(PyGroup::from_read(Arc::clone(file), path, addr) .into_pyobject(py)? .into_any() .unbind()), @@ -107,14 +125,6 @@ pub(crate) fn open( } } -/// Whether `path` names a dataset or group. -pub(crate) fn exists(file: &clawhdf5_rs::File, path: &str) -> bool { - header(file, path) - .ok() - .and_then(|h| kind(&h)) - .is_some_and(|k| k != Kind::Datatype) -} - /// The dataspace message of an object header. pub(crate) fn dataspace(file: &clawhdf5_rs::File, hdr: &ObjectHeader) -> PyResult { crate::no_panic(|| { @@ -166,12 +176,16 @@ pub(crate) fn is_null(space: &Dataspace) -> bool { space.space_type == DataspaceType::Null } -/// The attributes of the object at `path`, sorted by name (h5py's order). -/// Attributes whose messages cannot be parsed are left out, as the facade's -/// `attrs()` does. -pub(crate) fn attributes(file: &clawhdf5_rs::File, path: &str) -> PyResult> { +/// The attributes of the object at `addr` (whose path is `path`), sorted by +/// name (h5py's order). Attributes whose messages cannot be parsed are left +/// out, as the facade's `attrs()` does. +pub(crate) fn attributes( + file: &clawhdf5_rs::File, + addr: u64, + path: &str, +) -> PyResult> { + let hdr = header_at(file, addr, path)?; crate::no_panic(|| { - let hdr = header(file, path)?; let sb = file.superblock(); let (mut attrs, _errors) = clawhdf5_format::attribute::extract_attributes_tolerant( file.as_bytes(), diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py index f51c534..a02484c 100644 --- a/crates/clawhdf5-py/tests/test_read_vs_h5py.py +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -563,3 +563,36 @@ def test_a_long_index_list_decodes_each_chunk_once(h5py, tmp_path): took = time.perf_counter() - t0 assert_same(got, theirs[name][key], f"{name}[{len(key)}-key]") assert took < 2.0, f"{name}: {took:.2f} s" + + +@pytest.mark.parametrize("libver", ["earliest", "latest"]) +def test_big_groups_are_not_quadratic(h5py, tmp_path, libver): + """A dataset or group remembers where its object is, and a group its + links, so reads and walks over a large group do not resolve every path + from the root again (it was O(n) per access: O(n^2) to visit a group).""" + import time + + path = str(tmp_path / f"big_{libver}.h5") + n = 4000 + with h5py.File(path, "w", libver=libver) as f: + g = f.create_group("g") + for i in range(n): + g.create_dataset(f"d{i:05d}", data=np.int32(i)) + g.create_group("sub").create_dataset("leaf", data=np.arange(3)) + with h5py.File(path, "r") as theirs, clawhdf5.File(path, "r") as ours: + t0 = time.perf_counter() + g = ours["g"] + assert list(g.keys()) == list(theirs["g"].keys()) + total = sum(int(v[()]) for k, v in g.items() if k.startswith("d")) + assert total == n * (n - 1) // 2 + seen = 0 + for k in g: + if k.startswith("d"): + seen += int(g[k][()]) == int(k[1:]) + assert seen == n + ds = g["d00007"] + assert all(ds[()] == 7 for _ in range(2000)) + assert list(g["sub"]["leaf"][:]) == [0, 1, 2] + assert ours["/g/sub/leaf"][1] == 1 and g["/g/d00003"][()] == 3 + took = time.perf_counter() - t0 + assert took < 5.0, f"{took:.2f} s" diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 83cbfa8..2e86dfe 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -176,6 +176,22 @@ impl File { }) } + /// A `Dataset` handle for the object header at `address` (an address + /// from a group listing, or one kept from an earlier lookup), without + /// resolving a path. Resolving a path walks every group on it, which in + /// a large group costs a scan of its links; keep the address instead to + /// open the same dataset repeatedly. + pub fn dataset_at(&self, address: u64) -> Result, Error> { + let hdr = self.parse_header(address)?; + if !has_message(&hdr, MessageType::DataLayout) { + return Err(Error::NotADataset(format!("object at address {address}"))); + } + Ok(Dataset { + file: self, + header: hdr, + }) + } + /// Resolve a path and return a `Group` handle. /// /// The path uses `/` separators (e.g., `"sensors"`). diff --git a/crates/clawhdf5/tests/integration_tests.rs b/crates/clawhdf5/tests/integration_tests.rs index e8ddab9..47798e8 100644 --- a/crates/clawhdf5/tests/integration_tests.rs +++ b/crates/clawhdf5/tests/integration_tests.rs @@ -986,3 +986,35 @@ fn u64_data_roundtrip() { values ); } + +// --------------------------------------------------------------------------- +// Opening a dataset by address +// --------------------------------------------------------------------------- + +#[test] +fn dataset_at_opens_the_same_dataset_as_its_path() { + let mut b = FileBuilder::new(); + let mut g = b.create_group("grp"); + g.create_dataset("vals").with_f64_data(&[1.0, 2.5, -3.0]); + b.add_group(g.finish()); + let file = File::from_bytes(b.finish().unwrap()).unwrap(); + + let addr = + clawhdf5_format::group_v2::resolve_path_any(file.as_bytes(), file.superblock(), "grp/vals") + .unwrap(); + let by_addr = file.dataset_at(addr).unwrap(); + assert_eq!(by_addr.read_f64().unwrap(), vec![1.0, 2.5, -3.0]); + assert_eq!( + by_addr.shape().unwrap(), + file.dataset("grp/vals").unwrap().shape().unwrap() + ); + + // The group's own header is not a dataset. + let group_addr = + clawhdf5_format::group_v2::resolve_path_any(file.as_bytes(), file.superblock(), "grp") + .unwrap(); + assert!(matches!( + file.dataset_at(group_addr), + Err(clawhdf5::Error::NotADataset(_)) + )); +} -- 2.54.0 From 8bcae3c78e14c75b7edeb5927a74041c7bfde3d8 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:02:09 -0500 Subject: [PATCH 27/36] fix(format): a dataset attribute set again replaces the earlier value b0a1e4f fixed this for group and root attributes only. Setting a dataset attribute twice still wrote two attribute messages with one name, and h5py read back the first value: set_attr("a", 1) then set_attr("a", 2) read as 1, and list(attrs) was ["a", "a"]. DatasetBuilder::set_attr now replaces the earlier value, compact or dense. Likewise, a hand-set attribute named like a provenance attribute (_provenance_sha256, ...) is replaced by the computed one instead of being written next to it and read first. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/file_writer.rs | 5 +- crates/clawhdf5-format/src/type_builders.rs | 7 ++- .../clawhdf5/tests/writer_groups_interop.rs | 62 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 0b318a9..11345a1 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -1355,7 +1355,10 @@ fn flatten_ds(db: DatasetBuilder, refcount: u32) -> Result timestamp: prov.timestamp.clone(), source: prov.source.clone(), }; - attrs.extend(p.build_attrs(&raw)); + // The provenance attributes replace any the caller set by hand. + let prov = p.build_attrs(&raw); + attrs.retain(|a| prov.iter().all(|b| b.name != a.name)); + attrs.extend(prov); } let fill_message = fill_value_message(db.fill_time, db.fill_value.as_deref(), &dt)?; Ok(DsFlat { diff --git a/crates/clawhdf5-format/src/type_builders.rs b/crates/clawhdf5-format/src/type_builders.rs index fa64465..ce43f64 100644 --- a/crates/clawhdf5-format/src/type_builders.rs +++ b/crates/clawhdf5-format/src/type_builders.rs @@ -695,8 +695,13 @@ impl DatasetBuilder { self } + /// Set attribute `name`. Setting it again replaces the earlier value, + /// as `attrs[name] = v` does in h5py. pub fn set_attr(&mut self, name: &str, value: AttrValue) -> &mut Self { - self.attrs.push((name.to_string(), value)); + match self.attrs.iter_mut().find(|(n, _)| n == name) { + Some(slot) => slot.1 = value, + None => self.attrs.push((name.to_string(), value)), + } self } diff --git a/crates/clawhdf5/tests/writer_groups_interop.rs b/crates/clawhdf5/tests/writer_groups_interop.rs index e8488c9..97e3544 100644 --- a/crates/clawhdf5/tests/writer_groups_interop.rs +++ b/crates/clawhdf5/tests/writer_groups_interop.rs @@ -876,3 +876,65 @@ fn chained_hard_links_resolve_in_linear_time() { assert_eq!(f.dataset("g/s59/s0/v").unwrap().read_i32().unwrap(), [5]); } } + +#[test] +fn a_dataset_attribute_set_again_takes_the_new_value() { + skip_if_no_python!(); + // Setting a dataset attribute twice wrote two attribute messages with + // one name, and h5py read back the first value. Also with dense + // attribute storage (more than 8). + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.create_dataset("x") + .with_f64_data(&[1.0]) + .set_attr("a", AttrValue::I64(1)) + .set_attr("a", AttrValue::I64(2)); + let d = b.create_dataset("dense"); + d.with_i32_data(&[1]); + for i in 0..12 { + d.set_attr(&format!("k{i:02}"), AttrValue::I64(i)); + } + d.set_attr("k03", AttrValue::String("three".into())); + let path = write(&dir, "ds_attrs.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 a, d = f['x'].attrs, f['dense'].attrs\n\ + \x20 print(json.dumps([list(a), int(a['a']), len(d), d['k03'].decode(), int(d['k04'])]))", + ); + assert_eq!(out, r#"[["a"], 2, 12, "three", 4]"#); + let f = File::open(&path).unwrap(); + assert!(matches!( + f.dataset("x").unwrap().attrs().unwrap()["a"], + AttrValue::I64(2) + )); +} + +#[cfg(feature = "provenance")] +#[test] +fn provenance_attributes_replace_ones_set_by_hand() { + skip_if_no_python!(); + // A hand-set attribute with a provenance attribute's name was written + // next to the computed one, and h5py read the hand-set value. + let dir = tempfile::tempdir().unwrap(); + let mut b = FileBuilder::new(); + b.create_dataset("p") + .with_i32_data(&[1, 2]) + .with_provenance("me", "2026-09-26T00:00:00Z", None) + .set_attr("_provenance_sha256", AttrValue::String("forged".into())); + let path = write(&dir, "prov.h5", b); + let out = h5py( + &path, + "with h5py.File(path, 'r') as f:\n\ + \x20 a = f['p'].attrs\n\ + \x20 h = a['_provenance_sha256']\n\ + \x20 h = h.decode() if isinstance(h, bytes) else h\n\ + \x20 print(json.dumps([list(a).count('_provenance_sha256'), h != 'forged']))", + ); + assert_eq!(out, "[1, true]"); + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("p").unwrap().verify_provenance().unwrap(), + clawhdf5_format::provenance::VerifyResult::Ok + ); +} -- 2.54.0 From 05b0192a6060438840787c87b54fd95d9b7b5537 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:02:28 -0500 Subject: [PATCH 28/36] fix(py): a 0-d integer array indexes like an int ds[np.array(1)] went down the index-list path, where tolist() returns a scalar and extracting a list of indices raised a confusing TypeError. h5py treats it as an integer index; so do we now. The h5py comparison keys include 0-d arrays (signed and unsigned) on each axis; they failed before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 +++ crates/clawhdf5-py/src/select.rs | 8 ++++++++ crates/clawhdf5-py/tests/test_read_vs_h5py.py | 5 ++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e61de0..78a3628 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,9 @@ API: `File::dataset_at(address)` opens a dataset without resolving a path. libhdf5's `h5stat_newgrat.h5` (35001 members in the root): listing takes 0.03 s and 2000 opens 1 ms (h5py: 0.022 s). +- **`ds[np.array(1)]` is an integer index**, as in h5py; a 0-d integer + array went down the index-list path and raised a confusing `TypeError`. + The h5py comparison keys now include 0-d arrays on every axis. - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/crates/clawhdf5-py/src/select.rs b/crates/clawhdf5-py/src/select.rs index 6b358b9..7c52442 100644 --- a/crates/clawhdf5-py/src/select.rs +++ b/crates/clawhdf5-py/src/select.rs @@ -361,6 +361,14 @@ fn parse_axis(py: Python<'_>, a: &Bound<'_, PyAny>, n: u64) -> PyResult { let is_array_like = a.is_instance(&np.getattr("ndarray")?)? || a.is_instance_of::() || a.is_instance_of::(); + // A 0-d integer array (`ds[np.array(1)]`) is an integer index, as in h5py. + if a.is_instance(&np.getattr("ndarray")?)? && a.getattr("ndim")?.extract::()? == 0 { + let kind: String = a.getattr("dtype")?.getattr("kind")?.extract()?; + if kind == "i" || kind == "u" { + let i: i128 = a.call_method0("item")?.extract()?; + return Ok(Axis::Index(normalize(i, n)?)); + } + } if !is_bool && !is_array_like && a.hasattr("__index__")? { let i: i128 = a.call_method0("__index__")?.extract()?; return Ok(Axis::Index(normalize(i, n)?)); diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py index a02484c..cbac1e4 100644 --- a/crates/clawhdf5-py/tests/test_read_vs_h5py.py +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -234,7 +234,8 @@ def assert_same(ours, theirs, what=""): def keys_for(shape): if shape == (): return [(), Ellipsis] - keys = [(), Ellipsis, 0, -1, slice(None), slice(None, None, 2), slice(1, None, 3), slice(0, 0), np.int64(0)] + keys = [(), Ellipsis, 0, -1, slice(None), slice(None, None, 2), slice(1, None, 3), slice(0, 0), np.int64(0), + np.array(0), np.array(-1, dtype="i1")] n0 = shape[0] if n0 == 0: return [(), Ellipsis, slice(None), slice(None, None, 2), slice(0, 0)] @@ -252,6 +253,8 @@ def keys_for(shape): (slice(None), [0, n1 - 1] if n1 > 1 else [0]), (slice(None, None, 2), 1), (slice(0, 2), slice(3, 1)), + (np.array(1) if n0 > 1 else np.array(0), slice(None)), + (slice(None), np.array(n1 - 1, dtype="u2")), ] if len(shape) >= 3: keys += [(0, slice(None), -1), (slice(1, None, 2), 2, slice(None, None, 3)), (Ellipsis, 0, 0), (0, Ellipsis, 1)] -- 2.54.0 From 546fdb84fab84d5d65ef3baba6bf567f1dad26c7 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:02:47 -0500 Subject: [PATCH 29/36] docs: dense storage fixes and the real limits of big groups The changelog, known issues and README said a group holds up to 65 535 links while a group of about 17 000 was already unreadable. Record the fixes (child indirect blocks, the next-block offset, the index leaf cap, refusing oversized dense messages, hard-link memoisation, dataset attribute overwrite) and the limits that remain true: 65 535 links or dense attributes per object, and 65 515 bytes per dense message. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++++++++---- README.md | 3 ++- docs/known-issues.md | 15 ++++++++++++--- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 080045d..d1f6b8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,19 +28,55 @@ Link Info message carries the flags, each link its order, and a dense group a creation-order B-tree (type 6). h5py then lists members in insertion order. Attribute creation order is not tracked. -- A group holds at most 65 535 links (its link index is one B-tree leaf); - more is an error. `GroupBuilder`'s fields changed (they were +- A group holds at most 65 535 links (its link index is one B-tree leaf), + and in a group of more than 8 links (dense storage) each link message + must be at most 65 515 bytes (one fractal heap block; huge heap objects + are not written); more is an error. Measured at the limit: 65 535 links + with 100-byte names (a 7 MB heap) read in h5py, h5dump and clawhdf5, and + h5py can add to the group. `GroupBuilder`'s fields changed (they were crate-private); `FinishedGroup` is unchanged for callers. - Files that use one level of groups and no new link kinds are laid out as before: byte-identical to the writer with the Group Info fix below (compared on simple, mixed dense/chunked/compact/external-link and paged files). Tests: h5py and clawhdf5 read the same tree (every path, attribute and value) from a 5-level file; soft, hard, - external and cyclic hard links; 10 000 links in one group, with and - without creation order; libhdf5 adding and deleting links in our groups; + external and cyclic hard links; 10 000, 20 000 and 65 535 links in one + group, with and without creation order; libhdf5 adding and deleting links + in our groups; `h5rs check` passes and `h5rs dump` equals h5dump (`crates/clawhdf5/tests/writer_groups_interop.rs`, `crates/clawhdf5-tools/tests/h5rs_interop.rs`). +- **Big dense groups and attribute sets were unreadable.** The fractal heap + holding dense links or attributes wrote every doubling-table row as + direct blocks, but past the 512 KiB the root's direct blocks hold, rows + are child indirect blocks, and libhdf5 and `h5rs check` read them as + such: a group with 20 000 links of 20-byte names was written without + error and h5py could not list it ("incorrect metadata checksum"); 150 + dense attributes of up to 56 KB could not be opened. This was in 2.7.0 + too. The heap writer now writes child indirect blocks, nested as deep as + needed. Found on the way: an object bigger than the next block's space + was cut off (it now goes in the first block big enough), and h5py adding + a link to a heap over 64 KiB overwrote its first block (the header's + next-block offset was 0). +- **h5py crashed adding a link to a group of more than about 47 700 + links** (35 000 with creation order tracked). The link index leaf's node size gave libhdf5 room for more than + 65 535 records, which overflows the leaf's 2-byte count. The node is now + capped at 65 535 records. Dense attributes use the same index builder: + more than 65 535 on one object used to be written with the count modulo + 65 536, and are now an error. +- **A dense link or attribute message over 65 515 bytes** (e.g. a soft link + with a long target in a group of more than 8 links) was written cut off, + and libhdf5 could not list the group ("object overruns end of direct + block"). It is now an error. +- **Chained hard links took exponential time to resolve.** A hard-link + target going through other hard links resolved them again on every path + through them: 26 links whose targets each named the previous one twice + took 46 s. Each hard link is now resolved once, and a cycle is reported + by the link's name. +- **A dataset attribute set twice read back as its first value**, as for + groups below (h5py listed the name twice). The later value now replaces + the earlier one; a hand-set attribute named like a provenance attribute + is replaced by the computed one. - **A group attribute set twice read back as its first value.** Setting a group (or root) attribute again wrote a second attribute message with the same name, and h5py returned the first value. The later value now replaces diff --git a/README.md b/README.md index 3b3fd22..20294ec 100644 --- a/README.md +++ b/README.md @@ -429,7 +429,8 @@ b.add_external_link("raw", "raw.h5", "/data"); b.write("groups.h5")?; ``` -A group holds at most 65 535 links; more is an error. +A group holds at most 65 535 links; more is an error, as is a link over +65 515 bytes (a very long soft-link target) in a group of more than 8 links. ### Agent Memory diff --git a/docs/known-issues.md b/docs/known-issues.md index 76fc86e..cd40971 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -209,11 +209,20 @@ fill-value item that did is fixed). h5py, h5dump and `h5rs check --data` read them (`crates/clawhdf5/tests/writer_groups_interop.rs`, `crates/clawhdf5-tools/tests/h5rs_interop.rs`). Still missing: a group - with more than 65 535 links (its link index is one B-tree leaf) is an - error, and attribute creation order is not tracked. + with more than 65 535 links, or an object with more than 65 535 dense + attributes, is an error (the index is one B-tree leaf), and attribute + creation order is not tracked. + - ~~Dense link or attribute storage past 512 KiB of messages was written + unreadable (child indirect blocks of the fractal heap written as direct + blocks).~~ **Fixed 2026-09-26** (it affected 2.7.0 too): tested with + 20 000 and 65 535 links and with 8 MB of dense attributes, read by + h5py, h5dump, `h5rs check` and clawhdf5, and h5py can add links to + such groups. - ~~libhdf5 could not add a link to a group we wrote (no Group Info message).~~ **Fixed 2026-09-26.** - - Dense attribute storage for attributes over 64 KiB. + - Huge fractal heap objects: in dense storage (more than 8 attributes on + an object, or more than 8 links in a group) one attribute or link + message over 65 515 bytes is an error. - Output that HDF5 1.8 can read. - A B-tree v2 chunk index larger than one leaf, so datasets with several unlimited dimensions are limited to 65 535 chunks. -- 2.54.0 From 17edfe2cf0c2e56311f43337016e4bc72902df14 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:03:53 -0500 Subject: [PATCH 30/36] test(py): detect a held GIL, and errors h5py does not raise test_threads_read_the_same_file passed with the GIL held. The new test_reads_release_the_gil measures the longest stall of a spinning Python thread while another reads: with py.detach removed from the read it stalled 0.062 s of a 0.064 s read and failed; with it, about 3 ms. test_errors_match_h5py now compares the result whenever h5py reads the key, instead of only checking that we raise when h5py raises, over a longer key list. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 +++ crates/clawhdf5-py/tests/test_read_vs_h5py.py | 62 +++++++++++++++++-- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a3628..e175c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,14 @@ - **`ds[np.array(1)]` is an integer index**, as in h5py; a 0-d integer array went down the index-list path and raised a confusing `TypeError`. The h5py comparison keys now include 0-d arrays on every axis. +- **Tests that would notice a held GIL, and our extra errors.** + `test_reads_release_the_gil` times a Python thread spinning while another + reads: with the read made to hold the GIL it stalls for the whole read + (0.062 s of a 0.064 s read) and the test fails; released, its longest + stall is about 3 ms. (The existing threads test only checked values.) + `test_errors_match_h5py` now also requires that every key h5py reads + reads here too, with the same result, and covers more keys (0-d arrays, + repeated and empty lists, `()`, `...`). - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/crates/clawhdf5-py/tests/test_read_vs_h5py.py b/crates/clawhdf5-py/tests/test_read_vs_h5py.py index cbac1e4..6a3b6cb 100644 --- a/crates/clawhdf5-py/tests/test_read_vs_h5py.py +++ b/crates/clawhdf5-py/tests/test_read_vs_h5py.py @@ -266,7 +266,10 @@ def keys_for(shape): # checked against the values written instead (test_vlen_big_endian). H5PY_MISREADS = {"vlen/f8_be"} -ERROR_KEYS_1D = [slice(None, None, -1), 10**6, -(10**6), None, (0, 0, 0, 0, 0), [3, 1], (Ellipsis, Ellipsis), 1.5, "nope"] +ERROR_KEYS_1D = [ + slice(None, None, -1), 10**6, -(10**6), None, (0, 0, 0, 0, 0), [3, 1], (Ellipsis, Ellipsis), 1.5, "nope", + np.array(1.0), np.array(True), np.array(10**6), [0, 0], [], (), Ellipsis, (0,), [-1], np.array([1, 2]), +] # --------------------------------------------------------------------------- @@ -322,12 +325,13 @@ def test_errors_match_h5py(h5py, pair): for name in ["num/le_i4_1d", "num/be_f8_2d_gzip", "str/vlen", "num/scalar_f8"]: for key in ERROR_KEYS_1D: try: - theirs[name][key] + expected = theirs[name][key] except Exception as e: # noqa: BLE001 with pytest.raises(type(e)): ours[name][key] else: - pass # valid for this shape; covered by the value test + # h5py reads it, so must we (and the same values). + assert_same(ours[name][key], expected, f"{name}[{key!r}]") def test_compound_fields_match_h5py(pair): @@ -454,8 +458,7 @@ def test_only_the_selected_chunks_are_read(h5py, tmp_path): def test_threads_read_the_same_file(pair): - """Reads from many threads at once (the GIL is released during each - read) return exactly what h5py returns.""" + """Reads from many threads at once return exactly what h5py returns.""" ours, theirs, _ = pair names = ["num/le_f8_2d_gzip", "num/i4_3d_shuffle", "str/vlen_2d_gzip", "cmp/padded_chunked", "num/be_i8_1d"] expected = {n: theirs[n][()] for n in names} @@ -477,6 +480,55 @@ def test_threads_read_the_same_file(pair): assert not errors, errors[:3] +def test_reads_release_the_gil(h5py, tmp_path): + """While one thread is inside a long read, another Python thread keeps + running. With the GIL held for the read, the other thread would stall + for the whole read; the test measures its longest stall.""" + import sys + import time + + path = str(tmp_path / "gil.h5") + data = np.arange(2048 * 4096, dtype=" 0.03, f"a read took only {one_read:.3f} s; too short to measure" + + old = sys.getswitchinterval() + sys.setswitchinterval(0.001) + stop = threading.Event() + gaps = [] + + def spin(): + last = time.perf_counter() + worst = 0.0 + while not stop.is_set(): + now = time.perf_counter() + worst = max(worst, now - last) + last = now + gaps.append(worst) + + try: + t = threading.Thread(target=spin) + t.start() + time.sleep(0.01) + t0 = time.perf_counter() + for _ in range(2): + ds[key] + reading = time.perf_counter() - t0 + stop.set() + t.join() + finally: + sys.setswitchinterval(old) + np.testing.assert_array_equal(ds[key], data[:1000]) + # Held, the spinner would stall for about one read. + assert gaps[0] < one_read / 3, f"spinner stalled {gaps[0]:.3f} s during reads of {one_read:.3f} s ({reading:.3f} s)" + + def _v4_index_fixture(h5py, path): """One 2-D dataset per v4 chunk index (HDF5 1.10+ layout, libver='latest').""" data = (np.arange(37 * 23, dtype=" Date: Sat, 26 Sep 2026 09:04:21 -0500 Subject: [PATCH 31/36] fix(tools,wasm): resolve VL data through the library's VlResolver h5rs (dump, ls, diff, check --data) kept its own lenient VL decoder: a heap object longer than its element was cut to the element's length (libhdf5 and h5py refuse it), a null string printed "" where h5dump prints NULL, the stored element size was trusted, and every heap collection was kept as an owned copy for the whole run. It now resolves each element with VlResolver::element / string_element (new: one element in place, borrowing from the file), and refuses a VL type whose stored element size is not 4 + offset size + 4, as File does. H5::heap_object and its cache are gone. h5diff compares a null VL string equal to an empty one; so does h5rs diff. clawhdf5-wasm already resolved VL strings with read_vl_strings; it now uses VlResolver and checks the stored element size before reading, as File::read_string does. Tests (h5py writes the files, patched for "a\0b", a null element and mis-sized heap objects, with 8- and 4-byte offsets): - h5rs_interop dump_prints_vl_data_like_h5dump: byte-identical to h5dump; - dump_json_vl_values_match_h5py: h5py's values, errors where h5py fails; - check_data_flags_mis_sized_vl_heap_objects; - clawhdf5-wasm tests/vl_strings.rs: wasm, File and h5py agree. All four fail before. check --data over the 150 cve_hdf5 CVE and fuzzer files now passes 15 (h5dump rejects 8 of them), was 16 and 9: the stored-size check flags cve-2024-32608. h5rs-check-ok-files.sh --data: 0 of 422 flagged; h5rs-fuzz.sh: clean on 180 files. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 20 ++- crates/clawhdf5-tools/README.md | 22 ++- crates/clawhdf5-tools/src/check.rs | 82 +++++------ crates/clawhdf5-tools/src/diff.rs | 3 + crates/clawhdf5-tools/src/h5.rs | 27 ---- crates/clawhdf5-tools/src/value.rs | 98 ++++++------- crates/clawhdf5-tools/tests/gen_vl_files.py | 121 ++++++++++++++++ crates/clawhdf5-tools/tests/h5rs_interop.rs | 122 ++++++++++++++++ crates/clawhdf5-wasm/src/core.rs | 28 ++-- crates/clawhdf5-wasm/tests/vl_strings.rs | 150 ++++++++++++++++++++ docs/known-issues.md | 12 +- 11 files changed, 533 insertions(+), 152 deletions(-) create mode 100644 crates/clawhdf5-tools/tests/gen_vl_files.py create mode 100644 crates/clawhdf5-wasm/tests/vl_strings.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 847f3c6..fc22849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,22 @@ object running past its collection. Conformance unchanged at 575 of 697 (`crates/clawhdf5-format/tests/vl_heap_bounds.rs`). +- **Every reader resolves VL data the same way.** `h5rs` (`dump`, `ls`, + `diff`, `check --data`) had its own lenient VL decoder: a heap object + longer than the element's length was cut to it (h5py refuses it), a null + string printed `""` where h5dump prints `NULL`, the stored element size + was trusted, and each heap collection was kept as a copy for the whole + run. It now resolves through `VlResolver`, so `dump` matches h5dump byte + for byte on VL strings (`"a\0b"` as `"a"`, null as `NULL`), VL sequences + and 4-byte-offset files, `dump --json` gives h5py's values, and + `check --data` reports any heap object whose size is not exactly the + element's length × base size. `clawhdf5-wasm` already resolved VL strings + with `read_vl_strings`; it now uses `VlResolver` and refuses a VL type + whose stored element size disagrees with the file, as `File` does + (`crates/clawhdf5-tools/tests/h5rs_interop.rs`, + `crates/clawhdf5-wasm/tests/vl_strings.rs`). New + `VlResolver::element` / `string_element` resolve one element in place. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's @@ -264,10 +280,10 @@ printed with its address; exit 1 when there are any. libhdf5's h5check reads only the 1.8 format. On the conformance corpus it passes all 418 files that both clawhdf5 and h5py read in full, and `check --data` flags - 134 of the 150 CVE and fuzzer files of the `cve_hdf5` corpus (tank, + 135 of the 150 CVE and fuzzer files of the `cve_hdf5` corpus (tank, 2026-09-26). `--data` also follows variable-length data into its global heap collections and reports a damaged one at its address. It inherits - the library's tolerance, though: 9 of the 16 it passes are files h5dump + the library's tolerance, though: 8 of the 15 it passes are files h5dump 1.14.6 rejects (see `docs/known-issues.md`, header checks). - Values over `--max-bytes` (default 1 GiB) are reported instead of read; a panic is caught and reported as an internal error (exit 3). diff --git a/crates/clawhdf5-tools/README.md b/crates/clawhdf5-tools/README.md index 79f775d..8f089c3 100644 --- a/crates/clawhdf5-tools/README.md +++ b/crates/clawhdf5-tools/README.md @@ -76,8 +76,13 @@ print the same bytes as h5dump 1.14.6 and as Debian's h5dump 1.14.5 (the was run in that image on 2026-09-26) — `dump_matches_h5dump` in `tests/h5rs_interop.rs` checks this, and `dump_shows_nul_padding_in_nested_strings` that null-padded strings show their NULs (`"a\000b"`) at any depth, as -h5dump's do. Not covered by those tests: references, opaque, bitfield, -variable-length sequences and virtual datasets. Known differences from +h5dump's do. `dump_prints_vl_data_like_h5dump` covers variable-length +strings (one with an embedded NUL, which prints up to the NUL; empty; null, +which prints `NULL`), variable-length sequences, a VL compound member and +a VL attribute, with 8- and 4-byte offsets. Not covered by those tests: +references, opaque, bitfield, non-ASCII UTF-8 (h5dump prints each byte +above 0x7f as a sign-extended octal escape, h5rs the character) and +virtual datasets. Known differences from h5dump: - Floats print at their own precision (a `float32` 0.1 prints as `0.1`), @@ -233,9 +238,11 @@ extension) and checks: (which catches corrupt compressed data and Fletcher-32 mismatches), and follows every variable-length element (strings and sequences, also inside compounds and arrays) of every dataset and attribute into its global heap -collection: a collection that does not parse, a missing heap object, or a -sequence longer than its heap object is a problem at the collection's -address. Data the +collection: a collection that does not parse or overlaps another, a missing +heap object, or a heap object whose size is not exactly the element's +length times its base size (libhdf5 refuses such an element) is a problem +at the collection's address. Variable-length elements are resolved by the +library's `VlResolver`, as `clawhdf5::File` resolves them. Data the tool cannot decode (a filter it does not implement, such as szip, or a dataset over `--max-bytes`) is a `note:`, not a problem. Every problem is printed with the address of the structure involved; the exit status is 0 @@ -252,9 +259,10 @@ none at all without `--data`), and objects reachable only by external links. It clawhdf5's parsers, so it accepts what they accept: some header damage that libhdf5 refuses goes unreported. Of the 150 CVE and fuzzer files of the HDF Group's `cve_hdf5` corpus (`cvefiles/` and `fuzzerfiles/`), -`check --data` passes 16, and h5dump 1.14.6 rejects 9 of those (tank, +`check --data` passes 15, and h5dump 1.14.6 rejects 8 of those (tank, 2026-09-26, `h5rs check --data F` and `h5dump F` per file; before the -library's header checks it passed 28, of which h5dump rejects 21). +library's header checks it passed 28, of which h5dump rejects 21, and 16 +and 9 before a VL type's stored element size was checked). ## Robustness diff --git a/crates/clawhdf5-tools/src/check.rs b/crates/clawhdf5-tools/src/check.rs index 8c56771..da90247 100644 --- a/crates/clawhdf5-tools/src/check.rs +++ b/crates/clawhdf5-tools/src/check.rs @@ -15,11 +15,13 @@ use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::dataspace::{Dataspace, DataspaceType}; use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::error::FormatError; use clawhdf5_format::group_info::GroupInfoMessage; use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::symbol_table::SymbolTableMessage; +use clawhdf5_format::vl_data::{VlResolver, check_element_size, parse_vl_references}; use crate::cli::{Args, Out}; use crate::h5::{Error, ErrorKind, H5, Kind}; @@ -49,6 +51,15 @@ found, 3 internal error."; const MAX_CHUNKS_CHECKED: usize = 10_000_000; +/// A variable-length element's problem, worded as `check` reports heap +/// problems ("global heap ..."). +fn heap_problem(e: FormatError) -> String { + match e { + FormatError::VlDataError(m) if m.starts_with("global heap") => m, + e => format!("global heap: {e}"), + } +} + /// Whether values of `dt` hold variable-length data (in the global heap). fn has_vl(dt: &Datatype, depth: u32) -> bool { if depth > 32 { @@ -104,6 +115,8 @@ struct Checker<'a> { btrees_seen: HashSet, /// Global heap collections already read (with --data). gcols_seen: HashSet, + /// Resolves variable-length elements (with --data), for the whole file. + vl: VlResolver<'a>, panicked: bool, } @@ -157,6 +170,7 @@ pub fn run(args: &mut Args, out: &mut Out) -> std::io::Result { heaps_seen: HashSet::new(), btrees_seen: HashSet::new(), gcols_seen: HashSet::new(), + vl: VlResolver::new(h5.data(), h5.os(), h5.ls()), panicked: false, }; c.superblock(); @@ -707,62 +721,48 @@ impl Checker<'_> { } match dt { Datatype::VariableLength { + size, is_string, base_type, .. } => { - let os = usize::from(self.h5.os()); - let (Some(lenb), Some(addrb), Some(idxb)) = - (b.get(..4), b.get(4..4 + os), b.get(4 + os..8 + os)) - else { + // Resolved by the library's VlResolver, as every other + // reader resolves them (and as libhdf5 does): a heap object + // whose size is not the element's length × base size, a + // collection that overlaps another, or a missing object is + // a problem at the collection's address. + let Ok(vl) = parse_vl_references(b, 1, self.h5.os()) else { return; }; - let le = |x: &[u8]| { - x.iter() - .enumerate() - .fold(0u64, |a, (i, &v)| a | (u64::from(v) << (8 * i))) - }; - let (len, gcol, idx) = (le(lenb), le(addrb), le(idxb)); - let undef = if os >= 8 { - u64::MAX - } else { - (1u64 << (8 * os)) - 1 - }; - if len == 0 || gcol == 0 || gcol == undef || bad.contains_key(&gcol) { + let gcol = vl[0].collection_address; + if gcol == 0 || bad.contains_key(&gcol) { return; } - let obj = match self.h5.heap_object(gcol, idx as u32) { - Ok(o) => o, + if let Err(e) = check_element_size(*size, self.h5.os()) { + bad.insert(gcol, e.to_string()); + return; + } + let bs = if *is_string { + 1 + } else { + base_type.type_size() as usize + }; + if bs == 0 { + return; + } + let obj = match self.vl.element(b, bs) { + Ok(o) => o.unwrap_or(&[]), Err(e) => { - bad.insert(e.addr.unwrap_or(gcol), e.msg); + bad.insert(gcol, heap_problem(e)); return; } }; if self.gcols_seen.insert(gcol) { self.counts.global_heaps += 1; } - let bs = if *is_string { - 1 - } else { - u64::from(base_type.type_size()) - }; - if len - .checked_mul(bs) - .is_none_or(|need| need > obj.len() as u64) - { - bad.insert( - gcol, - format!( - "global heap object {idx} holds {} bytes; the element needs {len} x {bs}", - obj.len() - ), - ); - return; - } - if !*is_string && bs > 0 && has_vl(base_type, depth + 1) { - let bs = bs as usize; - for k in 0..len as usize { - self.vl_element(base_type, &obj[k * bs..(k + 1) * bs], depth + 1, bad); + if !*is_string && has_vl(base_type, depth + 1) { + for eb in obj.chunks_exact(bs) { + self.vl_element(base_type, eb, depth + 1, bad); } } } diff --git a/crates/clawhdf5-tools/src/diff.rs b/crates/clawhdf5-tools/src/diff.rs index c7ad723..98d4ba5 100644 --- a/crates/clawhdf5-tools/src/diff.rs +++ b/crates/clawhdf5-tools/src/diff.rs @@ -699,6 +699,9 @@ impl Diff { } match (x, y) { (Value::Str(p), Value::Str(q)) => p == q, + // h5diff compares a null VL string equal to an empty one. + (Value::NullStr, Value::NullStr) => true, + (Value::NullStr, Value::Str(s)) | (Value::Str(s), Value::NullStr) => s.is_empty(), (Value::Bytes(p), Value::Bytes(q)) | (Value::OtherRef(p), Value::OtherRef(q)) => p == q, (Value::Compound(p), Value::Compound(q)) => { p.len() == q.len() diff --git a/crates/clawhdf5-tools/src/h5.rs b/crates/clawhdf5-tools/src/h5.rs index e2e7931..1f4354a 100644 --- a/crates/clawhdf5-tools/src/h5.rs +++ b/crates/clawhdf5-tools/src/h5.rs @@ -8,7 +8,6 @@ use std::cell::RefCell; use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::rc::Rc; use clawhdf5::File; use clawhdf5_format::attribute::{AttributeMessage, extract_attributes_tolerant}; @@ -20,7 +19,6 @@ use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; use clawhdf5_format::fractal_heap::FractalHeapHeader; -use clawhdf5_format::global_heap::GlobalHeapCollection; use clawhdf5_format::group_v1; use clawhdf5_format::link_info::LinkInfoMessage; use clawhdf5_format::link_message::{LinkMessage, LinkTarget}; @@ -191,7 +189,6 @@ pub struct H5 { pub path: PathBuf, pub file: File, pub max_bytes: u64, - heaps: RefCell, String>>>, /// Fractal heaps whose blocks were verified: `None` = sound. verified_heaps: RefCell>>, } @@ -211,7 +208,6 @@ impl H5 { path: path.to_path_buf(), file, max_bytes: DEFAULT_MAX_BYTES, - heaps: RefCell::new(HashMap::new()), verified_heaps: RefCell::new(HashMap::new()), }) } @@ -433,29 +429,6 @@ impl H5 { r.map_or(Ok(()), Err) } - /// The global heap object `idx` of the collection at `addr` (cached per - /// collection). - pub fn heap_object(&self, addr: u64, idx: u32) -> Result> { - let coll = { - let mut cache = self.heaps.borrow_mut(); - cache - .entry(addr) - .or_insert_with(|| match usize::try_from(addr) { - Ok(a) => GlobalHeapCollection::parse(self.data(), a, self.ls()) - .map(Rc::new) - .map_err(|e| e.to_string()), - Err(_) => Err("address out of range".into()), - }) - .clone() - .map_err(|e| Error::at(addr, format!("global heap: {e}")))? - }; - let idx16 = u16::try_from(idx) - .map_err(|_| Error::at(addr, format!("global heap object index {idx} out of range")))?; - coll.get_object(idx16) - .map(|o| o.data.clone()) - .ok_or_else(|| Error::at(addr, format!("global heap has no object {idx}"))) - } - /// The dataspace of the dataset at `path` with a virtual dataset's /// extent resolved from its sources (as libhdf5 reports it) instead of /// the stored one. diff --git a/crates/clawhdf5-tools/src/value.rs b/crates/clawhdf5-tools/src/value.rs index 8160574..d9255f1 100644 --- a/crates/clawhdf5-tools/src/value.rs +++ b/crates/clawhdf5-tools/src/value.rs @@ -3,7 +3,10 @@ //! Decoding never panics: a short buffer, an unknown byte order or a //! dangling heap reference becomes [`Value::Error`]. +use std::cell::RefCell; + use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder, ReferenceType, StringPadding}; +use clawhdf5_format::vl_data::{VlResolver, check_element_size}; use serde_json::Value as J; use crate::dtype; @@ -16,6 +19,9 @@ pub enum Value { /// its own precision. Float(f64, u8), Str(String), + /// A null variable-length string (heap address 0): h5dump prints it as + /// `NULL`, h5py reads it as empty. + NullStr, /// Opaque, bitfield, time and oversized integers. Bytes(Vec), /// An enum member (name, when the value matches one) and its value. @@ -129,10 +135,10 @@ fn decode_float(dt: &Datatype, b: &[u8]) -> Value { } } -fn trim_string(b: &[u8], pad: Option<&StringPadding>) -> String { +fn trim_string(b: &[u8], pad: &StringPadding) -> String { let cut = b.iter().position(|&c| c == 0).unwrap_or(b.len()); let mut s = &b[..cut]; - if matches!(pad, Some(StringPadding::SpacePad)) { + if matches!(pad, StringPadding::SpacePad) { while let [rest @ .., b' '] = s { s = rest; } @@ -140,22 +146,20 @@ fn trim_string(b: &[u8], pad: Option<&StringPadding>) -> String { String::from_utf8_lossy(s).into_owned() } -/// Little-endian unsigned integer of `b` (up to 8 bytes). -fn le(b: &[u8]) -> u64 { - b.iter() - .take(8) - .enumerate() - .fold(0u64, |a, (i, &x)| a | (u64::from(x) << (8 * i))) -} - /// Decodes elements of one file. pub struct Decoder<'a> { pub h5: &'a H5, + /// Variable-length elements are resolved as the library resolves them + /// (so as libhdf5 does), not by a decoder of our own. + vl: RefCell>, } impl<'a> Decoder<'a> { pub fn new(h5: &'a H5) -> Self { - Self { h5 } + Self { + h5, + vl: RefCell::new(VlResolver::new(h5.data(), h5.os(), h5.ls())), + } } /// Decode element `i` of `raw`, an array of `dt` elements. @@ -187,7 +191,7 @@ impl<'a> Decoder<'a> { Datatype::Time { .. } | Datatype::BitField { .. } | Datatype::Opaque { .. } => { Value::Bytes(b.to_vec()) } - Datatype::String { padding, .. } => Value::Str(trim_string(b, Some(padding))), + Datatype::String { padding, .. } => Value::Str(trim_string(b, padding)), Datatype::Compound { members, .. } => { let mut out = Vec::with_capacity(members.len()); for m in members { @@ -246,61 +250,43 @@ impl<'a> Decoder<'a> { Value::Array(out) } Datatype::VariableLength { + size, is_string, - padding, base_type, .. - } => self.decode_vlen(*is_string, padding.as_ref(), base_type, b, depth), + } => match check_element_size(*size, self.h5.os()) { + Ok(()) => self.decode_vlen(*is_string, base_type, b, depth), + Err(e) => Value::Error(e.to_string()), + }, } } - fn decode_vlen( - &self, - is_string: bool, - padding: Option<&StringPadding>, - base: &Datatype, - b: &[u8], - depth: u32, - ) -> Value { - let os = usize::from(self.h5.os()); - let (Some(lenb), Some(addrb), Some(idxb)) = - (b.get(..4), b.get(4..4 + os), b.get(4 + os..8 + os)) - else { - return Value::Error("short VL element".into()); - }; - let len = le(lenb) as usize; - let addr = le(addrb); - let idx = le(idxb) as u32; - let undef = if os >= 8 { - u64::MAX - } else { - (1u64 << (8 * os)) - 1 - }; - let obj = if len == 0 || addr == 0 || addr == undef { - Vec::new() - } else { - match self.h5.heap_object(addr, idx) { - Ok(o) => o, - Err(e) => return Value::Error(e.to_string()), - } - }; + /// A variable-length element, resolved by the library's + /// [`VlResolver`]: a string ends at its first NUL, a heap object whose + /// size is not the element's length × base size is an error, and a + /// heap address of 0 is null — all as libhdf5 (and so h5dump and h5py) + /// has it. + fn decode_vlen(&self, is_string: bool, base: &Datatype, b: &[u8], depth: u32) -> Value { if is_string { - let l = len.min(obj.len()); - return Value::Str(trim_string(&obj[..l], padding)); + return match self.vl.borrow_mut().string_element(b) { + Ok(Some(s)) => Value::Str(String::from_utf8_lossy(s).into_owned()), + Ok(None) => Value::NullStr, + Err(e) => Value::Error(e.to_string()), + }; } let bs = base.type_size() as usize; if bs == 0 { return Value::Error("VL base type of size 0".into()); } - match len.checked_mul(bs) { - Some(need) if need <= obj.len() => {} - _ => return Value::Error("VL sequence longer than its heap object".into()), - } - let mut out = Vec::with_capacity(len); - for k in 0..len { - out.push(self.decode(base, &obj[k * bs..], depth + 1)); - } - Value::Seq(out) + let obj = match self.vl.borrow_mut().element(b, bs) { + Ok(o) => o.unwrap_or(&[]), + Err(e) => return Value::Error(e.to_string()), + }; + Value::Seq( + obj.chunks_exact(bs) + .map(|e| self.decode(base, e, depth + 1)) + .collect(), + ) } } @@ -351,6 +337,7 @@ pub fn text(v: &Value, h5paths: &dyn Fn(u64) -> Option) -> String { Value::Int(i) => i.to_string(), Value::Float(f, w) => fmt_float(*f, *w), Value::Str(s) => format!("\"{}\"", escape(s)), + Value::NullStr => "NULL".into(), Value::Bytes(b) => hex(b), Value::Enum(Some(n), _) => n.clone(), Value::Enum(None, i) => i.to_string(), @@ -411,6 +398,7 @@ pub fn to_json(v: &Value, h5paths: &dyn Fn(u64) -> Option) -> J { } } Value::Str(s) => J::from(s.as_str()), + Value::NullStr => J::from(""), Value::Bytes(b) | Value::OtherRef(b) => J::from(hex(b)), Value::Enum(_, i) => to_json(&Value::Int(*i), h5paths), Value::Compound(ms) => J::Array(ms.iter().map(|(_, v)| to_json(v, h5paths)).collect()), diff --git a/crates/clawhdf5-tools/tests/gen_vl_files.py b/crates/clawhdf5-tools/tests/gen_vl_files.py new file mode 100644 index 0000000..3937c19 --- /dev/null +++ b/crates/clawhdf5-tools/tests/gen_vl_files.py @@ -0,0 +1,121 @@ +"""Write the variable-length data files the h5rs VL tests run on. + +usage: gen_vl_files.py OUTDIR + +For 8-byte (`vl8`) and 4-byte (`vl4`) offsets, writes OUTDIR/vl8.h5 and +OUTDIR/vl4.h5, which libhdf5 reads in full, and OUTDIR/bad8.h5 and +OUTDIR/bad4.h5, whose `bad` and `badseq` elements 0 have a length that +disagrees with their global heap object (libhdf5: "Expected global heap +object size does not match"). h5py cannot write a VL string with a NUL in +it or a null element in a contiguous dataset, so those are patched in. + +Prints one JSON object: for each file, each dataset's values as h5py reads +them one element at a time (strings as text, sequences as lists, a compound +as a list of its fields), with null for an element h5py cannot read; the +root attribute `va`; and the addresses of the `bad` elements' collections. +""" + +import json +import os +import struct +import sys + +import h5py +import numpy as np + +out = sys.argv[1] +S = h5py.string_dtype("utf-8") +I4 = h5py.vlen_dtype(np.dtype(" null + open(path, "wb").write(bytes(b)) + + +def bad(path, sizes): + os_ = 8 if sizes is None else sizes[0] + with create(path, sizes) as f: + f.create_dataset("bad", data=np.array(["cdefgh", "ok"], dtype=object), dtype=S) + s = f.create_dataset("badseq", shape=(2,), dtype=I4) + s[0] = [1, 2, 3] + s[1] = [4] + off, soff = f["bad"].id.get_offset(), f["badseq"].id.get_offset() + b = bytearray(open(path, "rb").read()) + gcol = int.from_bytes(b[off + 4 : off + 4 + os_], "little") + struct.pack_into(" 3 + struct.pack_into(" 2 + open(path, "wb").write(bytes(b)) + return gcol + + +def value(v): + if isinstance(v, bytes): + return v.decode() + if isinstance(v, str): + return v + if isinstance(v, np.void): + return [value(x) for x in v] + if isinstance(v, np.ndarray): + return [value(x) for x in v] + return v.item() if hasattr(v, "item") else v + + +def read(ds): + got = [] + for i in range(ds.shape[0]): + try: + got.append(value(ds[i])) + except OSError: + got.append(None) + return got + + +result = {} +for tag, sizes in (("8", None), ("4", (4, 4))): + g, x = os.path.join(out, f"vl{tag}.h5"), os.path.join(out, f"bad{tag}.h5") + good(g, sizes) + gcol = bad(x, sizes) + with h5py.File(g, "r") as f: + result[f"vl{tag}"] = {n: read(f[n]) for n in ("d", "u", "seq", "sequ", "cmp")} + result[f"vl{tag}"]["va"] = [value(s) for s in f.attrs["va"]] + with h5py.File(x, "r") as f: + result[f"bad{tag}"] = {n: read(f[n]) for n in ("bad", "badseq")} + result[f"bad{tag}"]["gcol"] = gcol +json.dump(result, sys.stdout) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index 7d97cc5..a1b4492 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -773,3 +773,125 @@ fn every_subcommand_rejects_a_non_hdf5_file_cleanly() { assert_eq!(code(&h5rs(&["ls"])), 2); assert_eq!(code(&h5rs(&["--help"])), 0); } + +// --------------------------------------------------------------------------- +// variable-length data +// --------------------------------------------------------------------------- + +/// Runs `tests/gen_vl_files.py`: VL strings (with an embedded NUL, empty +/// and null elements), VL sequences, a VL compound member and a VL +/// attribute, with 8- and 4-byte offsets, plus files whose heap objects +/// disagree with their elements' lengths. +fn generate_vl() -> Option { + if missing(python_available(), "python3 with h5py") { + return None; + } + let dir = tempfile::tempdir().unwrap(); + let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/gen_vl_files.py"); + let out = Command::new(python()) + .arg(&script) + .arg(dir.path()) + .output() + .expect("run gen_vl_files.py"); + assert!( + out.status.success(), + "gen_vl_files.py failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + let values = serde_json::from_slice(&out.stdout).expect("gen_vl_files.py output"); + Some(Files { dir, values }) +} + +/// `dump` resolves VL elements through the library's `VlResolver`, as +/// libhdf5 does: "a\0b" prints as "a", a null string as NULL (it printed +/// ""), and with 4-byte offsets too; the output is h5dump's byte for byte. +#[test] +fn dump_prints_vl_data_like_h5dump() { + let Some(f) = generate_vl() else { return }; + for name in ["vl8.h5", "vl4.h5"] { + let p = f.p(name); + let ours = stdout(&h5rs(&["dump", &p])); + assert!( + ours.contains(r#"(0): "a", "", NULL, "zz", "hello""#), + "{name}:\n{ours}" + ); + assert!(ours.contains(r#"(0): NULL, "w", NULL, NULL"#), "{name}"); + assert!(ours.contains("(0): (1, 2, 3), (), (-5)"), "{name}"); + if missing(tool_available("h5dump"), "h5dump") { + continue; + } + let reference = run("h5dump", &[&p]); + assert!(reference.status.success(), "{name}: {reference:?}"); + assert_eq!(ours, stdout(&reference).replacen(&p, name, 1), "{name}"); + } +} + +/// `dump --json` gives the values h5py reads, element by element; and an +/// element whose heap object is not its length × base size is an error, as +/// in h5py, not a truncated value (it printed "cde" and (1, 2)). +#[test] +fn dump_json_vl_values_match_h5py() { + let Some(f) = generate_vl() else { return }; + for tag in ["8", "4"] { + let (good, bad) = (format!("vl{tag}"), format!("bad{tag}")); + let o = h5rs(&["dump", "--json", &f.p(&format!("{good}.h5"))]); + assert!(o.status.success(), "{good}: {o:?}"); + let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap(); + let want = &f.values[&good]; + for d in doc["datasets"].as_object().unwrap().values() { + let path = d["alias"][0].as_str().unwrap(); + assert_eq!(d["value"], want[&path[1..]], "{good}: {path}"); + } + let attrs = &doc["groups"][doc["root"].as_str().unwrap()]["attributes"]; + assert_eq!(attrs[0]["name"], "va"); + assert_eq!(attrs[0]["value"], want["va"], "{good}: va"); + + let o = h5rs(&["dump", "--json", &f.p(&format!("{bad}.h5"))]); + let doc: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap(); + let want = &f.values[&bad]; + for d in doc["datasets"].as_object().unwrap().values() { + let path = d["alias"][0].as_str().unwrap(); + let got = d["value"].as_array().unwrap(); + let want = want[&path[1..]].as_array().unwrap(); + assert_eq!(got.len(), want.len(), "{bad}: {path}"); + for (g, w) in got.iter().zip(want) { + if w.is_null() { + // h5py cannot read it: neither can we. + let e = g["error"] + .as_str() + .unwrap_or_else(|| panic!("{bad}: {path}: {g}")); + assert!(e.contains("holds"), "{bad}: {path}: {e}"); + } else { + assert_eq!(g, w, "{bad}: {path}"); + } + } + } + } +} + +/// `check --data` holds VL elements to libhdf5's rule: a heap object whose +/// size is not exactly the element's length × base size is a problem (it +/// only caught objects shorter than the element). +#[test] +fn check_data_flags_mis_sized_vl_heap_objects() { + let Some(f) = generate_vl() else { return }; + for tag in ["8", "4"] { + let o = h5rs(&["check", "--data", &f.p(&format!("vl{tag}.h5"))]); + let s = stdout(&o); + assert_eq!(code(&o), 0, "vl{tag}: {s}"); + assert!( + s.contains("global heap collections read: 1"), + "vl{tag}: {s}" + ); + + let o = h5rs(&["check", "--data", &f.p(&format!("bad{tag}.h5"))]); + let s = stdout(&o); + assert_eq!(code(&o), 1, "bad{tag}: {s}"); + let at = f.values[format!("bad{tag}")]["gcol"].as_u64().unwrap(); + for (path, what) in [("/bad", "6 bytes"), ("/badseq", "12 bytes")] { + let want = format!("problem: {at:#x} {path}: variable-length data: global heap object"); + assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}"); + assert!(s.contains(what), "bad{tag}: {s}"); + } + } +} diff --git a/crates/clawhdf5-wasm/src/core.rs b/crates/clawhdf5-wasm/src/core.rs index baadcda..4c6beab 100644 --- a/crates/clawhdf5-wasm/src/core.rs +++ b/crates/clawhdf5-wasm/src/core.rs @@ -9,6 +9,7 @@ use clawhdf5::{AttrValue, File, Selection}; use clawhdf5_format::data_read; use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; +use clawhdf5_format::vl_data::{VlResolver, check_element_size}; /// Errors are reported to JavaScript as messages. pub type Result = std::result::Result; @@ -221,6 +222,12 @@ impl Reader { None => (Selection::All, shape.clone()), Some(h) => hyperslab_selection(h, &shape)?, }; + // A VL type whose stored element size is not the one the file's + // offset size implies is refused before its data is read, as + // `File::read_string` refuses it. + if let Datatype::VariableLength { size, .. } = array_base(&dt) { + check_element_size(*size, self.file.superblock().offset_size).map_err(err)?; + } let raw = ds.read_selection(&selection).map_err(err)?; let data = self.decode(&raw, &dt)?; out_shape.extend(element_shape(&dt)); @@ -270,23 +277,14 @@ impl Reader { Datatype::VariableLength { is_string: true, .. } if !is_array => { - let size = dt.type_size() as usize; - if size == 0 || !raw.len().is_multiple_of(size) { - return Err(format!( - "{} bytes is not a whole number of {size}-byte string references", - raw.len() - )); - } + // The library's resolver, as File::read_string uses: a + // string ends at its first NUL and a heap object of the + // wrong size is an error, as in libhdf5 and h5py. let sb = self.file.superblock(); Data::Strings( - clawhdf5_format::vl_data::read_vl_strings( - self.file.as_bytes(), - raw, - (raw.len() / size) as u64, - sb.offset_size, - sb.length_size, - ) - .map_err(err)?, + VlResolver::new(self.file.as_bytes(), sb.offset_size, sb.length_size) + .strings(raw) + .map_err(err)?, ) } Datatype::Enumeration { .. } if !is_array => { diff --git a/crates/clawhdf5-wasm/tests/vl_strings.rs b/crates/clawhdf5-wasm/tests/vl_strings.rs new file mode 100644 index 0000000..1baddd0 --- /dev/null +++ b/crates/clawhdf5-wasm/tests/vl_strings.rs @@ -0,0 +1,150 @@ +//! The wasm reader resolves VL strings with the library's `VlResolver`, so +//! it returns what `File::read_string` and h5py return: a string ends at +//! its first NUL, a null element is empty, a heap object of the wrong size +//! is an error, and a VL datatype whose stored element size disagrees with +//! the file's offset size is refused. Checked with 8- and 4-byte offsets. +//! +//! Skipped when python3 with h5py is missing, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter. + +use std::process::Command; + +use clawhdf5::File; +use clawhdf5_wasm::core::{Data, Reader}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn h5py_available() -> bool { + let ok = Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if !ok { + assert!( + std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"), + "CLAWHDF5_REQUIRE_INTEROP=1 but python with h5py is not available" + ); + eprintln!("SKIP: python with h5py not available"); + } + ok +} + +/// For each offset size: `vl{8,4}.h5` with dataset `d` = "a\0b", "", null, +/// "zz" (patched: h5py writes neither a NUL nor a null element); +/// `bad{8,4}.h5` whose element 0 claims 3 bytes of a 6-byte heap object; +/// and `size{8,4}.h5` whose VL datatype message stores a 24-byte element. +/// Prints h5py's reading of each element as hex, or "error". +const SCRIPT: &str = r#" +import struct, sys, h5py, numpy as np +out = sys.argv[1] +S = h5py.string_dtype('utf-8') +def create(path, os_): + if os_ == 8: + return h5py.File(path, 'w', libver='earliest') + # The earliest format, so the patched object header has no checksum. + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE); fcpl.set_sizes(4, 4) + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) + fapl.set_libver_bounds(h5py.h5f.LIBVER_EARLIEST, h5py.h5f.LIBVER_V18) + return h5py.File(h5py.h5f.create(path.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)) +def elem(length, addr, index, os_): + return struct.pack(' = String::from_utf8(out.stdout) + .unwrap() + .lines() + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_string(), v.to_string()) + }) + .collect(); + let read = |name: &str| { + let bytes = std::fs::read(dir.path().join(format!("{name}.h5"))).unwrap(); + let wasm = Reader::open(bytes).unwrap().read("/d", None); + let file = File::open(dir.path().join(format!("{name}.h5"))) + .unwrap() + .dataset("d") + .unwrap() + .read_string(); + (wasm, file) + }; + for os in [8, 4] { + // h5py: "a\0b" is "a"; the null element (address 0) is empty. + assert_eq!(h5py[&format!("vl{os}")], "61,,,7a7a"); + let (wasm, file) = read(&format!("vl{os}")); + let Data::Strings(wasm) = wasm.unwrap().data else { + panic!("vl{os}: not strings") + }; + assert_eq!(wasm, ["a", "", "", "zz"], "vl{os}"); + assert_eq!(wasm, file.unwrap(), "vl{os}"); + + // h5py refuses the mis-sized element; so do both readers. + assert_eq!(h5py[&format!("bad{os}")], "error,6f6b"); + let (wasm, file) = read(&format!("bad{os}")); + assert!(wasm.unwrap_err().contains("holds 6 bytes"), "bad{os}"); + assert!(file.is_err(), "bad{os}"); + + // libhdf5 ignores the stored element size and reads the values; + // File refuses the datatype rather than guess its layout, and the + // wasm reader now does the same (it read with the stored size). + assert_eq!(h5py[&format!("size{os}")], "78,7979"); + let (wasm, file) = read(&format!("size{os}")); + let e = wasm.unwrap_err(); + assert!(e.contains("stores 24-byte elements"), "size{os}: {e}"); + assert!(file.is_err(), "size{os}"); + } +} diff --git a/docs/known-issues.md b/docs/known-issues.md index ceb6ab6..5f43bdb 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -207,9 +207,10 @@ fill-value item that did is fixed). - (`cve-2024-32616` `/group1/dset3` and `cve-2025-2309`'s `Comp_OBJREF` attribute are h5py/numpy type-mapping failures, not libhdf5 refusals.) - `h5rs check` validates with the library's parsers, so it inherits what - they accept: of the 150 CVE and fuzzer files, `check --data` passes 16, - and h5dump 1.14.6 rejects 9 of those (tank, 2026-09-26; 28 and 21 - before these checks). + they accept: of the 150 CVE and fuzzer files, `check --data` passes 15, + and h5dump 1.14.6 rejects 8 of those (tank, 2026-09-26; 28 and 21 + before these checks, 16 and 9 before a VL type's stored element size + was checked, which flags `cve-2024-32608`). - **Writer:** - Nested groups beyond one level: path-like names are now refused, not created. @@ -537,8 +538,9 @@ which is what libhdf5 itself writes. followed (no file system). - Variable-length string datasets are read by decoding `read_selection`'s bytes with `clawhdf5_format::vl_data` in the wasm crate; `File` itself still - cannot (see the audit gaps above). (`File` can since 2026-09-26; the wasm - crate still decodes them itself.) + cannot (see the audit gaps above). (`File` can since 2026-09-26. Since + 2026-09-26 the wasm crate resolves them with the same `VlResolver` as + `File` and `h5rs`, so all three return h5py's values.) ## The Node.js package (`packages/clawhdf5-node`) does not work -- 2.54.0 From 45d617c39e804453e8f4738932b322e0c1d9d64e Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:04:48 -0500 Subject: [PATCH 32/36] docs: say when a selection read decodes more than the selection The READMEs said ds[...] reads only the selected elements, and the facade's read_selection docs that only intersecting chunks are decompressed. The bounding-box path runs only when the box covers at most half the dataset; larger boxes (any strided slice across the dataset), compact, virtual and unwritten datasets and chunked ones with a non-default fill value decode the whole dataset. The READMEs, the facade and format docs, the bindings' docstrings and known-issues now say so, and how index lists are read. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 11 +++++++++-- README.md | 13 ++++++++++--- crates/clawhdf5-format/src/data_read.rs | 8 +++++--- crates/clawhdf5-py/README.md | 14 ++++++++++++-- crates/clawhdf5-py/src/dataset.rs | 9 ++++++--- crates/clawhdf5-py/src/select.rs | 4 +++- crates/clawhdf5/src/reader.rs | 12 ++++++++++-- docs/known-issues.md | 17 +++++++++++++++++ 8 files changed, 72 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e175c02..b6fc54e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,9 @@ tests imported `rustyhdf5`, so they failed at collection. Distribution, module and tests now all say `clawhdf5`, and the module has `__version__`. -- **h5py-style reads that read only what is selected.** `ds[...]` used to - read the whole dataset and slice it in numpy, and knew six dtypes. Now +- **h5py-style reads that read the selection, not the dataset.** `ds[...]` + used to read the whole dataset and slice it in numpy, and knew six + dtypes. Now integers (negative from the end), slices with positive steps, `...`, one increasing list of integers per key and compound field names map onto the facade's hyperslab selection (a list is read one group of neighbouring @@ -102,6 +103,12 @@ `test_errors_match_h5py` now also requires that every key h5py reads reads here too, with the same result, and covers more keys (0-d arrays, repeated and empty lists, `()`, `...`). +- **Docs say when a selection reads more than itself.** The README and + the package README said `ds[...]` reads only the selected elements, + without condition. The library decodes the whole dataset when the + selection's bounding box covers more than half of it, and for compact, + virtual, unwritten and non-default-fill chunked datasets; the READMEs, + the facade's `read_selection` docs and `docs/known-issues.md` now say so. - **CI builds and tests the Python package.** It was excluded from CI. `scripts/ci-test.sh` now lints `clawhdf5-py`, builds the wheel with maturin, unpacks it under `target/` and runs the pytest suite; skipped diff --git a/README.md b/README.md index c266f3b..86da933 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,9 @@ breaking change, are in [CHANGELOG.md](CHANGELOG.md). - Default fusion weights are now the measured 0.4 / 0.6 (v2.5.0). Re-ranking had been discarding the retrieval score, costing the Markdown backend 40.6pp of Hit@1; fixed in v2.6.0. -- Selection reads decode only the chunks they touch (a 64×64 window: 105 ms to - 0.39 ms), and full reads are 1.2–1.9× faster (v2.5.0). +- Selection reads whose bounding box covers at most half the dataset decode + only the chunks they touch (a 64×64 window: 105 ms to 0.39 ms), and full + reads are 1.2–1.9× faster (v2.5.0). **Memory** - A loaded store holds ~30% less (embeddings stored once, v2.6.0), and the @@ -428,7 +429,7 @@ with clawhdf5.File("data.h5", "r") as f: print(list(f.keys())) # sorted member names, like h5py ds = f["group/temperatures"] # relative or absolute ("/group/...") paths print(ds.shape, ds.dtype) # dtype is the numpy dtype h5py reports - block = ds[100:200, ::4] # reads only the selected elements + block = ds[100:200, ::4] # a small selection reads only its chunks row = ds[-1] # integers drop the axis picked = ds[[1, 5, 9], :] # one increasing index list per key units = ds.attrs["units"] # attributes come back as h5py returns them @@ -444,6 +445,12 @@ sequences, opaque, HDF5 array types and compounds; other types (references, bitfields, ...) raise `TypeError` instead of returning guessed data. Keys follow h5py (negative steps, `None` and boolean masks are refused). The read itself runs with the GIL released, so Python threads read in parallel. +A selection whose bounding box covers at most half the dataset decodes only +the chunks (or contiguous rows) that box overlaps; a larger one — including +a strided slice across the whole dataset — decodes the whole dataset, as +do datasets that are compact, virtual, unwritten, or chunked with a +non-default fill value (`docs/known-issues.md`). An index list is read one +group of neighbouring chunks at a time. Writing (`File(path, "w")`, `create_dataset`, `create_group`, `attrs[...] =`) covers `float64`, `float32`, `int64`, `int32` and `uint8` arrays. The tests in `crates/clawhdf5-py/tests` compare every read with h5py; run them with diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index d10f381..f9e0ac1 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -286,9 +286,11 @@ pub fn read_raw_data_indexed( /// Read raw bytes for only the selected elements of a dataset. /// -/// For chunked layouts, only chunks that intersect the selection are read -/// and decompressed. For compact/contiguous layouts, the full data is read -/// and then the selection is extracted. +/// When the selection's bounding box covers at most half the dataset, only +/// that box is materialised — the overlapping rows of a contiguous dataset, +/// the overlapping chunks of a chunked one, whatever its chunk index (see +/// [`crate::partial_read`]). Otherwise, and for compact and virtual +/// layouts, the whole dataset is decoded and the selection extracted. #[allow(clippy::too_many_arguments)] pub fn read_raw_data_selection( file_data: &[u8], diff --git a/crates/clawhdf5-py/README.md b/crates/clawhdf5-py/README.md index 9cff9ab..95cae12 100644 --- a/crates/clawhdf5-py/README.md +++ b/crates/clawhdf5-py/README.md @@ -29,7 +29,7 @@ with clawhdf5.File("data.h5", "r") as f: f.keys(), f["group"].items(), "group/data" in f ds = f["group/data"] # or f["/group/data"], f["group"]["data"] ds.shape, ds.dtype, ds.attrs["units"] - ds[10:20, ::2] # only the selected elements are read + ds[10:20, ::2] # a small selection reads only its chunks ds[-1], ds[..., 0], ds[[1, 4, 7]] np.asarray(ds) f["table"]["id"] # a compound field @@ -45,9 +45,19 @@ with clawhdf5.File("data.h5", "r") as f: increasing list of integers, compound field names. Each maps onto a hyperslab selection. `None`, negative steps and boolean masks are refused with h5py's errors. +- What is read from the file: a selection whose bounding box covers at + most half the dataset decodes only the chunks (or contiguous rows) the box + overlaps. The library decodes the whole dataset for a larger box + (including a strided slice such as `ds[::100]` across a chunked dataset), + and for compact, virtual and unwritten datasets and chunked ones with a + non-default fill value. An index list is read one group of neighbouring + chunks at a time (a new group only past a chunk with no selected index), + so each chunk is decoded once. `ds[()]`, `ds[...]` and `np.asarray(ds)` + use the file's chunk cache; other selections do not. - The bytes the library reads become the numpy array's buffer without a copy, and the read runs with the GIL released, so threads read in - parallel. + parallel. A bug in the library (a Rust panic) raises + `clawhdf5.InternalError`, a `RuntimeError`. - Attributes return what h5py returns; `clawhdf5.Empty` stands for a null dataspace (h5py's `Empty`). diff --git a/crates/clawhdf5-py/src/dataset.rs b/crates/clawhdf5-py/src/dataset.rs index c20f293..bcfd852 100644 --- a/crates/clawhdf5-py/src/dataset.rs +++ b/crates/clawhdf5-py/src/dataset.rs @@ -1,7 +1,9 @@ //! PyDataset — h5py-style read access to HDF5 datasets. //! //! `ds[key]` parses the key into hyperslab selections (see `select`) and -//! reads only those elements through the facade's `read_selection`; the +//! reads them through the facade's `read_selection`, which decodes only the +//! chunks a small selection touches (see its docs for when it decodes the +//! whole dataset instead); the //! bytes it returns become the numpy array's buffer without a copy (see //! `convert`). All file access and decoding runs with the GIL released, so //! Python threads reading the same or different datasets run in parallel. @@ -24,7 +26,7 @@ use crate::{PyEmpty, node, to_py_err}; /// ```python /// ds = f['group/dataset'] /// ds.shape, ds.dtype, ds.attrs['units'] -/// block = ds[10:20, ::2] # reads only the selected elements +/// block = ds[10:20, ::2] # a small selection reads only its chunks /// ``` #[pyclass(name = "Dataset")] pub struct PyDataset { @@ -299,7 +301,8 @@ impl PyDataset { /// Read with h5py indexing: integers, slices with positive steps, /// `...`, one increasing list of integers, and compound field names. - /// Only the selected elements are read from the file. + /// A selection whose bounding box covers at most half the dataset reads + /// only the chunks (or contiguous rows) it overlaps. fn __getitem__<'py>( &self, py: Python<'py>, diff --git a/crates/clawhdf5-py/src/select.rs b/crates/clawhdf5-py/src/select.rs index 7c52442..7d11f3c 100644 --- a/crates/clawhdf5-py/src/select.rs +++ b/crates/clawhdf5-py/src/select.rs @@ -1,5 +1,7 @@ //! h5py-style indexing (`ds[1, 2:10:3, ...]`) mapped onto hyperslab -//! selections, so only the selected elements are read. +//! selections, so the library reads the selection rather than the whole +//! dataset (it still decodes everything for large selections; see the +//! facade's `Dataset::read_selection`). //! //! The rules and error messages follow h5py's `selections.py`: integers //! (negative from the end) drop their axis, slices must have a positive diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 2e86dfe..c7d1fe6 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -525,8 +525,16 @@ impl<'f> Dataset<'f> { /// Read selected elements as raw bytes. /// - /// Only the elements matching the [`clawhdf5_format::selection::Selection`] are returned. For chunked - /// datasets, only intersecting chunks are decompressed. + /// Only the elements matching the [`clawhdf5_format::selection::Selection`] are returned. + /// + /// What is read to get them: when the selection's bounding box covers at + /// most half the dataset, only that box — the chunks overlapping it, or + /// the rows of a contiguous dataset. The whole dataset is decoded instead + /// when the box covers more than half (a strided selection spanning the + /// dataset does), for compact and virtual layouts, for a dataset with no + /// storage, and for a chunked dataset with a non-default fill value. + /// [`Selection::All`](clawhdf5_format::selection::Selection::All) goes + /// through the file's chunk cache; other selections do not. pub fn read_selection( &self, selection: &clawhdf5_format::selection::Selection, diff --git a/docs/known-issues.md b/docs/known-issues.md index bf8f77a..320ed2f 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -7,6 +7,23 @@ deleting it. --- +## Selection reads that decode more than the selection + +**Status:** open (documented 2026-09-26). `Dataset::read_selection` (and so +the Python `ds[...]`) materialises only the selection's bounding box when +that box covers at most half the dataset (`partial_read`). It decodes the +whole dataset and extracts the selection instead when: +- the bounding box covers more than half the dataset — which a strided + selection across a chunked dataset (`ds[::100]`) always does, although + it may touch few chunks; +- the dataset is compact or virtual, or has no storage; +- it is chunked with a non-default fill value (the box path does not fill + unallocated chunks, so the fill-aware full read is used). +Values are correct in every case; this is cost only. Selections other than +`Selection::All` also bypass the file's chunk cache. The bounding-box +heuristic's other cost is measured under "Concurrent and contiguous read +performance" below. + ## Concurrent and contiguous read performance (measured 2026-09-26) **Status:** open. Measured on tank with `concurrent_read` against h5py -- 2.54.0 From 8dcce084ca29d41aaee30a01ec160c22eb9d4eaa Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:05:31 -0500 Subject: [PATCH 33/36] test: the v4 chunk-index selection test passes clippy -D warnings A type alias for the hyperslab tuple, and as_chunks for the i32 decode. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/tests/v4_chunk_index_selection.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5/tests/v4_chunk_index_selection.rs b/crates/clawhdf5/tests/v4_chunk_index_selection.rs index 97cc7c9..23786bd 100644 --- a/crates/clawhdf5/tests/v4_chunk_index_selection.rs +++ b/crates/clawhdf5/tests/v4_chunk_index_selection.rs @@ -122,7 +122,10 @@ const CASES: &[Case] = &[ /// `(start, stride, count, block)` per dimension; the first few stay below /// half the dataset (bounding-box path), the rest exceed it (full path). -fn selections() -> Vec<([u64; 2], [u64; 2], [u64; 2], [u64; 2])> { +/// `(start, stride, count, block)` of a 2-D hyperslab. +type Hyperslab2 = ([u64; 2], [u64; 2], [u64; 2], [u64; 2]); + +fn selections() -> Vec { vec![ ([0, 0], [1, 1], [3, 23], [1, 1]), // ds[0:3] ([7, 3], [1, 1], [9, 6], [1, 1]), // interior window across chunks @@ -220,8 +223,10 @@ fn partial_hyperslabs_of_every_v4_chunk_index_match_h5py() { .read_selection(&sel) .unwrap_or_else(|e| panic!("{}: read_selection {sel:?} failed: {e}", case.name)); let got: Vec = raw - .chunks_exact(4) - .map(|b| i32::from_le_bytes(b.try_into().unwrap())) + .as_chunks::<4>() + .0 + .iter() + .map(|b| i32::from_le_bytes(*b)) .collect(); assert_eq!(got, want, "{}: selection {sel:?}", case.name); assert_eq!( -- 2.54.0 From 5a202f3791c3365af73f1ee6b8d56ae9dd86866d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:06:46 -0500 Subject: [PATCH 34/36] fix(format): a VL element at the undefined heap address is an error libhdf5 fails to read a VL element whose global heap address is undefined (all 0xff), even at length 0 ("addr undefined"); we returned "" (or an empty sequence) in every reader. Checked with h5py first: libhdf5 writes a null element with address 0, which still reads as empty, and h5py writes "" as a zero-size heap object at a real address, so no file they write relies on the old behaviour. read_vl_bytes now treats address 0 as null whatever the length, as VlResolver does. Tests, each failing before: vl_data unit test (8- and 4-byte offsets, lengths 0 and 1); clawhdf5 vl_data_interop a_vl_element_at_the_undefined_heap_address_fails_like_h5py (also checks where h5py writes ""); h5rs dump --json and check --data on the patched `undef` dataset; clawhdf5-wasm vl_strings. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 +++++ crates/clawhdf5-format/src/vl_data.rs | 49 +++++++++++++------- crates/clawhdf5-tools/tests/gen_vl_files.py | 11 ++++- crates/clawhdf5-tools/tests/h5rs_interop.rs | 19 ++++++-- crates/clawhdf5-wasm/tests/vl_strings.rs | 27 +++++++++-- crates/clawhdf5/tests/vl_data_interop.rs | 50 +++++++++++++++++++++ docs/known-issues.md | 4 +- 7 files changed, 143 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc22849..e656dff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,16 @@ `crates/clawhdf5-wasm/tests/vl_strings.rs`). New `VlResolver::element` / `string_element` resolve one element in place. +- **A VL element at the undefined heap address is an error**, as in + libhdf5 ("addr undefined"). One of length 0 read as `""` in every reader + (`File`, `h5rs`, `clawhdf5-wasm`, `read_vl_strings`, `read_vl_bytes`). + libhdf5 writes a null element with heap address 0, which still reads as + empty, and h5py writes `""` as a zero-size heap object at a real address, + so no file libhdf5 or h5py writes is affected + (`a_vl_element_at_the_undefined_heap_address_fails_like_h5py` in + `crates/clawhdf5/tests/vl_data_interop.rs`). `read_vl_bytes` now also + treats address 0 as null whatever the length, as `VlResolver` does. + ### Plugin filters (2026-09-26) - **LZF, bitshuffle, bzip2 and Blosc read and write, in pure Rust.** Files written by h5py with `compression="lzf"`, or with hdf5plugin's diff --git a/crates/clawhdf5-format/src/vl_data.rs b/crates/clawhdf5-format/src/vl_data.rs index f183ef0..3a54c1a 100644 --- a/crates/clawhdf5-format/src/vl_data.rs +++ b/crates/clawhdf5-format/src/vl_data.rs @@ -239,9 +239,6 @@ impl<'a> VlResolver<'a> { if addr == 0 { return Ok(None); } - if vl.length == 0 && is_undefined_address(addr, self.offset_size) { - return Ok(Some(&[])); - } let data = self.object(vl)?; let expected = (vl.length as usize) .checked_mul(base_size) @@ -372,10 +369,8 @@ pub fn read_vl_bytes( let mut result = Vec::with_capacity(refs.len()); for vl in &refs { - if vl.length == 0 - && (is_undefined_address(vl.collection_address, offset_size) - || vl.collection_address == 0) - { + // A heap address of 0 is a null element, as in VlResolver. + if vl.collection_address == 0 { result.push(Vec::new()); continue; } @@ -394,6 +389,15 @@ impl<'a> VlResolver<'a> { /// parsed on first use. fn object(&mut self, vl: &VlElement) -> Result<&'a [u8], FormatError> { let addr = vl.collection_address; + // libhdf5 writes a null element with address 0, never the undefined + // address, and fails to read one ("addr undefined") even when its + // length is 0; we returned an empty value. + if is_undefined_address(addr, self.offset_size) { + return Err(FormatError::VlDataError(format!( + "variable-length element (length {}) has the undefined global heap address", + vl.length + ))); + } if !self.cache.contains_key(&addr) { let offset = usize::try_from(addr).map_err(|_| FormatError::UnexpectedEof { expected: usize::MAX, @@ -546,16 +550,27 @@ mod tests { } #[test] - fn null_vl_element_empty_string() { - // length=0, address=undefined - let mut raw = Vec::new(); - raw.extend_from_slice(&0u32.to_le_bytes()); // length=0 - raw.extend_from_slice(&u64::MAX.to_le_bytes()); // undefined address - raw.extend_from_slice(&0u32.to_le_bytes()); // index - - let file_data = vec![0u8; 16]; - let strings = read_vl_strings(&file_data, &raw, 1, 8, 8).unwrap(); - assert_eq!(strings, vec![""]); + fn an_undefined_heap_address_is_an_error_even_at_length_0() { + // libhdf5 fails the read ("addr undefined"); h5py and libhdf5 write + // a null element with address 0. We returned "". + let mut file_data = vec![0u8; 256]; + build_gcol_at(&mut file_data, 64, &[(1, b"x")]); + for (os, undef) in [(8u8, u64::MAX), (4, 0xFFFF_FFFF)] { + for length in [0, 1] { + let mut raw = element(1, 64, 1, os); + raw.extend(element(length, undef, 1, os)); + let mut r = VlResolver::new(&file_data, os, 8); + let e = r.string_bytes(&raw).unwrap_err().to_string(); + assert!(e.contains("undefined"), "{e}"); + assert!(r.sequences(&raw, 1).is_err()); + assert!(r.string_element(&raw[raw.len() / 2..]).is_err()); + let n = 2; + assert!(read_vl_strings(&file_data, &raw, n, os, 8).is_err()); + assert!(read_vl_bytes(&file_data, &raw, n, os, 8).is_err()); + // The defined element alone still reads. + assert_eq!(r.strings(&raw[..raw.len() / 2]).unwrap(), ["x"]); + } + } } #[test] diff --git a/crates/clawhdf5-tools/tests/gen_vl_files.py b/crates/clawhdf5-tools/tests/gen_vl_files.py index 3937c19..cb50f2d 100644 --- a/crates/clawhdf5-tools/tests/gen_vl_files.py +++ b/crates/clawhdf5-tools/tests/gen_vl_files.py @@ -6,7 +6,8 @@ For 8-byte (`vl8`) and 4-byte (`vl4`) offsets, writes OUTDIR/vl8.h5 and OUTDIR/vl4.h5, which libhdf5 reads in full, and OUTDIR/bad8.h5 and OUTDIR/bad4.h5, whose `bad` and `badseq` elements 0 have a length that disagrees with their global heap object (libhdf5: "Expected global heap -object size does not match"). h5py cannot write a VL string with a NUL in +object size does not match"), and whose `undef` element 1 has length 0 and +the undefined heap address (libhdf5: "addr undefined"). h5py cannot write a VL string with a NUL in it or a null element in a contiguous dataset, so those are patched in. Prints one JSON object: for each file, each dataset's values as h5py reads @@ -76,11 +77,17 @@ def bad(path, sizes): s = f.create_dataset("badseq", shape=(2,), dtype=I4) s[0] = [1, 2, 3] s[1] = [4] + f.create_dataset("undef", data=np.array(["x", "", "yz"], dtype=object), dtype=S) off, soff = f["bad"].id.get_offset(), f["badseq"].id.get_offset() + uoff = f["undef"].id.get_offset() b = bytearray(open(path, "rb").read()) gcol = int.from_bytes(b[off + 4 : off + 4 + os_], "little") struct.pack_into(" 3 struct.pack_into(" 2 + # "": length 0 at the undefined address (all 0xff), which libhdf5 fails + # to read ("addr undefined"); it writes a null element as address 0. + es = 8 + os_ + b[uoff + es : uoff + 2 * es] = element(0, (1 << (8 * os_)) - 1, 1, os_) open(path, "wb").write(bytes(b)) return gcol @@ -116,6 +123,6 @@ for tag, sizes in (("8", None), ("4", (4, 4))): result[f"vl{tag}"] = {n: read(f[n]) for n in ("d", "u", "seq", "sequ", "cmp")} result[f"vl{tag}"]["va"] = [value(s) for s in f.attrs["va"]] with h5py.File(x, "r") as f: - result[f"bad{tag}"] = {n: read(f[n]) for n in ("bad", "badseq")} + result[f"bad{tag}"] = {n: read(f[n]) for n in ("bad", "badseq", "undef")} result[f"bad{tag}"]["gcol"] = gcol json.dump(result, sys.stdout) diff --git a/crates/clawhdf5-tools/tests/h5rs_interop.rs b/crates/clawhdf5-tools/tests/h5rs_interop.rs index a1b4492..202606f 100644 --- a/crates/clawhdf5-tools/tests/h5rs_interop.rs +++ b/crates/clawhdf5-tools/tests/h5rs_interop.rs @@ -828,7 +828,8 @@ fn dump_prints_vl_data_like_h5dump() { /// `dump --json` gives the values h5py reads, element by element; and an /// element whose heap object is not its length × base size is an error, as -/// in h5py, not a truncated value (it printed "cde" and (1, 2)). +/// in h5py, not a truncated value (it printed "cde" and (1, 2)); so is a +/// length-0 element at the undefined heap address (it printed ""). #[test] fn dump_json_vl_values_match_h5py() { let Some(f) = generate_vl() else { return }; @@ -860,7 +861,12 @@ fn dump_json_vl_values_match_h5py() { let e = g["error"] .as_str() .unwrap_or_else(|| panic!("{bad}: {path}: {g}")); - assert!(e.contains("holds"), "{bad}: {path}: {e}"); + let why = if path == "/undef" { + "undefined" + } else { + "holds" + }; + assert!(e.contains(why), "{bad}: {path}: {e}"); } else { assert_eq!(g, w, "{bad}: {path}"); } @@ -871,7 +877,8 @@ fn dump_json_vl_values_match_h5py() { /// `check --data` holds VL elements to libhdf5's rule: a heap object whose /// size is not exactly the element's length × base size is a problem (it -/// only caught objects shorter than the element). +/// only caught objects shorter than the element), and so is an element at +/// the undefined heap address. #[test] fn check_data_flags_mis_sized_vl_heap_objects() { let Some(f) = generate_vl() else { return }; @@ -893,5 +900,11 @@ fn check_data_flags_mis_sized_vl_heap_objects() { assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}"); assert!(s.contains(what), "bad{tag}: {s}"); } + // A length-0 element at the undefined heap address: libhdf5 fails + // to read it; check skipped it. + let undef: u64 = if tag == "8" { u64::MAX } else { 0xffff_ffff }; + let want = format!("problem: {undef:#x} /undef: variable-length data: global heap:"); + assert!(s.contains(&want), "bad{tag}: no {want:?} in\n{s}"); + assert!(s.contains("undefined global heap address"), "bad{tag}: {s}"); } } diff --git a/crates/clawhdf5-wasm/tests/vl_strings.rs b/crates/clawhdf5-wasm/tests/vl_strings.rs index 1baddd0..096cc16 100644 --- a/crates/clawhdf5-wasm/tests/vl_strings.rs +++ b/crates/clawhdf5-wasm/tests/vl_strings.rs @@ -1,8 +1,9 @@ //! The wasm reader resolves VL strings with the library's `VlResolver`, so //! it returns what `File::read_string` and h5py return: a string ends at //! its first NUL, a null element is empty, a heap object of the wrong size -//! is an error, and a VL datatype whose stored element size disagrees with -//! the file's offset size is refused. Checked with 8- and 4-byte offsets. +//! is an error, an element at the undefined heap address is an error, and a +//! VL datatype whose stored element size disagrees with the file's offset +//! size is refused. Checked with 8- and 4-byte offsets. //! //! Skipped when python3 with h5py is missing, unless //! `CLAWHDF5_REQUIRE_INTEROP=1`. `CLAWHDF5_PYTHON` names the interpreter. @@ -35,7 +36,9 @@ fn h5py_available() -> bool { /// For each offset size: `vl{8,4}.h5` with dataset `d` = "a\0b", "", null, /// "zz" (patched: h5py writes neither a NUL nor a null element); /// `bad{8,4}.h5` whose element 0 claims 3 bytes of a 6-byte heap object; -/// and `size{8,4}.h5` whose VL datatype message stores a 24-byte element. +/// `size{8,4}.h5` whose VL datatype message stores a 24-byte element; and +/// `undef{8,4}.h5` whose element 1 has length 0 and the undefined heap +/// address. /// Prints h5py's reading of each element as hex, or "error". const SCRIPT: &str = r#" import struct, sys, h5py, numpy as np @@ -77,7 +80,12 @@ for os_ in (8, 4): i = b.index(pat) struct.pack_into('()`, and VL values inside compounds or `AttrValue::Raw` attributes decode with `File::decode_strings` / `File::decode_vlen` (`crates/clawhdf5/tests/vl_data_interop.rs`). -- 2.54.0 From 408f69ec1dd2c8147a37f26a0580b482f8efbaaf Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:18:54 -0500 Subject: [PATCH 35/36] docs: conformance report after the perf and coverage merges (575 of 697 ok) Co-Authored-By: Claude Opus 5.5 (1M context) --- CONFORMANCE.md | 8 ++++---- conformance/baseline.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index fd6fbed..07ce59b 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -13,15 +13,15 @@ fatal. This file is generated by `conformance/run.sh`; do not edit it by hand. | | | |---|---| -| date | 2026-09-26 06:50 UTC | -| clawhdf5 commit | `72306c601399748616bc9d061be2ebc4c1bea9e0` | +| date | 2026-09-26 14:18 UTC | +| clawhdf5 commit | `73a01f1256fb9bf1b1e7601f755af9e8273cec4e` | | machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 | | command | `conformance/run.sh --no-fetch --update-baseline` | | rustc | rustc 1.98.1 (48a229cea 2026-09-01) | | reference | h5py 3.16.0, HDF5 2.0.0, numpy 2.5.3, hdf5plugin 7.1.0, Python 3.14.4 | | h5dump | Version 1.14.6 (CVE corpus only) | | limits | 20 s timeout (SIGKILL), 4096 MiB address space, per process; 16 files in parallel | -| runtime | 22 s probing + comparing (0 s fetch/build before it) | +| runtime | 23 s probing + comparing (0 s fetch/build before it) | ## Results @@ -191,7 +191,7 @@ columns are. | cvefiles/cve-2024-32606.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2024-32607-1.h5 | ok | read 10 obj | read 10 obj | ok | | cvefiles/cve-2024-32607-2.h5 | error exit | read 9 obj, 1 errors | read 9 obj, 1 errors | ok | -| cvefiles/cve-2024-32608.h5 | error exit | read 6 obj, 1 errors | read 6 obj | ok | +| cvefiles/cve-2024-32608.h5 | error exit | read 6 obj, 1 errors | read 6 obj, 1 errors | ok | | cvefiles/cve-2024-32609.h5 | error exit | SIGSEGV | read 3 obj, 1 errors | h5py-cannot-read | | cvefiles/cve-2024-32610.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2024-32611.h5 | ok | read 6 obj | read 6 obj | ok | diff --git a/conformance/baseline.json b/conformance/baseline.json index eb3747e..df3cb4a 100644 --- a/conformance/baseline.json +++ b/conformance/baseline.json @@ -1,7 +1,7 @@ { "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.", - "commit": "72306c601399748616bc9d061be2ebc4c1bea9e0", - "date": "2026-09-26 06:50 UTC", + "commit": "73a01f1256fb9bf1b1e7601f755af9e8273cec4e", + "date": "2026-09-26 14:18 UTC", "reference": "h5py 3.16.0 / HDF5 2.0.0", "files": 697, "ok": 575, -- 2.54.0 From dda28d6c7208af2e0544e56635eb2a0c99d8a283 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 09:28:51 -0500 Subject: [PATCH 36/36] bench: concurrent reads re-measured after the read fixes Idle tank at 408f69e, h5py re-run in the same session. Contiguous reads went from 0.25x to 1.44x h5py (full) and 0.12x to 6.3x (256x256 hyperslabs) on one thread; deflate full reads at 8 threads 887 -> 2943 MB/s (h5py processes 3042). Full chunked reads at 16 threads are still 0.69x-0.76x h5py processes; the issue stays open. Co-Authored-By: Claude Opus 5.5 (1M context) --- BENCHMARKS.md | 47 +++++++++++++++++++++++++++++++++++++++++++- docs/known-issues.md | 15 ++++++++------ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index c4e1705..0bc5fe7 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -484,7 +484,52 @@ explain the slower windows. ## Concurrent reads -### Results (2026-09-26, tank) +### Results after the read fixes (2026-09-26, tank, `408f69e`) + +Same machine, files and commands as the first run below, re-run on an idle +tank (load average 1.60 at the start; the 1-minute figure rose to about 5 +during the clawhdf5 runs, mostly their own threads) after two fixes: +contiguous reads back their output with transparent huge pages and copy +hyperslabs run by run, and full chunked reads no longer queue behind a +one-thread rayon pool. h5py was re-run in the same session. + +Each read decoding on its calling thread (`--decode-threads 1`, like h5py): + +| layout | mode | threads | clawhdf5 MB/s (eff) | h5py threads MB/s (eff) | h5py processes MB/s (eff) | +|---|---|---:|---:|---:|---:| +| deflate | distinct | 1 | 606 (1.00) | 432 (1.00) | 421 (1.00) | +| deflate | distinct | 4 | 1816 (0.75) | 428 (0.25) | 1654 (0.98) | +| deflate | distinct | 8 | 2943 (0.61) | 428 (0.12) | 3042 (0.90) | +| deflate | distinct | 16 | 2142 (0.22) | 375 (0.05) | 3083 (0.46) | +| deflate | same | 1 | 154 (1.00) | 130 (1.00) | 129 (1.00) | +| deflate | same | 4 | 599 (0.98) | 129 (0.25) | 499 (0.97) | +| deflate | same | 16 | 1592 (0.65) | 128 (0.06) | 1399 (0.68) | +| contiguous | distinct | 1 | 13665 (1.00) | 9490 (1.00) | 8781 (1.00) | +| contiguous | distinct | 16 | 12674 (0.06) | 2285 (0.02) | 6942 (0.05) | +| contiguous | same | 1 | 31991 (1.00) | 5087 (1.00) | 5078 (1.00) | +| contiguous | same | 16 | 237151 (0.46) | 4304 (0.05) | 35772 (0.44) | + +With the default rayon pool: deflate `distinct` 2117 MB/s at 1 thread (4.9x +h5py), 3163 at 4, 2341 at 16 (0.76x h5py processes); deflate `same` 1439 MB/s +at 16; contiguous as above within a few percent. + +Before -> after for clawhdf5 (`--decode-threads 1` unless noted): +contiguous full read at 1 thread 2495 -> 13665 MB/s (0.25x -> 1.44x h5py); +contiguous 256 x 256 hyperslabs at 1 thread 624 -> 31991 MB/s (0.12x -> +6.3x); deflate full reads at 8 threads 887 -> 2943 MB/s; deflate +hyperslabs at 16 threads 1244 -> 1592 MB/s. + +Read with care: +- `contiguous same` reads 1024 slabs of one 64 MiB dataset over and over, so + it mostly measures copies out of the CPU's caches (the 7800X3D has 96 MiB + of L3); the per-call overhead is what differs (h5py's is about 50 us). +- At 16 threads every tool dropped in this run (h5py threads on contiguous + data from 8002 to 2285 MB/s, processes from 12846 to 6942), so the + 16-thread rows are noisier than the others. +- Still behind: full reads of chunked data at 16 threads (0.69x-0.76x h5py + processes). See `docs/known-issues.md`. + +### First run, before the read fixes (2026-09-26, tank, `91644d8`) Measured on tank (AMD Ryzen 7 7800X3D, 8 cores / 16 threads, 61 GiB, Linux 7.0) at commit `91644d8`, load average 1.84 when the run started (the diff --git a/docs/known-issues.md b/docs/known-issues.md index 63a5fa2..2657cdf 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -40,11 +40,12 @@ tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, worker (per-thread CPU time: one thread did all the decoding, the 16 readers almost none). Hyperslab reads touch one chunk each and never used the pool. Reads now decode on the calling thread when the pool has one - thread (`tests/single_thread_decode_pool.rs`). **Still open:** this - fixes only a one-thread pool. With the default pool, 16 reader threads - ran at about 2900 MB/s before and after the change, still short of 16 - h5py processes (4424 MB/s); with a small pool (2-4 threads) readers - outside it still wait on its workers. Datasets larger than the + thread (`tests/single_thread_decode_pool.rs`); re-measured at + `408f69e`, 8 threads went from 887 to 2943 MB/s (h5py processes: 3042). + **Still open:** at 16 threads full chunked reads reach 2142-2341 MB/s, + 0.69x-0.76x 16 h5py processes (3083 MB/s in the same run), with the + default pool as with a one-thread one; with a small pool (2-4 threads) + readers outside it still wait on its workers. Datasets larger than the cache's budget were already read without inserting into it, and skipping its lookups entirely gained only a few percent at 16 threads. Remaining per-read overhead, not yet addressed: each full `read_f32` of a chunked @@ -52,7 +53,9 @@ tank with `concurrent_read` against h5py 3.16 / HDF5 2.0 (`BENCHMARKS.md`, the `f32` copy of it, and a new buffer per decoded chunk). - Contiguous datasets read 4x slower than h5py on one thread (2.5 vs 9.8 GB/s full, 0.12x for 256 x 256 hyperslabs). - **Fixed 2026-09-26** (not yet re-measured for `BENCHMARKS.md`): full + **Fixed 2026-09-26** (re-measured on tank at `408f69e`: 13665 MB/s + full and 31991 MB/s for 256 x 256 hyperslabs on one thread, 1.44x and + 6.3x h5py; `BENCHMARKS.md`): full reads were dominated by 4 KiB page faults on the fresh output buffer, which is now backed by transparent huge pages as numpy's is; hyperslab reads copied the selection three times, element by element, and now copy -- 2.54.0