diff --git a/CHANGELOG.md b/CHANGELOG.md index 784d1ef..20b04e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## v2.2.0 (2026-09-18) ### Security - `clawhdf5-format`: bounded decompression output (`MAX_DECOMPRESS_SIZE`) for @@ -245,6 +245,16 @@ reading compound types and — critically — every chunked/compressed dataset written by HDF5 2.0. Found by running the h5py interop tests against h5py 3.16 / HDF5 2.0. + Independently reported (with a patch) against the v2.1.0 tag by + M. Scot Breitenfeld (The HDF Group) — v2.1.0 predates this fix. +- `clawhdf5-format`: parse HDF5 2.0 native complex datatypes (class 11, + datatype version 5, e.g. `H5T_COMPLEX_IEEE_F64LE`). The properties are a + single base floating-point datatype, not a compound-style member list; the + old parser read the base type's bytes as member names, producing a garbage + datatype, and failed with `UnexpectedEof` when a complex type was nested in + a compound. It is now surfaced as the equivalent `{r, i}` compound (the + shape h5py writes for numpy complex dtypes), with a size check against the + base type. Validated end-to-end against an HDF5 2.0-written file. ### Performance - `clawhdf5-format`: chunked writes now compress all chunks up front via diff --git a/Cargo.toml b/Cargo.toml index 7fbd4a6..d2a6d2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,10 +21,10 @@ members = [ resolver = "2" [workspace.package] -version = "2.1.0" +version = "2.2.0" edition = "2024" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" [workspace.dependencies] tempfile = "3" diff --git a/crates/clawhdf5-accel/Cargo.toml b/crates/clawhdf5-accel/Cargo.toml index 25c89a9..7dbba6f 100644 --- a/crates/clawhdf5-accel/Cargo.toml +++ b/crates/clawhdf5-accel/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-accel" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "SIMD-accelerated operations for rustyhdf5" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "simd", "acceleration", "performance"] categories = ["science", "algorithms"] diff --git a/crates/clawhdf5-agent/Cargo.toml b/crates/clawhdf5-agent/Cargo.toml index 066a416..d7ba0ff 100644 --- a/crates/clawhdf5-agent/Cargo.toml +++ b/crates/clawhdf5-agent/Cargo.toml @@ -1,21 +1,21 @@ [package] name = "clawhdf5-agent" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "HDF5-backed persistent memory store for on-device AI agents" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["agent", "memory", "hdf5", "vector-search", "embedding"] categories = ["database", "science", "algorithms"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0", features = ["parallel", "fast-checksum"] } -clawhdf5 = { path = "../clawhdf5", version = "2.1.0" } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.1.0", features = ["mmap"] } -clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.1.0" } -clawhdf5-ann = { path = "../clawhdf5-ann", version = "2.1.0", optional = true } -clawhdf5-gpu = { path = "../clawhdf5-gpu", version = "2.1.0", optional = true, default-features = false } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0", features = ["parallel", "fast-checksum"] } +clawhdf5 = { path = "../clawhdf5", version = "2.2.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.2.0", features = ["mmap"] } +clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.2.0" } +clawhdf5-ann = { path = "../clawhdf5-ann", version = "2.2.0", optional = true } +clawhdf5-gpu = { path = "../clawhdf5-gpu", version = "2.2.0", optional = true, default-features = false } serde = { workspace = true } byteorder = "1" half = { workspace = true, optional = true } diff --git a/crates/clawhdf5-android/Cargo.toml b/crates/clawhdf5-android/Cargo.toml index 92f71fd..e5be0e8 100644 --- a/crates/clawhdf5-android/Cargo.toml +++ b/crates/clawhdf5-android/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-android" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Android JNI bridge for edgehdf5-memory HDF5 backend" license = "MIT" diff --git a/crates/clawhdf5-ann/Cargo.toml b/crates/clawhdf5-ann/Cargo.toml index 9abfcf8..9e5fcc6 100644 --- a/crates/clawhdf5-ann/Cargo.toml +++ b/crates/clawhdf5-ann/Cargo.toml @@ -1,18 +1,18 @@ [package] name = "clawhdf5-ann" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "HNSW approximate nearest neighbor index stored as HDF5" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "ann", "hnsw", "nearest-neighbor"] categories = ["algorithms", "science"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.1.0" } -clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.1.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.2.0" } +clawhdf5-accel = { path = "../clawhdf5-accel", version = "2.2.0" } rayon = { version = "1", optional = true } [features] diff --git a/crates/clawhdf5-bench/Cargo.toml b/crates/clawhdf5-bench/Cargo.toml index 36c8e60..f4ebdfe 100644 --- a/crates/clawhdf5-bench/Cargo.toml +++ b/crates/clawhdf5-bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clawhdf5-bench" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Benchmark harnesses for clawhdf5-agent (Track 8)" license = "MIT" diff --git a/crates/clawhdf5-cli/Cargo.toml b/crates/clawhdf5-cli/Cargo.toml index 0aaa640..beecd03 100644 --- a/crates/clawhdf5-cli/Cargo.toml +++ b/crates/clawhdf5-cli/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-cli" -version = "2.1.0" +version = "2.2.0" edition = "2024" license = "MIT" description = "CLI for clawhdf5 agent memory — create, save, search, recall, stats" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" keywords = ["hdf5", "ai", "memory", "agent", "cli"] categories = ["command-line-utilities", "science"] readme = "../../README.md" @@ -14,7 +14,7 @@ name = "clawhdf5" path = "src/main.rs" [dependencies] -clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.1.0" } +clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.2.0" } clap = { version = "4", features = ["derive", "env"] } serde_json = "1" serde = { workspace = true } diff --git a/crates/clawhdf5-derive/Cargo.toml b/crates/clawhdf5-derive/Cargo.toml index 8278093..1743a18 100644 --- a/crates/clawhdf5-derive/Cargo.toml +++ b/crates/clawhdf5-derive/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-derive" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Derive macros for rustyhdf5 HDF5 traits" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "derive", "macros", "science"] categories = ["development-tools::procedural-macro-helpers"] diff --git a/crates/clawhdf5-filters/Cargo.toml b/crates/clawhdf5-filters/Cargo.toml index 90a5ebf..0fb0e0b 100644 --- a/crates/clawhdf5-filters/Cargo.toml +++ b/crates/clawhdf5-filters/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-filters" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Filter and compression pipeline for clawhdf5" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "compression", "deflate", "filters"] categories = ["compression", "science"] diff --git a/crates/clawhdf5-format/Cargo.toml b/crates/clawhdf5-format/Cargo.toml index af2583f..2cd3dfc 100644 --- a/crates/clawhdf5-format/Cargo.toml +++ b/crates/clawhdf5-format/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-format" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Pure-Rust HDF5 binary format parsing and writing — no C dependencies" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "science", "data", "binary", "no-std"] categories = ["parser-implementations", "science", "encoding", "no-std"] @@ -25,7 +25,7 @@ pco = { version = "1.0", optional = true } [dev-dependencies] serde_json = "1" criterion = { workspace = true } -clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.1.0" } +clawhdf5-derive = { path = "../clawhdf5-derive", version = "2.2.0" } [[bench]] name = "bench" diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index c31d516..40b186e 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -546,27 +546,39 @@ impl Datatype { } } 11 => { - // Complex number — store as compound of two floats internally - // Parse like compound with version 3 and 2 members - // But actually class 11 has no special properties beyond class 6 compound. - // It's just recognized as a separate class. For now parse the 2 members - // as compound. - let num_members = (bf0 as u16) | ((bf1 as u16) << 8); - let mut members = Vec::with_capacity(num_members as usize); - let ob = offset_bytes_for_size(size); - for _ in 0..num_members { - let (name, name_len) = read_null_terminated_string(data, pos)?; - pos += name_len; - let byte_offset = read_uint(data, pos, ob)?; - pos += ob; - let (member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?; - pos += consumed; - members.push(CompoundMember { - name, - byte_offset, - datatype: member_dt, + // Complex number (HDF5 2.0, datatype version 5). The properties + // are a single base floating-point datatype message; an element + // is two consecutive base-type values (real, imaginary). There + // is no member list. Surface it as the equivalent two-member + // compound `{r, i}` — the same shape h5py writes for numpy + // complex dtypes — so downstream compound readers work as-is. + if version != 5 { + return Err(FormatError::InvalidDatatypeVersion { + class: class_id, + version, }); } + let (base_type, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?; + pos += consumed; + let base_size = base_type.type_size(); + if base_size.checked_mul(2) != Some(size) { + return Err(FormatError::DataSizeMismatch { + expected: (base_size as usize).saturating_mul(2), + actual: size as usize, + }); + } + let members = vec![ + CompoundMember { + name: String::from("r"), + byte_offset: 0, + datatype: base_type.clone(), + }, + CompoundMember { + name: String::from("i"), + byte_offset: base_size as u64, + datatype: base_type, + }, + ]; Ok((Datatype::Compound { size, members }, pos)) } _ => Err(FormatError::InvalidDatatypeClass(class_id)), @@ -1126,6 +1138,75 @@ mod tests { } } + /// Real datatype message bytes emitted by HDF5 2.0 for the native complex + /// type `H5T_COMPLEX_IEEE_F64LE`: class 11, version 5, size 16, followed by + /// the base IEEE f64 datatype message. + const COMPLEX_F64_HDF5_2_0: [u8; 28] = [ + 0x5b, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x11, 0x20, 0x3f, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x34, 0x0b, 0x00, 0x34, 0xff, 0x03, 0x00, 0x00, + ]; + + #[test] + fn test_complex_v5_from_hdf5_2_0() { + let (dt, consumed) = Datatype::parse(&COMPLEX_F64_HDF5_2_0).unwrap(); + assert_eq!(consumed, COMPLEX_F64_HDF5_2_0.len()); + match dt { + Datatype::Compound { size, members } => { + assert_eq!(size, 16); + assert_eq!(members.len(), 2); + assert_eq!((members[0].name.as_str(), members[0].byte_offset), ("r", 0)); + assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("i", 8)); + for m in &members { + assert!(matches!( + m.datatype, + Datatype::FloatingPoint { size: 8, .. } + )); + } + } + other => panic!("expected Compound, got {other:?}"), + } + } + + #[test] + fn test_compound_with_complex_member_from_hdf5_2_0() { + // Compound { z: complex f64 @0, k: i64 @16 } as written by HDF5 2.0. + // Regression guard: the complex member must consume exactly its own + // bytes so the following member parses. + let mut bytes = vec![0x56, 0x02, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, b'z', 0x00, 0x00]; + bytes.extend_from_slice(&COMPLEX_F64_HDF5_2_0); + bytes.extend_from_slice(&[b'k', 0x00, 0x10]); + bytes.extend_from_slice(&[ + 0x10, 0x08, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, + ]); + let (dt, consumed) = Datatype::parse(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + match dt { + Datatype::Compound { size, members } => { + assert_eq!(size, 24); + assert_eq!(members.len(), 2); + assert!(matches!( + &members[0].datatype, + Datatype::Compound { size: 16, members } if members.len() == 2 + )); + assert_eq!((members[1].name.as_str(), members[1].byte_offset), ("k", 16)); + } + other => panic!("expected Compound, got {other:?}"), + } + } + + #[test] + fn test_complex_size_mismatch_rejected() { + let mut bytes = COMPLEX_F64_HDF5_2_0; + bytes[4] = 0x0c; // claims 12 bytes, base type is 8 + assert!(matches!( + Datatype::parse(&bytes), + Err(FormatError::DataSizeMismatch { + expected: 16, + actual: 12 + }) + )); + } + #[test] fn test_reference_object() { let buf = build_dt_header(7, 1, [0, 0, 0], 8); diff --git a/crates/clawhdf5-format/tests/writer_h5py_tests.rs b/crates/clawhdf5-format/tests/writer_h5py_tests.rs index bcccbb1..7506717 100644 --- a/crates/clawhdf5-format/tests/writer_h5py_tests.rs +++ b/crates/clawhdf5-format/tests/writer_h5py_tests.rs @@ -292,6 +292,74 @@ f.close() assert_eq!(x_vals, vec![1.0, 3.0]); } +#[test] +#[ignore = "requires Python h5py module"] +fn read_h5py_generated_native_complex() { + // HDF5 2.0 native complex (datatype class 11, version 5), written through + // h5py's low-level API. Skips when the linked HDF5 predates 2.0. + let path = std::env::temp_dir().join("clawhdf5_h5py_native_complex.h5"); + let gen_script = format!( + r#" +import h5py, numpy as np +from h5py import h5t, h5s, h5d, h5f, h5p +if not getattr(h5py.get_config(), 'has_native_complex', False): + print('SKIP') +else: + fapl = h5p.create(h5p.FILE_ACCESS) + fapl.set_libver_bounds(h5f.LIBVER_LATEST, h5f.LIBVER_LATEST) + fid = h5f.create(b'{}', h5f.ACC_TRUNC, fapl=fapl) + t = h5t.COMPLEX_IEEE_F64LE + d = h5d.create(fid, b'z', t, h5s.create_simple((2,))) + d.write(h5s.ALL, h5s.ALL, np.array([1+2j, 3+4j], dtype=np.complex128), mtype=t) + fid.close() +"#, + path.display() + ); + if h5py_read(&path, &gen_script) == "SKIP" { + eprintln!("HDF5 < 2.0: no native complex support, skipping"); + return; + } + + let bytes = std::fs::read(&path).unwrap(); + let sig = clawhdf5_format::signature::find_signature(&bytes).unwrap(); + let sb = clawhdf5_format::superblock::Superblock::parse(&bytes, sig).unwrap(); + let addr = clawhdf5_format::group_v2::resolve_path_any(&bytes, &sb, "z").unwrap(); + let hdr = clawhdf5_format::object_header::ObjectHeader::parse( + &bytes, + addr as usize, + sb.offset_size, + sb.length_size, + ) + .unwrap(); + let msg = |t: clawhdf5_format::message_type::MessageType| { + &hdr.messages.iter().find(|m| m.msg_type == t).unwrap().data + }; + let (dt, _) = clawhdf5_format::datatype::Datatype::parse(msg( + clawhdf5_format::message_type::MessageType::Datatype, + )) + .unwrap(); + let ds = clawhdf5_format::dataspace::Dataspace::parse( + msg(clawhdf5_format::message_type::MessageType::Dataspace), + sb.length_size, + ) + .unwrap(); + let dl = clawhdf5_format::data_layout::DataLayout::parse( + msg(clawhdf5_format::message_type::MessageType::DataLayout), + sb.offset_size, + sb.length_size, + ) + .unwrap(); + let raw = clawhdf5_format::data_read::read_raw_data(&bytes, &dl, &ds, &dt).unwrap(); + let fields = clawhdf5_format::data_read::read_compound_fields(&raw, &dt).unwrap(); + assert_eq!(fields.len(), 2); + let re = + clawhdf5_format::data_read::read_as_f64(&fields[0].raw_data, &fields[0].datatype).unwrap(); + let im = + clawhdf5_format::data_read::read_as_f64(&fields[1].raw_data, &fields[1].datatype).unwrap(); + assert_eq!((fields[0].name.as_str(), re), ("r", vec![1.0, 3.0])); + assert_eq!((fields[1].name.as_str(), im), ("i", vec![2.0, 4.0])); +} + #[test] #[ignore = "requires Python h5py module"] fn read_h5py_generated_enum() { diff --git a/crates/clawhdf5-gpu/Cargo.toml b/crates/clawhdf5-gpu/Cargo.toml index d938320..2750505 100644 --- a/crates/clawhdf5-gpu/Cargo.toml +++ b/crates/clawhdf5-gpu/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-gpu" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "GPU-accelerated vector operations for rustyhdf5 using wgpu compute shaders" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "gpu", "wgpu", "compute"] categories = ["science", "graphics"] diff --git a/crates/clawhdf5-io/Cargo.toml b/crates/clawhdf5-io/Cargo.toml index 574d4f0..fec0700 100644 --- a/crates/clawhdf5-io/Cargo.toml +++ b/crates/clawhdf5-io/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "clawhdf5-io" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "I/O abstraction layer for rustyhdf5" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "io", "science", "data"] categories = ["filesystem", "science"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0" } memmap2 = { version = "0.9", optional = true } libc = { version = "0.2", optional = true } tokio = { version = "1", features = ["fs", "io-util"], optional = true } diff --git a/crates/clawhdf5-migrate/Cargo.toml b/crates/clawhdf5-migrate/Cargo.toml index a7e21c3..8960f62 100644 --- a/crates/clawhdf5-migrate/Cargo.toml +++ b/crates/clawhdf5-migrate/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-migrate" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "CLI to migrate SQLite agent memory databases to HDF5 format" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["sqlite", "hdf5", "migration", "agent", "memory"] categories = ["command-line-utilities", "database"] @@ -14,9 +14,9 @@ name = "clawhdf5-migrate" path = "src/main.rs" [dependencies] -clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.1.0" } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } -clawhdf5 = { path = "../clawhdf5", version = "2.1.0" } +clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.2.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0" } +clawhdf5 = { path = "../clawhdf5", version = "2.2.0" } rusqlite = { version = "0.31", features = ["bundled"] } clap = { version = "4", features = ["derive"] } half = { workspace = true } diff --git a/crates/clawhdf5-napi/Cargo.toml b/crates/clawhdf5-napi/Cargo.toml index 21062a7..5817720 100644 --- a/crates/clawhdf5-napi/Cargo.toml +++ b/crates/clawhdf5-napi/Cargo.toml @@ -1,16 +1,16 @@ [package] name = "clawhdf5-napi" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Node.js native addon (napi-rs) exposing clawhdf5-agent to TypeScript/JavaScript" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" [lib] crate-type = ["cdylib"] [dependencies] -clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.1.0" } +clawhdf5-agent = { path = "../clawhdf5-agent", version = "2.2.0" } napi = { version = "2", default-features = false, features = ["napi9"] } napi-derive = "2" diff --git a/crates/clawhdf5-netcdf4/Cargo.toml b/crates/clawhdf5-netcdf4/Cargo.toml index ba0568f..73bedd9 100644 --- a/crates/clawhdf5-netcdf4/Cargo.toml +++ b/crates/clawhdf5-netcdf4/Cargo.toml @@ -1,17 +1,17 @@ [package] name = "clawhdf5-netcdf4" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "NetCDF-4 read support built on rustyhdf5 — pure Rust, no C dependencies" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["netcdf", "netcdf4", "hdf5", "science", "climate"] categories = ["parser-implementations", "science"] [dependencies] -clawhdf5 = { path = "../clawhdf5", version = "2.1.0" } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } +clawhdf5 = { path = "../clawhdf5", version = "2.2.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0" } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/clawhdf5-py/Cargo.toml b/crates/clawhdf5-py/Cargo.toml index 1c611f8..1f2d8ad 100644 --- a/crates/clawhdf5-py/Cargo.toml +++ b/crates/clawhdf5-py/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "clawhdf5-py" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "python", "bindings", "science"] categories = ["api-bindings", "science"] @@ -14,8 +14,8 @@ name = "clawhdf5" crate-type = ["cdylib", "rlib"] [dependencies] -clawhdf5_rs = { path = "../clawhdf5", version = "2.1.0", package = "clawhdf5" } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } +clawhdf5_rs = { path = "../clawhdf5", version = "2.2.0", package = "clawhdf5" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0" } pyo3 = "0.29" numpy = "0.29" diff --git a/crates/clawhdf5-py/pyproject.toml b/crates/clawhdf5-py/pyproject.toml index ad2a4ee..51a88ae 100644 --- a/crates/clawhdf5-py/pyproject.toml +++ b/crates/clawhdf5-py/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "rustyhdf5" -version = "2.1.0" +version = "2.2.0" description = "Python bindings for rustyhdf5 — a pure-Rust HDF5 library" requires-python = ">=3.8" license = { text = "MIT" } diff --git a/crates/clawhdf5/Cargo.toml b/crates/clawhdf5/Cargo.toml index 2837b16..656c468 100644 --- a/crates/clawhdf5/Cargo.toml +++ b/crates/clawhdf5/Cargo.toml @@ -1,25 +1,25 @@ [package] name = "clawhdf5" -version = "2.1.0" +version = "2.2.0" edition = "2024" description = "Pure-Rust HDF5 reader/writer — no C dependencies" license = "MIT" -repository = "https://github.com/redclawsystems/clawhdf5" +repository = "https://git.redclaw.dev/quantumclaw/clawhdf5" readme = "README.md" keywords = ["hdf5", "science", "data", "binary"] categories = ["parser-implementations", "science", "encoding"] [dependencies] -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0" } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.1.0" } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.2.0" } rayon = { version = "1", optional = true } [dev-dependencies] tempfile = { workspace = true } criterion = { workspace = true } -clawhdf5-io = { path = "../clawhdf5-io", version = "2.1.0", features = ["mmap"] } -clawhdf5-format = { path = "../clawhdf5-format", version = "2.1.0", features = ["parallel", "fast-checksum"] } -clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.1.0" } +clawhdf5-io = { path = "../clawhdf5-io", version = "2.2.0", features = ["mmap"] } +clawhdf5-format = { path = "../clawhdf5-format", version = "2.2.0", features = ["parallel", "fast-checksum"] } +clawhdf5-filters = { path = "../clawhdf5-filters", version = "2.2.0" } [[bench]] name = "mmap_bench" diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 2d11a02..e776244 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -556,7 +556,7 @@ let final_results = confidence::reject_low_confidence( - **[BENCHMARKS.md](../BENCHMARKS.md)** — Full performance numbers - **[ROADMAP.md](../ROADMAP.md)** — What's coming next -- **[GitHub](https://github.com/redclawsystems/clawhdf5)** — Source code +- **[Source](https://git.redclaw.dev/quantumclaw/clawhdf5)** — Source code - **[ClawBrainHub](https://clawbrainhub.com)** — The `.brain` marketplace (coming soon) --- diff --git a/docs/known-issues.md b/docs/known-issues.md new file mode 100644 index 0000000..4191157 --- /dev/null +++ b/docs/known-issues.md @@ -0,0 +1,82 @@ +# Known Issues + +Bugs found during development or downstream use, tracked here because this +repository's issue tracker is disabled. One entry per bug; when an entry is +fixed, record the fix in `CHANGELOG.md` and update its status here rather than +deleting it. + +--- + +## Compound datatype message version 5 is not parsed (HDF5 2.0) + +**Status:** fixed on `main` in `a13ff51` (2026-06-03); **not in the v2.1.0 +tag**, which was cut five commits earlier. Ships in the next release. + +**Reported by:** M. Scot Breitenfeld (The HDF Group), 2026-09-08, against v2.1.0. + +**Summary:** `clawhdf5-format` v2.1.0 rejects any dataset with a compound +(struct) datatype written by an HDF5 2.0 library in `libver='latest'` mode: +`InvalidDatatypeVersion { class: 6, version: 5 }`. + +**Reproduction** (h5py 3.16.0 / HDF5 2.0.0): + +```python +import h5py, numpy as np +dt = np.dtype([('x', 'f8'), ('y', 'f8'), ('id', 'i4')]) +data = np.array([(1.0, 2.0, 10), (3.0, 4.0, 20)], dtype=dt) +f = h5py.File('compound.h5', 'w', libver='latest') +f.create_dataset('particles', data=data) +f.close() +``` + +Committed as `crates/clawhdf5-format/tests/writer_h5py_tests.rs::read_h5py_generated_compound` +(`#[ignore]`d; needs `python3` with h5py on `PATH`). Run with +`cargo test -p clawhdf5-format --test writer_h5py_tests -- --include-ignored`: +v2.1.0 gives 25 passed / 1 failed; `main` passes everything. + +**Root cause:** the compound (class 6) branch of `Datatype::parse` +(`crates/clawhdf5-format/src/datatype.rs`) accepted only versions 1–4. Datatype +message versions 4 and 5 changed only the Reference and Complex classes, so a +v5-tagged compound uses the unchanged v3 member-list layout. + +**Fix:** versions 3–5 are accepted for compound (class 6) and array (class 10) +datatypes, and data layout message version 5 is accepted too (needed for every +chunked dataset written by HDF5 2.0). Byte-level regression tests: +`test_compound_v5_from_hdf5_2_0`, `test_array_v5_from_hdf5_2_0`. + +## Native complex datatype (class 11) is mis-parsed (HDF5 2.0) + +**Status:** fixed 2026-09-18. Found while validating the report above. + +**Summary:** HDF5 2.0 native complex types (`H5T_COMPLEX_IEEE_F64LE` etc.) +were parsed as if they carried a compound-style member list. The properties are +actually a single base floating-point datatype, so the parser produced a garbage +datatype, or `UnexpectedEof` when the complex type was a compound member. h5py's +default numpy-complex mapping is unaffected (it writes a `{r, i}` compound); +only files using the native type through the C API / h5py low-level API hit this. + +**Fix:** class 11 parses its base type and is surfaced as the equivalent +`{r, i}` compound. Tests: `test_complex_v5_from_hdf5_2_0`, +`test_compound_with_complex_member_from_hdf5_2_0`, +`writer_h5py_tests.rs::read_h5py_generated_native_complex`. + +## Revised reference datatype (class 7, version 4) is not parsed + +**Status:** open, unconfirmed against a real file. + +**Summary:** HDF5 1.12+ `H5T_STD_REF` references use datatype version 4 with +reference types 2–4 (object2 / region2 / attribute), which `Datatype::parse` +rejects with `InvalidReferenceType`. h5py still writes the legacy v1 +object/region references, which read correctly, so no reproducing file has been +generated yet; one written with the C API (`H5T_STD_REF`) is needed. + +## `clawhdf5-gpu` `gpu_tests` can hang under the default parallel test runner + +**Status:** open. Observed 2026-09-18 (RTX 5060 Ti, Linux). + +**Summary:** during `cargo test --workspace`, the `gpu_tests` binary sat idle +(~1% CPU) for 25+ minutes and had to be killed. Run single-threaded it passes +in seconds (20/20): `cargo test -p clawhdf5-gpu --test gpu_tests -- --test-threads=1`. +Suspected cause: several tests creating wgpu devices concurrently (possibly +compounded by the rest of the workspace's tests loading the machine). Not yet +root-caused; workaround is `--test-threads=1` for that crate. diff --git a/packages/clawhdf5-node/package.json b/packages/clawhdf5-node/package.json index e27c985..bb15124 100644 --- a/packages/clawhdf5-node/package.json +++ b/packages/clawhdf5-node/package.json @@ -1,13 +1,13 @@ { "name": "@redclaw/clawhdf5", - "version": "2.1.0", + "version": "2.2.0", "description": "Node.js bindings for clawhdf5 — HDF5-backed agent memory with hippocampal consolidation", "main": "index.js", "types": "index.d.ts", "license": "MIT", "repository": { "type": "git", - "url": "https://github.com/redclawsystems/clawhdf5" + "url": "https://git.redclaw.dev/quantumclaw/clawhdf5" }, "keywords": [ "agent",