feat: add with_u64_data builder; fix read_selection cache bypass
CI / test (push) Failing after 2s

Found via a real-world integration audit against omni-cortex (a JEPA-based
cognitive architecture built on clawhdf5 as its tiered Working/Episodic/
Semantic memory store).

- Add DatasetBuilder::with_u64_data (crates/clawhdf5-format/type_builders.rs).
  The read side already has read_u64/read_as_u64, but there was no
  symmetric write-side builder — only signed with_i32_data/with_i64_data
  existed. Every consumer needing full-range u64 (timestamps, IDs) had to
  bit-cast through i64 via `i64::from_ne_bytes(v.to_ne_bytes())` on write
  and reverse it on read. omni-cortex does this in at least 6 places
  across its writer/reader/mmap-reader/consolidate crates. Confirmed the
  new builder round-trips full-range u64 (including values with the high
  bit set) end-to-end in a standalone sanity check mirroring their usage.
- Fix Dataset::read_selection(&Selection::All) to route through the same
  per-file chunk cache read_raw()/read_f64() etc. already use, instead of
  the uncached read_chunked_data path. Selection::All is semantically a
  full read; there's no reason two ways of asking for "everything" should
  have different caching behavior. Also gains read_raw()'s virtual-dataset
  resolver support for free. omni-cortex's Reader/mmap-reader/consolidate
  crates all call read_selection(&Selection::All) for their chunked/
  compressed dataset reads, so this was a real, if currently low-traffic
  (single-pass read pattern), inconsistency in the public API's behavior.
- README: fix a stale crate-map claim that clawhdf5-filters supports
  "blosc" compression — it never did (the crate only ever held
  fast_deflate.rs; lz4/zstd/pcodec/szip filters live in clawhdf5-format).

New tests: u64_data_roundtrip, read_selection_all_matches_read_raw_on_chunked_dataset.
This commit is contained in:
Omar Sobh
2026-08-06 09:24:43 -07:00
parent 429c29b76b
commit dfae9e2cc1
4 changed files with 87 additions and 1 deletions
+1 -1
View File
@@ -352,7 +352,7 @@ clawhdf5 workspace (16 crates, ~92K lines of Rust; plus libaec-sys, an
├── Core HDF5 ├── Core HDF5
│ ├── clawhdf5-format — Binary parser/writer (no_std), shared type definitions │ ├── clawhdf5-format — Binary parser/writer (no_std), shared type definitions
│ ├── clawhdf5-io — I/O abstraction (buffered, mmap, async) │ ├── clawhdf5-io — I/O abstraction (buffered, mmap, async)
│ ├── clawhdf5-filters — Compression (deflate, lz4, zstd, blosc) │ ├── clawhdf5-filters — Fast deflate path (zlib-ng); lz4/zstd/pcodec/szip filters live in clawhdf5-format
│ ├── clawhdf5-derive — Proc macros │ ├── clawhdf5-derive — Proc macros
│ ├── clawhdf5 — High-level API │ ├── clawhdf5 — High-level API
│ ├── clawhdf5-netcdf4 — NetCDF-4 support │ ├── clawhdf5-netcdf4 — NetCDF-4 support
@@ -90,6 +90,16 @@ pub fn make_i64_type() -> Datatype {
} }
} }
pub fn make_u64_type() -> Datatype {
Datatype::FixedPoint {
size: 8,
byte_order: DatatypeByteOrder::LittleEndian,
signed: false,
bit_offset: 0,
bit_precision: 64,
}
}
pub fn make_u8_type() -> Datatype { pub fn make_u8_type() -> Datatype {
Datatype::FixedPoint { Datatype::FixedPoint {
size: 1, size: 1,
@@ -444,6 +454,25 @@ impl DatasetBuilder {
self self
} }
/// Write a native unsigned 64-bit integer dataset. Pairs with the
/// read side's `read_u64`/`read_as_u64`, which already support this
/// datatype — this was the missing symmetric write-side builder
/// (callers previously had to bit-cast through `with_i64_data` /
/// `i64::from_ne_bytes(v.to_ne_bytes())` to round-trip full-range u64
/// values like timestamps or IDs).
pub fn with_u64_data(&mut self, data: &[u64]) -> &mut Self {
self.datatype = Some(make_u64_type());
let mut b = Vec::with_capacity(data.len() * 8);
for &v in data {
b.extend_from_slice(&v.to_le_bytes());
}
self.data = Some(b);
if self.shape.is_none() {
self.shape = Some(vec![data.len() as u64]);
}
self
}
pub fn with_u8_data(&mut self, data: &[u8]) -> &mut Self { pub fn with_u8_data(&mut self, data: &[u8]) -> &mut Self {
self.datatype = Some(make_u8_type()); self.datatype = Some(make_u8_type());
self.data = Some(data.to_vec()); self.data = Some(data.to_vec());
+8
View File
@@ -436,6 +436,14 @@ impl<'f> Dataset<'f> {
&self, &self,
selection: &clawhdf5_format::selection::Selection, selection: &clawhdf5_format::selection::Selection,
) -> Result<Vec<u8>, Error> { ) -> Result<Vec<u8>, Error> {
// `Selection::All` is semantically a full read — route it through
// the same per-file chunk cache `read_raw()` uses instead of the
// selection path's uncached `read_chunked_data`, so callers get
// consistent caching behavior regardless of which method they used
// to ask for "everything".
if matches!(selection, clawhdf5_format::selection::Selection::All) {
return self.read_raw();
}
let dt = self.datatype()?; let dt = self.datatype()?;
let ds = self.dataspace()?; let ds = self.dataspace()?;
let dl = self.data_layout()?; let dl = self.data_layout()?;
@@ -935,3 +935,52 @@ fn dense_links_multiblock_fractal_heap_roundtrip() {
); );
} }
} }
#[test]
fn read_selection_all_matches_read_raw_on_chunked_dataset() {
// read_selection(&Selection::All) is semantically a full read and must
// go through the same cached path as read_raw()/read_f64() — not a
// separate uncached code path that happens to return the same bytes.
use clawhdf5_format::selection::Selection;
let data: Vec<f64> = (0..500).map(|i| i as f64 * 0.5).collect();
let mut b = FileBuilder::new();
b.create_dataset("chunked")
.with_f64_data(&data)
.with_chunks(&[100])
.with_deflate(6);
let file = File::from_bytes(b.finish().unwrap()).unwrap();
let ds = file.dataset("chunked").unwrap();
let via_read_f64 = ds.read_f64().unwrap();
let via_selection_bytes = ds.read_selection(&Selection::All).unwrap();
let via_selection: Vec<f64> = via_selection_bytes
.chunks_exact(8)
.map(|c| f64::from_le_bytes(c.try_into().unwrap()))
.collect();
assert_eq!(via_read_f64, data);
assert_eq!(via_selection, data);
}
#[test]
fn u64_data_roundtrip() {
// Values spanning the full u64 range, including ones with the high bit
// set that would come back negative (and wrong) if bit-cast through
// an i64 dataset instead of a native unsigned one.
let values: Vec<u64> = vec![
0,
1,
u64::MAX,
u64::MAX / 2,
1 << 63,
1_700_000_000_000_000_000,
];
let mut b = FileBuilder::new();
b.create_dataset("timestamps").with_u64_data(&values);
let file = File::from_bytes(b.finish().unwrap()).unwrap();
assert_eq!(
file.dataset("timestamps").unwrap().read_u64().unwrap(),
values
);
}