Found via a real-world integration audit against omni-cortex (a JEPA-based cognitive architecture built on clawhdf5 as its tiered Working/Episodic/ Semantic memory store). - Add DatasetBuilder::with_u64_data (crates/clawhdf5-format/type_builders.rs). The read side already has read_u64/read_as_u64, but there was no symmetric write-side builder — only signed with_i32_data/with_i64_data existed. Every consumer needing full-range u64 (timestamps, IDs) had to bit-cast through i64 via `i64::from_ne_bytes(v.to_ne_bytes())` on write and reverse it on read. omni-cortex does this in at least 6 places across its writer/reader/mmap-reader/consolidate crates. Confirmed the new builder round-trips full-range u64 (including values with the high bit set) end-to-end in a standalone sanity check mirroring their usage. - Fix Dataset::read_selection(&Selection::All) to route through the same per-file chunk cache read_raw()/read_f64() etc. already use, instead of the uncached read_chunked_data path. Selection::All is semantically a full read; there's no reason two ways of asking for "everything" should have different caching behavior. Also gains read_raw()'s virtual-dataset resolver support for free. omni-cortex's Reader/mmap-reader/consolidate crates all call read_selection(&Selection::All) for their chunked/ compressed dataset reads, so this was a real, if currently low-traffic (single-pass read pattern), inconsistency in the public API's behavior. - README: fix a stale crate-map claim that clawhdf5-filters supports "blosc" compression — it never did (the crate only ever held fast_deflate.rs; lz4/zstd/pcodec/szip filters live in clawhdf5-format). New tests: u64_data_roundtrip, read_selection_all_matches_read_raw_on_chunked_dataset.
This commit is contained in:
@@ -352,7 +352,7 @@ clawhdf5 workspace (16 crates, ~92K lines of Rust; plus libaec-sys, an
|
|||||||
├── Core HDF5
|
├── Core HDF5
|
||||||
│ ├── clawhdf5-format — Binary parser/writer (no_std), shared type definitions
|
│ ├── clawhdf5-format — Binary parser/writer (no_std), shared type definitions
|
||||||
│ ├── clawhdf5-io — I/O abstraction (buffered, mmap, async)
|
│ ├── clawhdf5-io — I/O abstraction (buffered, mmap, async)
|
||||||
│ ├── clawhdf5-filters — Compression (deflate, lz4, zstd, blosc)
|
│ ├── clawhdf5-filters — Fast deflate path (zlib-ng); lz4/zstd/pcodec/szip filters live in clawhdf5-format
|
||||||
│ ├── clawhdf5-derive — Proc macros
|
│ ├── clawhdf5-derive — Proc macros
|
||||||
│ ├── clawhdf5 — High-level API
|
│ ├── clawhdf5 — High-level API
|
||||||
│ ├── clawhdf5-netcdf4 — NetCDF-4 support
|
│ ├── clawhdf5-netcdf4 — NetCDF-4 support
|
||||||
|
|||||||
@@ -90,6 +90,16 @@ pub fn make_i64_type() -> Datatype {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn make_u64_type() -> Datatype {
|
||||||
|
Datatype::FixedPoint {
|
||||||
|
size: 8,
|
||||||
|
byte_order: DatatypeByteOrder::LittleEndian,
|
||||||
|
signed: false,
|
||||||
|
bit_offset: 0,
|
||||||
|
bit_precision: 64,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn make_u8_type() -> Datatype {
|
pub fn make_u8_type() -> Datatype {
|
||||||
Datatype::FixedPoint {
|
Datatype::FixedPoint {
|
||||||
size: 1,
|
size: 1,
|
||||||
@@ -444,6 +454,25 @@ impl DatasetBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write a native unsigned 64-bit integer dataset. Pairs with the
|
||||||
|
/// read side's `read_u64`/`read_as_u64`, which already support this
|
||||||
|
/// datatype — this was the missing symmetric write-side builder
|
||||||
|
/// (callers previously had to bit-cast through `with_i64_data` /
|
||||||
|
/// `i64::from_ne_bytes(v.to_ne_bytes())` to round-trip full-range u64
|
||||||
|
/// values like timestamps or IDs).
|
||||||
|
pub fn with_u64_data(&mut self, data: &[u64]) -> &mut Self {
|
||||||
|
self.datatype = Some(make_u64_type());
|
||||||
|
let mut b = Vec::with_capacity(data.len() * 8);
|
||||||
|
for &v in data {
|
||||||
|
b.extend_from_slice(&v.to_le_bytes());
|
||||||
|
}
|
||||||
|
self.data = Some(b);
|
||||||
|
if self.shape.is_none() {
|
||||||
|
self.shape = Some(vec![data.len() as u64]);
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn with_u8_data(&mut self, data: &[u8]) -> &mut Self {
|
pub fn with_u8_data(&mut self, data: &[u8]) -> &mut Self {
|
||||||
self.datatype = Some(make_u8_type());
|
self.datatype = Some(make_u8_type());
|
||||||
self.data = Some(data.to_vec());
|
self.data = Some(data.to_vec());
|
||||||
|
|||||||
@@ -436,6 +436,14 @@ impl<'f> Dataset<'f> {
|
|||||||
&self,
|
&self,
|
||||||
selection: &clawhdf5_format::selection::Selection,
|
selection: &clawhdf5_format::selection::Selection,
|
||||||
) -> Result<Vec<u8>, Error> {
|
) -> Result<Vec<u8>, Error> {
|
||||||
|
// `Selection::All` is semantically a full read — route it through
|
||||||
|
// the same per-file chunk cache `read_raw()` uses instead of the
|
||||||
|
// selection path's uncached `read_chunked_data`, so callers get
|
||||||
|
// consistent caching behavior regardless of which method they used
|
||||||
|
// to ask for "everything".
|
||||||
|
if matches!(selection, clawhdf5_format::selection::Selection::All) {
|
||||||
|
return self.read_raw();
|
||||||
|
}
|
||||||
let dt = self.datatype()?;
|
let dt = self.datatype()?;
|
||||||
let ds = self.dataspace()?;
|
let ds = self.dataspace()?;
|
||||||
let dl = self.data_layout()?;
|
let dl = self.data_layout()?;
|
||||||
|
|||||||
@@ -935,3 +935,52 @@ fn dense_links_multiblock_fractal_heap_roundtrip() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn read_selection_all_matches_read_raw_on_chunked_dataset() {
|
||||||
|
// read_selection(&Selection::All) is semantically a full read and must
|
||||||
|
// go through the same cached path as read_raw()/read_f64() — not a
|
||||||
|
// separate uncached code path that happens to return the same bytes.
|
||||||
|
use clawhdf5_format::selection::Selection;
|
||||||
|
|
||||||
|
let data: Vec<f64> = (0..500).map(|i| i as f64 * 0.5).collect();
|
||||||
|
let mut b = FileBuilder::new();
|
||||||
|
b.create_dataset("chunked")
|
||||||
|
.with_f64_data(&data)
|
||||||
|
.with_chunks(&[100])
|
||||||
|
.with_deflate(6);
|
||||||
|
let file = File::from_bytes(b.finish().unwrap()).unwrap();
|
||||||
|
|
||||||
|
let ds = file.dataset("chunked").unwrap();
|
||||||
|
let via_read_f64 = ds.read_f64().unwrap();
|
||||||
|
let via_selection_bytes = ds.read_selection(&Selection::All).unwrap();
|
||||||
|
let via_selection: Vec<f64> = via_selection_bytes
|
||||||
|
.chunks_exact(8)
|
||||||
|
.map(|c| f64::from_le_bytes(c.try_into().unwrap()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
assert_eq!(via_read_f64, data);
|
||||||
|
assert_eq!(via_selection, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn u64_data_roundtrip() {
|
||||||
|
// Values spanning the full u64 range, including ones with the high bit
|
||||||
|
// set that would come back negative (and wrong) if bit-cast through
|
||||||
|
// an i64 dataset instead of a native unsigned one.
|
||||||
|
let values: Vec<u64> = vec![
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
u64::MAX,
|
||||||
|
u64::MAX / 2,
|
||||||
|
1 << 63,
|
||||||
|
1_700_000_000_000_000_000,
|
||||||
|
];
|
||||||
|
let mut b = FileBuilder::new();
|
||||||
|
b.create_dataset("timestamps").with_u64_data(&values);
|
||||||
|
let file = File::from_bytes(b.finish().unwrap()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
file.dataset("timestamps").unwrap().read_u64().unwrap(),
|
||||||
|
values
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user