feat: compress fixed-length string datasets (+ fix shared chunk-cache bug)

clawhdf5-agent: fixed-length string datasets (memory text chunks, session
summaries, ids, tags, entity/relation names) were stored uncompressed behind
a stale "chunked compound not yet supported" comment. Chunked writes work for
fixed-size string/compound datatypes like any other, so write_string_dataset
now chunks + deflates once a dataset's payload reaches 4 KiB — large,
redundant NullPad content compresses well while tiny metadata stays
contiguous (no chunk-overhead bloat). The dead `compress` parameter is
removed in favor of this size heuristic.

clawhdf5-format: enabling string compression exposed a latent bug — the
per-file ChunkCache built its chunk index once and reused it for every
chunked dataset in the file, keyed only by chunk coordinate with no dataset
discrimination. With one chunked dataset per file this never surfaced; with
two of different rank (a 1-D compressed string array and the 2-D embeddings
matrix) the first dataset's rank-1 index was reused for the second, panicking
with an out-of-bounds chunk coordinate. The cache now binds to a dataset by
its chunk-index address and rebinds — dropping the index, chunk-index map,
layout, and decompressed slots — whenever the dataset being read changes,
while still caching repeated/sequential access to the same dataset.

Tests: facade regression reading a 1-D compressed string dataset and a 2-D
compressed f32 dataset through one shared File cache (verified to panic
without the fix); existing agent e2e tests (large text chunks, migration
round-trip) now pass with compression on.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

CHANGELOG.md

@@ -3,6 +3,14 @@
 ## Unreleased
 
 ### New Features
+- `clawhdf5-agent`: **compress fixed-length string datasets** (memory text
+  chunks, session summaries, ids, tags, entity/relation names, …). These were
+  always stored uncompressed with a "chunked compound not yet supported" note
+  that was simply stale — chunked writes work for fixed-size string/compound
+  datatypes like any other. `write_string_dataset` now chunks + deflates a
+  string dataset once its payload reaches 4 KiB, so large, highly-redundant
+  NullPad content shrinks substantially while tiny metadata stays contiguous
+  (no chunk-overhead bloat).
 - `clawhdf5-format`: decode the **scale-offset filter** (id 6) — both the
   integer variant (`H5Z_SO_INT`) and the floating-point **D-scale** variant
   (`H5Z_SO_FLOAT_DSCALE`). Handles signed/unsigned int sizes, f32/f64, negative
@@ -48,6 +56,16 @@
   fixture produced via the HDF5 low-level API; no E-scale decoder is needed.
 
 ### Bug Fixes
+- `clawhdf5-format`: scope the per-file **chunk cache by dataset**. The shared
+  `ChunkCache` built its chunk index once and reused it for every chunked
+  dataset in the file, keyed only by chunk coordinate with no dataset
+  discrimination. With a single chunked dataset per file this was latent; once a
+  file holds two chunked datasets of different rank (e.g. a 1-D compressed
+  string array and the 2-D embeddings matrix), the first dataset's index was
+  reused for the second, panicking with an out-of-bounds chunk coordinate. The
+  cache now rebinds (dropping its index, chunk-index map, layout, and
+  decompressed slots) whenever the dataset being read changes, while still
+  caching repeated/sequential access to the same dataset.
 - `clawhdf5-format`: read **paged Fixed Array** chunk indexes. A filtered,
   fixed-dimension dataset with more than one data-block page (>1024 chunks by
   default) previously failed with "paged Fixed Array data blocks not yet

crates/clawhdf5-agent/src/schema.rs

@@ -65,7 +65,7 @@ fn build_memory_group(
     let mut group = builder.create_group("memory");
 
     // chunks: fixed-length string array
-    write_string_dataset(&mut group, "chunks", &cache.chunks, false);
+    write_string_dataset(&mut group, "chunks", &cache.chunks);
 
     // embeddings: f32 [N x D]
     let n = cache.embeddings.len() as u64;
@@ -101,7 +101,7 @@ fn build_memory_group(
     }
 
     // source_channel: fixed-length string array
-    write_string_dataset(&mut group, "source_channel", &cache.source_channels, false);
+    write_string_dataset(&mut group, "source_channel", &cache.source_channels);
 
     // timestamps: f64 array
     group
@@ -109,11 +109,11 @@ fn build_memory_group(
         .with_f64_data(&cache.timestamps)
         .fill_time(FillTime::Never);
 
-    // session_ids: fixed-length string array (no compression — chunked compound not yet supported)
+    // session_ids: fixed-length string array (auto-compressed when large)
-    write_string_dataset(&mut group, "session_ids", &cache.session_ids, false);
+    write_string_dataset(&mut group, "session_ids", &cache.session_ids);
 
-    // tags: fixed-length string array (no compression — chunked compound not yet supported)
+    // tags: fixed-length string array (auto-compressed when large)
-    write_string_dataset(&mut group, "tags", &cache.tags, false);
+    write_string_dataset(&mut group, "tags", &cache.tags);
 
     // tombstones: u8 array — use compact if small
     {
@@ -150,7 +150,7 @@ fn build_sessions_group(
     let mut group = builder.create_group("sessions");
 
     let ids: Vec<String> = sessions.entries.iter().map(|e| e.id.clone()).collect();
-    write_string_dataset(&mut group, "ids", &ids, false);
+    write_string_dataset(&mut group, "ids", &ids);
 
     let start_idxs: Vec<i64> = sessions
         .entries
@@ -165,14 +165,14 @@ fn build_sessions_group(
     group.create_dataset("end_idxs").with_i64_data(&end_idxs);
 
     let channels: Vec<String> = sessions.entries.iter().map(|e| e.channel.clone()).collect();
-    write_string_dataset(&mut group, "channels", &channels, false);
+    write_string_dataset(&mut group, "channels", &channels);
 
     let timestamps: Vec<f64> = sessions.entries.iter().map(|e| e.ts).collect();
     group
         .create_dataset("timestamps")
         .with_f64_data(&timestamps);
 
-    write_string_dataset(&mut group, "summaries", &sessions.summaries, false);
+    write_string_dataset(&mut group, "summaries", &sessions.summaries);
 
     let finished = group.finish();
     builder.add_group(finished);
@@ -192,14 +192,14 @@ fn build_knowledge_group(
         .with_i64_data(&entity_ids);
 
     let entity_names: Vec<String> = knowledge.entities.iter().map(|e| e.name.clone()).collect();
-    write_string_dataset(&mut group, "entity_names", &entity_names, false);
+    write_string_dataset(&mut group, "entity_names", &entity_names);
 
     let entity_types: Vec<String> = knowledge
         .entities
         .iter()
         .map(|e| e.entity_type.clone())
         .collect();
-    write_string_dataset(&mut group, "entity_types", &entity_types, false);
+    write_string_dataset(&mut group, "entity_types", &entity_types);
 
     let emb_idxs: Vec<i64> = knowledge.entities.iter().map(|e| e.embedding_idx).collect();
     group
@@ -222,7 +222,7 @@ fn build_knowledge_group(
         .iter()
         .map(|r| r.relation.clone())
         .collect();
-    write_string_dataset(&mut group, "relation_types", &rel_types, false);
+    write_string_dataset(&mut group, "relation_types", &rel_types);
 
     let rel_weights: Vec<f32> = knowledge.relations.iter().map(|r| r.weight).collect();
     group
@@ -234,7 +234,7 @@ fn build_knowledge_group(
 
     // Aliases
     if !knowledge.alias_strings.is_empty() {
-        write_string_dataset(&mut group, "alias_strings", &knowledge.alias_strings, false);
+        write_string_dataset(&mut group, "alias_strings", &knowledge.alias_strings);
         group
             .create_dataset("alias_entity_ids")
             .with_i64_data(&knowledge.alias_entity_ids);
@@ -252,11 +252,15 @@ fn build_knowledge_group(
 ///
 /// When `compress` is true, uses chunked storage with deflate(6) —
 /// NullPad strings have high redundancy and compress very well.
+/// Payload size (bytes) at or above which a fixed-length string dataset is
+/// stored chunked + deflate-compressed. Below this, the chunk B-tree/heap
+/// overhead outweighs the savings, so the data is left contiguous.
+const STRING_COMPRESS_THRESHOLD: usize = 4096;
+
 fn write_string_dataset(
     group: &mut clawhdf5_format::type_builders::GroupBuilder,
     name: &str,
     strings: &[String],
-    compress: bool,
 ) {
     if strings.is_empty() {
         // Empty dataset: use 1-byte string type with no data
@@ -278,6 +282,7 @@ fn write_string_dataset(
         bytes.resize(max_len, 0);
         raw.extend_from_slice(&bytes);
     }
+    let raw_len = raw.len();
 
     let dtype = Datatype::String {
         size: max_len as u32,
@@ -288,9 +293,12 @@ fn write_string_dataset(
         .create_dataset(name)
         .with_compound_data(dtype, raw, strings.len() as u64);
 
-    // Deflate compression for string datasets — NullPad has high redundancy
+    // Fixed-length NullPad strings have high redundancy (padding + repeated
-    if compress && strings.len() > 1 {
+    // content), so deflate pays off once the payload is large enough to absorb
-        // Chunk size: target ~64KB chunks for string data
+    // the chunking overhead. Fixed-length string datasets are chunkable like
+    // any other fixed-size datatype.
+    if strings.len() > 1 && raw_len >= STRING_COMPRESS_THRESHOLD {
+        // Target ~64KB chunks for string data.
         let elem_size = max_len as u64;
         let target_chunk = 64 * 1024;
         let rows_per_chunk = (target_chunk / elem_size).max(1).min(strings.len() as u64);

crates/clawhdf5-format/src/chunk_cache.rs

@@ -256,6 +256,14 @@ struct CacheInner {
     /// Populated once per dataset on first access.
     index: Option<HashMap<ChunkCoord, ChunkInfo>>,
 
+    /// Address of the dataset (its chunk-index base address) that the cached
+    /// index, chunk index, layout, and decompressed slots currently belong to.
+    /// The cache is shared per file across datasets, so every cached-read entry
+    /// checks this and resets the per-dataset state when the dataset changes —
+    /// otherwise one dataset's chunk index (with its own rank) would be reused
+    /// for another, corrupting reads.
+    index_addr: Option<u64>,
+
     /// LRU cache of decompressed chunk data.
     slots: Vec<CachedChunk>,
 
@@ -334,6 +342,7 @@ impl ChunkCache {
         Self {
             inner: std::sync::Mutex::new(CacheInner {
                 index: None,
+                index_addr: None,
                 slots: Vec::with_capacity(max_slots.min(64)),
                 current_bytes: 0,
                 max_bytes,
@@ -349,6 +358,29 @@ impl ChunkCache {
 
     // ----- Index operations -----
 
+    /// Bind the cache to the dataset at chunk-index address `addr`.
+    ///
+    /// The cache is shared per file across all of its datasets. If the cache
+    /// currently holds state for a different dataset, all per-dataset state
+    /// (chunk index, chunk-index map, layout, and decompressed slots) is
+    /// dropped so the next access rebuilds it for this dataset. Reading the
+    /// same dataset again is a no-op, preserving the cache's benefit for
+    /// repeated/sequential access. Returns `true` if a reset occurred.
+    pub fn ensure_dataset(&self, addr: u64) -> bool {
+        let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
+        if inner.index_addr == Some(addr) {
+            return false;
+        }
+        inner.index = None;
+        inner.chunk_index = None;
+        inner.chunk_layout = None;
+        inner.slots.clear();
+        inner.current_bytes = 0;
+        inner.last_coord = None;
+        inner.index_addr = Some(addr);
+        true
+    }
+
     /// Returns `true` if the chunk index has been built.
     pub fn has_index(&self) -> bool {
         self.inner
@@ -565,6 +597,7 @@ impl ChunkCache {
     pub fn clear(&self) {
         let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
         inner.index = None;
+        inner.index_addr = None;
         inner.slots.clear();
         inner.current_bytes = 0;
         inner.tick = 0;

crates/clawhdf5-format/src/chunked_read.rs

@@ -593,6 +593,10 @@ pub fn read_chunked_data_cached(
         )));
     }
 
+    // The per-file cache is shared across datasets; bind it to this one so a
+    // different dataset's chunk index is never reused for this read.
+    cache.ensure_dataset(addr);
+
     // Populate chunk index on first access
     if !cache.has_index() {
         let chunks = match (version, chunk_index_type) {
@@ -946,6 +950,10 @@ pub fn read_chunked_data_sweep(
         )));
     }
 
+    // The per-file cache is shared across datasets; bind it to this one so a
+    // different dataset's chunk index is never reused for this read.
+    cache.ensure_dataset(addr);
+
     // Populate chunk index on first access
     if !cache.has_index() {
         let chunks = match (version, chunk_index_type) {
@@ -1169,6 +1177,10 @@ pub fn read_chunked_data_indexed(
         )));
     }
 
+    // The per-file cache is shared across datasets; bind it to this one so a
+    // different dataset's chunk index is never reused for this read.
+    cache.ensure_dataset(addr);
+
     // Build chunk index on first access
     if !cache.has_chunk_index() {
         let chunks = match (version, chunk_index_type) {

crates/clawhdf5/tests/integration_tests.rs

@@ -706,6 +706,58 @@ fn fletcher32_roundtrip() {
 // 15. Multiple groups with same-named datasets
 // ---------------------------------------------------------------------------
 
+#[test]
+fn multiple_chunked_datasets_share_file_cache() {
+    // The per-file ChunkCache is shared across datasets. Two chunked datasets
+    // of *different rank* must each read correctly: a 1-D dataset's chunk index
+    // (rank 1) must not be reused for a 2-D dataset (rank 2). Read the 1-D one
+    // first so it seeds the shared cache, then the 2-D one.
+    use clawhdf5_format::datatype::{CharacterSet, Datatype, StringPadding};
+
+    // 1-D chunked + compressed fixed-length strings (payload > compress threshold).
+    let strings: Vec<String> = (0..64).map(|i| format!("entry-{i:06}-{}", "x".repeat(80))).collect();
+    let max_len = strings.iter().map(|s| s.len()).max().unwrap();
+    let mut sraw = Vec::new();
+    for s in &strings {
+        let mut b = s.as_bytes().to_vec();
+        b.resize(max_len, 0);
+        sraw.extend_from_slice(&b);
+    }
+    let sdt = Datatype::String {
+        size: max_len as u32,
+        padding: StringPadding::NullPad,
+        charset: CharacterSet::Utf8,
+    };
+
+    // 2-D chunked + compressed f32 matrix.
+    let (n, d) = (40usize, 8usize);
+    let mat: Vec<f32> = (0..n * d).map(|i| i as f32).collect();
+
+    let mut b = FileBuilder::new();
+    {
+        let ds = b.create_dataset("strs");
+        ds.with_compound_data(sdt, sraw, strings.len() as u64);
+        ds.with_chunks(&[16]);
+        ds.with_deflate(6);
+    }
+    {
+        let ds = b.create_dataset("mat");
+        ds.with_f32_data(&mat).with_shape(&[n as u64, d as u64]);
+        ds.with_chunks(&[10, d as u64]).with_shuffle().with_deflate(6);
+    }
+    let bytes = b.finish().unwrap();
+    let file = File::from_bytes(bytes).unwrap();
+
+    // Read the 1-D dataset first (seeds the shared cache with a rank-1 index),
+    // then the 2-D dataset through the same File/cache.
+    let got_strs = file.dataset("strs").unwrap().read_string().unwrap();
+    assert_eq!(got_strs, strings);
+    let got_mat = file.dataset("mat").unwrap().read_f32().unwrap();
+    assert_eq!(got_mat, mat);
+    // Read the 1-D one again to confirm the cache rebinds back correctly.
+    assert_eq!(file.dataset("strs").unwrap().read_string().unwrap(), strings);
+}
+
 #[test]
 fn virtual_dataset_external_file_auto_resolved() {
     // The facade resolves external Virtual Dataset sources relative to the