fix(format): key the shared chunk cache by dataset
A File is Send + Sync and keeps one ChunkCache for all its datasets. The cached readers bound that cache to "the current dataset" with ensure_dataset(addr), then checked, built and read its index and its decompressed chunks in separate lock acquisitions. Two threads reading two chunked datasets interleaved those steps, so one could store its chunk index under the other's binding, or get the other's decompressed chunk for the same coordinate: wrong data, or an index-out-of-bounds panic when the ranks differed (16 threads x 40 reads over 24 datasets panicked on every run). The cache now keeps per-dataset state keyed by chunk-index address: the chunk index, ChunkIndex and ChunkLayout per dataset (held as Arcs, built outside the lock, first writer wins), and decompressed chunks keyed by (address, coordinate). The chunked readers use the new addr-taking methods (chunks_for, chunk_layout_for, get/put_decompressed_in, prefetch_hint_in) exclusively. Memory stays bounded: decompressed data by the existing byte/slot budget across datasets, indexes by at most 64 datasets and 2^20 index entries in total, dropping the least recently used dataset's index first. Switching datasets no longer throws away the other datasets' cached chunks. The address-less methods remain and act on the dataset last bound with ensure_dataset; they are documented as not for concurrent readers. Regression: threads_reading_different_datasets_get_their_own_chunks (crates/clawhdf5/tests/concurrent_chunk_cache.rs), plus cache unit tests datasets_sharing_coordinates_stay_separate, dataset_indexes_are_bounded and concurrent_readers_of_different_datasets_see_their_own_chunks. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -223,13 +223,32 @@ pub const DEFAULT_CACHE_BYTES: usize = 16 * 1024 * 1024; // 16 MiB
|
||||
/// coordinate map and reduces collision chains compared to power-of-two sizes.
|
||||
pub const DEFAULT_MAX_SLOTS: usize = 521;
|
||||
|
||||
/// Most datasets whose chunk index a [`ChunkCache`] keeps at once.
|
||||
pub const MAX_INDEXED_DATASETS: usize = 64;
|
||||
|
||||
/// Most chunk-index entries, summed over all datasets, a [`ChunkCache`] keeps.
|
||||
/// Least-recently-used datasets' indexes are dropped past this (the dataset
|
||||
/// being read is always kept), so a file with many or huge chunked datasets
|
||||
/// cannot grow the cache without bound.
|
||||
pub const MAX_INDEXED_CHUNKS: usize = 1 << 20;
|
||||
|
||||
/// The dataset key the address-less (legacy) methods use when
|
||||
/// [`ChunkCache::ensure_dataset`] has not been called.
|
||||
#[cfg(feature = "std")]
|
||||
const UNBOUND_DATASET: u64 = u64::MAX;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LRU entry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Decompressed chunks are keyed by dataset *and* coordinate: every chunked
|
||||
/// dataset has a chunk at (0, 0, ...), so the coordinate alone is ambiguous.
|
||||
#[cfg(feature = "std")]
|
||||
type SlotKey = (u64, ChunkCoord);
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
struct CachedChunk {
|
||||
coord: ChunkCoord,
|
||||
key: SlotKey,
|
||||
/// Shared so a cache hit is a refcount bump, not a copy of the whole
|
||||
/// (potentially large) decompressed chunk.
|
||||
data: Arc<CacheAlignedBuffer>,
|
||||
@@ -237,21 +256,48 @@ struct CachedChunk {
|
||||
last_access: u64,
|
||||
}
|
||||
|
||||
/// Per-dataset index state.
|
||||
#[cfg(feature = "std")]
|
||||
#[derive(Default)]
|
||||
struct DatasetEntry {
|
||||
/// Chunk coordinate -> ChunkInfo (offset + size in file).
|
||||
index: Option<Arc<HashMap<ChunkCoord, ChunkInfo>>>,
|
||||
/// Pre-built chunk index for O(1) coordinate lookups.
|
||||
chunk_index: Option<Arc<ChunkIndex>>,
|
||||
/// Pre-computed chunk layout for fast assembly.
|
||||
chunk_layout: Option<Arc<ChunkLayout>>,
|
||||
/// Tick of the last use, for dropping the least recently used dataset.
|
||||
last_used: u64,
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
impl DatasetEntry {
|
||||
fn weight(&self) -> usize {
|
||||
self.index.as_ref().map_or(0, |m| m.len())
|
||||
+ self.chunk_index.as_ref().map_or(0, |c| c.num_chunks())
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ChunkCache
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A per-dataset chunk cache with hash-based index and LRU eviction.
|
||||
/// A per-file chunk cache: chunk indexes per dataset, plus an LRU of
|
||||
/// decompressed chunks, all keyed by dataset.
|
||||
///
|
||||
/// # Usage
|
||||
/// A dataset is identified by the address of its chunk index (B-tree, fixed
|
||||
/// or extensible array, ...), which is unique within a file. Every method
|
||||
/// that takes an `addr` works on that dataset only, so threads reading
|
||||
/// different datasets through one shared cache never see each other's
|
||||
/// chunks. The address-less methods (`has_index`, `populate_index`,
|
||||
/// `get_decompressed`, ...) act on the dataset last bound with
|
||||
/// [`Self::ensure_dataset`]; that binding is shared state, so concurrent
|
||||
/// readers must use the `*_in` / `*_for` methods instead (the chunked
|
||||
/// readers in [`crate::chunked_read`] do).
|
||||
///
|
||||
/// ```ignore
|
||||
/// let cache = ChunkCache::new();
|
||||
/// // Pass &cache to read_chunked_data — it will populate the index lazily.
|
||||
/// ```
|
||||
///
|
||||
/// The cache is wrapped in `Mutex` internally so it can be mutated through
|
||||
/// shared references (thread-safe).
|
||||
/// Memory is bounded: decompressed data by `max_bytes`/`max_slots` across
|
||||
/// all datasets, indexes by [`MAX_INDEXED_DATASETS`] and
|
||||
/// [`MAX_INDEXED_CHUNKS`].
|
||||
///
|
||||
/// Only available with the `std` feature because it requires `std::sync::Mutex`.
|
||||
#[cfg(feature = "std")]
|
||||
@@ -261,26 +307,20 @@ pub struct ChunkCache {
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
struct CacheInner {
|
||||
/// Hash index: chunk coordinate -> ChunkInfo (offset + size in file).
|
||||
/// Populated once per dataset on first access.
|
||||
index: Option<HashMap<ChunkCoord, ChunkInfo>>,
|
||||
/// Per-dataset chunk indexes, keyed by chunk-index address.
|
||||
datasets: HashMap<u64, DatasetEntry>,
|
||||
|
||||
/// Address of the dataset (its chunk-index base address) that the cached
|
||||
/// index, chunk index, layout, and decompressed slots currently belong to.
|
||||
/// The cache is shared per file across datasets, so every cached-read entry
|
||||
/// checks this and resets the per-dataset state when the dataset changes —
|
||||
/// otherwise one dataset's chunk index (with its own rank) would be reused
|
||||
/// for another, corrupting reads.
|
||||
index_addr: Option<u64>,
|
||||
/// Dataset the address-less methods act on (see `ensure_dataset`).
|
||||
current: Option<u64>,
|
||||
|
||||
/// LRU cache of decompressed chunk data.
|
||||
slots: Vec<CachedChunk>,
|
||||
|
||||
/// Coordinate -> index into `slots`, for O(1) lookup instead of a linear
|
||||
/// Key -> index into `slots`, for O(1) lookup instead of a linear
|
||||
/// scan. Kept in sync with `slots` on every insert/evict/clear — in
|
||||
/// particular, `slots.swap_remove(i)` moves the last element into slot
|
||||
/// `i`, so the moved element's index entry must be updated too.
|
||||
slot_index: HashMap<ChunkCoord, usize>,
|
||||
slot_index: HashMap<SlotKey, usize>,
|
||||
|
||||
/// Current total bytes of cached decompressed data.
|
||||
current_bytes: usize,
|
||||
@@ -294,17 +334,145 @@ struct CacheInner {
|
||||
/// Monotonic counter for LRU ordering.
|
||||
tick: u64,
|
||||
|
||||
/// Last accessed chunk coordinate (for sequential detection).
|
||||
last_coord: Option<ChunkCoord>,
|
||||
/// Last accessed chunk (for sequential detection).
|
||||
last_coord: Option<SlotKey>,
|
||||
|
||||
/// Access pattern statistics.
|
||||
stats: AccessStats,
|
||||
}
|
||||
|
||||
/// Pre-built chunk index for O(1) coordinate lookups.
|
||||
chunk_index: Option<ChunkIndex>,
|
||||
#[cfg(feature = "std")]
|
||||
impl CacheInner {
|
||||
fn current(&self) -> u64 {
|
||||
self.current.unwrap_or(UNBOUND_DATASET)
|
||||
}
|
||||
|
||||
/// Pre-computed chunk layout for fast assembly.
|
||||
chunk_layout: Option<ChunkLayout>,
|
||||
fn touch(&mut self, addr: u64) -> &mut DatasetEntry {
|
||||
self.tick += 1;
|
||||
let tick = self.tick;
|
||||
let entry = self.datasets.entry(addr).or_default();
|
||||
entry.last_used = tick;
|
||||
entry
|
||||
}
|
||||
|
||||
fn entry(&self, addr: u64) -> Option<&DatasetEntry> {
|
||||
self.datasets.get(&addr)
|
||||
}
|
||||
|
||||
/// Drop least-recently-used datasets' indexes (never `keep`'s) until the
|
||||
/// dataset and chunk-entry budgets hold.
|
||||
fn trim_datasets(&mut self, keep: u64) {
|
||||
loop {
|
||||
let total: usize = self.datasets.values().map(DatasetEntry::weight).sum();
|
||||
if self.datasets.len() <= MAX_INDEXED_DATASETS && total <= MAX_INDEXED_CHUNKS {
|
||||
return;
|
||||
}
|
||||
let victim = self
|
||||
.datasets
|
||||
.iter()
|
||||
.filter(|(a, _)| **a != keep)
|
||||
.min_by_key(|(_, e)| e.last_used)
|
||||
.map(|(a, _)| *a);
|
||||
match victim {
|
||||
Some(a) => {
|
||||
self.datasets.remove(&a);
|
||||
}
|
||||
None => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_decompressed(&mut self, addr: u64, coord: &[u64]) -> Option<Arc<CacheAlignedBuffer>> {
|
||||
self.tick += 1;
|
||||
let tick = self.tick;
|
||||
|
||||
// Track sequential vs random access
|
||||
let is_sequential = self.last_coord.as_ref().is_some_and(|(prev_addr, prev)| {
|
||||
// Sequential if exactly one dimension changed
|
||||
let changes: usize = prev
|
||||
.iter()
|
||||
.zip(coord.iter())
|
||||
.filter(|(a, b)| a != b)
|
||||
.count();
|
||||
*prev_addr == addr && changes <= 1
|
||||
});
|
||||
if is_sequential {
|
||||
self.stats.sequential_count += 1;
|
||||
} else if self.last_coord.is_some() {
|
||||
self.stats.random_count += 1;
|
||||
}
|
||||
let key: SlotKey = (addr, coord.to_vec());
|
||||
let found = if let Some(&idx) = self.slot_index.get(&key) {
|
||||
self.slots[idx].last_access = tick;
|
||||
Some(Arc::clone(&self.slots[idx].data))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
self.last_coord = Some(key);
|
||||
if let Some(ref data) = found {
|
||||
self.stats.hits += 1;
|
||||
self.stats.bytes_read += data.len() as u64;
|
||||
} else {
|
||||
self.stats.misses += 1;
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
fn put_decompressed(
|
||||
&mut self,
|
||||
key: SlotKey,
|
||||
data: Arc<CacheAlignedBuffer>,
|
||||
) -> Arc<CacheAlignedBuffer> {
|
||||
let data_len = data.len();
|
||||
|
||||
// Don't cache if single chunk exceeds budget — still return the data
|
||||
// to the caller, just don't retain it.
|
||||
if data_len > self.max_bytes {
|
||||
return data;
|
||||
}
|
||||
|
||||
// Check if already present
|
||||
self.tick += 1;
|
||||
let tick = self.tick;
|
||||
if let Some(&idx) = self.slot_index.get(&key) {
|
||||
self.slots[idx].last_access = tick;
|
||||
return Arc::clone(&self.slots[idx].data); // already cached
|
||||
}
|
||||
|
||||
// Evict until we have room
|
||||
while self.slots.len() >= self.max_slots
|
||||
|| (self.current_bytes + data_len > self.max_bytes && !self.slots.is_empty())
|
||||
{
|
||||
// Find LRU slot
|
||||
let lru_idx = self
|
||||
.slots
|
||||
.iter()
|
||||
.enumerate()
|
||||
.min_by_key(|(_, s)| s.last_access)
|
||||
.map(|(i, _)| i)
|
||||
.unwrap();
|
||||
let removed = self.slots.swap_remove(lru_idx);
|
||||
self.slot_index.remove(&removed.key);
|
||||
// swap_remove moved the former last element into `lru_idx` (unless
|
||||
// it *was* the last element) — fix up that element's index entry.
|
||||
if lru_idx < self.slots.len() {
|
||||
let moved_key = self.slots[lru_idx].key.clone();
|
||||
self.slot_index.insert(moved_key, lru_idx);
|
||||
}
|
||||
self.current_bytes -= removed.data.len();
|
||||
self.stats.evictions += 1;
|
||||
}
|
||||
|
||||
self.current_bytes += data_len;
|
||||
let new_idx = self.slots.len();
|
||||
self.slot_index.insert(key.clone(), new_idx);
|
||||
self.slots.push(CachedChunk {
|
||||
key,
|
||||
data: Arc::clone(&data),
|
||||
last_access: tick,
|
||||
});
|
||||
data
|
||||
}
|
||||
}
|
||||
|
||||
/// Access pattern statistics tracked by the chunk cache.
|
||||
@@ -356,8 +524,8 @@ impl ChunkCache {
|
||||
pub fn with_capacity(max_bytes: usize, max_slots: usize) -> Self {
|
||||
Self {
|
||||
inner: std::sync::Mutex::new(CacheInner {
|
||||
index: None,
|
||||
index_addr: None,
|
||||
datasets: HashMap::new(),
|
||||
current: None,
|
||||
slots: Vec::with_capacity(max_slots.min(64)),
|
||||
slot_index: HashMap::with_capacity(max_slots.min(64)),
|
||||
current_bytes: 0,
|
||||
@@ -366,340 +534,331 @@ impl ChunkCache {
|
||||
tick: 0,
|
||||
last_coord: None,
|
||||
stats: AccessStats::default(),
|
||||
chunk_index: None,
|
||||
chunk_layout: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// ----- Index operations -----
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, CacheInner> {
|
||||
self.inner.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// The most decompressed bytes this cache will hold.
|
||||
pub fn max_bytes(&self) -> usize {
|
||||
self.inner.lock().map(|g| g.max_bytes).unwrap_or(0)
|
||||
self.lock().max_bytes
|
||||
}
|
||||
|
||||
/// Bind the cache to the dataset at chunk-index address `addr`.
|
||||
// ----- Dataset-keyed operations (safe to use concurrently) -----
|
||||
|
||||
/// The chunk list of the dataset whose chunk index is at `addr`.
|
||||
///
|
||||
/// The cache is shared per file across all of its datasets. If the cache
|
||||
/// currently holds state for a different dataset, all per-dataset state
|
||||
/// (chunk index, chunk-index map, layout, and decompressed slots) is
|
||||
/// dropped so the next access rebuilds it for this dataset. Reading the
|
||||
/// same dataset again is a no-op, preserving the cache's benefit for
|
||||
/// repeated/sequential access. Returns `true` if a reset occurred.
|
||||
/// On the first call for a dataset, `build` scans its chunk index; the
|
||||
/// result is kept (offsets truncated to `rank` for the lookup key), so
|
||||
/// later calls skip the scan. `build` runs without the cache lock held;
|
||||
/// if two threads race to build the same dataset's index, the first
|
||||
/// stored one wins and both return equivalent lists.
|
||||
pub fn chunks_for<E>(
|
||||
&self,
|
||||
addr: u64,
|
||||
rank: usize,
|
||||
build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
|
||||
) -> Result<Vec<ChunkInfo>, E> {
|
||||
Ok(self
|
||||
.index_for(addr, rank, build)?
|
||||
.values()
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn index_for<E>(
|
||||
&self,
|
||||
addr: u64,
|
||||
rank: usize,
|
||||
build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
|
||||
) -> Result<Arc<HashMap<ChunkCoord, ChunkInfo>>, E> {
|
||||
if let Some(index) = self.lock().touch(addr).index.clone() {
|
||||
return Ok(index);
|
||||
}
|
||||
let chunks = build()?;
|
||||
let map: HashMap<ChunkCoord, ChunkInfo> = chunks
|
||||
.into_iter()
|
||||
.map(|ci| (ci.offsets.iter().take(rank).copied().collect(), ci))
|
||||
.collect();
|
||||
let mut inner = self.lock();
|
||||
let entry = inner.touch(addr);
|
||||
let index = Arc::clone(entry.index.get_or_insert_with(|| Arc::new(map)));
|
||||
inner.trim_datasets(addr);
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
/// The pre-computed assembly layout of the dataset at `addr`, building
|
||||
/// its chunk index (via `build`, as in [`Self::chunks_for`]) and layout on
|
||||
/// first use.
|
||||
pub fn chunk_layout_for<E>(
|
||||
&self,
|
||||
addr: u64,
|
||||
rank: usize,
|
||||
build: impl FnOnce() -> Result<Vec<ChunkInfo>, E>,
|
||||
ds_dims: &[usize],
|
||||
chunk_dims: &[usize],
|
||||
elem_size: usize,
|
||||
) -> Result<Arc<ChunkLayout>, E> {
|
||||
let (layout, chunk_index) = {
|
||||
let mut inner = self.lock();
|
||||
let entry = inner.touch(addr);
|
||||
(entry.chunk_layout.clone(), entry.chunk_index.clone())
|
||||
};
|
||||
if let Some(layout) = layout {
|
||||
return Ok(layout);
|
||||
}
|
||||
let chunk_index = match chunk_index {
|
||||
Some(ci) => ci,
|
||||
None => {
|
||||
let index = self.index_for(addr, rank, build)?;
|
||||
let chunks: Vec<ChunkInfo> = index.values().cloned().collect();
|
||||
Arc::new(ChunkIndex::build(&chunks, rank))
|
||||
}
|
||||
};
|
||||
let layout = ChunkLayout::build(&chunk_index, ds_dims, chunk_dims, elem_size);
|
||||
let mut inner = self.lock();
|
||||
let entry = inner.touch(addr);
|
||||
entry.chunk_index.get_or_insert(chunk_index);
|
||||
let layout = Arc::clone(entry.chunk_layout.get_or_insert_with(|| Arc::new(layout)));
|
||||
inner.trim_datasets(addr);
|
||||
Ok(layout)
|
||||
}
|
||||
|
||||
/// Cached decompressed chunk at `coord` of the dataset at `addr`.
|
||||
///
|
||||
/// O(1) lookup; the clone is an `Arc` refcount bump, not a copy of the
|
||||
/// underlying decompressed data.
|
||||
pub fn get_decompressed_in(&self, addr: u64, coord: &[u64]) -> Option<Arc<CacheAlignedBuffer>> {
|
||||
self.lock().get_decompressed(addr, coord)
|
||||
}
|
||||
|
||||
/// Cache decompressed chunk data for `coord` of the dataset at `addr`.
|
||||
/// Returns the `Arc`-shared buffer now cached (or already cached).
|
||||
pub fn put_decompressed_in(
|
||||
&self,
|
||||
addr: u64,
|
||||
coord: ChunkCoord,
|
||||
data: Vec<u8>,
|
||||
) -> Arc<CacheAlignedBuffer> {
|
||||
self.put_decompressed_aligned_in(addr, coord, CacheAlignedBuffer::from_vec(data))
|
||||
}
|
||||
|
||||
/// [`Self::put_decompressed_in`] for an already-aligned buffer.
|
||||
pub fn put_decompressed_aligned_in(
|
||||
&self,
|
||||
addr: u64,
|
||||
coord: ChunkCoord,
|
||||
data: CacheAlignedBuffer,
|
||||
) -> Arc<CacheAlignedBuffer> {
|
||||
let data = Arc::new(data);
|
||||
self.lock().put_decompressed((addr, coord), data)
|
||||
}
|
||||
|
||||
/// Record that the given chunk coordinates of the dataset at `addr` are
|
||||
/// predicted to be accessed soon (bookkeeping only).
|
||||
///
|
||||
/// This does **not** prefetch or pre-decompress anything — it only
|
||||
/// checks whether each coordinate is already in the chunk index and
|
||||
/// updates access-pattern stats accordingly.
|
||||
pub fn prefetch_hint_in(&self, addr: u64, next_coords: &[ChunkCoord]) {
|
||||
let mut inner = self.lock();
|
||||
let Some(index) = inner.entry(addr).and_then(|e| e.index.clone()) else {
|
||||
return;
|
||||
};
|
||||
let known = next_coords
|
||||
.iter()
|
||||
.filter(|c| index.contains_key(*c))
|
||||
.count();
|
||||
inner.stats.sequential_count += known as u64;
|
||||
}
|
||||
|
||||
// ----- Address-less operations on the bound dataset -----
|
||||
|
||||
/// Bind the address-less methods to the dataset at chunk-index address
|
||||
/// `addr`. Returns `true` if this changed the bound dataset.
|
||||
///
|
||||
/// Each dataset's state is kept separately, so switching loses nothing
|
||||
/// and never exposes one dataset's index or chunks to another. The
|
||||
/// binding itself is shared, though: concurrent readers should use the
|
||||
/// `addr`-taking methods rather than bind and then call these.
|
||||
pub fn ensure_dataset(&self, addr: u64) -> bool {
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if inner.index_addr == Some(addr) {
|
||||
return false;
|
||||
}
|
||||
inner.index = None;
|
||||
inner.chunk_index = None;
|
||||
inner.chunk_layout = None;
|
||||
inner.slots.clear();
|
||||
inner.slot_index.clear();
|
||||
inner.current_bytes = 0;
|
||||
inner.last_coord = None;
|
||||
inner.index_addr = Some(addr);
|
||||
true
|
||||
let mut inner = self.lock();
|
||||
let changed = inner.current != Some(addr);
|
||||
inner.current = Some(addr);
|
||||
changed
|
||||
}
|
||||
|
||||
/// Returns `true` if the chunk index has been built.
|
||||
/// Returns `true` if the bound dataset's chunk index has been built.
|
||||
pub fn has_index(&self) -> bool {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.index
|
||||
.is_some()
|
||||
let inner = self.lock();
|
||||
inner
|
||||
.entry(inner.current())
|
||||
.is_some_and(|e| e.index.is_some())
|
||||
}
|
||||
|
||||
/// Build the chunk index from a pre-collected list of `ChunkInfo`.
|
||||
/// Build the bound dataset's chunk index from a pre-collected list of
|
||||
/// `ChunkInfo`.
|
||||
///
|
||||
/// The `rank` parameter is used to truncate offsets to spatial dims only
|
||||
/// (B-tree v1 stores rank+1 offsets).
|
||||
pub fn populate_index(&self, chunks: &[ChunkInfo], rank: usize) {
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if inner.index.is_some() {
|
||||
return; // already populated
|
||||
}
|
||||
let mut map = HashMap::with_capacity(chunks.len());
|
||||
|
||||
for ci in chunks {
|
||||
let coord: ChunkCoord = ci.offsets.iter().take(rank).copied().collect();
|
||||
map.insert(coord, ci.clone());
|
||||
}
|
||||
inner.index = Some(map);
|
||||
let addr = self.lock().current();
|
||||
let _ = self.index_for::<core::convert::Infallible>(addr, rank, || Ok(chunks.to_vec()));
|
||||
}
|
||||
|
||||
/// Look up a chunk by its spatial coordinate in the index.
|
||||
/// Look up a chunk by its spatial coordinate in the bound dataset's index.
|
||||
pub fn lookup_index(&self, coord: &[u64]) -> Option<ChunkInfo> {
|
||||
let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
inner.index.as_ref()?.get(coord).cloned()
|
||||
let inner = self.lock();
|
||||
inner
|
||||
.entry(inner.current())?
|
||||
.index
|
||||
.as_ref()?
|
||||
.get(coord)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
/// Return all indexed chunks as a `Vec<ChunkInfo>` (order unspecified).
|
||||
/// Return all of the bound dataset's indexed chunks (order unspecified).
|
||||
pub fn all_indexed_chunks(&self) -> Option<Vec<ChunkInfo>> {
|
||||
let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
inner.index.as_ref().map(|m| m.values().cloned().collect())
|
||||
let inner = self.lock();
|
||||
let index = inner.entry(inner.current())?.index.as_ref()?;
|
||||
Some(index.values().cloned().collect())
|
||||
}
|
||||
|
||||
// ----- Chunk index (pre-built coordinate → ChunkInfo map) -----
|
||||
|
||||
/// Returns `true` if the chunk B-tree index has been built.
|
||||
/// Returns `true` if the bound dataset's `ChunkIndex` has been built.
|
||||
pub fn has_chunk_index(&self) -> bool {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.chunk_index
|
||||
.is_some()
|
||||
let inner = self.lock();
|
||||
inner
|
||||
.entry(inner.current())
|
||||
.is_some_and(|e| e.chunk_index.is_some())
|
||||
}
|
||||
|
||||
/// Build and store the chunk B-tree index from a pre-collected list of `ChunkInfo`.
|
||||
/// Build and store the bound dataset's `ChunkIndex`.
|
||||
pub fn populate_chunk_index(&self, chunks: &[ChunkInfo], rank: usize) {
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if inner.chunk_index.is_some() {
|
||||
return;
|
||||
}
|
||||
inner.chunk_index = Some(ChunkIndex::build(chunks, rank));
|
||||
let built = Arc::new(ChunkIndex::build(chunks, rank));
|
||||
let mut inner = self.lock();
|
||||
let addr = inner.current();
|
||||
inner.touch(addr).chunk_index.get_or_insert(built);
|
||||
inner.trim_datasets(addr);
|
||||
}
|
||||
|
||||
// ----- Chunk layout (pre-computed assembly plan) -----
|
||||
|
||||
/// Returns `true` if the chunk layout has been computed.
|
||||
/// Returns `true` if the bound dataset's chunk layout has been computed.
|
||||
pub fn has_chunk_layout(&self) -> bool {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.chunk_layout
|
||||
.is_some()
|
||||
let inner = self.lock();
|
||||
inner
|
||||
.entry(inner.current())
|
||||
.is_some_and(|e| e.chunk_layout.is_some())
|
||||
}
|
||||
|
||||
/// Build and store the pre-computed chunk layout for fast assembly.
|
||||
/// Build and store the bound dataset's chunk layout (needs its
|
||||
/// `ChunkIndex`; does nothing without one).
|
||||
pub fn populate_chunk_layout(&self, ds_dims: &[usize], chunk_dims: &[usize], elem_size: usize) {
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if inner.chunk_layout.is_some() {
|
||||
let mut inner = self.lock();
|
||||
let addr = inner.current();
|
||||
let entry = inner.touch(addr);
|
||||
if entry.chunk_layout.is_some() {
|
||||
return;
|
||||
}
|
||||
if let Some(ref idx) = inner.chunk_index {
|
||||
inner.chunk_layout = Some(ChunkLayout::build(idx, ds_dims, chunk_dims, elem_size));
|
||||
if let Some(idx) = entry.chunk_index.clone() {
|
||||
entry.chunk_layout = Some(Arc::new(ChunkLayout::build(
|
||||
&idx, ds_dims, chunk_dims, elem_size,
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a function with a reference to the chunk layout.
|
||||
///
|
||||
/// Returns `None` if the layout hasn't been computed yet.
|
||||
/// Execute a function with a reference to the bound dataset's chunk
|
||||
/// layout. Returns `None` if the layout hasn't been computed yet.
|
||||
pub fn with_chunk_layout<F, R>(&self, f: F) -> Option<R>
|
||||
where
|
||||
F: FnOnce(&ChunkLayout) -> R,
|
||||
{
|
||||
let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
inner.chunk_layout.as_ref().map(f)
|
||||
let layout = {
|
||||
let inner = self.lock();
|
||||
inner.entry(inner.current())?.chunk_layout.clone()?
|
||||
};
|
||||
Some(f(&layout))
|
||||
}
|
||||
|
||||
// ----- Decompressed data cache (LRU) -----
|
||||
|
||||
/// Try to get cached decompressed data for a chunk coordinate.
|
||||
/// Try to get cached decompressed data for a chunk of the bound dataset.
|
||||
///
|
||||
/// O(1) lookup. Returns an owned copy for API compatibility with callers
|
||||
/// that need a `Vec<u8>`; prefer [`Self::get_decompressed_aligned`] when
|
||||
/// an `Arc`-shared buffer works for the caller, since that avoids the
|
||||
/// copy entirely.
|
||||
/// Returns an owned copy; prefer [`Self::get_decompressed_aligned`] when
|
||||
/// an `Arc`-shared buffer works for the caller.
|
||||
pub fn get_decompressed(&self, coord: &[u64]) -> Option<Vec<u8>> {
|
||||
self.get_decompressed_aligned(coord)
|
||||
.map(|arc| arc.as_slice().to_vec())
|
||||
}
|
||||
|
||||
/// Try to get a reference-counted clone of the aligned buffer for a chunk.
|
||||
///
|
||||
/// O(1) index lookup; the clone is an `Arc` refcount bump, not a copy of
|
||||
/// the underlying decompressed data.
|
||||
/// Reference-counted cached buffer for a chunk of the bound dataset.
|
||||
pub fn get_decompressed_aligned(&self, coord: &[u64]) -> Option<Arc<CacheAlignedBuffer>> {
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
inner.tick += 1;
|
||||
let tick = inner.tick;
|
||||
|
||||
// Track sequential vs random access
|
||||
let is_sequential = inner.last_coord.as_ref().is_some_and(|prev| {
|
||||
// Sequential if exactly one dimension changed
|
||||
let changes: usize = prev
|
||||
.iter()
|
||||
.zip(coord.iter())
|
||||
.filter(|(a, b)| a != b)
|
||||
.count();
|
||||
changes <= 1
|
||||
});
|
||||
if is_sequential {
|
||||
inner.stats.sequential_count += 1;
|
||||
} else if inner.last_coord.is_some() {
|
||||
inner.stats.random_count += 1;
|
||||
}
|
||||
inner.last_coord = Some(coord.to_vec());
|
||||
|
||||
let found = if let Some(&idx) = inner.slot_index.get(coord) {
|
||||
inner.slots[idx].last_access = tick;
|
||||
Some(Arc::clone(&inner.slots[idx].data))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(ref data) = found {
|
||||
inner.stats.hits += 1;
|
||||
inner.stats.bytes_read += data.len() as u64;
|
||||
} else {
|
||||
inner.stats.misses += 1;
|
||||
}
|
||||
found
|
||||
let mut inner = self.lock();
|
||||
let addr = inner.current();
|
||||
inner.get_decompressed(addr, coord)
|
||||
}
|
||||
|
||||
/// Insert decompressed chunk data into the LRU cache.
|
||||
///
|
||||
/// The data is stored in a [`CacheAlignedBuffer`] so subsequent reads
|
||||
/// return cache-line-aligned memory. Returns the `Arc`-shared buffer that
|
||||
/// is now cached (or already was), so the caller can reuse it directly
|
||||
/// instead of holding a separate copy of the same data.
|
||||
/// Insert decompressed chunk data for the bound dataset into the LRU
|
||||
/// cache, returning the `Arc`-shared buffer now cached.
|
||||
pub fn put_decompressed(&self, coord: ChunkCoord, data: Vec<u8>) -> Arc<CacheAlignedBuffer> {
|
||||
let aligned = CacheAlignedBuffer::from_vec(data);
|
||||
self.put_decompressed_aligned(coord, aligned)
|
||||
self.put_decompressed_aligned(coord, CacheAlignedBuffer::from_vec(data))
|
||||
}
|
||||
|
||||
/// Insert an already-aligned buffer into the LRU cache.
|
||||
///
|
||||
/// Returns the `Arc`-shared buffer now held by the cache (the one just
|
||||
/// inserted, or the existing cached copy if `coord` was already present).
|
||||
/// Insert an already-aligned buffer for the bound dataset.
|
||||
pub fn put_decompressed_aligned(
|
||||
&self,
|
||||
coord: ChunkCoord,
|
||||
data: CacheAlignedBuffer,
|
||||
) -> Arc<CacheAlignedBuffer> {
|
||||
let data = Arc::new(data);
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let data_len = data.len();
|
||||
|
||||
// Don't cache if single chunk exceeds budget — still return the data
|
||||
// to the caller, just don't retain it.
|
||||
if data_len > inner.max_bytes {
|
||||
return data;
|
||||
let mut inner = self.lock();
|
||||
let addr = inner.current();
|
||||
inner.put_decompressed((addr, coord), data)
|
||||
}
|
||||
|
||||
// Check if already present
|
||||
inner.tick += 1;
|
||||
let tick = inner.tick;
|
||||
if let Some(&idx) = inner.slot_index.get(&coord) {
|
||||
inner.slots[idx].last_access = tick;
|
||||
return Arc::clone(&inner.slots[idx].data); // already cached
|
||||
/// [`Self::prefetch_hint_in`] for the bound dataset.
|
||||
pub fn prefetch_hint(&self, next_coords: &[ChunkCoord]) {
|
||||
let addr = self.lock().current();
|
||||
self.prefetch_hint_in(addr, next_coords);
|
||||
}
|
||||
|
||||
// Evict until we have room
|
||||
while inner.slots.len() >= inner.max_slots
|
||||
|| (inner.current_bytes + data_len > inner.max_bytes && !inner.slots.is_empty())
|
||||
{
|
||||
// Find LRU slot
|
||||
let lru_idx = inner
|
||||
.slots
|
||||
.iter()
|
||||
.enumerate()
|
||||
.min_by_key(|(_, s)| s.last_access)
|
||||
.map(|(i, _)| i)
|
||||
.unwrap();
|
||||
let removed = inner.slots.swap_remove(lru_idx);
|
||||
inner.slot_index.remove(&removed.coord);
|
||||
// swap_remove moved the former last element into `lru_idx` (unless
|
||||
// it *was* the last element) — fix up that element's index entry.
|
||||
if lru_idx < inner.slots.len() {
|
||||
let moved_coord = inner.slots[lru_idx].coord.clone();
|
||||
inner.slot_index.insert(moved_coord, lru_idx);
|
||||
}
|
||||
inner.current_bytes -= removed.data.len();
|
||||
inner.stats.evictions += 1;
|
||||
}
|
||||
// ----- Whole-cache operations -----
|
||||
|
||||
inner.current_bytes += data_len;
|
||||
let new_idx = inner.slots.len();
|
||||
inner.slot_index.insert(coord.clone(), new_idx);
|
||||
inner.slots.push(CachedChunk {
|
||||
coord,
|
||||
data: Arc::clone(&data),
|
||||
last_access: tick,
|
||||
});
|
||||
data
|
||||
}
|
||||
|
||||
/// Clear the entire cache (index + decompressed data).
|
||||
/// Clear the entire cache (indexes + decompressed data + stats).
|
||||
pub fn clear(&self) {
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
inner.index = None;
|
||||
inner.index_addr = None;
|
||||
let mut inner = self.lock();
|
||||
inner.datasets.clear();
|
||||
inner.current = None;
|
||||
inner.slots.clear();
|
||||
inner.slot_index.clear();
|
||||
inner.current_bytes = 0;
|
||||
inner.tick = 0;
|
||||
inner.last_coord = None;
|
||||
inner.stats = AccessStats::default();
|
||||
inner.chunk_index = None;
|
||||
inner.chunk_layout = None;
|
||||
}
|
||||
|
||||
/// Record that the given chunk coordinates are predicted to be accessed
|
||||
/// soon (bookkeeping only).
|
||||
///
|
||||
/// This does **not** prefetch or pre-decompress anything — it only
|
||||
/// checks whether each coordinate is already in the chunk index and
|
||||
/// updates access-pattern stats accordingly. Real prefetching (e.g.
|
||||
/// background pre-decompression) is not implemented.
|
||||
pub fn prefetch_hint(&self, next_coords: &[ChunkCoord]) {
|
||||
let inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if inner.index.is_none() {
|
||||
return;
|
||||
}
|
||||
drop(inner);
|
||||
// For each predicted coordinate, verify it exists in the index.
|
||||
// The index is already populated, so this is a no-op for known chunks.
|
||||
// The purpose is to signal intent — callers can pre-decompress if needed.
|
||||
// We touch the stats to record that prefetch hints were issued.
|
||||
let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for coord in next_coords {
|
||||
let exists = inner
|
||||
.index
|
||||
.as_ref()
|
||||
.map(|idx| idx.contains_key(coord))
|
||||
.unwrap_or(false);
|
||||
if exists {
|
||||
inner.stats.sequential_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the current access pattern statistics.
|
||||
pub fn access_stats(&self) -> AccessStats {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.stats
|
||||
.clone()
|
||||
self.lock().stats.clone()
|
||||
}
|
||||
|
||||
/// Update the sweep direction label in the access stats.
|
||||
pub fn set_sweep_direction(&self, direction: &'static str) {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.stats
|
||||
.sweep_direction = Some(direction);
|
||||
self.lock().stats.sweep_direction = Some(direction);
|
||||
}
|
||||
|
||||
/// Number of decompressed chunks currently cached.
|
||||
/// Number of decompressed chunks currently cached (all datasets).
|
||||
pub fn cached_chunk_count(&self) -> usize {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.slots
|
||||
.len()
|
||||
self.lock().slots.len()
|
||||
}
|
||||
|
||||
/// Total bytes of decompressed data currently cached.
|
||||
/// Total bytes of decompressed data currently cached (all datasets).
|
||||
pub fn cached_bytes(&self) -> usize {
|
||||
self.inner
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.current_bytes
|
||||
self.lock().current_bytes
|
||||
}
|
||||
|
||||
/// Number of datasets whose chunk index is currently kept.
|
||||
pub fn indexed_dataset_count(&self) -> usize {
|
||||
self.lock().datasets.len()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -808,6 +967,92 @@ mod tests {
|
||||
assert_eq!(cache.cached_bytes(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn datasets_sharing_coordinates_stay_separate() {
|
||||
let cache = ChunkCache::new();
|
||||
let a = vec![make_chunk(vec![0, 0], 0x100, 8)];
|
||||
let b = vec![make_chunk(vec![0, 0], 0x900, 8)];
|
||||
let got_a = cache.chunks_for::<()>(1, 1, || Ok(a.clone())).unwrap();
|
||||
let got_b = cache.chunks_for::<()>(2, 1, || Ok(b.clone())).unwrap();
|
||||
assert_eq!(got_a[0].address, 0x100);
|
||||
assert_eq!(got_b[0].address, 0x900);
|
||||
// Built once per dataset: a second lookup doesn't call the builder.
|
||||
let again = cache
|
||||
.chunks_for::<()>(1, 1, || panic!("index rebuilt"))
|
||||
.unwrap();
|
||||
assert_eq!(again[0].address, 0x100);
|
||||
|
||||
cache.put_decompressed_in(1, vec![0], vec![1; 4]);
|
||||
cache.put_decompressed_in(2, vec![0], vec![2; 4]);
|
||||
assert_eq!(
|
||||
cache.get_decompressed_in(1, &[0]).unwrap().as_slice(),
|
||||
&[1; 4]
|
||||
);
|
||||
assert_eq!(
|
||||
cache.get_decompressed_in(2, &[0]).unwrap().as_slice(),
|
||||
&[2; 4]
|
||||
);
|
||||
assert!(cache.get_decompressed_in(3, &[0]).is_none());
|
||||
assert_eq!(cache.cached_chunk_count(), 2);
|
||||
|
||||
// The bound-dataset methods see only the bound dataset.
|
||||
cache.ensure_dataset(2);
|
||||
assert_eq!(cache.lookup_index(&[0]).unwrap().address, 0x900);
|
||||
assert_eq!(cache.get_decompressed(&[0]).unwrap(), vec![2; 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dataset_indexes_are_bounded() {
|
||||
let cache = ChunkCache::new();
|
||||
for addr in 0..(MAX_INDEXED_DATASETS as u64 + 10) {
|
||||
cache
|
||||
.chunks_for::<()>(addr, 1, || Ok(vec![make_chunk(vec![0], addr, 8)]))
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(cache.indexed_dataset_count(), MAX_INDEXED_DATASETS);
|
||||
|
||||
// One huge index evicts the others but is itself kept.
|
||||
let huge: Vec<ChunkInfo> = (0..MAX_INDEXED_CHUNKS as u64)
|
||||
.map(|i| make_chunk(vec![i], i, 8))
|
||||
.collect();
|
||||
let got = cache.chunks_for::<()>(9999, 1, || Ok(huge)).unwrap();
|
||||
assert_eq!(got.len(), MAX_INDEXED_CHUNKS);
|
||||
assert_eq!(cache.indexed_dataset_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_readers_of_different_datasets_see_their_own_chunks() {
|
||||
let cache = std::sync::Arc::new(ChunkCache::with_capacity(1 << 20, 64));
|
||||
let handles: Vec<_> = (0..8u64)
|
||||
.map(|t| {
|
||||
let cache = std::sync::Arc::clone(&cache);
|
||||
std::thread::spawn(move || {
|
||||
for round in 0..500u64 {
|
||||
let addr = (t + round) % 16;
|
||||
let coord = vec![round % 4];
|
||||
let chunks = cache
|
||||
.chunks_for::<()>(addr, 1, || {
|
||||
Ok((0..4).map(|c| make_chunk(vec![c], addr, 8)).collect())
|
||||
})
|
||||
.unwrap();
|
||||
assert!(chunks.iter().all(|c| c.address == addr));
|
||||
let want = vec![addr as u8; 8];
|
||||
let got = match cache.get_decompressed_in(addr, &coord) {
|
||||
Some(hit) => hit.to_vec(),
|
||||
None => cache
|
||||
.put_decompressed_in(addr, coord, want.clone())
|
||||
.to_vec(),
|
||||
};
|
||||
assert_eq!(got, want);
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_insert_is_noop() {
|
||||
let cache = ChunkCache::new();
|
||||
|
||||
@@ -836,24 +836,20 @@ pub fn read_chunked_data_cached(
|
||||
)));
|
||||
}
|
||||
|
||||
// The per-file cache is shared across datasets; bind it to this one so a
|
||||
// different dataset's chunk index is never reused for this read.
|
||||
cache.ensure_dataset(addr);
|
||||
|
||||
// Populate chunk index on first access
|
||||
if !cache.has_index() {
|
||||
let (chunks, _) = list_chunks(
|
||||
// The per-file cache is shared across datasets (and threads); every
|
||||
// lookup is keyed by this dataset's chunk-index address, so another
|
||||
// dataset's index or chunks are never used for this read.
|
||||
let chunks = cache.chunks_for(addr, rank, || {
|
||||
list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
cache.populate_index(&chunks, rank);
|
||||
}
|
||||
|
||||
let chunks = cache.all_indexed_chunks().unwrap_or_default();
|
||||
)
|
||||
.map(|(chunks, _)| chunks)
|
||||
})?;
|
||||
|
||||
// Assemble output
|
||||
let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?;
|
||||
@@ -920,7 +916,7 @@ pub fn read_chunked_data_cached(
|
||||
continue;
|
||||
}
|
||||
let coord: Vec<u64> = chunk_info.offsets.iter().take(rank).copied().collect();
|
||||
match cache.get_decompressed_aligned(&coord) {
|
||||
match cache.get_decompressed_in(addr, &coord) {
|
||||
Some(cached) => place(&cached, chunk_info),
|
||||
None => misses.push(chunk_info),
|
||||
}
|
||||
@@ -957,7 +953,7 @@ pub fn read_chunked_data_cached(
|
||||
let data = data?;
|
||||
if cache_them {
|
||||
let coord: Vec<u64> = chunk_info.offsets.iter().take(rank).copied().collect();
|
||||
let cached = cache.put_decompressed(coord, data);
|
||||
let cached = cache.put_decompressed_in(addr, coord, data);
|
||||
place(&cached, chunk_info);
|
||||
} else {
|
||||
place(&data, chunk_info);
|
||||
@@ -1161,24 +1157,20 @@ pub fn read_chunked_data_sweep(
|
||||
)));
|
||||
}
|
||||
|
||||
// The per-file cache is shared across datasets; bind it to this one so a
|
||||
// different dataset's chunk index is never reused for this read.
|
||||
cache.ensure_dataset(addr);
|
||||
|
||||
// Populate chunk index on first access
|
||||
if !cache.has_index() {
|
||||
let (chunks, _) = list_chunks(
|
||||
// The per-file cache is shared across datasets (and threads); every
|
||||
// lookup is keyed by this dataset's chunk-index address, so another
|
||||
// dataset's index or chunks are never used for this read.
|
||||
let chunks = cache.chunks_for(addr, rank, || {
|
||||
list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)?;
|
||||
cache.populate_index(&chunks, rank);
|
||||
}
|
||||
|
||||
let chunks = cache.all_indexed_chunks().unwrap_or_default();
|
||||
)
|
||||
.map(|(chunks, _)| chunks)
|
||||
})?;
|
||||
|
||||
// Assemble output
|
||||
let total_bytes = checked_byte_len(dataspace.checked_num_elements()?, elem_size)?;
|
||||
@@ -1209,12 +1201,12 @@ pub fn read_chunked_data_sweep(
|
||||
|
||||
// Issue prefetch hint for predicted next chunks
|
||||
if !sweep.predicted_next.is_empty() {
|
||||
cache.prefetch_hint(&sweep.predicted_next);
|
||||
cache.prefetch_hint_in(addr, &sweep.predicted_next);
|
||||
cache.set_sweep_direction(sweep.direction);
|
||||
}
|
||||
|
||||
// Try decompressed cache first
|
||||
let decompressed = if let Some(cached) = cache.get_decompressed_aligned(&coord) {
|
||||
let decompressed = if let Some(cached) = cache.get_decompressed_in(addr, &coord) {
|
||||
cached
|
||||
} else {
|
||||
// Decompress from file
|
||||
@@ -1233,7 +1225,7 @@ pub fn read_chunked_data_sweep(
|
||||
} else {
|
||||
raw_chunk.to_vec()
|
||||
};
|
||||
cache.put_decompressed(coord, dec)
|
||||
cache.put_decompressed_in(addr, coord, dec)
|
||||
};
|
||||
|
||||
let chunk_offsets: Vec<usize> = chunk_info
|
||||
@@ -1317,48 +1309,34 @@ pub fn read_chunked_data_indexed(
|
||||
)));
|
||||
}
|
||||
|
||||
// The per-file cache is shared across datasets; bind it to this one so a
|
||||
// different dataset's chunk index is never reused for this read.
|
||||
cache.ensure_dataset(addr);
|
||||
|
||||
// Build chunk index on first access
|
||||
if !cache.has_chunk_index() {
|
||||
let (chunks, _) = list_chunks(
|
||||
// Chunk index and assembly plan for this dataset, built on first access
|
||||
// and kept per dataset (keyed by chunk-index address) in the shared cache.
|
||||
let plan = cache.chunk_layout_for(
|
||||
addr,
|
||||
rank,
|
||||
|| {
|
||||
list_chunks(
|
||||
file_data,
|
||||
layout,
|
||||
dataspace,
|
||||
elem_size,
|
||||
offset_size,
|
||||
length_size,
|
||||
)
|
||||
.map(|(chunks, _)| chunks)
|
||||
},
|
||||
&ds_dims,
|
||||
&chunk_dims,
|
||||
elem_size,
|
||||
)?;
|
||||
cache.populate_chunk_index(&chunks, rank);
|
||||
// Also populate the legacy index for compatibility
|
||||
if !cache.has_index() {
|
||||
cache.populate_index(&chunks, rank);
|
||||
}
|
||||
}
|
||||
|
||||
// Build chunk layout on first access
|
||||
if !cache.has_chunk_layout() {
|
||||
cache.populate_chunk_layout(&ds_dims, &chunk_dims, elem_size);
|
||||
}
|
||||
|
||||
// Get the layout info (mappings, output size, chunk total bytes)
|
||||
let (mappings_info, output_bytes, chunk_total_bytes) = cache
|
||||
.with_chunk_layout(|layout| {
|
||||
let info: Vec<_> = layout
|
||||
.mappings
|
||||
.iter()
|
||||
.map(|m| (m.coord.clone(), m.file_offset, m.file_size, m.filter_mask))
|
||||
.collect();
|
||||
(info, layout.output_bytes, layout.chunk_total_bytes)
|
||||
})
|
||||
.ok_or_else(|| FormatError::ChunkedReadError("chunk layout not available".into()))?;
|
||||
let chunk_total_bytes = plan.chunk_total_bytes;
|
||||
|
||||
// Decompress chunks (using LRU cache where possible)
|
||||
let mut chunk_buffers: Vec<Arc<CacheAlignedBuffer>> = Vec::with_capacity(mappings_info.len());
|
||||
for (coord, file_offset, file_size, filter_mask) in &mappings_info {
|
||||
if let Some(cached) = cache.get_decompressed_aligned(coord) {
|
||||
let mut chunk_buffers: Vec<Arc<CacheAlignedBuffer>> = Vec::with_capacity(plan.mappings.len());
|
||||
for m in &plan.mappings {
|
||||
let (coord, file_offset, file_size, filter_mask) =
|
||||
(&m.coord, &m.file_offset, &m.file_size, &m.filter_mask);
|
||||
if let Some(cached) = cache.get_decompressed_in(addr, coord) {
|
||||
chunk_buffers.push(cached);
|
||||
} else {
|
||||
let c_addr = *file_offset as usize;
|
||||
@@ -1377,17 +1355,15 @@ pub fn read_chunked_data_indexed(
|
||||
raw_chunk.to_vec()
|
||||
};
|
||||
let aligned = CacheAlignedBuffer::from_vec(decompressed);
|
||||
let arc = cache.put_decompressed_aligned(coord.clone(), aligned);
|
||||
let arc = cache.put_decompressed_aligned_in(addr, coord.clone(), aligned);
|
||||
chunk_buffers.push(arc);
|
||||
}
|
||||
}
|
||||
|
||||
// Assemble using pre-computed layout
|
||||
let mut output = vec![0u8; output_bytes];
|
||||
let mut output = vec![0u8; plan.output_bytes];
|
||||
let data_refs: Vec<&[u8]> = chunk_buffers.iter().map(|b| b.as_slice()).collect();
|
||||
cache.with_chunk_layout(|layout| {
|
||||
layout.assemble(&data_refs, &mut output);
|
||||
});
|
||||
plan.assemble(&data_refs, &mut output);
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
@@ -2307,12 +2283,12 @@ mod tests {
|
||||
let datatype = make_f64_type();
|
||||
let cache = ChunkCache::new();
|
||||
|
||||
assert!(!cache.has_index());
|
||||
assert_eq!(cache.indexed_dataset_count(), 0);
|
||||
let raw = read_chunked_data_cached(
|
||||
&file_data, &layout, &dataspace, &datatype, None, 8, 8, &cache,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(cache.has_index());
|
||||
assert_eq!(cache.indexed_dataset_count(), 1);
|
||||
assert_eq!(raw.len(), 20 * 8);
|
||||
for i in 0..20 {
|
||||
let val = f64::from_le_bytes(raw[i * 8..(i + 1) * 8].try_into().unwrap());
|
||||
@@ -2334,7 +2310,7 @@ mod tests {
|
||||
&file_data, &layout, &dataspace, &datatype, None, 8, 8, &cache,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(cache.has_index());
|
||||
assert_eq!(cache.indexed_dataset_count(), 1);
|
||||
assert_eq!(cache.cached_chunk_count(), 0);
|
||||
|
||||
// Second read — reuses the cached index
|
||||
@@ -2343,6 +2319,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(raw1, raw2);
|
||||
assert_eq!(cache.indexed_dataset_count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
//! A `File` is `Send + Sync` and keeps one chunk cache for all its datasets.
|
||||
//! Threads reading different chunked datasets through the same `File` must
|
||||
//! each get their own dataset's data.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use clawhdf5::{File, FileBuilder};
|
||||
|
||||
const DATASETS: usize = 24;
|
||||
const THREADS: usize = 16;
|
||||
const ROUNDS: usize = 40;
|
||||
|
||||
/// Contents of dataset `k`: distinct from every other dataset's, element for
|
||||
/// element, so any chunk served from the wrong dataset shows.
|
||||
fn values(k: usize, n: usize) -> Vec<f64> {
|
||||
(0..n).map(|i| (k * 100_000 + i) as f64).collect()
|
||||
}
|
||||
|
||||
fn build() -> File {
|
||||
let mut b = FileBuilder::new();
|
||||
for k in 0..DATASETS {
|
||||
let ds = b.create_dataset(&format!("d{k:02}"));
|
||||
match k % 3 {
|
||||
// 1-D, compressed: chunk offsets 0, 8, 16, ... in every dataset.
|
||||
0 => {
|
||||
ds.with_f64_data(&values(k, 64)).with_shape(&[64]);
|
||||
ds.with_chunks(&[8]).with_deflate(1);
|
||||
}
|
||||
// 1-D, shuffle + compressed, a different length.
|
||||
1 => {
|
||||
ds.with_f64_data(&values(k, 40)).with_shape(&[40]);
|
||||
ds.with_chunks(&[8]).with_shuffle().with_deflate(1);
|
||||
}
|
||||
// 2-D, compressed: coordinates (0,0), (0,4), (4,0), ... overlap
|
||||
// the other datasets' in the first dimension.
|
||||
_ => {
|
||||
ds.with_f64_data(&values(k, 64)).with_shape(&[8, 8]);
|
||||
ds.with_chunks(&[4, 4]).with_deflate(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
File::from_bytes(b.finish().unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn expected(k: usize) -> Vec<f64> {
|
||||
values(k, if k % 3 == 1 { 40 } else { 64 })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threads_reading_different_datasets_get_their_own_chunks() {
|
||||
let file = Arc::new(build());
|
||||
// Sequential sanity check first.
|
||||
for k in 0..DATASETS {
|
||||
let got = file.dataset(&format!("d{k:02}")).unwrap().read_f64();
|
||||
assert_eq!(got.unwrap(), expected(k), "sequential d{k:02}");
|
||||
}
|
||||
|
||||
let handles: Vec<_> = (0..THREADS)
|
||||
.map(|t| {
|
||||
let file = Arc::clone(&file);
|
||||
std::thread::spawn(move || {
|
||||
let mut wrong = Vec::new();
|
||||
for round in 0..ROUNDS {
|
||||
let k = (t * 7 + round * 5) % DATASETS;
|
||||
let name = format!("d{k:02}");
|
||||
match file.dataset(&name).unwrap().read_f64() {
|
||||
Ok(v) if v == expected(k) => {}
|
||||
Ok(v) => wrong.push(format!("{name}: wrong data, first {:?}", &v[..4])),
|
||||
Err(e) => wrong.push(format!("{name}: {e}")),
|
||||
}
|
||||
}
|
||||
wrong
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let failures: Vec<String> = handles
|
||||
.into_iter()
|
||||
.flat_map(|h| h.join().unwrap())
|
||||
.collect();
|
||||
assert!(
|
||||
failures.is_empty(),
|
||||
"{} of {} concurrent reads were wrong, e.g. {:?}",
|
||||
failures.len(),
|
||||
THREADS * ROUNDS,
|
||||
&failures[..failures.len().min(5)]
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user