Omar Sobh
eebc62d87b
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Successful in 10s
Closes the retention loop between snapshots and pin-aware LRU eviction. A snapshot is not just a "list of blobs at time T" any more — it's a *retention pin* on every blob it captures. Operators can guarantee a build stays on disk for N days by snapshotting it and pruning the snapshot when the window is up. Additions: * SnapshotStore::pinned_blob_ids() → union of blob_ids across all live snapshots. Cheap: one JSON read per snapshot. * cmd_cluster_gc extends the tag-pin set with snapshot pins before handing it to evict_to_size_cap_with_pins. Output line now reads "pinned blobs: N (M from snapshots)". * ClusterServices auto-GC ticker does the same on every tick; log fields include snapshot_pins so ops see the retention set size at a glance. +2 tests: - pinned_blob_ids_unions_all_snapshots (overlap dedupe) - pinned_blob_ids_empty_when_no_snapshots 355 tests pass (+2). Pre-existing macOS hot::tests::test_project_target_size_bytes failure unchanged.
415 lines
16 KiB
Rust
415 lines
16 KiB
Rust
//! Snapshot store (Phase 7d).
|
||
//!
|
||
//! A **snapshot** is a named, immutable record of every blob live in
|
||
//! the store at the moment it was created. It is *not* a copy of the
|
||
//! data — blobs are content-addressed and already live under
|
||
//! `blobs/`. A snapshot is a list of `blob_id`s + wall-clock creation
|
||
//! time, stored as JSON at
|
||
//! `<root>/snapshots/<name>.json`.
|
||
//!
|
||
//! Why this exists:
|
||
//! * **Rollback anchor** — before a risky migration, snapshot the
|
||
//! current graph; if things go sideways the snapshot lists exactly
|
||
//! which blobs must survive.
|
||
//! * **Retention pin** — combined with the pin-aware LRU eviction
|
||
//! from Phase 4a, the operator can guarantee "these blobs stay on
|
||
//! disk for the next N days" without hand-listing them.
|
||
//! * **Audit** — "which blobs existed at release time?"
|
||
//!
|
||
//! Layout:
|
||
//!
|
||
//! ```text
|
||
//! <root>/snapshots/<name>.json # SnapshotManifest, JSON
|
||
//! ```
|
||
//!
|
||
//! Snapshot names are operator-supplied strings; the same
|
||
//! `validate_name` rules that guard `TagStore` apply — printable
|
||
//! non-slash characters, bounded length.
|
||
|
||
use anyhow::{bail, Context, Result};
|
||
use serde::{Deserialize, Serialize};
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use crate::cluster::blob::{BlobId, BlobStore};
|
||
|
||
/// Longest snapshot name we accept. Same rationale as
|
||
/// [`crate::cluster::tags::MAX_TAG_KEY_BYTES`] but tighter — names
|
||
/// end up in filesystem paths.
|
||
pub const MAX_SNAPSHOT_NAME_BYTES: usize = 512;
|
||
|
||
/// On-disk JSON representation of a snapshot.
|
||
///
|
||
/// `created_at_unix` is wall-clock seconds at the moment the snapshot
|
||
/// was written. Stored explicitly so listing doesn't have to `stat`
|
||
/// files, and so restore/report tools can render "created 3 days ago"
|
||
/// without a filesystem call.
|
||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||
pub struct SnapshotManifest {
|
||
pub name: String,
|
||
pub created_at_unix: u64,
|
||
/// Blob ids that were live in the store at snapshot time.
|
||
/// Duplicates deduped by construction. Order is arbitrary.
|
||
pub blob_ids: Vec<BlobId>,
|
||
}
|
||
|
||
/// Compact form returned by [`SnapshotStore::list`]. Enough to render
|
||
/// a table without slurping every snapshot's full blob list into
|
||
/// memory.
|
||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||
pub struct SnapshotSummary {
|
||
pub name: String,
|
||
pub created_at_unix: u64,
|
||
pub blob_count: usize,
|
||
pub file_bytes: u64,
|
||
}
|
||
|
||
/// Filesystem-backed snapshot store rooted at a directory.
|
||
#[derive(Debug, Clone)]
|
||
pub struct SnapshotStore {
|
||
root: PathBuf,
|
||
}
|
||
|
||
impl SnapshotStore {
|
||
/// Open (create if missing) the snapshot store under `root`.
|
||
/// `root` is typically the same directory the blob store lives
|
||
/// under — snapshots land in `<root>/snapshots/`.
|
||
pub fn open(root: PathBuf) -> Result<Self> {
|
||
std::fs::create_dir_all(root.join("snapshots"))
|
||
.with_context(|| format!("creating snapshots dir under {}", root.display()))?;
|
||
Ok(Self { root })
|
||
}
|
||
|
||
/// Point-in-time snapshot of every blob currently in `store`.
|
||
/// Reads the live manifest set via `BlobStore::list_blob_ids`
|
||
/// and writes a single JSON file. Fails if a snapshot with the
|
||
/// same name already exists — snapshots are meant to be
|
||
/// immutable checkpoints, not mutable pointers. Use `delete`
|
||
/// then `create` if you really want to overwrite.
|
||
pub async fn create(
|
||
&self,
|
||
name: &str,
|
||
store: &BlobStore,
|
||
created_at_unix: u64,
|
||
) -> Result<SnapshotManifest> {
|
||
validate_name(name)?;
|
||
let path = self.snapshot_path(name);
|
||
if tokio::fs::metadata(&path).await.is_ok() {
|
||
bail!("snapshot {:?} already exists at {}", name, path.display());
|
||
}
|
||
let mut blob_ids = store.list_blob_ids().await?;
|
||
// Stable order = stable JSON. Blob_ids from list_blob_ids come
|
||
// in filesystem walk order, which is not portable.
|
||
blob_ids.sort();
|
||
blob_ids.dedup();
|
||
let manifest = SnapshotManifest {
|
||
name: name.to_string(),
|
||
created_at_unix,
|
||
blob_ids,
|
||
};
|
||
let bytes = serde_json::to_vec_pretty(&manifest)
|
||
.context("serializing snapshot manifest")?;
|
||
atomic_write(&path, &bytes).await?;
|
||
Ok(manifest)
|
||
}
|
||
|
||
/// Read a snapshot back. `None` if unknown; `Err` if the file is
|
||
/// present but malformed (visibility over silent skip).
|
||
pub async fn get(&self, name: &str) -> Result<Option<SnapshotManifest>> {
|
||
validate_name(name)?;
|
||
let path = self.snapshot_path(name);
|
||
let bytes = match tokio::fs::read(&path).await {
|
||
Ok(b) => b,
|
||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||
Err(e) => return Err(anyhow::Error::from(e)),
|
||
};
|
||
let m: SnapshotManifest = serde_json::from_slice(&bytes)
|
||
.with_context(|| format!("decoding {}", path.display()))?;
|
||
Ok(Some(m))
|
||
}
|
||
|
||
/// Enumerate every snapshot. Cheap: one file read per snapshot
|
||
/// (the summary field is 3 numbers + a name). Returns entries
|
||
/// sorted by `created_at_unix` ascending — oldest first — so an
|
||
/// operator scanning a long list can find the oldest to prune.
|
||
pub async fn list(&self) -> Result<Vec<SnapshotSummary>> {
|
||
let dir = self.root.join("snapshots");
|
||
let mut out = Vec::new();
|
||
let mut entries = match tokio::fs::read_dir(&dir).await {
|
||
Ok(e) => e,
|
||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(out),
|
||
Err(e) => return Err(anyhow::Error::from(e)),
|
||
};
|
||
while let Some(entry) = entries.next_entry().await? {
|
||
if !entry.file_type().await?.is_file() {
|
||
continue;
|
||
}
|
||
let name = entry.file_name();
|
||
let name_str = match name.to_str() {
|
||
Some(s) => s,
|
||
None => continue,
|
||
};
|
||
if !name_str.ends_with(".json") {
|
||
continue;
|
||
}
|
||
let bytes = match tokio::fs::read(entry.path()).await {
|
||
Ok(b) => b,
|
||
Err(_) => continue,
|
||
};
|
||
let file_bytes = bytes.len() as u64;
|
||
if let Ok(m) = serde_json::from_slice::<SnapshotManifest>(&bytes) {
|
||
out.push(SnapshotSummary {
|
||
name: m.name,
|
||
created_at_unix: m.created_at_unix,
|
||
blob_count: m.blob_ids.len(),
|
||
file_bytes,
|
||
});
|
||
}
|
||
}
|
||
out.sort_by_key(|s| s.created_at_unix);
|
||
Ok(out)
|
||
}
|
||
|
||
/// Remove a snapshot. Returns `true` if a file was removed,
|
||
/// `false` if none existed. Never touches blob storage —
|
||
/// deleting a snapshot only forgets the reference set, not the
|
||
/// blobs themselves.
|
||
pub async fn delete(&self, name: &str) -> Result<bool> {
|
||
validate_name(name)?;
|
||
let path = self.snapshot_path(name);
|
||
match tokio::fs::remove_file(&path).await {
|
||
Ok(()) => Ok(true),
|
||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
|
||
Err(e) => Err(anyhow::Error::from(e)),
|
||
}
|
||
}
|
||
|
||
/// Phase 7d follow-on: union of every `blob_id` referenced by ANY
|
||
/// snapshot. Feeds pin-aware eviction — every blob captured by a
|
||
/// live snapshot survives the LRU cap. Semantically "snapshots
|
||
/// act as immortal retention pins until the operator deletes
|
||
/// them".
|
||
///
|
||
/// Cheap: one file read + JSON parse per snapshot. A fleet with
|
||
/// 100 snapshots × 10k blobs each is ~1 MB of JSON I/O.
|
||
pub async fn pinned_blob_ids(
|
||
&self,
|
||
) -> Result<std::collections::HashSet<BlobId>> {
|
||
let mut out = std::collections::HashSet::new();
|
||
for summary in self.list().await? {
|
||
if let Some(m) = self.get(&summary.name).await? {
|
||
for id in m.blob_ids {
|
||
out.insert(id);
|
||
}
|
||
}
|
||
}
|
||
Ok(out)
|
||
}
|
||
|
||
fn snapshot_path(&self, name: &str) -> PathBuf {
|
||
// Names are validated: no slashes, printable only. Safe to
|
||
// use directly as a filename fragment. We still normalize
|
||
// via hex-of-name if the operator ever passes something
|
||
// exotic like a unicode dash — but that's a future concern.
|
||
self.root
|
||
.join("snapshots")
|
||
.join(format!("{name}.json"))
|
||
}
|
||
}
|
||
|
||
fn validate_name(name: &str) -> Result<()> {
|
||
if name.is_empty() {
|
||
bail!("snapshot name cannot be empty");
|
||
}
|
||
if name.len() > MAX_SNAPSHOT_NAME_BYTES {
|
||
bail!(
|
||
"snapshot name length {} exceeds cap {}",
|
||
name.len(),
|
||
MAX_SNAPSHOT_NAME_BYTES
|
||
);
|
||
}
|
||
for c in name.chars() {
|
||
if c.is_control() {
|
||
bail!("snapshot name has control character");
|
||
}
|
||
if c == '/' || c == '\\' || c == '\0' {
|
||
bail!("snapshot name may not contain / \\ or NUL");
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
static RANDOM_SUFFIX: std::sync::atomic::AtomicU64 =
|
||
std::sync::atomic::AtomicU64::new(0);
|
||
|
||
async fn atomic_write(final_path: &Path, bytes: &[u8]) -> Result<()> {
|
||
use tokio::io::AsyncWriteExt;
|
||
let parent = final_path
|
||
.parent()
|
||
.context("snapshot path had no parent")?;
|
||
let tmp_name = format!(
|
||
".tmp.{}.{}",
|
||
std::process::id(),
|
||
RANDOM_SUFFIX.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||
);
|
||
let tmp_path = parent.join(tmp_name);
|
||
{
|
||
let mut f = tokio::fs::File::create(&tmp_path)
|
||
.await
|
||
.with_context(|| format!("creating tmp snapshot {}", tmp_path.display()))?;
|
||
f.write_all(bytes).await?;
|
||
f.sync_all().await?;
|
||
}
|
||
tokio::fs::rename(&tmp_path, final_path)
|
||
.await
|
||
.with_context(|| {
|
||
format!(
|
||
"renaming {} → {}",
|
||
tmp_path.display(),
|
||
final_path.display()
|
||
)
|
||
})?;
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use tempfile::TempDir;
|
||
|
||
fn open() -> (TempDir, BlobStore, SnapshotStore) {
|
||
let tmp = TempDir::new().unwrap();
|
||
let blob = BlobStore::open(tmp.path().to_path_buf()).unwrap();
|
||
let snap = SnapshotStore::open(tmp.path().to_path_buf()).unwrap();
|
||
(tmp, blob, snap)
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn create_captures_all_live_blob_ids() {
|
||
// Three blobs in, snapshot should reference all three.
|
||
let (_tmp, blob, snap) = open();
|
||
let a = blob.put_bytes(b"alpha").await.unwrap();
|
||
let b = blob.put_bytes(b"beta").await.unwrap();
|
||
let c = blob.put_bytes(b"gamma").await.unwrap();
|
||
let m = snap.create("v1", &blob, 1_700_000_000).await.unwrap();
|
||
assert_eq!(m.name, "v1");
|
||
assert_eq!(m.created_at_unix, 1_700_000_000);
|
||
assert_eq!(m.blob_ids.len(), 3);
|
||
let ids: std::collections::HashSet<_> = m.blob_ids.iter().collect();
|
||
assert!(ids.contains(&a));
|
||
assert!(ids.contains(&b));
|
||
assert!(ids.contains(&c));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn create_is_immutable_second_call_errors() {
|
||
// Snapshots are meant to be pin-in-time. Silent overwrite
|
||
// would be a footgun. Second create with same name must
|
||
// error, not clobber.
|
||
let (_tmp, blob, snap) = open();
|
||
blob.put_bytes(b"x").await.unwrap();
|
||
snap.create("v1", &blob, 1).await.unwrap();
|
||
let err = snap.create("v1", &blob, 2).await.unwrap_err();
|
||
assert!(err.to_string().contains("already exists"));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn get_returns_none_for_missing() {
|
||
let (_tmp, _blob, snap) = open();
|
||
assert!(snap.get("nope").await.unwrap().is_none());
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn list_sorts_by_creation_time_ascending() {
|
||
// Oldest first — operator triaging a growing list wants
|
||
// the pruning candidate at the top.
|
||
let (_tmp, blob, snap) = open();
|
||
blob.put_bytes(b"seed").await.unwrap();
|
||
snap.create("newer", &blob, 2000).await.unwrap();
|
||
snap.create("older", &blob, 1000).await.unwrap();
|
||
snap.create("newest", &blob, 3000).await.unwrap();
|
||
let entries = snap.list().await.unwrap();
|
||
let names: Vec<_> = entries.iter().map(|s| s.name.as_str()).collect();
|
||
assert_eq!(names, ["older", "newer", "newest"]);
|
||
assert_eq!(entries[0].blob_count, 1);
|
||
assert!(entries[0].file_bytes > 0);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn delete_true_when_present_false_when_not() {
|
||
let (_tmp, blob, snap) = open();
|
||
blob.put_bytes(b"seed").await.unwrap();
|
||
snap.create("v1", &blob, 1).await.unwrap();
|
||
assert!(snap.delete("v1").await.unwrap());
|
||
assert!(!snap.delete("v1").await.unwrap());
|
||
assert!(snap.get("v1").await.unwrap().is_none());
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn deleting_snapshot_does_not_touch_blobs() {
|
||
// A snapshot is a reference set, not a copy. Deleting one
|
||
// must not affect blob data — otherwise operators could
|
||
// accidentally nuke live data by pruning snapshots.
|
||
let (_tmp, blob, snap) = open();
|
||
let id = blob.put_bytes(b"survivor").await.unwrap();
|
||
snap.create("temp", &blob, 1).await.unwrap();
|
||
snap.delete("temp").await.unwrap();
|
||
// Blob still readable.
|
||
let back = blob.get_bytes(&id).await.unwrap();
|
||
assert_eq!(back.as_deref(), Some(&b"survivor"[..]));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn validate_name_rejects_slash_and_control() {
|
||
assert!(validate_name("with/slash").is_err());
|
||
assert!(validate_name("with\\backslash").is_err());
|
||
assert!(validate_name("with\0null").is_err());
|
||
assert!(validate_name("with\x01ctrl").is_err());
|
||
assert!(validate_name("").is_err());
|
||
assert!(validate_name("ok-name").is_ok());
|
||
assert!(validate_name("v2026.07.14-pre-release").is_ok());
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn pinned_blob_ids_unions_all_snapshots() {
|
||
// Two snapshots, some overlap. Union must dedupe.
|
||
let (_tmp, blob, snap) = open();
|
||
let a = blob.put_bytes(b"pin-alpha").await.unwrap();
|
||
let b = blob.put_bytes(b"pin-beta").await.unwrap();
|
||
let c = blob.put_bytes(b"pin-gamma").await.unwrap();
|
||
// Snapshot 1: captures a, b, c
|
||
snap.create("s1", &blob, 1).await.unwrap();
|
||
// Snapshot 2 (later): same content, still captures a, b, c
|
||
snap.create("s2", &blob, 2).await.unwrap();
|
||
|
||
let pins = snap.pinned_blob_ids().await.unwrap();
|
||
assert_eq!(pins.len(), 3);
|
||
assert!(pins.contains(&a));
|
||
assert!(pins.contains(&b));
|
||
assert!(pins.contains(&c));
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn pinned_blob_ids_empty_when_no_snapshots() {
|
||
let (_tmp, blob, snap) = open();
|
||
blob.put_bytes(b"blob-with-no-snapshot").await.unwrap();
|
||
assert!(snap.pinned_blob_ids().await.unwrap().is_empty());
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn round_trip_preserves_blob_ids_sorted() {
|
||
// list_blob_ids order is filesystem-dependent. Snapshot
|
||
// must sort before writing so different nodes taking a
|
||
// snapshot of the same content get byte-identical files.
|
||
let (_tmp, blob, snap) = open();
|
||
for word in ["one", "two", "three", "four", "five"] {
|
||
blob.put_bytes(word.as_bytes()).await.unwrap();
|
||
}
|
||
let m = snap.create("v1", &blob, 42).await.unwrap();
|
||
let mut sorted = m.blob_ids.clone();
|
||
sorted.sort();
|
||
assert_eq!(m.blob_ids, sorted, "blob_ids must be sorted on write");
|
||
let m2 = snap.get("v1").await.unwrap().unwrap();
|
||
assert_eq!(m2.blob_ids, m.blob_ids);
|
||
}
|
||
}
|