clawverse/clawstor
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 7f follow-on: Gitea live-refs adapter + cluster-ref-sweep CLI #60

Merged
osobh merged 1 commits from phase-7f-gitea-sweep into main 2026-07-14 17:58:32 +00:00
Conversation 0 Commits 1 Files Changed 5
+694 -2
5 changed files with 694 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7bc5ba987c - Show all commits
Cargo.lock
Generated
+381 -1
View File
@@ -383,6 +383,7 @@ dependencies = [
"libc",
"quinn",
"rcgen",
"reqwest",
"rustls",
"rustls-pemfile",
"serde",
@@ -394,7 +395,7 @@ dependencies = [
"tokio-stream",
"toml",
"tower",
"tower-http",
"tower-http 0.5.2",
"tracing",
"tracing-subscriber",
"zstd",
@@ -717,6 +718,23 @@ dependencies = [
"pin-project-lite",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
"http",
"hyper",
"hyper-util",
"rustls",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
]
[[package]]
@@ -725,13 +743,21 @@ version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
"base64",
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
"hyper",
"ipnet",
"libc",
"percent-encoding",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
]
[[package]]
@@ -758,12 +784,115 @@ dependencies = [
"cc",
]
[[package]]
name = "icu_collections"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
dependencies = [
"displaydoc",
"potential_utf",
"utf8_iter",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_locale_core"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
dependencies = [
"displaydoc",
"litemap",
"tinystr",
"writeable",
"zerovec",
]
[[package]]
name = "icu_normalizer"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
dependencies = [
"icu_collections",
"icu_normalizer_data",
"icu_properties",
"icu_provider",
"smallvec",
"zerovec",
]
[[package]]
name = "icu_normalizer_data"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
[[package]]
name = "icu_properties"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
dependencies = [
"icu_collections",
"icu_locale_core",
"icu_properties_data",
"icu_provider",
"zerotrie",
"zerovec",
]
[[package]]
name = "icu_properties_data"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
[[package]]
name = "icu_provider"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
dependencies = [
"displaydoc",
"icu_locale_core",
"writeable",
"yoke",
"zerofrom",
"zerotrie",
"zerovec",
]
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "idna"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
dependencies = [
"idna_adapter",
"smallvec",
"utf8_iter",
]
[[package]]
name = "idna_adapter"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
dependencies = [
"icu_normalizer",
"icu_properties",
]
[[package]]
name = "indexmap"
version = "2.14.0"
@@ -776,6 +905,12 @@ dependencies = [
"serde_core",
]
[[package]]
name = "ipnet"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
@@ -842,6 +977,12 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litemap"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -1060,6 +1201,15 @@ version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "potential_utf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
dependencies = [
"zerovec",
]
[[package]]
name = "powerfmt"
version = "0.2.0"
@@ -1242,6 +1392,44 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "reqwest"
version = "0.12.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64",
"bytes",
"futures-core",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-util",
"js-sys",
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tower",
"tower-http 0.6.11",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
]
[[package]]
name = "ring"
version = "0.17.14"
@@ -1474,6 +1662,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -1502,6 +1696,9 @@ name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
@@ -1632,6 +1829,16 @@ dependencies = [
"time-core",
]
[[package]]
name = "tinystr"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
dependencies = [
"displaydoc",
"zerovec",
]
[[package]]
name = "tinyvec"
version = "1.12.0"
@@ -1675,6 +1882,16 @@ dependencies = [
"syn",
]
[[package]]
name = "tokio-rustls"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.18"
@@ -1782,6 +1999,24 @@ dependencies = [
"tracing",
]
[[package]]
name = "tower-http"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"pin-project-lite",
"tower",
"tower-layer",
"tower-service",
"url",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -1856,6 +2091,12 @@ dependencies = [
"tracing-log",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "unicase"
version = "2.9.0"
@@ -1880,6 +2121,24 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
"serde",
]
[[package]]
name = "utf8_iter"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -1892,6 +2151,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -1929,6 +2197,16 @@ dependencies = [
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.125"
@@ -1995,6 +2273,16 @@ dependencies = [
"semver",
]
[[package]]
name = "web-sys"
version = "0.3.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
@@ -2005,6 +2293,15 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "winapi"
version = "0.3.9"
@@ -2290,6 +2587,12 @@ dependencies = [
"wasmparser",
]
[[package]]
name = "writeable"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
[[package]]
name = "x509-parser"
version = "0.16.0"
@@ -2327,12 +2630,89 @@ dependencies = [
"time",
]
[[package]]
name = "yoke"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
dependencies = [
"stable_deref_trait",
"yoke-derive",
"zerofrom",
]
[[package]]
name = "yoke-derive"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
name = "zerofrom"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
dependencies = [
"zerofrom-derive",
]
[[package]]
name = "zerofrom-derive"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
name = "zeroize"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
[[package]]
name = "zerotrie"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
dependencies = [
"displaydoc",
"yoke",
"zerofrom",
]
[[package]]
name = "zerovec"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
dependencies = [
"yoke",
"zerofrom",
"zerovec-derive",
]
[[package]]
name = "zerovec-derive"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "zmij"
version = "1.0.21"
claw-store/Cargo.toml
+8 -1
View File
@@ -15,7 +15,7 @@ name = "claw-cargo"
path = "src/claw_cargo.rs"
[dependencies]
clap = { version = "4", features = ["derive"] }
clap = { version = "4", features = ["derive", "env"] }
serde = { version = "1", features = ["derive"] }
toml = "0.8"
tokio = { version = "1", features = ["full"] }
@@ -67,6 +67,13 @@ tar = "0.4"
# v0.13 — zstd wrapping around the tar stream. Level 3 is the default;
# gets 5-10× compression on cargo .rlib without noticeable CPU cost.
zstd = "0.13"
# v0.12 — HTTP client for the Phase 7f Gitea live-refs adapter.
# `rustls-tls` reuses the same rustls stack already pulled in by
# quinn — no additional TLS impls in the binary. `json` unlocks
# serde parsing of the /branches + /tags responses without a
# hand-rolled decoder. `default-features=false` drops native-tls
# and the openssl dep chain we don't need.
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
[dev-dependencies]
tempfile = "3"
claw-store/src/cluster.rs
+1
View File
@@ -17,6 +17,7 @@
pub mod blob;
pub mod build_cache;
pub mod client_config;
pub mod gitea;
pub mod gossip;
pub mod metrics;
pub mod prom;
claw-store/src/cluster/gitea.rs
+173
View File
@@ -0,0 +1,173 @@
//! Gitea live-refs adapter (Phase 7f).
//!
//! Thin, focused client that answers exactly one question per repo:
//! "which branches and tags are live upstream right now?" It exists
//! only to feed [`crate::cluster::ref_tracking::RefTracking::stale_at`].
//!
//! Deliberately not a full Gitea SDK. If a second consumer needs
//! Gitea in the future, extract common bits then.
use anyhow::{bail, Context, Result};
use serde::Deserialize;
use std::collections::HashSet;
use std::time::Duration;
/// Live branches + tags for one repo, from Gitea.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LiveRefs {
/// Union of `branches` + `tags`. Names are as they appear in
/// the Gitea API — no `refs/heads/` or `refs/tags/` prefix.
/// Matches the format the caller records via
/// [`crate::cluster::ref_tracking::RefTracking::record`].
pub refs: HashSet<String>,
}
/// HTTP client bound to a single Gitea instance.
#[derive(Debug, Clone)]
pub struct GiteaClient {
base_url: String,
token: Option<String>,
http: reqwest::Client,
}
impl GiteaClient {
/// Construct against `base_url` (e.g. `https://git.redclaw.dev`)
/// with an optional bearer token. Public read-only endpoints
/// work without a token; private repos need one.
pub fn new(base_url: impl Into<String>, token: Option<String>) -> Result<Self> {
let http = reqwest::Client::builder()
.user_agent("clawstor-ref-sweep/0.1")
.timeout(Duration::from_secs(15))
.build()
.context("building reqwest client")?;
let base_url = base_url.into();
// Trim trailing slash so path joining stays predictable.
let base_url = base_url.trim_end_matches('/').to_string();
Ok(Self {
base_url,
token,
http,
})
}
/// Fetch every branch + every tag for `repo` (owner/name).
/// Returns the union — the caller's ref-tracking store stores
/// them unprefixed, so this matches directly.
///
/// Both endpoints are fetched concurrently. Pagination is
/// followed (Gitea caps page size at 50; a busy repo can have
/// hundreds of branches).
pub async fn live_refs(&self, repo: &str) -> Result<LiveRefs> {
validate_repo(repo)?;
let branches_path = format!("/api/v1/repos/{repo}/branches");
let tags_path = format!("/api/v1/repos/{repo}/tags");
let (branches, tags) = tokio::try_join!(
self.paginate::<Named>(&branches_path),
self.paginate::<Named>(&tags_path),
)?;
let mut refs: HashSet<String> = HashSet::new();
for b in branches {
refs.insert(b.name);
}
for t in tags {
refs.insert(t.name);
}
Ok(LiveRefs { refs })
}
async fn paginate<T: for<'de> Deserialize<'de>>(
&self,
path: &str,
) -> Result<Vec<T>> {
let mut out = Vec::new();
let mut page = 1u32;
// Small hard cap so a runaway server response can't lock
// us into an infinite loop.
const PAGE_LIMIT: u32 = 200;
loop {
if page > PAGE_LIMIT {
bail!(
"aborting after {} pages of {}; server may be misbehaving",
PAGE_LIMIT,
path
);
}
let url = format!(
"{}{}?limit=50&page={}",
self.base_url, path, page
);
let mut req = self.http.get(&url);
if let Some(tok) = &self.token {
req = req.header("Authorization", format!("token {tok}"));
}
let resp = req.send().await.with_context(|| format!("GET {url}"))?;
let status = resp.status();
if status == reqwest::StatusCode::NOT_FOUND {
// Deleted repo, or private + no token. Caller
// treats "repo missing from live-refs map" as
// "all refs dead", so surface the fact via an
// empty Vec + an early return.
return Ok(out);
}
if !status.is_success() {
bail!("GET {} returned {}", url, status);
}
let batch: Vec<T> = resp
.json()
.await
.with_context(|| format!("parsing JSON from {url}"))?;
let n = batch.len();
out.extend(batch);
if n < 50 {
// Short page = last page.
return Ok(out);
}
page += 1;
}
}
}
fn validate_repo(repo: &str) -> Result<()> {
if repo.is_empty() {
bail!("repo cannot be empty");
}
if !repo.contains('/') {
bail!("repo must be `owner/name`, got {:?}", repo);
}
if repo.contains("..") || repo.contains(' ') {
bail!("repo has forbidden characters: {:?}", repo);
}
Ok(())
}
/// Both `/branches` and `/tags` return objects with (at least) a
/// `name` field. Anything else in the payload is discarded — we
/// only need names for the stale-ref match.
#[derive(Debug, Deserialize)]
struct Named {
name: String,
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn validate_repo_shape() {
assert!(validate_repo("").is_err());
assert!(validate_repo("no-slash").is_err());
assert!(validate_repo("has spaces/bad").is_err());
assert!(validate_repo("../etc/passwd").is_err());
assert!(validate_repo("owner/name").is_ok());
assert!(validate_repo("clawverse/clawstor").is_ok());
}
#[tokio::test]
async fn client_builds_and_trims_trailing_slash() {
let c = GiteaClient::new("https://git.example/", None).unwrap();
assert_eq!(c.base_url, "https://git.example");
let c2 = GiteaClient::new("https://git.example", Some("t".into())).unwrap();
assert_eq!(c2.base_url, "https://git.example");
assert_eq!(c2.token.as_deref(), Some("t"));
}
}
claw-store/src/main.rs
+131
View File
@@ -172,6 +172,25 @@ enum Cmd {
#[arg(long)]
evict_to_gb: Option<u64>,
},
/// Phase 7f (2026-07-14): identify cache fingerprints whose
/// producing git refs are all gone upstream AND whose last-seen
/// age exceeds the retention window. Dry-run only in this cut —
/// prints the stale fingerprints grouped by repo. Deletion is a
/// separate operator step.
ClusterRefSweep {
/// Gitea base URL, e.g. https://git.redclaw.dev
#[arg(long)]
gitea_url: String,
/// Bearer token for private repos. Public read-only repos
/// work without one; typically supplied via env not flag.
#[arg(long, env = "GITEA_TOKEN")]
gitea_token: Option<String>,
/// Minimum age (in days) before a dead-ref fingerprint is
/// considered stale. Protects fresh CI builds from being
/// reaped before someone can rebuild against them.
#[arg(long, default_value_t = 14)]
retention_days: u64,
},
/// Phase 7d (2026-07-14): take a point-in-time snapshot of every
/// blob currently in the local store. Snapshots are cheap
/// reference sets (no data copy). Combine with pin-aware LRU
@@ -291,6 +310,11 @@ async fn main() -> Result<()> {
tls_dir,
dry_run,
} => cmd_cluster_repair(&cfg, &peer, rpc_addr, &tls_dir, dry_run).await?,
Cmd::ClusterRefSweep {
gitea_url,
gitea_token,
retention_days,
} => cmd_cluster_ref_sweep(&cfg, &gitea_url, gitea_token, retention_days).await?,
Cmd::ClusterSnapshotCreate { name } => cmd_cluster_snapshot_create(&cfg, &name).await?,
Cmd::ClusterSnapshotList => cmd_cluster_snapshot_list(&cfg).await?,
Cmd::ClusterSnapshotShow { name } => cmd_cluster_snapshot_show(&cfg, &name).await?,
@@ -677,6 +701,113 @@ async fn cmd_cluster_snapshot_delete(cfg: &Config, name: &str) -> Result<()> {
Ok(())
}
async fn cmd_cluster_ref_sweep(
cfg: &Config,
gitea_url: &str,
gitea_token: Option<String>,
retention_days: u64,
) -> Result<()> {
use cluster::gitea::GiteaClient;
use cluster::ref_tracking::RefTracking;
let root = cfg
.cluster
.as_ref()
.and_then(|c| c.blob_store_root.clone())
.context("cluster.blob_store_root not configured; nothing to sweep")?;
let rt = RefTracking::open(root.clone())
.with_context(|| format!("opening ref-tracking at {}", root.display()))?;
let started = std::time::Instant::now();
let all = rt.list_all().await.context("listing ref-tracking entries")?;
// Group repos to minimize Gitea calls.
let mut repos = std::collections::BTreeSet::new();
for e in &all {
repos.insert(e.repo.clone());
}
println!("── clawstor cluster-ref-sweep ──────────────────────");
println!("gitea: {}", gitea_url);
println!("retention days: {}", retention_days);
println!("tracked fps: {}", all.len());
println!("distinct repos: {}", repos.len());
if all.is_empty() {
println!("nothing to sweep.");
println!("total elapsed: {:?}", started.elapsed());
println!("────────────────────────────────────────────────────");
return Ok(());
}
let client = GiteaClient::new(gitea_url, gitea_token)
.context("building Gitea client")?;
let mut live: std::collections::HashMap<String, std::collections::HashSet<String>> =
std::collections::HashMap::new();
let mut missing_repos: Vec<String> = Vec::new();
for repo in &repos {
match client.live_refs(repo).await {
Ok(lr) => {
if lr.refs.is_empty() {
missing_repos.push(repo.clone());
} else {
live.insert(repo.clone(), lr.refs);
}
}
Err(e) => {
eprintln!("warn: gitea live_refs({repo}) failed: {e}");
// Leave repo out of `live` — stale_at treats missing
// as all-dead, which is the safer default for a
// repo we couldn't query.
}
}
}
println!(
"live refs fetched: {} repos ({} appeared empty/deleted)",
live.len(),
missing_repos.len()
);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let retention_secs = retention_days.saturating_mul(24 * 3600);
let stale = rt
.stale_at(now, &live, retention_secs)
.await
.context("computing stale set")?;
// Group stale fps by repo for readable output.
let mut by_repo: std::collections::BTreeMap<String, Vec<[u8; 32]>> =
std::collections::BTreeMap::new();
let stale_set: std::collections::HashSet<_> = stale.iter().collect();
for e in &all {
if stale_set.contains(&e.fingerprint) {
by_repo.entry(e.repo.clone()).or_default().push(e.fingerprint);
}
}
println!("stale fps: {}", stale.len());
println!();
if !by_repo.is_empty() {
for (repo, fps) in &by_repo {
println!(" {} — {} fp(s):", repo, fps.len());
for fp in fps {
let mut hex = String::with_capacity(64);
for b in fp {
hex.push_str(&format!("{b:02x}"));
}
println!(" {}", hex);
}
}
}
println!();
println!("dry-run: no records modified. Use `forget` per fp to prune records");
println!("(blob eviction on next cluster-gc handles the actual disk reclaim).");
println!("total elapsed: {:?}", started.elapsed());
println!("────────────────────────────────────────────────────");
Ok(())
}
async fn cmd_cluster_peer_status(
peer: &str,
rpc_addr: SocketAddr,