Phase 7f follow-on: Gitea live-refs adapter + cluster-ref-sweep CLI
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Failing after 6s

Wires the Phase 7f ref-tracking primitives to a real Gitea. New
CLI `claw-store cluster-ref-sweep --gitea-url <> [--gitea-token]
[--retention-days N]` queries every distinct repo we've recorded
against, fetches its live branches + tags, computes the stale set
via RefTracking::stale_at, and prints the stale fingerprints
grouped by repo.

Dry-run only in this cut. Deletion is separate — the operator
decides whether to call `forget` per fp, and whether to also
prune the corresponding blob/tag. Blob eviction happens via
cluster-gc as usual (dead refs no longer contribute to any pin).

New module cluster::gitea:
* GiteaClient::new(base_url, token) — reqwest with 15s timeout,
  rustls-tls (reuses the rustls stack quinn already pulls in).
* live_refs(repo) — fetches /branches + /tags concurrently,
  paginated (page 200 hard cap for safety), returns HashSet.
* 404 on either endpoint returns empty set — deleted repos then
  flow through stale_at as "all refs dead", the correct default.

Deps:
* reqwest 0.12 with rustls-tls + json, default-features off (no
  native-tls / openssl chain).
* clap 4 + "env" feature so --gitea-token can read GITEA_TOKEN.

+2 tests (validate_repo shape, client trims trailing slash).
Full test suite: 367 pass (+2). Pre-existing macOS
hot::tests::test_project_target_size_bytes failure unchanged.
This commit is contained in:
Omar Sobh
2026-07-14 10:58:27 -07:00
parent 294697d2f5
commit 7bc5ba987c
5 changed files with 694 additions and 2 deletions
+173
View File
@@ -0,0 +1,173 @@
//! Gitea live-refs adapter (Phase 7f).
//!
//! Thin, focused client that answers exactly one question per repo:
//! "which branches and tags are live upstream right now?" It exists
//! only to feed [`crate::cluster::ref_tracking::RefTracking::stale_at`].
//!
//! Deliberately not a full Gitea SDK. If a second consumer needs
//! Gitea in the future, extract common bits then.
use anyhow::{bail, Context, Result};
use serde::Deserialize;
use std::collections::HashSet;
use std::time::Duration;
/// Live branches + tags for one repo, from Gitea.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LiveRefs {
/// Union of `branches` + `tags`. Names are as they appear in
/// the Gitea API — no `refs/heads/` or `refs/tags/` prefix.
/// Matches the format the caller records via
/// [`crate::cluster::ref_tracking::RefTracking::record`].
pub refs: HashSet<String>,
}
/// HTTP client bound to a single Gitea instance.
#[derive(Debug, Clone)]
pub struct GiteaClient {
base_url: String,
token: Option<String>,
http: reqwest::Client,
}
impl GiteaClient {
/// Construct against `base_url` (e.g. `https://git.redclaw.dev`)
/// with an optional bearer token. Public read-only endpoints
/// work without a token; private repos need one.
pub fn new(base_url: impl Into<String>, token: Option<String>) -> Result<Self> {
let http = reqwest::Client::builder()
.user_agent("clawstor-ref-sweep/0.1")
.timeout(Duration::from_secs(15))
.build()
.context("building reqwest client")?;
let base_url = base_url.into();
// Trim trailing slash so path joining stays predictable.
let base_url = base_url.trim_end_matches('/').to_string();
Ok(Self {
base_url,
token,
http,
})
}
/// Fetch every branch + every tag for `repo` (owner/name).
/// Returns the union — the caller's ref-tracking store stores
/// them unprefixed, so this matches directly.
///
/// Both endpoints are fetched concurrently. Pagination is
/// followed (Gitea caps page size at 50; a busy repo can have
/// hundreds of branches).
pub async fn live_refs(&self, repo: &str) -> Result<LiveRefs> {
validate_repo(repo)?;
let branches_path = format!("/api/v1/repos/{repo}/branches");
let tags_path = format!("/api/v1/repos/{repo}/tags");
let (branches, tags) = tokio::try_join!(
self.paginate::<Named>(&branches_path),
self.paginate::<Named>(&tags_path),
)?;
let mut refs: HashSet<String> = HashSet::new();
for b in branches {
refs.insert(b.name);
}
for t in tags {
refs.insert(t.name);
}
Ok(LiveRefs { refs })
}
async fn paginate<T: for<'de> Deserialize<'de>>(
&self,
path: &str,
) -> Result<Vec<T>> {
let mut out = Vec::new();
let mut page = 1u32;
// Small hard cap so a runaway server response can't lock
// us into an infinite loop.
const PAGE_LIMIT: u32 = 200;
loop {
if page > PAGE_LIMIT {
bail!(
"aborting after {} pages of {}; server may be misbehaving",
PAGE_LIMIT,
path
);
}
let url = format!(
"{}{}?limit=50&page={}",
self.base_url, path, page
);
let mut req = self.http.get(&url);
if let Some(tok) = &self.token {
req = req.header("Authorization", format!("token {tok}"));
}
let resp = req.send().await.with_context(|| format!("GET {url}"))?;
let status = resp.status();
if status == reqwest::StatusCode::NOT_FOUND {
// Deleted repo, or private + no token. Caller
// treats "repo missing from live-refs map" as
// "all refs dead", so surface the fact via an
// empty Vec + an early return.
return Ok(out);
}
if !status.is_success() {
bail!("GET {} returned {}", url, status);
}
let batch: Vec<T> = resp
.json()
.await
.with_context(|| format!("parsing JSON from {url}"))?;
let n = batch.len();
out.extend(batch);
if n < 50 {
// Short page = last page.
return Ok(out);
}
page += 1;
}
}
}
fn validate_repo(repo: &str) -> Result<()> {
if repo.is_empty() {
bail!("repo cannot be empty");
}
if !repo.contains('/') {
bail!("repo must be `owner/name`, got {:?}", repo);
}
if repo.contains("..") || repo.contains(' ') {
bail!("repo has forbidden characters: {:?}", repo);
}
Ok(())
}
/// Both `/branches` and `/tags` return objects with (at least) a
/// `name` field. Anything else in the payload is discarded — we
/// only need names for the stale-ref match.
#[derive(Debug, Deserialize)]
struct Named {
name: String,
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn validate_repo_shape() {
assert!(validate_repo("").is_err());
assert!(validate_repo("no-slash").is_err());
assert!(validate_repo("has spaces/bad").is_err());
assert!(validate_repo("../etc/passwd").is_err());
assert!(validate_repo("owner/name").is_ok());
assert!(validate_repo("clawverse/clawstor").is_ok());
}
#[tokio::test]
async fn client_builds_and_trims_trailing_slash() {
let c = GiteaClient::new("https://git.example/", None).unwrap();
assert_eq!(c.base_url, "https://git.example");
let c2 = GiteaClient::new("https://git.example", Some("t".into())).unwrap();
assert_eq!(c2.base_url, "https://git.example");
assert_eq!(c2.token.as_deref(), Some("t"));
}
}