//! dashboard-v2 aggregator backend — additive `/api/v2/*` handlers. //! //! Fleet-single-pane-of-glass model: `claw-store serve` runs on the //! operator's laptop (typically). It holds a fleet-CA leaf cert + //! the peer list, connects to each daemon's cluster RPC port over //! mTLS, and issues `DashboardStatus` RPCs. Peers don't need to //! run any HTTP server of their own — RPC over the existing QUIC //! socket is enough. //! //! Config: peers come from `[[cluster.peers]]` in the aggregator's //! `config.toml`. TLS material via `[cluster.tls]`. //! //! Design doc: `docs/dashboard-v2.md`. use axum::{ extract::{Path, State}, http::StatusCode, routing::{get, post}, Json, Router, }; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; use tokio::task::JoinSet; use crate::cluster::rpc::{ call_delete_tag, call_put_tag, call_dashboard_status, call_dashboard_storage, CacheSummary, DashboardStatusReply, DashboardStorageReply, FilesystemUsage, HotTierUsage, MountStatus, TimerStatus, }; use crate::cluster::transport::{NodeIdentity, QuicClient}; use crate::config::{Config, PeerEntry, TokenEntry}; use crate::sessions::{LeasedTag, Session, SessionStore}; /// Aggregator runtime: one QuicClient, one peer list, one identity. /// /// The client is reused across every RPC (quinn holds one UDP /// endpoint + a pool of connections). Peer addresses are resolved /// per-request so a hot-swapped config doesn't require restart. pub struct V2State { pub aggregator_name: String, pub peers: Vec, /// QuicClient wraps a single UDP endpoint + a connection pool. /// Arc so the storage handler's per-peer JoinSet can hand the /// client to spawned tasks without re-creating the endpoint. pub client: std::sync::Arc, /// Fallback RPC port when a peer entry has `lan_addr = "10.x:7701"` /// (gossip port, not RPC). We assume RPC = gossip + 1 in that /// case unless the operator sets an explicit `rpc_addr`. pub default_rpc_port_offset: u16, /// Bearer token required on mutating endpoints (POST/DELETE). /// GET stays open — the dashboard loads without credentials. /// Sourced from the aggregator's own `config.toml` `api_token`. /// `None` disables the check (only appropriate on a trusted LAN, /// e.g. Tailscale-only + loopback bind). /// /// Treated as an **admin** token — no namespace constraint. For /// multi-tenant use, prefer the per-app tokens in `token_entries`. pub api_token: Option, /// Per-app tokens with optional namespace scoping (Phase 9 F4). /// Merged with `api_token`: `api_token` is admin (no namespace); /// entries here can be admin (no `namespace`) or scoped (must /// match `:*` on tag names). First match wins. pub token_entries: Vec, /// TTL-scoped session store (Phase 9 S1-S3). Sessions own tag /// leases and are reaped by a background sweeper when their /// `expires_at_unix` passes without a `renew` or `commit`. pub sessions: SessionStore, } impl V2State { /// Build an aggregator state from the aggregator's own config. /// Requires `[cluster]` + `[cluster.tls]` + at least one /// `[[cluster.peers]]`. The aggregator doesn't need a /// blob_store_root itself. pub fn from_config(cfg: &Config) -> anyhow::Result { let cluster = cfg .cluster .as_ref() .ok_or_else(|| anyhow::anyhow!("dashboard-v2 aggregator needs [cluster] config"))?; let identity = NodeIdentity::from_cluster_config(cluster) .map_err(|e| anyhow::anyhow!("loading fleet-CA identity: {e}"))?; let client = QuicClient::new("0.0.0.0:0".parse()?, identity) .map_err(|e| anyhow::anyhow!("building QUIC client: {e}"))?; // Session store persists next to warm.projects_path — the // aggregator already has a writable directory there. Fall // back to /tmp so tests don't need config.warm set. let sessions_path = std::path::PathBuf::from(&cfg.warm.projects_path) .join("aggregator-sessions.json"); let sessions = SessionStore::load(sessions_path) .map_err(|e| anyhow::anyhow!("loading session store: {e}"))?; // Bug fix 2026-07-31: the fleet view previously never included // the node actually serving the dashboard — `cluster.peers` is // by definition every *other* node, so hitting a given node's // `/api/v2/fleet` directly silently dropped that node from its // own view (looked like "node X is missing" from the UI, even // though X was perfectly healthy — it just never queried // itself). Fix: synthesize a self `PeerEntry` from our own // gossip bind address and include it in the fan-out list, same // as any other peer. `peer_rpc_addr` derives the RPC port from // `lan_addr`/`tailscale_addr` via the fleet's +1 convention, so // this resolves to the same `bind_rpc_lan`/`bind_rpc_tailscale` // the daemon actually listens on. let self_peer = PeerEntry { name: cfg.node.name.clone(), zone: cluster.zone.clone(), lan_addr: cluster.bind_lan, tailscale_addr: cluster.bind_tailscale, }; let mut peers = cluster.peers.clone(); peers.push(self_peer); Ok(Self { aggregator_name: cfg.node.name.clone(), peers, client: std::sync::Arc::new(client), default_rpc_port_offset: 1, api_token: cfg.api_token.clone(), token_entries: cfg .aggregator .as_ref() .map(|a| a.tokens.clone()) .unwrap_or_default(), sessions, }) } /// Resolve a peer name → RPC socket. Prefers `rpc_addr` when the /// peer entry sets one; otherwise takes `lan_addr` and adds the /// port offset (gossip → RPC = +1 by fleet convention). fn peer_rpc_addr(&self, peer: &PeerEntry) -> Option { // Prefer LAN (fast), fall back to tailnet. RPC port = // gossip port + 1 by fleet convention (see fleet configs). if let Some(lan) = peer.lan_addr { return Some(SocketAddr::new( lan.ip(), lan.port() + self.default_rpc_port_offset, )); } if let Some(ts) = peer.tailscale_addr { return Some(SocketAddr::new( ts.ip(), ts.port() + self.default_rpc_port_offset, )); } None } /// Fetch a single node's dashboard payload via RPC. async fn fetch_node(&self, peer: &PeerEntry) -> anyhow::Result { let conn = self.dial(peer).await?; let reply = call_dashboard_status(&conn).await?; conn.close(quinn::VarInt::from_u32(0), b"done"); Ok(reply) } /// Fetch a single node's storage inventory via RPC. async fn fetch_storage(&self, peer: &PeerEntry) -> anyhow::Result { let conn = self.dial(peer).await?; let reply = call_dashboard_storage(&conn).await?; conn.close(quinn::VarInt::from_u32(0), b"done"); Ok(reply) } async fn dial(&self, peer: &PeerEntry) -> anyhow::Result { let addr = self.peer_rpc_addr(peer).ok_or_else(|| { anyhow::anyhow!("peer {} has no reachable RPC address", peer.name) })?; let conn = tokio::time::timeout( Duration::from_secs(5), self.client.connect(addr, &peer.name), ) .await .map_err(|_| anyhow::anyhow!("connect to {} timed out", peer.name))??; Ok(conn) } } // ── response shapes ────────────────────────────────────────────── #[derive(Serialize, Deserialize)] pub struct NodeStatusV2 { pub node_name: String, pub zone: String, pub blob_store_root: Option, pub blob_count: usize, pub tag_count: usize, pub ref_count: usize, pub snapshot_count: usize, pub ref_tracking_count: usize, pub blob_store_bytes: u64, pub rustc_release: Option, #[serde(default)] pub filesystem: Option, #[serde(default)] pub hot: Option, #[serde(default)] pub mount: Option, #[serde(default)] pub cache: Option, #[serde(default)] pub timers: Vec, /// `true` when the aggregator successfully talked to the peer; /// `false` when the RPC failed. Frontend uses this to badge the /// card as offline. pub online: bool, /// Human-readable error when `online = false`. pub error: Option, } impl NodeStatusV2 { fn ok(name: &str, r: DashboardStatusReply) -> Self { let mut n = Self { node_name: r.node_name, zone: r.zone, blob_store_root: r.blob_store_root, blob_count: r.blob_count, tag_count: r.tag_count, ref_count: r.ref_count, snapshot_count: r.snapshot_count, ref_tracking_count: r.ref_tracking_count, blob_store_bytes: r.blob_store_bytes, rustc_release: r.rustc_release, filesystem: r.filesystem, hot: r.hot, mount: r.mount, cache: r.cache, timers: r.timers, online: true, error: None, }; if n.node_name.is_empty() { n.node_name = name.to_string(); } n } fn err(name: &str, e: String) -> Self { Self { node_name: name.to_string(), zone: String::new(), blob_store_root: None, blob_count: 0, tag_count: 0, ref_count: 0, snapshot_count: 0, ref_tracking_count: 0, blob_store_bytes: 0, rustc_release: None, filesystem: None, hot: None, mount: None, cache: None, timers: Vec::new(), online: false, error: Some(e), } } } #[derive(Serialize)] pub struct FleetSnapshot { pub aggregator_name: String, pub fetched_at_unix: u64, pub nodes: Vec, } /// One tag with the node it came from. Frontend renders `node` as a /// column so operators see where each artifact lives. #[derive(Serialize)] pub struct TagRow { pub node: String, pub key: String, pub value_hex: String, } #[derive(Serialize)] pub struct SnapshotRow { pub node: String, pub name: String, pub created_at_unix: u64, pub blob_count: usize, pub file_bytes: u64, } #[derive(Serialize)] pub struct RefTrackingRow { pub node: String, pub fingerprint_hex: String, pub repo: String, pub refs: Vec, pub first_seen_unix: u64, pub last_seen_unix: u64, } #[derive(Serialize)] pub struct BlobRow { pub node: String, pub blob_id_hex: String, pub size_bytes: u64, pub chunk_count: usize, } #[derive(Serialize)] pub struct RefRow { pub node: String, pub fingerprint_hex: String, pub blob_id_hex: String, } #[derive(Serialize)] pub struct ProjectRow { pub node: String, pub repo: String, pub cache_bytes: u64, pub fingerprint_count: usize, pub refs: Vec, pub first_seen_unix: u64, pub last_seen_unix: u64, pub tier: String, } // ── handlers ───────────────────────────────────────────────────── async fn handle_fleet(State(s): State>) -> Json { // Fan out concurrently. // Fan out with a JoinSet — bounded concurrency, no futures crate. let mut set = tokio::task::JoinSet::new(); for p in &s.peers { let name = p.name.clone(); let state = s.clone(); let peer = p.clone(); set.spawn(async move { match state.fetch_node(&peer).await { Ok(r) => NodeStatusV2::ok(&name, r), Err(e) => NodeStatusV2::err(&name, format!("{e:#}")), } }); } let mut nodes: Vec = Vec::new(); while let Some(res) = set.join_next().await { if let Ok(n) = res { nodes.push(n); } } // Deterministic ordering — by name — so the UI doesn't shuffle // between polls. nodes.sort_by(|a, b| a.node_name.cmp(&b.node_name)); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); Json(FleetSnapshot { aggregator_name: s.aggregator_name.clone(), fetched_at_unix: now, nodes, }) } async fn handle_node_status( State(s): State>, Path(name): Path, ) -> Json { let peer = s.peers.iter().find(|p| p.name == name).cloned(); let peer = match peer { Some(p) => p, None => { return Json(NodeStatusV2::err( &name, format!("no peer named {name} in aggregator config"), )) } }; let node = match s.fetch_node(&peer).await { Ok(r) => NodeStatusV2::ok(&name, r), Err(e) => NodeStatusV2::err(&name, format!("{e:#}")), }; Json(node) } // ── storage handlers (fleet-aggregated) ────────────────────────── /// Fan out DashboardStorage to every peer, return `(node, reply)` /// pairs where the RPC succeeded. Failures logged but skipped /// silently — a browser view of "N nodes, one is down" is more /// useful than a hard 500. async fn gather_storage(s: &V2State) -> Vec<(String, DashboardStorageReply)> { let mut set = tokio::task::JoinSet::new(); for p in &s.peers { let name = p.name.clone(); let peer = p.clone(); let state = Arc::new(s.clone_shallow()); set.spawn(async move { match state.fetch_storage(&peer).await { Ok(r) => Some((name, r)), Err(e) => { tracing::debug!(peer = %name, error = %e, "DashboardStorage RPC failed"); None } } }); } let mut out = Vec::new(); while let Some(res) = set.join_next().await { if let Ok(Some(pair)) = res { out.push(pair); } } out } async fn handle_tags(State(s): State>) -> Json> { let mut rows = Vec::new(); for (node, r) in gather_storage(&s).await { for t in r.tags { rows.push(TagRow { node: node.clone(), key: t.key, value_hex: t.value_hex, }); } } rows.sort_by(|a, b| a.key.cmp(&b.key).then_with(|| a.node.cmp(&b.node))); Json(rows) } async fn handle_snapshots(State(s): State>) -> Json> { let mut rows = Vec::new(); for (node, r) in gather_storage(&s).await { for snap in r.snapshots { rows.push(SnapshotRow { node: node.clone(), name: snap.name, created_at_unix: snap.created_at_unix, blob_count: snap.blob_count, file_bytes: snap.file_bytes, }); } } // Oldest first — matches CLI convention. rows.sort_by_key(|s| s.created_at_unix); Json(rows) } async fn handle_ref_tracking(State(s): State>) -> Json> { let mut rows = Vec::new(); for (node, r) in gather_storage(&s).await { for e in r.ref_tracking { rows.push(RefTrackingRow { node: node.clone(), fingerprint_hex: e.fingerprint_hex, repo: e.repo, refs: e.refs, first_seen_unix: e.first_seen_unix, last_seen_unix: e.last_seen_unix, }); } } rows.sort_by(|a, b| a.repo.cmp(&b.repo).then_with(|| a.node.cmp(&b.node))); Json(rows) } async fn handle_blobs(State(s): State>) -> Json> { let mut rows = Vec::new(); for (node, r) in gather_storage(&s).await { for b in r.blobs_sample { rows.push(BlobRow { node: node.clone(), blob_id_hex: b.blob_id_hex, size_bytes: b.size_bytes, chunk_count: b.chunk_count, }); } } rows.sort_by(|a, b| a.blob_id_hex.cmp(&b.blob_id_hex)); Json(rows) } async fn handle_projects(State(s): State>) -> Json> { let mut rows = Vec::new(); for (node, r) in gather_storage(&s).await { for p in r.projects { rows.push(ProjectRow { node: node.clone(), repo: p.repo, cache_bytes: p.cache_bytes, fingerprint_count: p.fingerprint_count, refs: p.refs, first_seen_unix: p.first_seen_unix, last_seen_unix: p.last_seen_unix, tier: p.tier, }); } } // Hottest first, then by node so ties from the same repo // group visually. rows.sort_by(|a, b| { b.last_seen_unix .cmp(&a.last_seen_unix) .then_with(|| a.repo.cmp(&b.repo)) }); Json(rows) } async fn handle_refs(State(s): State>) -> Json> { let mut rows = Vec::new(); for (node, r) in gather_storage(&s).await { for x in r.refs_sample { rows.push(RefRow { node: node.clone(), fingerprint_hex: x.fingerprint_hex, blob_id_hex: x.blob_id_hex, }); } } rows.sort_by(|a, b| a.fingerprint_hex.cmp(&b.fingerprint_hex)); Json(rows) } impl V2State { /// Cheap clone for JoinSet handoff — reuses the underlying /// QuicClient (Arc'd inside) + clones the small peer list. fn clone_shallow(&self) -> Self { Self { aggregator_name: self.aggregator_name.clone(), peers: self.peers.clone(), client: self.client.clone(), default_rpc_port_offset: self.default_rpc_port_offset, api_token: self.api_token.clone(), token_entries: self.token_entries.clone(), sessions: self.sessions.clone(), } } } /// The identity attached to an authenticated request. Either an admin /// (unrestricted) or a per-app token bound to a `namespace` prefix. /// Handlers use this to decide whether a tag name is in-scope. #[derive(Debug, Clone)] pub enum AuthedCaller { /// Any tag name allowed. Sourced from either `api_token` (legacy /// admin) or a `[[aggregator.tokens]]` entry with no namespace. Admin, /// Restricted to tags whose name is `:`. Namespaced { namespace: String }, /// No auth was configured on the aggregator at all. Trusted-LAN /// mode — everything allowed, no bearer required. Open, } impl AuthedCaller { /// Returns `true` iff this caller may pin/unpin/list a tag with /// this name. Admin + Open allow anything; namespaced callers /// require an exact `:` prefix (with the separator, /// so `workspace:42` cannot silently reach `workspace:420:*`). pub fn may_touch_tag(&self, tag: &str) -> bool { match self { AuthedCaller::Admin | AuthedCaller::Open => true, AuthedCaller::Namespaced { namespace } => { let prefix = format!("{namespace}:"); tag.starts_with(&prefix) } } } /// Phase 9 R1b: returns the workspace this caller may act on for /// repo-ensure operations. Namespaced callers are pinned to their /// own namespace; admin/open callers must provide one explicitly /// in the request body. Explicit user-supplied workspace is only /// honored for admin/open — a namespaced caller supplying a /// mismatched workspace is a forbidden write. /// Gate for fleet-infrastructure actions (shutdown-prep) that /// have nothing to do with a tag/repo namespace — a namespaced /// per-app token has no business stopping a node's services. pub fn require_admin(&self) -> Result<(), (StatusCode, String)> { match self { AuthedCaller::Admin | AuthedCaller::Open => Ok(()), AuthedCaller::Namespaced { .. } => Err(( StatusCode::FORBIDDEN, "this action requires an admin token".to_string(), )), } } pub fn resolve_workspace(&self, requested: Option<&str>) -> Result { match self { AuthedCaller::Admin | AuthedCaller::Open => match requested { Some(w) if !w.is_empty() => Ok(w.to_string()), _ => Err(( StatusCode::BAD_REQUEST, "workspace is required for unnamespaced callers".to_string(), )), }, AuthedCaller::Namespaced { namespace } => match requested { None => Ok(namespace.clone()), Some(w) if w == namespace => Ok(namespace.clone()), Some(w) => Err(( StatusCode::FORBIDDEN, format!( "workspace \"{w}\" is outside your namespace \"{namespace}\"" ), )), }, } } } // ── write-through fan-out (Phase 9 F1) ────────────────────────── // // The aggregator is the single choke-point for cross-fleet writes, // so external callers (clawmates, gitea runners, ops tooling) speak // one URL over ordinary HTTPS instead of minting a fleet-CA leaf // cert per app. The aggregator holds a leaf cert and dials every // peer over the existing mTLS QUIC transport. fn decode_blob_id(hex: &str) -> Option<[u8; 32]> { if hex.len() != 64 { return None; } let mut out = [0u8; 32]; for i in 0..32 { let byte = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).ok()?; out[i] = byte; } Some(out) } #[derive(Deserialize)] pub struct PutTagBody { /// 64-char hex-encoded BlobId (blake3 hash) to bind the tag to. pub blob_id: String, } #[derive(Serialize)] pub struct PeerResult { pub peer: String, pub ok: bool, pub error: Option, } #[derive(Serialize)] pub struct FanoutReply { pub tag: String, pub peers: Vec, /// Aggregate: `true` iff every peer succeeded. pub all_ok: bool, } /// `POST /api/v2/tags/:name` — bind a tag to a blob_id on **every** /// peer concurrently. Callers get a per-peer result so partial /// failures are diagnosable (single peer down doesn't hide the /// success of the others). async fn handle_put_tag( Path(name): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { check_tag_scope(&caller, &name)?; let value = decode_blob_id(&body.blob_id) .ok_or_else(|| (StatusCode::BAD_REQUEST, "blob_id must be 64-char hex".to_string()))?; let results = fanout_put_tag(&s, &name, &value).await; let all_ok = results.iter().all(|r| r.ok); Ok(Json(FanoutReply { tag: name, peers: results, all_ok, })) } /// `DELETE /api/v2/tags/:name` — remove a tag on every peer. /// Per-peer `ok=true` covers both "deleted" and "was already /// absent" (the underlying RPC returns `Ok(false)` for the latter; /// we still surface success — reversing a nonexistent tag is a /// no-op, not an error). async fn handle_delete_tag( Path(name): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, ) -> Result, (StatusCode, String)> { check_tag_scope(&caller, &name)?; let results = fanout_delete_tag(&s, &name).await; let all_ok = results.iter().all(|r| r.ok); Ok(Json(FanoutReply { tag: name, peers: results, all_ok, })) } /// Combined name-shape + namespace check. Empty name → 400; out-of- /// namespace → 403 with a clear message so multi-tenant callers can /// tell "you can't touch that" from "your token was bogus" (401). fn check_tag_scope(caller: &AuthedCaller, name: &str) -> Result<(), (StatusCode, String)> { if name.is_empty() { return Err((StatusCode::BAD_REQUEST, "tag name cannot be empty".into())); } if !caller.may_touch_tag(name) { let ns = match caller { AuthedCaller::Namespaced { namespace } => namespace.as_str(), _ => "(none)", }; return Err(( StatusCode::FORBIDDEN, format!("tag \"{name}\" is outside your namespace \"{ns}:\""), )); } Ok(()) } async fn fanout_put_tag(s: &V2State, name: &str, value: &[u8; 32]) -> Vec { let mut set = JoinSet::new(); for peer in &s.peers { let peer = peer.clone(); let name = name.to_string(); let value = *value; let s = s.clone_shallow(); set.spawn(async move { let peer_name = peer.name.clone(); let res = async { let conn = s.dial(&peer).await?; let r = call_put_tag(&conn, &name, &value).await; conn.close(quinn::VarInt::from_u32(0), b"done"); r } .await; match res { Ok(()) => PeerResult { peer: peer_name, ok: true, error: None }, Err(e) => PeerResult { peer: peer_name, ok: false, error: Some(e.to_string()) }, } }); } let mut out = Vec::new(); while let Some(joined) = set.join_next().await { match joined { Ok(r) => out.push(r), Err(e) => out.push(PeerResult { peer: "".into(), ok: false, error: Some(e.to_string()), }), } } // Deterministic ordering makes the response diff-friendly and // lets integration tests assert on shape without sorting. out.sort_by(|a, b| a.peer.cmp(&b.peer)); out } async fn fanout_delete_tag(s: &V2State, name: &str) -> Vec { let mut set = JoinSet::new(); for peer in &s.peers { let peer = peer.clone(); let name = name.to_string(); let s = s.clone_shallow(); set.spawn(async move { let peer_name = peer.name.clone(); let res = async { let conn = s.dial(&peer).await?; // Discard the bool — "was already gone" is not an error // at the fan-out layer (reversing an absent tag = no-op). let _ = call_delete_tag(&conn, &name).await?; conn.close(quinn::VarInt::from_u32(0), b"done"); Ok::<(), anyhow::Error>(()) } .await; match res { Ok(()) => PeerResult { peer: peer_name, ok: true, error: None }, Err(e) => PeerResult { peer: peer_name, ok: false, error: Some(e.to_string()) }, } }); } let mut out = Vec::new(); while let Some(joined) = set.join_next().await { match joined { Ok(r) => out.push(r), Err(e) => out.push(PeerResult { peer: "".into(), ok: false, error: Some(e.to_string()), }), } } out.sort_by(|a, b| a.peer.cmp(&b.peer)); out } // ── Phase 9 R1b: repo-ensure fan-out ──────────────────────────── #[derive(Deserialize)] pub struct RepoEnsureBody { pub url: String, pub git_ref: String, /// Optional for namespaced tokens (server pins to the caller's /// namespace); required for admin/open callers. #[serde(default)] pub workspace: Option, } #[derive(Serialize)] pub struct RepoPeerResult { pub peer: String, pub ok: bool, #[serde(skip_serializing_if = "Option::is_none")] pub path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub head_sha: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cached: Option, #[serde(skip_serializing_if = "Option::is_none")] pub removed: Option, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, } #[derive(Serialize)] pub struct RepoFanoutReply { pub url: String, pub git_ref: String, pub workspace: String, pub peers: Vec, pub all_ok: bool, } async fn handle_repos_ensure( State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { if body.url.trim().is_empty() { return Err((StatusCode::BAD_REQUEST, "url cannot be empty".into())); } if body.git_ref.trim().is_empty() { return Err((StatusCode::BAD_REQUEST, "git_ref cannot be empty".into())); } let workspace = caller.resolve_workspace(body.workspace.as_deref())?; let req = crate::cluster::repo_ensure::RepoEnsureRequest { url: body.url.clone(), git_ref: body.git_ref.clone(), workspace: workspace.clone(), }; let results = fanout_repo_ensure(&s, &req).await; let all_ok = results.iter().all(|r| r.ok); Ok(Json(RepoFanoutReply { url: body.url, git_ref: body.git_ref, workspace, peers: results, all_ok, })) } async fn handle_repos_release( State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { if body.url.trim().is_empty() { return Err((StatusCode::BAD_REQUEST, "url cannot be empty".into())); } if body.git_ref.trim().is_empty() { return Err((StatusCode::BAD_REQUEST, "git_ref cannot be empty".into())); } let workspace = caller.resolve_workspace(body.workspace.as_deref())?; let req = crate::cluster::repo_ensure::RepoReleaseRequest { url: body.url.clone(), git_ref: body.git_ref.clone(), workspace: workspace.clone(), }; let results = fanout_repo_release(&s, &req).await; let all_ok = results.iter().all(|r| r.ok); Ok(Json(RepoFanoutReply { url: body.url, git_ref: body.git_ref, workspace, peers: results, all_ok, })) } // ── shutdown-prep (targets one specific node, not a fan-out) ────── #[derive(Serialize)] pub struct ShutdownPrepCheckResponse { pub node: String, pub ready: bool, pub output: String, } async fn handle_shutdown_prep_check( State(s): State>, Path(name): Path, axum::extract::Extension(caller): axum::extract::Extension, ) -> Result, (StatusCode, String)> { caller.require_admin()?; let peer = s .peers .iter() .find(|p| p.name == name) .cloned() .ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?; let conn = s .dial(&peer) .await .map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?; let reply = crate::cluster::rpc::call_shutdown_prep_check(&conn).await; conn.close(quinn::VarInt::from_u32(0), b"done"); let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?; Ok(Json(ShutdownPrepCheckResponse { node: name, ready: reply.ready, output: reply.output, })) } #[derive(Deserialize)] pub struct ShutdownPrepExecuteBody { /// Must equal the target node's own name — a second, server-side /// confirmation beyond "the operator clicked the right button in /// the UI". Checked again on the peer itself in /// `shutdown_prep::execute`. pub confirm_node_name: String, } #[derive(Serialize)] pub struct ShutdownPrepExecuteResponse { pub node: String, pub started: bool, pub message: String, } async fn handle_shutdown_prep_execute( State(s): State>, Path(name): Path, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { caller.require_admin()?; if body.confirm_node_name != name { return Err(( StatusCode::BAD_REQUEST, format!( "confirm_node_name '{}' does not match target node '{name}'", body.confirm_node_name ), )); } let peer = s .peers .iter() .find(|p| p.name == name) .cloned() .ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?; let conn = s .dial(&peer) .await .map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?; let reply = crate::cluster::rpc::call_shutdown_prep_execute(&conn, &name).await; conn.close(quinn::VarInt::from_u32(0), b"done"); let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?; Ok(Json(ShutdownPrepExecuteResponse { node: name, started: reply.started, message: reply.message, })) } async fn fanout_repo_ensure( s: &V2State, req: &crate::cluster::repo_ensure::RepoEnsureRequest, ) -> Vec { let mut set = JoinSet::new(); for peer in &s.peers { let peer = peer.clone(); let req = req.clone(); let s = s.clone_shallow(); set.spawn(async move { let peer_name = peer.name.clone(); let res = async { let conn = s.dial(&peer).await?; let r = crate::cluster::rpc::call_repo_ensure(&conn, &req).await; conn.close(quinn::VarInt::from_u32(0), b"done"); r } .await; match res { Ok(reply) => RepoPeerResult { peer: peer_name, ok: true, path: Some(reply.path), head_sha: Some(reply.head_sha), cached: Some(reply.cached), removed: None, error: None, }, Err(e) => RepoPeerResult { peer: peer_name, ok: false, path: None, head_sha: None, cached: None, removed: None, error: Some(e.to_string()), }, } }); } let mut out = Vec::new(); while let Some(joined) = set.join_next().await { match joined { Ok(r) => out.push(r), Err(e) => out.push(RepoPeerResult { peer: "".into(), ok: false, path: None, head_sha: None, cached: None, removed: None, error: Some(e.to_string()), }), } } out.sort_by(|a, b| a.peer.cmp(&b.peer)); out } async fn fanout_repo_release( s: &V2State, req: &crate::cluster::repo_ensure::RepoReleaseRequest, ) -> Vec { let mut set = JoinSet::new(); for peer in &s.peers { let peer = peer.clone(); let req = req.clone(); let s = s.clone_shallow(); set.spawn(async move { let peer_name = peer.name.clone(); let res = async { let conn = s.dial(&peer).await?; let r = crate::cluster::rpc::call_repo_release(&conn, &req).await; conn.close(quinn::VarInt::from_u32(0), b"done"); r } .await; match res { Ok(reply) => RepoPeerResult { peer: peer_name, ok: true, path: None, head_sha: None, cached: None, removed: Some(reply.removed), error: None, }, Err(e) => RepoPeerResult { peer: peer_name, ok: false, path: None, head_sha: None, cached: None, removed: None, error: Some(e.to_string()), }, } }); } let mut out = Vec::new(); while let Some(joined) = set.join_next().await { match joined { Ok(r) => out.push(r), Err(e) => out.push(RepoPeerResult { peer: "".into(), ok: false, path: None, head_sha: None, cached: None, removed: None, error: Some(e.to_string()), }), } } out.sort_by(|a, b| a.peer.cmp(&b.peer)); out } // ── auth middleware ───────────────────────────────────────────── /// Resolve the request's bearer against the aggregator's known /// tokens and produce an [`AuthedCaller`]. On GET/HEAD, we still /// resolve so filtered list handlers can scope by namespace — but /// missing/bad auth just yields [`AuthedCaller::Open`] to preserve /// the "dashboard loads without credentials" property; open callers /// only see unrestricted views, so nothing sensitive leaks. fn resolve_caller(s: &V2State, headers: &axum::http::HeaderMap) -> Option { // No auth configured anywhere → trusted-LAN mode. if s.api_token.is_none() && s.token_entries.is_empty() { return Some(AuthedCaller::Open); } let provided = headers .get(axum::http::header::AUTHORIZATION) .and_then(|v| v.to_str().ok()) .and_then(|v| v.strip_prefix("Bearer "))?; if let Some(admin) = &s.api_token { if provided == admin.as_str() { return Some(AuthedCaller::Admin); } } for entry in &s.token_entries { if provided == entry.token.as_str() { return Some(match &entry.namespace { Some(ns) => AuthedCaller::Namespaced { namespace: ns.clone() }, None => AuthedCaller::Admin, }); } } None } /// Middleware: on mutating methods, require a resolved caller and /// stash it in request extensions for handlers. GET/HEAD stays open /// (see `resolve_caller` for the rationale). async fn v2_auth( State(s): State>, mut request: axum::extract::Request, next: axum::middleware::Next, ) -> axum::response::Response { let method = request.method().clone(); let is_read = method == axum::http::Method::GET || method == axum::http::Method::HEAD; let caller = resolve_caller(&s, request.headers()); match (is_read, caller) { // Reads always pass; auth is best-effort for future // namespace-scoped list filtering. (true, Some(c)) => { request.extensions_mut().insert(c); } (true, None) => { request.extensions_mut().insert(AuthedCaller::Open); } // Writes require a resolved caller. (false, Some(c)) => { request.extensions_mut().insert(c); } (false, None) => { return axum::response::Response::builder() .status(StatusCode::UNAUTHORIZED) .header("content-type", "application/json") .body(axum::body::Body::from(r#"{"error":"unauthorized"}"#)) .unwrap_or_default(); } } next.run(request).await } // ── session endpoints (Phase 9 S1-S3) ─────────────────────────── #[derive(Deserialize)] pub struct CreateSessionBody { /// Absolute-from-now TTL in seconds. Callers should send the /// same value on `renew` to reset the clock; there's no additive /// mode. Cap enforced at 24h to protect the sweeper from a /// caller pinning tags forever. pub ttl_secs: u64, /// Optional human note for operator/debugging visibility. #[serde(default)] pub note: Option, } #[derive(Deserialize)] pub struct PinInSessionBody { pub tag: String, pub blob_id: String, } #[derive(Deserialize)] pub struct RenewBody { pub ttl_secs: u64, } /// Max TTL a session may live before requiring renewal or commit. /// Prevents a caller from creating an effectively-immortal session /// with, say, ttl_secs = u64::MAX. const MAX_SESSION_TTL_SECS: u64 = 24 * 60 * 60; /// Namespace filter that pairs a caller with the sessions they may /// see or modify. Admin/Open see everything; namespaced callers only /// see their own namespace's sessions. fn caller_ns(caller: &AuthedCaller) -> Option { match caller { AuthedCaller::Admin | AuthedCaller::Open => None, AuthedCaller::Namespaced { namespace } => Some(namespace.clone()), } } /// True iff `caller` may inspect / mutate `session`. fn caller_may_access(caller: &AuthedCaller, session: &Session) -> bool { match caller { AuthedCaller::Admin | AuthedCaller::Open => true, AuthedCaller::Namespaced { namespace } => { session.namespace.as_deref() == Some(namespace.as_str()) } } } async fn handle_create_session( State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { if body.ttl_secs == 0 || body.ttl_secs > MAX_SESSION_TTL_SECS { return Err(( StatusCode::BAD_REQUEST, format!("ttl_secs must be 1..={MAX_SESSION_TTL_SECS}"), )); } let ns = caller_ns(&caller); let sess = s .sessions .create(ns, body.ttl_secs, body.note) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; Ok(Json(sess)) } async fn handle_list_sessions( State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, ) -> Json> { // caller_ns == None means admin/open → no filter → see all. let filter = caller_ns(&caller); let list = s.sessions.list(filter.as_deref()).await; Json(list) } async fn handle_get_session( Path(id): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, ) -> Result, StatusCode> { let sess = s.sessions.get(&id).await.ok_or(StatusCode::NOT_FOUND)?; if !caller_may_access(&caller, &sess) { // Preserve the "does it exist" secret from other tenants — // NOT_FOUND, not FORBIDDEN. Same trick the wizard-side // publish-approval endpoint uses. return Err(StatusCode::NOT_FOUND); } Ok(Json(sess)) } /// Pin a tag *within* a session: fan the pin out to peers, then /// record it in the session so a later commit/cancel/expiry can /// reverse it. Same tag scope rule as bare `POST /tags/:name` — /// namespaced callers must stay in their namespace. async fn handle_session_pin( Path(id): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { check_tag_scope(&caller, &body.tag)?; let value = decode_blob_id(&body.blob_id) .ok_or_else(|| (StatusCode::BAD_REQUEST, "blob_id must be 64-char hex".into()))?; let sess = s .sessions .get(&id) .await .ok_or((StatusCode::NOT_FOUND, "session not found".into()))?; if !caller_may_access(&caller, &sess) { return Err((StatusCode::NOT_FOUND, "session not found".into())); } if sess.committed { return Err(( StatusCode::CONFLICT, "cannot attach leases to a committed session".into(), )); } // Fan out the pin BEFORE attaching to the session — if the fleet // rejects it, we don't record a phantom lease that can never be // unpinned. Same shape as the standalone put-tag path. let peer_results = fanout_put_tag(&s, &body.tag, &value).await; if !peer_results.iter().all(|r| r.ok) { // Return the per-peer error detail so the caller can act. let err = serde_json::to_string(&peer_results).unwrap_or_default(); return Err((StatusCode::BAD_GATEWAY, err)); } let lease = LeasedTag { tag: body.tag, blob_id_hex: body.blob_id, pinned_at_unix: now_unix_u64(), }; let updated = s .sessions .attach_lease(&id, lease) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; Ok(Json(updated)) } async fn handle_renew_session( Path(id): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, Json(body): Json, ) -> Result, (StatusCode, String)> { if body.ttl_secs == 0 || body.ttl_secs > MAX_SESSION_TTL_SECS { return Err(( StatusCode::BAD_REQUEST, format!("ttl_secs must be 1..={MAX_SESSION_TTL_SECS}"), )); } let sess = s .sessions .get(&id) .await .ok_or((StatusCode::NOT_FOUND, "session not found".into()))?; if !caller_may_access(&caller, &sess) { return Err((StatusCode::NOT_FOUND, "session not found".into())); } let updated = s .sessions .renew(&id, body.ttl_secs) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; Ok(Json(updated)) } async fn handle_commit_session( Path(id): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, ) -> Result, (StatusCode, String)> { let sess = s .sessions .get(&id) .await .ok_or((StatusCode::NOT_FOUND, "session not found".into()))?; if !caller_may_access(&caller, &sess) { return Err((StatusCode::NOT_FOUND, "session not found".into())); } let updated = s .sessions .commit(&id) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; Ok(Json(updated)) } async fn handle_delete_session( Path(id): Path, State(s): State>, axum::extract::Extension(caller): axum::extract::Extension, ) -> Result>, StatusCode> { let sess = s.sessions.get(&id).await.ok_or(StatusCode::NOT_FOUND)?; if !caller_may_access(&caller, &sess) { return Err(StatusCode::NOT_FOUND); } // Unpin every tag on every peer FIRST — if we removed the session // first, a mid-deletion crash would strand tags on the fleet. let mut all_results = Vec::new(); for lease in &sess.leases { let r = fanout_delete_tag(&s, &lease.tag).await; all_results.extend(r); } let _ = s.sessions.remove(&id).await; Ok(Json(all_results)) } fn now_unix_u64() -> u64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0) } /// Reap callback for the sessions sweeper. Owns the actual fan-out. /// Kept a free function so `sessions::spawn_sweeper` doesn't have /// to know about `V2State` or its RPC client. async fn reap_expired(state: Arc, sess: Session) { for lease in &sess.leases { let results = fanout_delete_tag(&state, &lease.tag).await; // Log per-peer failures; the sweeper is best-effort. A // failing peer will be retried on the next lease that // touches it because the tag stays in the store until // successfully removed everywhere (well, the sweeper drops // the session either way — this is a known tradeoff: // durability of unpin vs. never-ending sweeper retries). for r in results { if !r.ok { eprintln!( "sweeper: session {} tag {} peer {} unpin failed: {}", sess.id, lease.tag, r.peer, r.error.as_deref().unwrap_or("") ); } } } } // ── route registration ────────────────────────────────────────── /// Assemble the aggregator router with state and middleware baked in. /// The auth middleware needs the concrete `Arc` at layer time /// (so it can read `api_token`), which is why this returns a fully- /// stated `Router<()>` instead of a state-generic router. /// /// Also spawns the background TTL sweeper (Phase 9 S1). The task is /// detached — its lifetime is the process lifetime. pub fn build(state: Arc) -> Router { // Spawn the sweeper. 15s tick is a reasonable balance: quick // enough that a mid-wizard-close cleanup feels prompt, slow // enough that idle aggregators aren't burning cycles. { let state_for_sweeper = state.clone(); let store = state.sessions.clone(); crate::sessions::spawn_sweeper( store, Duration::from_secs(15), move |sess| { let state = state_for_sweeper.clone(); async move { reap_expired(state, sess).await } }, ); } Router::new() .route("/api/v2/fleet", get(handle_fleet)) .route("/api/v2/node/:name/status", get(handle_node_status)) .route("/api/v2/storage/blobs", get(handle_blobs)) .route("/api/v2/storage/tags", get(handle_tags)) .route("/api/v2/storage/refs", get(handle_refs)) .route("/api/v2/storage/snapshots", get(handle_snapshots)) .route("/api/v2/storage/ref-tracking", get(handle_ref_tracking)) .route("/api/v2/projects", get(handle_projects)) .route( "/api/v2/tags/:name", post(handle_put_tag).delete(handle_delete_tag), ) .route( "/api/v2/sessions", post(handle_create_session).get(handle_list_sessions), ) .route( "/api/v2/sessions/:id", get(handle_get_session).delete(handle_delete_session), ) .route("/api/v2/sessions/:id/pin", post(handle_session_pin)) .route("/api/v2/sessions/:id/renew", post(handle_renew_session)) .route("/api/v2/sessions/:id/commit", post(handle_commit_session)) .route("/api/v2/repos/ensure", post(handle_repos_ensure)) .route("/api/v2/repos/release", post(handle_repos_release)) .route( "/api/v2/node/:name/shutdown-prep/check", post(handle_shutdown_prep_check), ) .route( "/api/v2/node/:name/shutdown-prep/execute", post(handle_shutdown_prep_execute), ) .route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth)) .with_state(state) }