1553 lines
62 KiB
Rust
1553 lines
62 KiB
Rust
mod actions;
|
|
mod cargo_init;
|
|
mod cluster;
|
|
mod config;
|
|
mod daemon;
|
|
mod head_watch;
|
|
mod hot;
|
|
mod manifest;
|
|
mod restore;
|
|
mod serve;
|
|
mod serve_v2;
|
|
mod sessions;
|
|
mod snapshot;
|
|
mod sync;
|
|
mod zfs;
|
|
|
|
use anyhow::{bail, Context, Result};
|
|
use clap::{Parser, Subcommand};
|
|
use config::Config;
|
|
use manifest::{Manifest, Project};
|
|
use std::net::SocketAddr;
|
|
use std::path::PathBuf;
|
|
use zfs::{SystemZfs, ZfsOps};
|
|
|
|
#[derive(Parser)]
|
|
#[command(name = "claw-store", about = "Fleet storage tier manager", version)]
|
|
struct Cli {
|
|
#[arg(long, default_value = "/etc/claw-store/config.toml")]
|
|
config: PathBuf,
|
|
#[command(subcommand)]
|
|
cmd: Cmd,
|
|
}
|
|
|
|
#[derive(Subcommand)]
|
|
enum Cmd {
|
|
/// Register a project and wire up hot tier (format: org/repo)
|
|
Activate {
|
|
project: String,
|
|
#[arg(long, help = "Clone from this URL if not present in warm tier")]
|
|
clone: Option<String>,
|
|
},
|
|
/// Sync to peer then remove from hot tier (keeps warm clone)
|
|
Deactivate { project: String },
|
|
/// List all repos in warm tier with activation status
|
|
List,
|
|
/// Push commits to origin then notify peer to pull
|
|
Sync { project: String },
|
|
/// Pull latest from origin into warm tier (also invoked by peer)
|
|
Pull { project: String },
|
|
/// Run the background daemon (snapshot cron, GC, replication, sync retry)
|
|
Daemon,
|
|
/// Show tier usage, active projects, recent snapshots
|
|
Status,
|
|
/// Run hot tier GC manually
|
|
Gc,
|
|
/// Take a ZFS snapshot of the warm tier now
|
|
Snapshot,
|
|
/// List available snapshots
|
|
ListSnapshots { project: String },
|
|
/// Restore a project from a snapshot
|
|
Restore { project: String, snapshot: String },
|
|
/// Trigger nightly replication to cold tier now
|
|
Replicate,
|
|
/// Start the HTTP API server for the dashboard
|
|
Serve {
|
|
#[arg(long, default_value = "7700")]
|
|
port: u16,
|
|
/// Legacy dashboard static assets (served under `/`).
|
|
#[arg(long)]
|
|
static_dir: Option<PathBuf>,
|
|
/// dashboard-v2 static assets (served under `/v2/*`).
|
|
/// See docs/dashboard-v2.md.
|
|
#[arg(long)]
|
|
v2_static_dir: Option<PathBuf>,
|
|
},
|
|
/// Mark a project as pinned — survives every GC pass (stale + LRU)
|
|
Pin { project: String },
|
|
/// Unmark a project — it's now a normal GC candidate again
|
|
Unpin { project: String },
|
|
/// Probe a cluster peer's LAN and Tailscale endpoints; print the winning route
|
|
ClusterProbe {
|
|
/// Peer name as listed in `[[cluster.peers]]` in the config
|
|
peer: String,
|
|
},
|
|
/// Bootstrap gossip, wait for convergence, print the peer membership table
|
|
ClusterStatus {
|
|
/// This node's name as advertised to peers.
|
|
#[arg(long)]
|
|
name: String,
|
|
/// How long to wait for convergence before printing.
|
|
#[arg(long, default_value = "3")]
|
|
wait_secs: u64,
|
|
},
|
|
/// Ping a peer over the QUIC RPC layer with an ephemeral fleet CA.
|
|
/// Demonstrates the mTLS handshake end-to-end against a live peer.
|
|
/// The `--rpc-addr` must match the peer's advertised RPC endpoint
|
|
/// (gossip port + 1 by default).
|
|
///
|
|
/// If `--tls-dir` is set, loads a persistent NodeIdentity from that
|
|
/// directory (produced by `fleet-ca sign`) — the peer must have
|
|
/// signed against the same CA.
|
|
ClusterPing {
|
|
/// This node's name — used as the SAN in the ephemeral leaf cert
|
|
/// when `--tls-dir` is not set.
|
|
#[arg(long)]
|
|
name: String,
|
|
/// Peer's node name — used as the expected server name (SAN) on
|
|
/// the incoming cert.
|
|
#[arg(long)]
|
|
peer: String,
|
|
/// Peer's RPC socket. Typically gossip_port + 1.
|
|
#[arg(long)]
|
|
rpc_addr: SocketAddr,
|
|
/// Phase 8c/8d: optional Tailscale RPC socket for
|
|
/// LAN-first-with-fallback routing. See `cluster-peer-status`.
|
|
#[arg(long)]
|
|
tailscale_addr: Option<SocketAddr>,
|
|
/// LAN probe deadline (ms) when `--tailscale-addr` is set.
|
|
#[arg(long, default_value_t = 200)]
|
|
lan_probe_ms: u64,
|
|
/// Payload to send. Echoed back with a "pong:" prefix.
|
|
#[arg(long, default_value = "hello")]
|
|
payload: String,
|
|
/// Optional: dir containing `ca.crt`, `node.crt`, `node.key`
|
|
/// (from `fleet-ca sign --out-dir <dir>`). When absent, falls
|
|
/// back to an ephemeral in-memory CA.
|
|
#[arg(long)]
|
|
tls_dir: Option<PathBuf>,
|
|
},
|
|
/// Generate a fresh fleet root CA and write it to disk. Run once per
|
|
/// cluster, on the primary node. Every other node needs only the
|
|
/// public `ca.crt` — but this dir also gets `ca.key`, which must
|
|
/// stay on the signing host.
|
|
FleetCaInit {
|
|
/// Directory to write `ca.crt` + `ca.key` into.
|
|
#[arg(long)]
|
|
dir: PathBuf,
|
|
/// Common Name embedded in the CA's distinguished name.
|
|
#[arg(long, default_value = "clawstor fleet CA")]
|
|
cn: String,
|
|
},
|
|
/// Sign a leaf certificate for a node under the fleet CA. Outputs
|
|
/// `ca.crt` (public), `node.crt` (public), `node.key` (private,
|
|
/// 0o600) into `--out-dir`. Copy those three files to the target
|
|
/// node and point `[cluster.tls]` at them.
|
|
/// Phase 8 (2026-07-14): sign a leaf cert using this node's
|
|
/// Tailscale identity as extra SANs. Queries the local
|
|
/// `tailscale` CLI for MagicDNS name + tailnet IPs and folds
|
|
/// them into the cert alongside `--node`. Zero-touch bootstrap
|
|
/// for laptops joining the fleet: they can be reached by
|
|
/// MagicDNS name from anywhere on the tailnet.
|
|
FleetCaTailscaleSign {
|
|
/// Directory holding the fleet CA (`ca.crt` + `ca.key`),
|
|
/// typically produced by `fleet-ca-init`.
|
|
#[arg(long)]
|
|
ca_dir: PathBuf,
|
|
/// Primary node name for the cert (CN + first SAN).
|
|
/// Defaults to the Tailscale short hostname.
|
|
#[arg(long)]
|
|
node: Option<String>,
|
|
/// Where to write `ca.crt` + `node.crt` + `node.key`.
|
|
#[arg(long)]
|
|
out_dir: PathBuf,
|
|
},
|
|
FleetCaSign {
|
|
/// Directory holding the CA (`ca.crt` + `ca.key`) — the same
|
|
/// dir passed to `fleet-ca init`.
|
|
#[arg(long)]
|
|
ca_dir: PathBuf,
|
|
/// Node name — becomes the SAN + Common Name on the leaf cert.
|
|
#[arg(long)]
|
|
node: String,
|
|
/// Output directory for the three per-node PEM files.
|
|
#[arg(long)]
|
|
out_dir: PathBuf,
|
|
},
|
|
/// Call the peer's PeerStatus RPC and print its local view of the
|
|
/// cluster. Requires that peer be running a daemon with
|
|
/// `[cluster.tls]` configured, and that our `--tls-dir` was signed
|
|
/// by the same CA as the peer's.
|
|
ClusterPeerStatus {
|
|
/// Peer's node name — must match the peer's cert SAN.
|
|
#[arg(long)]
|
|
peer: String,
|
|
/// Peer's RPC socket. Typically gossip_port + 1.
|
|
#[arg(long)]
|
|
rpc_addr: SocketAddr,
|
|
/// Phase 8c (2026-07-14): optional Tailscale RPC socket.
|
|
/// When set, `--rpc-addr` is tried first with a short
|
|
/// deadline (`--lan-probe-ms`); on failure or timeout we
|
|
/// fall through to this tailnet address. Roaming ops
|
|
/// (laptop on LTE, in-flight wifi) get connectivity
|
|
/// without hand-editing addresses per environment.
|
|
#[arg(long)]
|
|
tailscale_addr: Option<SocketAddr>,
|
|
/// LAN probe deadline in milliseconds. Only used when
|
|
/// `--tailscale-addr` is set. Default 200ms matches the
|
|
/// arch doc — long enough for a live LAN handshake, short
|
|
/// enough that roaming clients don't stall.
|
|
#[arg(long, default_value_t = 200)]
|
|
lan_probe_ms: u64,
|
|
/// Directory holding this node's mTLS material
|
|
/// (`ca.crt` + `node.crt` + `node.key` from `fleet-ca sign`).
|
|
#[arg(long)]
|
|
tls_dir: PathBuf,
|
|
},
|
|
/// Field finding 2026-07-12: sweep orphan chunks from this node's
|
|
/// blob store (chunks referenced by no manifest). Safe to run any
|
|
/// time — never touches chunks referenced by a live manifest.
|
|
/// Run manually or from cron; a future daemon-side ticker will
|
|
/// invoke this automatically (see `[cluster.gc_interval_hours]`).
|
|
///
|
|
/// With `--evict-to-gb <N>`, also runs LRU eviction: deletes blob
|
|
/// manifests oldest-first until the referenced-chunk footprint
|
|
/// is at or below `N` GiB.
|
|
ClusterGc {
|
|
/// Optional: evict oldest blobs until the store is `<= N` GiB.
|
|
/// Skip to run orphan-chunk sweep only.
|
|
#[arg(long)]
|
|
evict_to_gb: Option<u64>,
|
|
},
|
|
/// Phase 7f (2026-07-14): identify cache fingerprints whose
|
|
/// producing git refs are all gone upstream AND whose last-seen
|
|
/// age exceeds the retention window. Dry-run only in this cut —
|
|
/// prints the stale fingerprints grouped by repo. Deletion is a
|
|
/// separate operator step.
|
|
ClusterRefSweep {
|
|
/// Gitea base URL, e.g. https://git.redclaw.dev
|
|
#[arg(long)]
|
|
gitea_url: String,
|
|
/// Bearer token for private repos. Public read-only repos
|
|
/// work without one; typically supplied via env not flag.
|
|
#[arg(long, env = "GITEA_TOKEN")]
|
|
gitea_token: Option<String>,
|
|
/// Minimum age (in days) before a dead-ref fingerprint is
|
|
/// considered stale. Protects fresh CI builds from being
|
|
/// reaped before someone can rebuild against them.
|
|
#[arg(long, default_value_t = 14)]
|
|
retention_days: u64,
|
|
/// Polish (2026-07-14): actually forget the stale
|
|
/// ref-tracking records. Without this flag the command is
|
|
/// dry-run only. Blob data is untouched — eviction happens
|
|
/// on the next `cluster-gc` when the fingerprint's tag pin
|
|
/// disappears (nobody pins it any more).
|
|
#[arg(long)]
|
|
apply: bool,
|
|
},
|
|
/// Phase 7d (2026-07-14): take a point-in-time snapshot of every
|
|
/// blob currently in the local store. Snapshots are cheap
|
|
/// reference sets (no data copy). Combine with pin-aware LRU
|
|
/// eviction to guarantee blobs stay on disk for a retention
|
|
/// window.
|
|
ClusterSnapshotCreate {
|
|
/// Operator-supplied snapshot name (no `/`, `\`, or NUL).
|
|
#[arg(long)]
|
|
name: String,
|
|
},
|
|
/// List every snapshot, oldest first.
|
|
ClusterSnapshotList,
|
|
/// Show a snapshot's full blob-id list.
|
|
ClusterSnapshotShow {
|
|
#[arg(long)]
|
|
name: String,
|
|
},
|
|
/// Remove a snapshot. Does NOT touch the referenced blob data —
|
|
/// snapshots are pointer-sets, not copies.
|
|
ClusterSnapshotDelete {
|
|
#[arg(long)]
|
|
name: String,
|
|
},
|
|
/// Phase 7c (2026-07-14): fix corrupt/missing chunks by pulling
|
|
/// them from a peer. Runs scrub first; if nothing bad, exits
|
|
/// clean. Otherwise probes the peer (HasChunk) for each unique
|
|
/// bad chunk and pulls it (GetChunk) when the peer has it.
|
|
/// Bytes are re-hashed on write, so a lying peer cannot corrupt
|
|
/// us further. Unrecoverable chunks (peer doesn't have) are
|
|
/// listed in the report — operator's cue to try another peer.
|
|
ClusterRepair {
|
|
/// Peer's node name — must match the peer's cert SAN.
|
|
#[arg(long)]
|
|
peer: String,
|
|
/// Peer's RPC socket. Typically gossip_port + 1.
|
|
#[arg(long)]
|
|
rpc_addr: SocketAddr,
|
|
/// Phase 8c (2026-07-14): optional Tailscale RPC socket
|
|
/// for LAN-first-with-fallback routing. See
|
|
/// `cluster-peer-status` for details.
|
|
#[arg(long)]
|
|
tailscale_addr: Option<SocketAddr>,
|
|
/// LAN probe deadline (ms) when `--tailscale-addr` is set.
|
|
#[arg(long, default_value_t = 200)]
|
|
lan_probe_ms: u64,
|
|
/// Directory holding this node's mTLS material.
|
|
#[arg(long)]
|
|
tls_dir: PathBuf,
|
|
/// Scrub + report what WOULD be repaired without contacting
|
|
/// the peer or writing anything.
|
|
#[arg(long)]
|
|
dry_run: bool,
|
|
},
|
|
/// Phase 7a (2026-07-14): read-only fsck for the local blob store.
|
|
/// Walks every blob manifest, recomputes BLAKE3 for each chunk,
|
|
/// reports missing + corrupt chunks. Never mutates disk. Safe to
|
|
/// run against a live daemon.
|
|
ClusterScrub {
|
|
/// Print each (blob, chunk) mismatch instead of just totals.
|
|
#[arg(long)]
|
|
verbose: bool,
|
|
},
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()))
|
|
.init();
|
|
|
|
let cli = Cli::parse();
|
|
|
|
// Config-independent commands: filesystem-only, run before we touch
|
|
// /etc/claw-store/config.toml. Lets an operator bootstrap the CA on
|
|
// a fresh box that doesn't have a config yet.
|
|
match &cli.cmd {
|
|
Cmd::FleetCaInit { dir, cn } => return cmd_fleet_ca_init(dir, cn),
|
|
Cmd::FleetCaSign {
|
|
ca_dir,
|
|
node,
|
|
out_dir,
|
|
} => return cmd_fleet_ca_sign(ca_dir, node, out_dir),
|
|
Cmd::FleetCaTailscaleSign {
|
|
ca_dir,
|
|
node,
|
|
out_dir,
|
|
} => return cmd_fleet_ca_tailscale_sign(ca_dir, node.as_deref(), out_dir),
|
|
_ => {}
|
|
}
|
|
|
|
let cfg = Config::load(&cli.config)
|
|
.with_context(|| format!("loading config from {}", cli.config.display()))?;
|
|
let manifest_path = Manifest::default_path();
|
|
let mut manifest = Manifest::load(&manifest_path)?;
|
|
let zfs = SystemZfs;
|
|
|
|
match cli.cmd {
|
|
Cmd::Activate { project, clone } =>
|
|
cmd_activate(&cfg, &mut manifest, &manifest_path, &project, clone.as_deref())?,
|
|
Cmd::Deactivate { project } =>
|
|
cmd_deactivate(&cfg, &mut manifest, &manifest_path, &project)?,
|
|
Cmd::List => cmd_list(&cfg, &manifest)?,
|
|
Cmd::Sync { project } => cmd_sync(&cfg, &manifest, &project)?,
|
|
Cmd::Pull { project } => cmd_pull(&cfg, &mut manifest, &manifest_path, &project)?,
|
|
Cmd::Daemon => daemon::run(cfg, manifest).await?,
|
|
Cmd::Status => cmd_status(&cfg, &manifest, &zfs)?,
|
|
Cmd::Gc => cmd_gc(&cfg, &mut manifest)?,
|
|
Cmd::Snapshot => cmd_snapshot(&cfg, &zfs)?,
|
|
Cmd::ListSnapshots { project } => cmd_list_snapshots(&cfg, &zfs, &project)?,
|
|
Cmd::Restore { project, snapshot } => cmd_restore(&cfg, &zfs, &project, &snapshot)?,
|
|
Cmd::Replicate => cmd_replicate(&cfg, &zfs)?,
|
|
Cmd::Serve { port, static_dir, v2_static_dir } =>
|
|
serve::run_server(cfg, manifest, port, static_dir, v2_static_dir).await?,
|
|
Cmd::Pin { project } => cmd_set_pin(&manifest_path, &project, true)?,
|
|
Cmd::Unpin { project } => cmd_set_pin(&manifest_path, &project, false)?,
|
|
Cmd::ClusterProbe { peer } => cmd_cluster_probe(&cfg, &peer).await?,
|
|
Cmd::ClusterStatus { name, wait_secs } => cmd_cluster_status(&cfg, &name, wait_secs).await?,
|
|
Cmd::ClusterPing {
|
|
name,
|
|
peer,
|
|
rpc_addr,
|
|
tailscale_addr,
|
|
lan_probe_ms,
|
|
payload,
|
|
tls_dir,
|
|
} => cmd_cluster_ping(&name, &peer, rpc_addr, tailscale_addr, lan_probe_ms, &payload, tls_dir.as_deref()).await?,
|
|
Cmd::ClusterGc { evict_to_gb } => cmd_cluster_gc(&cfg, evict_to_gb).await?,
|
|
Cmd::ClusterScrub { verbose } => cmd_cluster_scrub(&cfg, verbose).await?,
|
|
Cmd::ClusterRepair {
|
|
peer,
|
|
rpc_addr,
|
|
tailscale_addr,
|
|
lan_probe_ms,
|
|
tls_dir,
|
|
dry_run,
|
|
} => cmd_cluster_repair(&cfg, &peer, rpc_addr, tailscale_addr, lan_probe_ms, &tls_dir, dry_run).await?,
|
|
Cmd::ClusterRefSweep {
|
|
gitea_url,
|
|
gitea_token,
|
|
retention_days,
|
|
apply,
|
|
} => cmd_cluster_ref_sweep(&cfg, &gitea_url, gitea_token, retention_days, apply).await?,
|
|
Cmd::ClusterSnapshotCreate { name } => cmd_cluster_snapshot_create(&cfg, &name).await?,
|
|
Cmd::ClusterSnapshotList => cmd_cluster_snapshot_list(&cfg).await?,
|
|
Cmd::ClusterSnapshotShow { name } => cmd_cluster_snapshot_show(&cfg, &name).await?,
|
|
Cmd::ClusterSnapshotDelete { name } => cmd_cluster_snapshot_delete(&cfg, &name).await?,
|
|
Cmd::ClusterPeerStatus {
|
|
peer,
|
|
rpc_addr,
|
|
tailscale_addr,
|
|
lan_probe_ms,
|
|
tls_dir,
|
|
} => cmd_cluster_peer_status(&peer, rpc_addr, tailscale_addr, lan_probe_ms, &tls_dir).await?,
|
|
Cmd::FleetCaInit { .. } | Cmd::FleetCaSign { .. } | Cmd::FleetCaTailscaleSign { .. } => {
|
|
// Handled by the config-independent short-circuit above.
|
|
unreachable!("fleet-ca commands short-circuit before config load");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
// ── cluster peer-status ───────────────────────────────────────────────────────
|
|
|
|
async fn cmd_cluster_gc(cfg: &Config, evict_to_gb: Option<u64>) -> Result<()> {
|
|
use cluster::blob::{BlobId, BlobStore};
|
|
use cluster::tags::TagStore;
|
|
|
|
let root = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.and_then(|c| c.blob_store_root.clone())
|
|
.context("cluster.blob_store_root not configured; nothing to GC")?;
|
|
if !root.is_dir() {
|
|
bail!("blob_store_root {} does not exist", root.display());
|
|
}
|
|
let store = BlobStore::open(root.clone())
|
|
.with_context(|| format!("opening blob store at {}", root.display()))?;
|
|
|
|
let started = std::time::Instant::now();
|
|
|
|
// Phase 1: orphan-chunk sweep (always safe).
|
|
let orphan = store
|
|
.gc_orphan_chunks()
|
|
.await
|
|
.context("gc_orphan_chunks failed")?;
|
|
|
|
// Phase 4 (2026-07-13): gather pin set from tag store so evictions
|
|
// respect them. Cheap even at fleet scale (one 32-byte value per
|
|
// tag).
|
|
let tags_dir = root.join("tags-db");
|
|
let now_unix = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|d| d.as_secs())
|
|
.unwrap_or(0);
|
|
let (mut pinned_blobs, expired_pruned) = if tags_dir.is_dir() {
|
|
let ts = TagStore::open(tags_dir.clone())
|
|
.with_context(|| format!("opening tag store at {}", tags_dir.display()))?;
|
|
// Phase 4b: prune first so the set we hand to the evictor is
|
|
// current as of `now_unix`.
|
|
let pruned = ts
|
|
.prune_expired_stamped_at(now_unix)
|
|
.await
|
|
.context("pruning expired pins")?;
|
|
let pins = ts
|
|
.pinned_blob_values_at(now_unix)
|
|
.await
|
|
.context("collecting pinned tag values")?
|
|
.into_iter()
|
|
.map(BlobId::from_bytes)
|
|
.collect::<std::collections::HashSet<_>>();
|
|
(pins, pruned)
|
|
} else {
|
|
(std::collections::HashSet::new(), 0)
|
|
};
|
|
// Phase 7d follow-on: snapshot references act as immortal pins.
|
|
// Any blob captured by ANY snapshot survives the LRU cap so
|
|
// operators can guarantee retention windows via snapshots alone,
|
|
// without hand-managing per-blob tags.
|
|
let snapshot_store = cluster::snapshot::SnapshotStore::open(root.clone())
|
|
.context("opening snapshot store")?;
|
|
let snapshot_pins = snapshot_store
|
|
.pinned_blob_ids()
|
|
.await
|
|
.context("collecting snapshot pins")?;
|
|
let snapshot_pin_count = snapshot_pins.len();
|
|
pinned_blobs.extend(snapshot_pins);
|
|
|
|
// Phase 2 (optional): pin-aware LRU eviction to hit a size cap.
|
|
let evict = if let Some(gb) = evict_to_gb {
|
|
let cap = gb.saturating_mul(1024 * 1024 * 1024);
|
|
Some(
|
|
store
|
|
.evict_to_size_cap_with_pins(cap, &pinned_blobs)
|
|
.await
|
|
.context("evict_to_size_cap_with_pins failed")?,
|
|
)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let elapsed = started.elapsed();
|
|
println!("── clawstor cluster-gc ─────────────────────────────");
|
|
println!("root: {}", root.display());
|
|
println!("orphan chunks:");
|
|
println!(" scanned: {}", orphan.chunks_scanned);
|
|
println!(" removed: {}", orphan.chunks_removed);
|
|
println!(" bytes reclaimed: {}", orphan.bytes_reclaimed);
|
|
if let Some(evict) = &evict {
|
|
println!("lru eviction (cap {} GiB):", evict_to_gb.unwrap_or(0));
|
|
println!(" chunks removed: {}", evict.chunks_removed);
|
|
println!(" bytes reclaimed: {}", evict.bytes_reclaimed);
|
|
println!(" pinned blobs: {} ({} from snapshots)", pinned_blobs.len(), snapshot_pin_count);
|
|
println!(" expired pins pruned: {}", expired_pruned);
|
|
}
|
|
println!("total elapsed: {:?}", elapsed);
|
|
println!("────────────────────────────────────────────────────");
|
|
Ok(())
|
|
}
|
|
|
|
async fn cmd_cluster_scrub(cfg: &Config, verbose: bool) -> Result<()> {
|
|
use cluster::blob::BlobStore;
|
|
|
|
let root = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.and_then(|c| c.blob_store_root.clone())
|
|
.context("cluster.blob_store_root not configured; nothing to scrub")?;
|
|
if !root.is_dir() {
|
|
bail!("blob_store_root {} does not exist", root.display());
|
|
}
|
|
let store = BlobStore::open(root.clone())
|
|
.with_context(|| format!("opening blob store at {}", root.display()))?;
|
|
|
|
let started = std::time::Instant::now();
|
|
let report = store
|
|
.scrub_all()
|
|
.await
|
|
.context("scrub_all failed")?;
|
|
let elapsed = started.elapsed();
|
|
|
|
println!("── clawstor cluster-scrub ──────────────────────────");
|
|
println!("root: {}", root.display());
|
|
println!("manifests scanned: {}", report.manifests_scanned);
|
|
println!("chunks scanned: {}", report.chunks_scanned);
|
|
println!(" ok: {}", report.chunks_ok);
|
|
println!(" missing: {}", report.chunks_missing);
|
|
println!(" corrupt: {}", report.chunks_corrupt);
|
|
if verbose {
|
|
if !report.missing_chunks.is_empty() {
|
|
println!();
|
|
println!("missing chunks:");
|
|
for (blob, chunk) in &report.missing_chunks {
|
|
println!(" blob {} chunk {}", blob, chunk.to_hex());
|
|
}
|
|
}
|
|
if !report.corrupt_chunks.is_empty() {
|
|
println!();
|
|
println!("corrupt chunks:");
|
|
for (blob, chunk) in &report.corrupt_chunks {
|
|
println!(" blob {} chunk {}", blob, chunk.to_hex());
|
|
}
|
|
}
|
|
} else if !report.missing_chunks.is_empty() || !report.corrupt_chunks.is_empty() {
|
|
println!();
|
|
println!("re-run with --verbose to list affected (blob, chunk) pairs");
|
|
}
|
|
println!("total elapsed: {:?}", elapsed);
|
|
println!("────────────────────────────────────────────────────");
|
|
// Non-zero exit when the store has any integrity issue so cron
|
|
// jobs and CI checks surface a real failure instead of a
|
|
// clean-looking log.
|
|
if report.chunks_corrupt > 0 || report.chunks_missing > 0 {
|
|
anyhow::bail!(
|
|
"scrub found {} corrupt + {} missing chunks",
|
|
report.chunks_corrupt,
|
|
report.chunks_missing
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
async fn cmd_cluster_repair(
|
|
cfg: &Config,
|
|
peer: &str,
|
|
rpc_addr: SocketAddr,
|
|
tailscale_addr: Option<SocketAddr>,
|
|
lan_probe_ms: u64,
|
|
tls_dir: &std::path::Path,
|
|
dry_run: bool,
|
|
) -> Result<()> {
|
|
use cluster::blob::BlobStore;
|
|
use cluster::rpc::{call_get_chunk, call_has_chunk};
|
|
use cluster::transport::{ConnectRoute, NodeIdentity, QuicClient};
|
|
|
|
let root = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.and_then(|c| c.blob_store_root.clone())
|
|
.context("cluster.blob_store_root not configured; nothing to repair")?;
|
|
if !root.is_dir() {
|
|
bail!("blob_store_root {} does not exist", root.display());
|
|
}
|
|
let store = BlobStore::open(root.clone())
|
|
.with_context(|| format!("opening blob store at {}", root.display()))?;
|
|
|
|
let started = std::time::Instant::now();
|
|
|
|
// Phase 1: scrub locally to identify the target set.
|
|
let scrub = store
|
|
.scrub_all()
|
|
.await
|
|
.context("initial scrub_all failed")?;
|
|
|
|
println!("── clawstor cluster-repair ─────────────────────────");
|
|
println!("root: {}", root.display());
|
|
println!("peer: {} @ {}", peer, rpc_addr);
|
|
if dry_run {
|
|
println!("mode: DRY-RUN (no peer contact, no writes)");
|
|
}
|
|
println!("scrub:");
|
|
println!(" manifests: {}", scrub.manifests_scanned);
|
|
println!(" chunks: {}", scrub.chunks_scanned);
|
|
println!(" ok: {}", scrub.chunks_ok);
|
|
println!(" missing: {}", scrub.chunks_missing);
|
|
println!(" corrupt: {}", scrub.chunks_corrupt);
|
|
|
|
if scrub.chunks_missing == 0 && scrub.chunks_corrupt == 0 {
|
|
println!("nothing to repair.");
|
|
println!("total elapsed: {:?}", started.elapsed());
|
|
println!("────────────────────────────────────────────────────");
|
|
return Ok(());
|
|
}
|
|
|
|
// Dedup to one entry per unique chunk-hash — scrub emits per-reference.
|
|
let mut unique: std::collections::HashSet<_> = std::collections::HashSet::new();
|
|
for (_blob, chunk) in scrub.missing_chunks.iter().chain(scrub.corrupt_chunks.iter()) {
|
|
unique.insert(*chunk);
|
|
}
|
|
let targets: Vec<_> = unique.into_iter().collect();
|
|
println!("unique bad chunks: {}", targets.len());
|
|
|
|
if dry_run {
|
|
println!("dry-run: skipping peer contact + writes");
|
|
println!("total elapsed: {:?}", started.elapsed());
|
|
println!("────────────────────────────────────────────────────");
|
|
return Ok(());
|
|
}
|
|
|
|
// Phase 2: connect to the peer. Phase 8c: LAN-first with
|
|
// optional tailnet fallback when the operator supplied one.
|
|
let identity = NodeIdentity::from_pem_dir(tls_dir)
|
|
.with_context(|| format!("loading identity from {}", tls_dir.display()))?;
|
|
let client = QuicClient::new("0.0.0.0:0".parse()?, identity)?;
|
|
let (conn, route) = client
|
|
.connect_lan_first(
|
|
peer,
|
|
Some(rpc_addr),
|
|
tailscale_addr,
|
|
std::time::Duration::from_millis(lan_probe_ms),
|
|
)
|
|
.await
|
|
.with_context(|| format!("connecting to {peer}"))?;
|
|
match route {
|
|
ConnectRoute::Lan(a) => println!("route: LAN ({a})"),
|
|
ConnectRoute::Tailscale(a) => println!("route: tailnet ({a})"),
|
|
}
|
|
|
|
// Phase 3: repair. Fetcher probes HasChunk first (cheap) so a
|
|
// peer that lacks the chunk is one round-trip, not a full pull
|
|
// attempt.
|
|
let report = store
|
|
.repair_chunks(&targets, |hash| {
|
|
let conn = &conn;
|
|
async move {
|
|
if !call_has_chunk(conn, &hash).await? {
|
|
return Ok(None);
|
|
}
|
|
call_get_chunk(conn, &hash).await
|
|
}
|
|
})
|
|
.await;
|
|
|
|
println!("repair:");
|
|
println!(" attempted: {}", report.attempted);
|
|
println!(" repaired: {}", report.repaired);
|
|
println!(" unrecoverable: {}", report.unrecoverable.len());
|
|
println!(" errors: {}", report.errors.len());
|
|
if !report.unrecoverable.is_empty() {
|
|
println!();
|
|
println!("unrecoverable (peer doesn't have them; try another peer):");
|
|
for chunk in &report.unrecoverable {
|
|
println!(" {}", chunk.to_hex());
|
|
}
|
|
}
|
|
if !report.errors.is_empty() {
|
|
println!();
|
|
println!("errors:");
|
|
for (chunk, e) in &report.errors {
|
|
println!(" {} {}", chunk.to_hex(), e);
|
|
}
|
|
}
|
|
println!("total elapsed: {:?}", started.elapsed());
|
|
println!("────────────────────────────────────────────────────");
|
|
|
|
// Non-zero exit when we couldn't fully repair — same rationale as
|
|
// cluster-scrub: cron/CI should notice, not gloss over.
|
|
if !report.unrecoverable.is_empty() || !report.errors.is_empty() {
|
|
anyhow::bail!(
|
|
"repair incomplete: {} unrecoverable, {} errors",
|
|
report.unrecoverable.len(),
|
|
report.errors.len()
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn open_blob_and_snapshot_stores(
|
|
cfg: &Config,
|
|
) -> Result<(cluster::blob::BlobStore, cluster::snapshot::SnapshotStore)> {
|
|
let root = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.and_then(|c| c.blob_store_root.clone())
|
|
.context("cluster.blob_store_root not configured")?;
|
|
if !root.is_dir() {
|
|
bail!("blob_store_root {} does not exist", root.display());
|
|
}
|
|
let blob = cluster::blob::BlobStore::open(root.clone())
|
|
.with_context(|| format!("opening blob store at {}", root.display()))?;
|
|
let snap = cluster::snapshot::SnapshotStore::open(root.clone())
|
|
.with_context(|| format!("opening snapshot store at {}", root.display()))?;
|
|
Ok((blob, snap))
|
|
}
|
|
|
|
async fn cmd_cluster_snapshot_create(cfg: &Config, name: &str) -> Result<()> {
|
|
let (blob, snap) = open_blob_and_snapshot_stores(cfg)?;
|
|
let now = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|d| d.as_secs())
|
|
.unwrap_or(0);
|
|
let started = std::time::Instant::now();
|
|
let m = snap.create(name, &blob, now).await?;
|
|
println!("── clawstor snapshot-create ────────────────────────");
|
|
println!("name: {}", m.name);
|
|
println!("created_at (unix): {}", m.created_at_unix);
|
|
println!("blob count: {}", m.blob_ids.len());
|
|
println!("elapsed: {:?}", started.elapsed());
|
|
println!("────────────────────────────────────────────────────");
|
|
Ok(())
|
|
}
|
|
|
|
async fn cmd_cluster_snapshot_list(cfg: &Config) -> Result<()> {
|
|
let (_blob, snap) = open_blob_and_snapshot_stores(cfg)?;
|
|
let entries = snap.list().await?;
|
|
println!("── clawstor snapshots ──────────────────────────────");
|
|
if entries.is_empty() {
|
|
println!("(no snapshots)");
|
|
} else {
|
|
println!(
|
|
"{:<20} {:<20} {:<10} {}",
|
|
"CREATED_AT", "NAME", "BLOBS", "SIZE"
|
|
);
|
|
for s in &entries {
|
|
println!(
|
|
"{:<20} {:<20} {:<10} {}",
|
|
s.created_at_unix, s.name, s.blob_count, s.file_bytes
|
|
);
|
|
}
|
|
}
|
|
println!("────────────────────────────────────────────────────");
|
|
Ok(())
|
|
}
|
|
|
|
async fn cmd_cluster_snapshot_show(cfg: &Config, name: &str) -> Result<()> {
|
|
let (_blob, snap) = open_blob_and_snapshot_stores(cfg)?;
|
|
let m = snap
|
|
.get(name)
|
|
.await?
|
|
.with_context(|| format!("snapshot {:?} not found", name))?;
|
|
println!("── clawstor snapshot show ──────────────────────────");
|
|
println!("name: {}", m.name);
|
|
println!("created_at (unix): {}", m.created_at_unix);
|
|
println!("blob count: {}", m.blob_ids.len());
|
|
println!("blob ids:");
|
|
for id in &m.blob_ids {
|
|
println!(" {}", id.to_hex());
|
|
}
|
|
println!("────────────────────────────────────────────────────");
|
|
Ok(())
|
|
}
|
|
|
|
async fn cmd_cluster_snapshot_delete(cfg: &Config, name: &str) -> Result<()> {
|
|
let (_blob, snap) = open_blob_and_snapshot_stores(cfg)?;
|
|
let removed = snap.delete(name).await?;
|
|
if removed {
|
|
println!("snapshot {:?} deleted (blob data untouched)", name);
|
|
} else {
|
|
println!("snapshot {:?} did not exist", name);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
async fn cmd_cluster_ref_sweep(
|
|
cfg: &Config,
|
|
gitea_url: &str,
|
|
gitea_token: Option<String>,
|
|
retention_days: u64,
|
|
apply: bool,
|
|
) -> Result<()> {
|
|
use cluster::gitea::GiteaClient;
|
|
use cluster::ref_tracking::RefTracking;
|
|
|
|
let root = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.and_then(|c| c.blob_store_root.clone())
|
|
.context("cluster.blob_store_root not configured; nothing to sweep")?;
|
|
let rt = RefTracking::open(root.clone())
|
|
.with_context(|| format!("opening ref-tracking at {}", root.display()))?;
|
|
|
|
let started = std::time::Instant::now();
|
|
let all = rt.list_all().await.context("listing ref-tracking entries")?;
|
|
// Group repos to minimize Gitea calls.
|
|
let mut repos = std::collections::BTreeSet::new();
|
|
for e in &all {
|
|
repos.insert(e.repo.clone());
|
|
}
|
|
|
|
println!("── clawstor cluster-ref-sweep ──────────────────────");
|
|
println!("gitea: {}", gitea_url);
|
|
println!("retention days: {}", retention_days);
|
|
println!("tracked fps: {}", all.len());
|
|
println!("distinct repos: {}", repos.len());
|
|
|
|
if all.is_empty() {
|
|
println!("nothing to sweep.");
|
|
println!("total elapsed: {:?}", started.elapsed());
|
|
println!("────────────────────────────────────────────────────");
|
|
return Ok(());
|
|
}
|
|
|
|
let client = GiteaClient::new(gitea_url, gitea_token)
|
|
.context("building Gitea client")?;
|
|
let mut live: std::collections::HashMap<String, std::collections::HashSet<String>> =
|
|
std::collections::HashMap::new();
|
|
let mut missing_repos: Vec<String> = Vec::new();
|
|
for repo in &repos {
|
|
match client.live_refs(repo).await {
|
|
Ok(lr) => {
|
|
if lr.refs.is_empty() {
|
|
missing_repos.push(repo.clone());
|
|
} else {
|
|
live.insert(repo.clone(), lr.refs);
|
|
}
|
|
}
|
|
Err(e) => {
|
|
eprintln!("warn: gitea live_refs({repo}) failed: {e}");
|
|
// Leave repo out of `live` — stale_at treats missing
|
|
// as all-dead, which is the safer default for a
|
|
// repo we couldn't query.
|
|
}
|
|
}
|
|
}
|
|
println!(
|
|
"live refs fetched: {} repos ({} appeared empty/deleted)",
|
|
live.len(),
|
|
missing_repos.len()
|
|
);
|
|
|
|
let now = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map(|d| d.as_secs())
|
|
.unwrap_or(0);
|
|
let retention_secs = retention_days.saturating_mul(24 * 3600);
|
|
let stale = rt
|
|
.stale_at(now, &live, retention_secs)
|
|
.await
|
|
.context("computing stale set")?;
|
|
// Group stale fps by repo for readable output.
|
|
let mut by_repo: std::collections::BTreeMap<String, Vec<[u8; 32]>> =
|
|
std::collections::BTreeMap::new();
|
|
let stale_set: std::collections::HashSet<_> = stale.iter().collect();
|
|
for e in &all {
|
|
if stale_set.contains(&e.fingerprint) {
|
|
by_repo.entry(e.repo.clone()).or_default().push(e.fingerprint);
|
|
}
|
|
}
|
|
|
|
println!("stale fps: {}", stale.len());
|
|
println!();
|
|
if !by_repo.is_empty() {
|
|
for (repo, fps) in &by_repo {
|
|
println!(" {} — {} fp(s):", repo, fps.len());
|
|
for fp in fps {
|
|
let mut hex = String::with_capacity(64);
|
|
for b in fp {
|
|
hex.push_str(&format!("{b:02x}"));
|
|
}
|
|
println!(" {}", hex);
|
|
}
|
|
}
|
|
}
|
|
println!();
|
|
if apply {
|
|
// Polish (2026-07-14): actually forget the stale records.
|
|
// Blob data untouched — the next cluster-gc reclaims disk
|
|
// once the fp's associated tag pins drop off.
|
|
let mut forgotten = 0usize;
|
|
let mut errors = 0usize;
|
|
for fp in &stale {
|
|
match rt.forget(fp).await {
|
|
Ok(true) => forgotten += 1,
|
|
Ok(false) => {} // already gone
|
|
Err(e) => {
|
|
errors += 1;
|
|
eprintln!("warn: forget({}) failed: {e}", hex_bytes(fp));
|
|
}
|
|
}
|
|
}
|
|
println!("applied: forgot {forgotten} ref-tracking records ({errors} errors)");
|
|
} else {
|
|
println!("dry-run: no records modified. Re-run with --apply to prune.");
|
|
}
|
|
println!("(blob eviction on next cluster-gc handles the actual disk reclaim).");
|
|
println!("total elapsed: {:?}", started.elapsed());
|
|
println!("────────────────────────────────────────────────────");
|
|
Ok(())
|
|
}
|
|
|
|
fn hex_bytes(bytes: &[u8; 32]) -> String {
|
|
let mut s = String::with_capacity(64);
|
|
for b in bytes {
|
|
s.push_str(&format!("{b:02x}"));
|
|
}
|
|
s
|
|
}
|
|
|
|
async fn cmd_cluster_peer_status(
|
|
peer: &str,
|
|
rpc_addr: SocketAddr,
|
|
tailscale_addr: Option<SocketAddr>,
|
|
lan_probe_ms: u64,
|
|
tls_dir: &std::path::Path,
|
|
) -> Result<()> {
|
|
use cluster::rpc::call_peer_status;
|
|
use cluster::transport::{ConnectRoute, NodeIdentity, QuicClient};
|
|
|
|
let identity = NodeIdentity::from_pem_dir(tls_dir)
|
|
.with_context(|| format!("loading identity from {}", tls_dir.display()))?;
|
|
let client = QuicClient::new("0.0.0.0:0".parse()?, identity)?;
|
|
// Phase 8c: LAN-first with optional tailnet fallback. When the
|
|
// caller didn't pass --tailscale-addr the behavior is
|
|
// byte-identical to the pre-8c path (single-addr dial).
|
|
let (conn, route) = client
|
|
.connect_lan_first(
|
|
peer,
|
|
Some(rpc_addr),
|
|
tailscale_addr,
|
|
std::time::Duration::from_millis(lan_probe_ms),
|
|
)
|
|
.await?;
|
|
match route {
|
|
ConnectRoute::Lan(a) => println!("route: LAN ({a})"),
|
|
ConnectRoute::Tailscale(a) => println!("route: tailnet ({a})"),
|
|
}
|
|
let status = call_peer_status(&conn).await?;
|
|
|
|
println!(
|
|
"peer: {} (zone: {})",
|
|
status.local_name, status.local_zone
|
|
);
|
|
println!();
|
|
if status.peers.is_empty() {
|
|
println!(" (no peers known)");
|
|
} else {
|
|
println!(
|
|
" {:<20} {:<14} {:<8} {:<22} {:<10} {:<20}",
|
|
"NAME", "ZONE", "STATE", "RPC LAN", "RUSTC", "HOT USED / MAX"
|
|
);
|
|
println!(" {}", "-".repeat(100));
|
|
// Field finding 2026-07-12: also render each peer's rustc
|
|
// release. Toolchain drift silently silos caches; showing it
|
|
// here means one glance surfaces the problem.
|
|
let local_rustc = status
|
|
.peers
|
|
.iter()
|
|
.filter_map(|p| p.rustc_release.clone())
|
|
.next()
|
|
.unwrap_or_default();
|
|
for p in &status.peers {
|
|
let state = if p.alive { "alive" } else { "dead" };
|
|
let hot = match (p.hot_used_bytes, p.hot_max_bytes) {
|
|
(Some(u), Some(m)) => format!("{u} / {m}"),
|
|
(Some(u), None) => format!("{u} / -"),
|
|
_ => "-".into(),
|
|
};
|
|
let rustc = p.rustc_release.as_deref().unwrap_or("-");
|
|
let mismatch = !local_rustc.is_empty()
|
|
&& !rustc.is_empty()
|
|
&& rustc != local_rustc
|
|
&& local_rustc != "-";
|
|
let rustc_col = if mismatch {
|
|
format!("{rustc}!")
|
|
} else {
|
|
rustc.to_string()
|
|
};
|
|
println!(
|
|
" {:<20} {:<14} {:<8} {:<22} {:<10} {:<20}",
|
|
p.name,
|
|
p.zone,
|
|
state,
|
|
p.rpc_lan
|
|
.map(|a| a.to_string())
|
|
.unwrap_or_else(|| "-".into()),
|
|
rustc_col,
|
|
hot,
|
|
);
|
|
}
|
|
if status
|
|
.peers
|
|
.iter()
|
|
.filter_map(|p| p.rustc_release.as_deref())
|
|
.collect::<std::collections::HashSet<_>>()
|
|
.len()
|
|
> 1
|
|
{
|
|
println!();
|
|
println!(
|
|
" ⚠ rustc release mismatch across peers → fingerprints will silo caches"
|
|
);
|
|
}
|
|
}
|
|
|
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
|
client.shutdown().await;
|
|
Ok(())
|
|
}
|
|
|
|
// ── fleet-ca init / sign ─────────────────────────────────────────────────────
|
|
|
|
fn cmd_fleet_ca_init(dir: &std::path::Path, cn: &str) -> Result<()> {
|
|
use cluster::transport::FleetCa;
|
|
if dir.join("ca.crt").exists() {
|
|
bail!(
|
|
"refusing to overwrite existing CA at {}: delete it first if you really want a new one",
|
|
dir.display()
|
|
);
|
|
}
|
|
let ca = FleetCa::generate(cn).context("generating fleet CA")?;
|
|
ca.save(dir).context("saving fleet CA to disk")?;
|
|
println!("fleet CA written:");
|
|
println!(" {}", dir.join("ca.crt").display());
|
|
println!(" {} (chmod 0600; keep on primary only)", dir.join("ca.key").display());
|
|
println!();
|
|
println!("Sign a per-node identity with:");
|
|
println!(
|
|
" claw-store fleet-ca-sign --ca-dir {} --node <name> --out-dir <dir>",
|
|
dir.display()
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_fleet_ca_sign(
|
|
ca_dir: &std::path::Path,
|
|
node: &str,
|
|
out_dir: &std::path::Path,
|
|
) -> Result<()> {
|
|
use cluster::transport::FleetCa;
|
|
let ca = FleetCa::load(ca_dir).context("loading fleet CA")?;
|
|
ca.sign_leaf_to_pem(node, out_dir)
|
|
.context("signing + writing per-node PEMs")?;
|
|
println!("signed leaf for {node}:");
|
|
println!(" {}", out_dir.join("ca.crt").display());
|
|
println!(" {}", out_dir.join("node.crt").display());
|
|
println!(" {} (chmod 0600; distribute securely)", out_dir.join("node.key").display());
|
|
println!();
|
|
println!("On {node}, point [cluster.tls] in the config at those three paths.");
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_fleet_ca_tailscale_sign(
|
|
ca_dir: &std::path::Path,
|
|
node: Option<&str>,
|
|
out_dir: &std::path::Path,
|
|
) -> Result<()> {
|
|
use cluster::tailscale;
|
|
use cluster::transport::FleetCa;
|
|
|
|
let ts = tailscale::read_self()
|
|
.context("reading Tailscale identity via `tailscale status --json`")?;
|
|
let primary = match node {
|
|
Some(n) if !n.is_empty() => n.to_string(),
|
|
_ => ts
|
|
.short_hostname
|
|
.clone()
|
|
.context("no --node given and Tailscale reports no HostName")?,
|
|
};
|
|
let sans = ts.suggested_sans();
|
|
|
|
let ca = FleetCa::load(ca_dir).context("loading fleet CA")?;
|
|
ca.sign_leaf_to_pem_with_sans(&primary, &sans, out_dir)
|
|
.context("signing + writing per-node PEMs (with Tailscale SANs)")?;
|
|
println!("── fleet-ca tailscale-sign ─────────────────────────");
|
|
println!("primary CN/SAN: {primary}");
|
|
if !sans.is_empty() {
|
|
println!("extra SANs:");
|
|
for s in &sans {
|
|
println!(" - {s}");
|
|
}
|
|
} else {
|
|
println!("extra SANs: (none — Tailscale reported no identity data)");
|
|
}
|
|
println!();
|
|
println!("written:");
|
|
println!(" {}", out_dir.join("ca.crt").display());
|
|
println!(" {}", out_dir.join("node.crt").display());
|
|
println!(
|
|
" {} (chmod 0600; distribute securely)",
|
|
out_dir.join("node.key").display()
|
|
);
|
|
println!();
|
|
println!("On this node, point [cluster.tls] in the config at those three paths.");
|
|
println!("────────────────────────────────────────────────────");
|
|
Ok(())
|
|
}
|
|
|
|
// ── cluster ping ─────────────────────────────────────────────────────────────
|
|
|
|
/// Round-trip a `ping` payload to `peer` over the QUIC RPC transport
|
|
/// using an ephemeral fleet CA. Both this node and the peer must have
|
|
/// invoked `cluster-ping` with a shared CA — the current implementation
|
|
/// generates one CA per invocation via `NodeIdentity::generate_test_pair`,
|
|
/// meaning peers can only talk to each other when they share the same
|
|
/// invocation (typical dev-loopback usage). Production identity loading
|
|
/// from persistent PEM files lands in Phase 1d.
|
|
async fn cmd_cluster_ping(
|
|
name: &str,
|
|
peer: &str,
|
|
rpc_addr: SocketAddr,
|
|
tailscale_addr: Option<SocketAddr>,
|
|
lan_probe_ms: u64,
|
|
payload: &str,
|
|
tls_dir: Option<&std::path::Path>,
|
|
) -> Result<()> {
|
|
use cluster::transport::{ping, ConnectRoute, NodeIdentity, QuicClient};
|
|
|
|
// Two identity paths:
|
|
// 1. `--tls-dir` present → load persisted PEM. The peer must have
|
|
// been signed by the same CA (via `fleet-ca sign`); otherwise
|
|
// the TLS handshake fails.
|
|
// 2. Absent → ephemeral CA. Only useful when both endpoints run
|
|
// in the same process (dev/loopback demo).
|
|
let id_self = match tls_dir {
|
|
Some(dir) => NodeIdentity::from_pem_dir(dir)
|
|
.with_context(|| format!("loading node identity from {}", dir.display()))?,
|
|
None => {
|
|
let (id_self, _id_peer) = NodeIdentity::generate_test_pair(name, peer)?;
|
|
id_self
|
|
}
|
|
};
|
|
|
|
let client = QuicClient::new("0.0.0.0:0".parse()?, id_self)?;
|
|
// Phase 8: LAN-first with optional tailnet fallback. Zero flag
|
|
// = identical to pre-8 single-addr dial.
|
|
let (conn, route) = client
|
|
.connect_lan_first(
|
|
peer,
|
|
Some(rpc_addr),
|
|
tailscale_addr,
|
|
std::time::Duration::from_millis(lan_probe_ms),
|
|
)
|
|
.await?;
|
|
match route {
|
|
ConnectRoute::Lan(a) => println!("route: LAN ({a})"),
|
|
ConnectRoute::Tailscale(a) => println!("route: tailnet ({a})"),
|
|
}
|
|
let response = ping(&conn, payload.as_bytes()).await?;
|
|
println!("→ sent: {}", payload);
|
|
println!("← recv: {}", String::from_utf8_lossy(&response));
|
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
|
client.shutdown().await;
|
|
Ok(())
|
|
}
|
|
|
|
// ── cluster status ───────────────────────────────────────────────────────────
|
|
|
|
async fn cmd_cluster_status(cfg: &Config, name: &str, wait_secs: u64) -> Result<()> {
|
|
let cluster_cfg = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.context("no [cluster] section in config; cannot start gossip")?;
|
|
let gossip = cluster::gossip::ClusterGossip::bootstrap(cluster_cfg, name).await?;
|
|
println!("gossip bootstrapped as {}; waiting {}s for convergence...", name, wait_secs);
|
|
tokio::time::sleep(std::time::Duration::from_secs(wait_secs)).await;
|
|
|
|
let mut peers = gossip.peers().await;
|
|
peers.sort_by(|a, b| a.name.cmp(&b.name));
|
|
|
|
println!();
|
|
println!(
|
|
"{:<20} {:<14} {:<8} {:<22} {:<22} {:<20}",
|
|
"PEER", "ZONE", "STATE", "RPC LAN", "RPC TAILSCALE", "HOT USED / MAX"
|
|
);
|
|
println!("{}", "-".repeat(110));
|
|
if peers.is_empty() {
|
|
println!("(no peers yet — either alone in the fleet or gossip still converging)");
|
|
} else {
|
|
for p in &peers {
|
|
let state = if p.alive { "alive" } else { "dead" };
|
|
let hot = match (p.hot_used_bytes, p.hot_max_bytes) {
|
|
(Some(u), Some(m)) => format!("{u} / {m}"),
|
|
(Some(u), None) => format!("{u} / -"),
|
|
_ => "-".into(),
|
|
};
|
|
println!(
|
|
"{:<20} {:<14} {:<8} {:<22} {:<22} {:<20}",
|
|
p.name,
|
|
p.zone,
|
|
state,
|
|
p.rpc_lan.map(|a| a.to_string()).unwrap_or_else(|| "-".into()),
|
|
p.rpc_tailscale.map(|a| a.to_string()).unwrap_or_else(|| "-".into()),
|
|
hot,
|
|
);
|
|
}
|
|
}
|
|
println!();
|
|
gossip.shutdown();
|
|
Ok(())
|
|
}
|
|
|
|
// ── cluster probe ────────────────────────────────────────────────────────────
|
|
|
|
async fn cmd_cluster_probe(cfg: &Config, peer_name: &str) -> Result<()> {
|
|
let cluster_cfg = cfg
|
|
.cluster
|
|
.as_ref()
|
|
.context("no [cluster] section in config; cannot probe peers")?;
|
|
let peer = cluster_cfg
|
|
.peer(peer_name)
|
|
.with_context(|| format!("peer {peer_name:?} not found in cluster.peers"))?;
|
|
let probe = cluster::LanFirstProbe::new();
|
|
let winner = probe.probe(peer).await?;
|
|
println!("peer: {}", peer.name);
|
|
println!("zone: {}", peer.zone);
|
|
println!("route: {}", winner.kind.as_str());
|
|
println!("addr: {}", winner.addr);
|
|
println!("elapsed: {:?}", winner.elapsed);
|
|
Ok(())
|
|
}
|
|
|
|
// ── pin / unpin ──────────────────────────────────────────────────────────────
|
|
|
|
fn cmd_set_pin(manifest_path: &std::path::Path, project: &str, pinned: bool) -> Result<()> {
|
|
let m = Manifest::update(manifest_path, |m| {
|
|
match m.get_mut(project) {
|
|
Some(p) => {
|
|
p.pinned = pinned;
|
|
Ok(())
|
|
}
|
|
None => bail!(
|
|
"no such project in manifest: {project}\n\
|
|
(only activated projects can be pinned; run `claw-store activate {project}` first)"
|
|
),
|
|
}
|
|
})?;
|
|
let _ = m; // suppress unused — we just want the side effect
|
|
println!(
|
|
"{}: {}",
|
|
if pinned { "pinned" } else { "unpinned" },
|
|
project
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
// ── activate ─────────────────────────────────────────────────────────────────
|
|
|
|
fn cmd_activate(
|
|
cfg: &Config,
|
|
manifest: &mut Manifest,
|
|
manifest_path: &std::path::Path,
|
|
project: &str,
|
|
clone_url: Option<&str>,
|
|
) -> Result<()> {
|
|
let warm = warm_path(cfg, project);
|
|
let hot_target = hot_path(cfg, project);
|
|
|
|
if !warm.exists() {
|
|
match clone_url {
|
|
Some(url) => {
|
|
println!("Cloning {} → {}", url, warm.display());
|
|
let warm_str = warm.to_str()
|
|
.with_context(|| format!("non-UTF-8 path: {}", warm.display()))?;
|
|
let status = std::process::Command::new("git")
|
|
.args(["clone", url, warm_str])
|
|
.status()?;
|
|
anyhow::ensure!(status.success(), "git clone failed");
|
|
}
|
|
None => bail!(
|
|
"warm path does not exist: {}\nUse --clone <url> to clone it first.",
|
|
warm.display()
|
|
),
|
|
}
|
|
}
|
|
|
|
std::fs::create_dir_all(&hot_target)?;
|
|
cargo_init::write_cargo_config(&warm, &hot_target)?;
|
|
|
|
// Audit followup — use locked, atomic update so a concurrent
|
|
// dashboard activation (POSTing to /api/activate which shells out
|
|
// back to this same CLI) can't race the writer. `upsert` preserves
|
|
// an existing row's `pinned` flag — re-activating a pinned project
|
|
// doesn't silently unpin it.
|
|
let new_row = Project {
|
|
name: project.to_string(),
|
|
warm_path: warm.clone(),
|
|
hot_target_path: hot_target.clone(),
|
|
last_build: None,
|
|
last_active: None,
|
|
last_sync: None,
|
|
pinned: false,
|
|
};
|
|
Manifest::update(manifest_path, |m| {
|
|
let preserved_pin = m.get(project).map(|p| p.pinned).unwrap_or(false);
|
|
let mut row = new_row.clone();
|
|
row.pinned = preserved_pin;
|
|
m.upsert(row);
|
|
Ok(())
|
|
})?;
|
|
// Keep the in-memory `manifest` in sync with disk so downstream
|
|
// commands in the same process see the new row.
|
|
*manifest = Manifest::load(manifest_path)?;
|
|
|
|
println!("activated: {}", project);
|
|
println!(" source → {}", warm.display());
|
|
println!(" target/ → {}", hot_target.display());
|
|
Ok(())
|
|
}
|
|
|
|
// ── deactivate ────────────────────────────────────────────────────────────────
|
|
|
|
/// CLI-facing deactivate: thin wrapper around `actions::deactivate_project`
|
|
/// that adds println! feedback for the terminal. The daemon calls the
|
|
/// shared function directly with tracing::info! instead.
|
|
fn cmd_deactivate(
|
|
cfg: &Config,
|
|
manifest: &mut Manifest,
|
|
manifest_path: &std::path::Path,
|
|
project: &str,
|
|
) -> Result<()> {
|
|
if cfg.peer.is_some() {
|
|
println!("Syncing {} to peer before deactivate...", project);
|
|
}
|
|
let outcome = actions::deactivate_project(cfg, manifest_path, project)?;
|
|
if let Some(err) = &outcome.sync_error {
|
|
eprintln!(" warning: sync failed ({}), queued for retry", err);
|
|
}
|
|
// Refresh the caller's in-memory manifest to match the on-disk state.
|
|
*manifest = Manifest::load(manifest_path)?;
|
|
let warm = warm_path(cfg, project);
|
|
let freed = if outcome.freed_bytes > 0 {
|
|
format!(", freed {:.1} MB", outcome.freed_mb())
|
|
} else {
|
|
String::new()
|
|
};
|
|
println!("deactivated: {} (warm clone kept at {}{})", project, warm.display(), freed);
|
|
Ok(())
|
|
}
|
|
|
|
// ── list ──────────────────────────────────────────────────────────────────────
|
|
|
|
fn cmd_list(cfg: &Config, manifest: &Manifest) -> Result<()> {
|
|
let base = &cfg.warm.projects_path;
|
|
println!("{:<45} {:<10} {}", "PROJECT", "STATUS", "LAST ACTIVE");
|
|
println!("{}", "-".repeat(75));
|
|
|
|
let mut rows: Vec<(String, bool, String)> = Vec::new();
|
|
|
|
if let Ok(top_entries) = std::fs::read_dir(base) {
|
|
for org_entry in top_entries.flatten() {
|
|
let org_path = org_entry.path();
|
|
if !org_path.is_dir() { continue; }
|
|
let org = org_entry.file_name().to_string_lossy().to_string();
|
|
|
|
if org_path.join(".git").exists() {
|
|
// Top-level git repo (flat layout)
|
|
let active = manifest.get(&org);
|
|
let last = active
|
|
.and_then(|p| p.last_active)
|
|
.map(|t| t.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "—".into());
|
|
rows.push((org, active.is_some(), last));
|
|
} else {
|
|
// Org directory — scan for repos inside it
|
|
if let Ok(repos) = std::fs::read_dir(&org_path) {
|
|
for repo_entry in repos.flatten() {
|
|
let repo_path = repo_entry.path();
|
|
if !repo_path.is_dir() || !repo_path.join(".git").exists() { continue; }
|
|
let repo = repo_entry.file_name().to_string_lossy().to_string();
|
|
let key = format!("{}/{}", org, repo);
|
|
let active = manifest.get(&key);
|
|
let last = active
|
|
.and_then(|p| p.last_active)
|
|
.map(|t| t.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "—".into());
|
|
rows.push((key, active.is_some(), last));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
rows.sort_by(|a, b| a.0.cmp(&b.0));
|
|
rows.dedup_by(|a, b| a.0 == b.0);
|
|
|
|
let active_count = rows.iter().filter(|r| r.1).count();
|
|
for (name, is_active, last) in &rows {
|
|
let status = if *is_active { "active" } else { "warm" };
|
|
println!("{:<45} {:<10} {}", name, status, last);
|
|
}
|
|
println!("\n{} repos ({} active, {} warm-only)", rows.len(), active_count, rows.len() - active_count);
|
|
Ok(())
|
|
}
|
|
|
|
// ── sync ─────────────────────────────────────────────────────────────────────
|
|
|
|
fn cmd_sync(cfg: &Config, manifest: &Manifest, project: &str) -> Result<()> {
|
|
let peer = cfg.peer.as_ref()
|
|
.context("no [peer] configured — add peer.host and peer.user to config.toml")?;
|
|
|
|
let warm = manifest.get(project)
|
|
.map(|p| p.warm_path.clone())
|
|
.unwrap_or_else(|| warm_path(cfg, project));
|
|
|
|
println!("Pushing {}...", project);
|
|
sync::sync_project(&warm, project, &peer.user, &peer.host)?;
|
|
println!(" pushed to origin");
|
|
println!(" notified {}@{} to pull", peer.user, peer.host);
|
|
Ok(())
|
|
}
|
|
|
|
// ── pull ─────────────────────────────────────────────────────────────────────
|
|
|
|
fn cmd_pull(
|
|
cfg: &Config,
|
|
manifest: &mut Manifest,
|
|
manifest_path: &std::path::Path,
|
|
project: &str,
|
|
) -> Result<()> {
|
|
let warm = manifest.get(project)
|
|
.map(|p| p.warm_path.clone())
|
|
.unwrap_or_else(|| warm_path(cfg, project));
|
|
|
|
sync::pull_project(&warm)?;
|
|
|
|
// Stamp last_sync on the manifest entry if it exists.
|
|
Manifest::update(manifest_path, |m| {
|
|
if let Some(p) = m.get_mut(project) {
|
|
p.last_sync = Some(chrono::Utc::now());
|
|
}
|
|
Ok(())
|
|
})?;
|
|
*manifest = Manifest::load(manifest_path)?;
|
|
println!("pulled: {}", project);
|
|
Ok(())
|
|
}
|
|
|
|
// ── existing commands ─────────────────────────────────────────────────────────
|
|
|
|
fn cmd_status(cfg: &Config, manifest: &Manifest, zfs: &SystemZfs) -> Result<()> {
|
|
println!("=== claw-store status — {} ===\n", cfg.node.name);
|
|
println!("HOT tier: {}", cfg.hot.path.display());
|
|
let used = hot::total_used_gb(manifest).unwrap_or(0.0);
|
|
println!(" used: {:.1} GB / {} GB max\n", used, cfg.hot.max_gb);
|
|
println!("WARM tier: {}", cfg.warm.projects_path.display());
|
|
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset).unwrap_or_default();
|
|
println!(" snapshots: {}\n", snaps.len());
|
|
if let Some(cold) = &cfg.cold {
|
|
println!("COLD tier: {}\n", cold.archive_path.display());
|
|
}
|
|
if let Some(peer) = &cfg.peer {
|
|
println!("PEER: {}@{}\n", peer.user, peer.host);
|
|
}
|
|
let pinned_count = manifest.projects.iter().filter(|p| p.pinned).count();
|
|
println!(
|
|
"Active projects ({}{}):",
|
|
manifest.projects.len(),
|
|
if pinned_count > 0 { format!(", {pinned_count} pinned") } else { String::new() },
|
|
);
|
|
for p in &manifest.projects {
|
|
let active = p.last_active
|
|
.map(|t| t.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "never".into());
|
|
let synced = p.last_sync
|
|
.map(|t| t.format("%Y-%m-%d %H:%M").to_string())
|
|
.unwrap_or_else(|| "never".into());
|
|
let pin_marker = if p.pinned { " 📌" } else { "" };
|
|
println!(" {}{} (active: {}, synced: {})", p.name, pin_marker, active, synced);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_gc(cfg: &Config, manifest: &mut Manifest) -> Result<()> {
|
|
let stale = hot::gc_stale_targets(manifest, cfg.hot.stale_hours)?;
|
|
for name in &stale { println!(" evicted (stale): {}", name); }
|
|
let space = hot::gc_by_space(manifest, cfg.hot.max_gb as f64)?;
|
|
for name in &space { println!(" evicted (space): {}", name); }
|
|
if stale.is_empty() && space.is_empty() {
|
|
println!("Nothing to evict.");
|
|
return Ok(());
|
|
}
|
|
// Persist via the locked atomic path — `manifest.save` is now
|
|
// flock+rename internally, so this is safe under concurrent writers.
|
|
manifest.save(&Manifest::default_path())?;
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_snapshot(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
|
let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
|
|
snapshot::run_snapshot_cycle(
|
|
zfs, &cfg.warm.zfs_dataset, &ts,
|
|
cfg.warm.snapshot_retain_hours as usize,
|
|
cfg.warm.snapshot_retain_days as usize,
|
|
cfg.warm.snapshot_retain_weeks as usize,
|
|
)?;
|
|
println!("Snapshot {} taken.", ts);
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_list_snapshots(cfg: &Config, zfs: &SystemZfs, _project: &str) -> Result<()> {
|
|
let snaps = zfs.list_snapshots(&cfg.warm.zfs_dataset)?;
|
|
if snaps.is_empty() { println!("No snapshots found."); return Ok(()); }
|
|
for s in &snaps { println!(" {}", s); }
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_restore(cfg: &Config, zfs: &SystemZfs, project: &str, snap: &str) -> Result<()> {
|
|
let full_snap = format!("{}@{}", cfg.warm.zfs_dataset, snap);
|
|
let found = restore::find_snapshot(zfs, &cfg.warm.zfs_dataset, snap)?;
|
|
anyhow::ensure!(found.is_some(), "snapshot '{}' not found", snap);
|
|
let path = restore::restore_project(zfs, project, &full_snap, &cfg.warm.zfs_dataset)?;
|
|
println!("Restored to: {}", path.display());
|
|
println!("Cleanup: zfs destroy {}/{}-restore-{}", cfg.warm.zfs_dataset, project, snap);
|
|
Ok(())
|
|
}
|
|
|
|
fn cmd_replicate(cfg: &Config, zfs: &SystemZfs) -> Result<()> {
|
|
let rep = cfg.replication.as_ref()
|
|
.context("no replication config — this node does not replicate")?;
|
|
let host = rep.send_to_host.as_ref().context("send_to_host not set")?;
|
|
let user = rep.send_to_user.as_ref().context("send_to_user not set")?;
|
|
let dest = rep.cold_dataset_on_peer.as_ref().context("cold_dataset_on_peer not set")?;
|
|
snapshot::replicate_to_cold(zfs, &cfg.warm.zfs_dataset, user, host, dest)?;
|
|
println!("Replication to {}@{} complete.", user, host);
|
|
Ok(())
|
|
}
|
|
|
|
// ── path helpers ──────────────────────────────────────────────────────────────
|
|
|
|
fn warm_path(cfg: &Config, project: &str) -> PathBuf {
|
|
cfg.warm.projects_path.join(project)
|
|
}
|
|
|
|
fn hot_path(cfg: &Config, project: &str) -> PathBuf {
|
|
cfg.hot.path.join(project)
|
|
}
|