Add shutdown-prep button to dashboard-v2 NodeDetail
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Failing after 3s
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Failing after 3s
Wires safe-shutdown-prep.sh into the dashboard so an operator can prep a node for hardware maintenance from a browser instead of SSH. New RPC methods (0x20/0x21): - ShutdownPrepCheck runs `--dry-run` to completion and returns the full report. Never stops anything, safe to call repeatedly. - ShutdownPrepExecute starts the real run detached (`systemd-run --user --scope --collect`), placing it in a cgroup outside claw-store.service's own -- the script's own step 6 stops that service, i.e. the process that would otherwise be running it, so it has to survive its own parent dying. Returns immediately with a "started" message; full output lands in /var/lib/claw-store/shutdown-prep.log for whoever's at the machine once it's gone dark, since there's no way to stream a live result past the point the daemon stops itself. - Execute double-checks confirm_node_name against the peer's own configured name server-side, on top of the aggregator's own path match -- defense in depth for a highly consequential action. Aggregator endpoints (admin-token gated, AuthedCaller::require_admin): POST /api/v2/node/:name/shutdown-prep/check POST /api/v2/node/:name/shutdown-prep/execute Frontend: ShutdownPrepPanel on NodeDetail. Check button always enabled; the real "stop services" button only unlocks after a ready check, and additionally requires typing the exact node name to confirm before it's clickable. Also fixes a script bug found while testing this against the live daemon process (not caught in manual interactive-shell testing): the zpool-detection line parsed raw `mount` output positionally, which returned the wrong field under the daemon's process context for reasons that didn't reproduce interactively. Switched to `df --output=source`, which is stable across both. Verified end-to-end against tank, architect, and morpheus, including cross-node targeting (tank's dashboard successfully triggered a check on morpheus over the fleet RPC layer). Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
@@ -570,6 +570,19 @@ impl AuthedCaller {
|
||||
/// in the request body. Explicit user-supplied workspace is only
|
||||
/// honored for admin/open — a namespaced caller supplying a
|
||||
/// mismatched workspace is a forbidden write.
|
||||
/// Gate for fleet-infrastructure actions (shutdown-prep) that
|
||||
/// have nothing to do with a tag/repo namespace — a namespaced
|
||||
/// per-app token has no business stopping a node's services.
|
||||
pub fn require_admin(&self) -> Result<(), (StatusCode, String)> {
|
||||
match self {
|
||||
AuthedCaller::Admin | AuthedCaller::Open => Ok(()),
|
||||
AuthedCaller::Namespaced { .. } => Err((
|
||||
StatusCode::FORBIDDEN,
|
||||
"this action requires an admin token".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_workspace(&self, requested: Option<&str>) -> Result<String, (StatusCode, String)> {
|
||||
match self {
|
||||
AuthedCaller::Admin | AuthedCaller::Open => match requested {
|
||||
@@ -866,6 +879,93 @@ async fn handle_repos_release(
|
||||
}))
|
||||
}
|
||||
|
||||
// ── shutdown-prep (targets one specific node, not a fan-out) ──────
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ShutdownPrepCheckResponse {
|
||||
pub node: String,
|
||||
pub ready: bool,
|
||||
pub output: String,
|
||||
}
|
||||
|
||||
async fn handle_shutdown_prep_check(
|
||||
State(s): State<Arc<V2State>>,
|
||||
Path(name): Path<String>,
|
||||
axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
|
||||
) -> Result<Json<ShutdownPrepCheckResponse>, (StatusCode, String)> {
|
||||
caller.require_admin()?;
|
||||
let peer = s
|
||||
.peers
|
||||
.iter()
|
||||
.find(|p| p.name == name)
|
||||
.cloned()
|
||||
.ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
|
||||
let conn = s
|
||||
.dial(&peer)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
|
||||
let reply = crate::cluster::rpc::call_shutdown_prep_check(&conn).await;
|
||||
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||
let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
|
||||
Ok(Json(ShutdownPrepCheckResponse {
|
||||
node: name,
|
||||
ready: reply.ready,
|
||||
output: reply.output,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ShutdownPrepExecuteBody {
|
||||
/// Must equal the target node's own name — a second, server-side
|
||||
/// confirmation beyond "the operator clicked the right button in
|
||||
/// the UI". Checked again on the peer itself in
|
||||
/// `shutdown_prep::execute`.
|
||||
pub confirm_node_name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ShutdownPrepExecuteResponse {
|
||||
pub node: String,
|
||||
pub started: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
async fn handle_shutdown_prep_execute(
|
||||
State(s): State<Arc<V2State>>,
|
||||
Path(name): Path<String>,
|
||||
axum::extract::Extension(caller): axum::extract::Extension<AuthedCaller>,
|
||||
Json(body): Json<ShutdownPrepExecuteBody>,
|
||||
) -> Result<Json<ShutdownPrepExecuteResponse>, (StatusCode, String)> {
|
||||
caller.require_admin()?;
|
||||
if body.confirm_node_name != name {
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"confirm_node_name '{}' does not match target node '{name}'",
|
||||
body.confirm_node_name
|
||||
),
|
||||
));
|
||||
}
|
||||
let peer = s
|
||||
.peers
|
||||
.iter()
|
||||
.find(|p| p.name == name)
|
||||
.cloned()
|
||||
.ok_or((StatusCode::NOT_FOUND, format!("no peer named {name}")))?;
|
||||
let conn = s
|
||||
.dial(&peer)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("connecting to {name}: {e:#}")))?;
|
||||
let reply = crate::cluster::rpc::call_shutdown_prep_execute(&conn, &name).await;
|
||||
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||
let reply = reply.map_err(|e| (StatusCode::BAD_GATEWAY, format!("{e:#}")))?;
|
||||
Ok(Json(ShutdownPrepExecuteResponse {
|
||||
node: name,
|
||||
started: reply.started,
|
||||
message: reply.message,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn fanout_repo_ensure(
|
||||
s: &V2State,
|
||||
req: &crate::cluster::repo_ensure::RepoEnsureRequest,
|
||||
@@ -1347,6 +1447,14 @@ pub fn build(state: Arc<V2State>) -> Router {
|
||||
.route("/api/v2/sessions/:id/commit", post(handle_commit_session))
|
||||
.route("/api/v2/repos/ensure", post(handle_repos_ensure))
|
||||
.route("/api/v2/repos/release", post(handle_repos_release))
|
||||
.route(
|
||||
"/api/v2/node/:name/shutdown-prep/check",
|
||||
post(handle_shutdown_prep_check),
|
||||
)
|
||||
.route(
|
||||
"/api/v2/node/:name/shutdown-prep/execute",
|
||||
post(handle_shutdown_prep_execute),
|
||||
)
|
||||
.route_layer(axum::middleware::from_fn_with_state(state.clone(), v2_auth))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user