fix(missions): make an unrunnable test suite legible, and check the runtime at boot
Two changes against the same defect: the platform could not tell a missing capability from a legitimate negative result. verify_tests returned Option<bool>, collapsing four outcomes into None: no suite found, docker unreachable, exec failed, and no exit status. When clawmates-runtime shipped without cargo, every on_green_tests phase returned None and landed on -wip — identical to the reading for "this repo has no tests", which is the conclusion I drew and reported. The gate was correct throughout; it simply could not say why it was unproven. TestOutcome now names the four cases. Gating is unchanged (only Passed clears, unproven is never a pass), and tests_verified keeps its tri-state meaning for existing readers. tests_status and tests_detail are new, so an artifact distinguishes no_suite from could_not_run, and a CouldNotRun is logged as the infrastructure fault it is rather than passing quietly. runtime_preflight probes the runtime container at boot for every tool the platform invokes inside it and names what each absence disables. This is the check that was missing: the Dockerfile gained a toolchain, the image was never built, gw-04 ran the old one for days, and the only symptoms were an ungated suite and a security scan that scanned nothing. A report, not a gate — a missing scanner should stop us believing a scan, not stop the server. Its test guards the probes themselves, since a typo would produce a permanent false "missing" and train operators to ignore it. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
9bdc3cd89b
commit
f7e336ff5f
@@ -0,0 +1,190 @@
|
||||
//! Does the mission runtime actually carry the tools we depend on?
|
||||
//!
|
||||
//! Every capability in this codebase is written twice: once as code that
|
||||
//! invokes a binary, and once as a Dockerfile line that installs it. The two
|
||||
//! are only connected by someone having built and shipped the image, and
|
||||
//! nothing checked that they agreed.
|
||||
//!
|
||||
//! They did not. `deploy/clawmates-runtime/Dockerfile` gained a Rust
|
||||
//! toolchain, `gitleaks`, `trivy`, `semgrep` and `cargo-audit`; the image was
|
||||
//! never built, and gw-04 kept running the previous one for days. The
|
||||
//! consequences were all silent:
|
||||
//!
|
||||
//! - `verify_tests` could not launch `cargo test`, so every `on_green_tests`
|
||||
//! phase landed on `-wip` — indistinguishable from "no test suite here"
|
||||
//! - `security_scan` emitted `tool_error` rows and reported completion
|
||||
//! - the evaluator's allow-listed checks could not run the scanners
|
||||
//!
|
||||
//! No error, no log line, no failing test. The code was right and the machine
|
||||
//! was not. This module makes that specific disagreement observable: it asks
|
||||
//! the running container what it has and says so plainly at boot.
|
||||
//!
|
||||
//! It is a report, not a gate. A missing scanner should not stop the server
|
||||
//! from serving — it should stop us believing a scan that scanned nothing.
|
||||
|
||||
use crate::container_exec;
|
||||
use std::time::Duration;
|
||||
|
||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// A tool the platform invokes inside the runtime container, and what breaks
|
||||
/// without it. The consequence text is the point: a bare list of missing
|
||||
/// binaries does not tell an operator what is now quietly not happening.
|
||||
struct Dependency {
|
||||
argv: &'static [&'static str],
|
||||
needed_for: &'static str,
|
||||
}
|
||||
|
||||
const DEPENDENCIES: &[Dependency] = &[
|
||||
Dependency {
|
||||
argv: &["cargo", "--version"],
|
||||
needed_for: "the on_green_tests gate for Rust repos; without it every \
|
||||
phase is unverified and lands on -wip",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["git", "--version"],
|
||||
needed_for: "agent-side git operations in the mission checkout",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["gitleaks", "version"],
|
||||
needed_for: "secret scanning in security_scan phases and evaluator checks",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["trivy", "--version"],
|
||||
needed_for: "vulnerability scanning in security_scan phases",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["semgrep", "--version"],
|
||||
needed_for: "static analysis in security_scan phases",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["cargo-audit", "--version"],
|
||||
needed_for: "dependency advisories in security_scan phases",
|
||||
},
|
||||
];
|
||||
|
||||
/// One tool's availability, as reported by the container itself.
|
||||
pub struct ToolStatus {
|
||||
pub program: String,
|
||||
pub present: bool,
|
||||
/// Version string when present, error when not.
|
||||
pub detail: String,
|
||||
pub needed_for: &'static str,
|
||||
}
|
||||
|
||||
/// Probe the runtime container for everything we invoke inside it.
|
||||
///
|
||||
/// Returns an empty vec if Docker itself is unreachable — that is a different
|
||||
/// and louder failure which the caller reports separately, and emitting six
|
||||
/// "missing" lines for it would be misleading.
|
||||
pub async fn probe(container: &str) -> Result<Vec<ToolStatus>, String> {
|
||||
let docker = container_exec::connect().map_err(|e| format!("docker unreachable: {e}"))?;
|
||||
let mut out = Vec::with_capacity(DEPENDENCIES.len());
|
||||
for dep in DEPENDENCIES {
|
||||
let argv: Vec<String> = dep.argv.iter().map(|s| s.to_string()).collect();
|
||||
let status =
|
||||
match container_exec::exec(&docker, container, None, &argv, PROBE_TIMEOUT).await {
|
||||
Ok(r) if r.success() => ToolStatus {
|
||||
program: dep.argv[0].to_string(),
|
||||
present: true,
|
||||
detail: r
|
||||
.combined()
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.chars()
|
||||
.take(80)
|
||||
.collect(),
|
||||
needed_for: dep.needed_for,
|
||||
},
|
||||
Ok(r) => ToolStatus {
|
||||
program: dep.argv[0].to_string(),
|
||||
present: false,
|
||||
detail: r.combined().trim().chars().take(160).collect(),
|
||||
needed_for: dep.needed_for,
|
||||
},
|
||||
Err(e) => ToolStatus {
|
||||
program: dep.argv[0].to_string(),
|
||||
present: false,
|
||||
detail: e.chars().take(160).collect(),
|
||||
needed_for: dep.needed_for,
|
||||
},
|
||||
};
|
||||
out.push(status);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Probe at startup and write the result to stderr.
|
||||
///
|
||||
/// Spawned rather than awaited so a slow or absent Docker socket cannot delay
|
||||
/// the server coming up — the report is diagnostic, and the platform has to
|
||||
/// keep working without it.
|
||||
pub fn report_at_boot() {
|
||||
tokio::spawn(async {
|
||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||
match probe(&container).await {
|
||||
Err(e) => eprintln!(
|
||||
"runtime_preflight: could not probe `{container}` ({e}) — mission \
|
||||
test gating and security scans may silently do nothing"
|
||||
),
|
||||
Ok(tools) => {
|
||||
let missing: Vec<&ToolStatus> = tools.iter().filter(|t| !t.present).collect();
|
||||
if missing.is_empty() {
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.program.as_str()).collect();
|
||||
eprintln!(
|
||||
"runtime_preflight: `{container}` has all {} expected tools ({})",
|
||||
tools.len(),
|
||||
names.join(", ")
|
||||
);
|
||||
return;
|
||||
}
|
||||
eprintln!(
|
||||
"runtime_preflight: `{container}` is MISSING {} of {} tools the \
|
||||
platform invokes. The image on this host is behind \
|
||||
deploy/clawmates-runtime/Dockerfile — rebuild and redeploy it.",
|
||||
missing.len(),
|
||||
tools.len()
|
||||
);
|
||||
for t in missing {
|
||||
eprintln!(
|
||||
"runtime_preflight: {} — absent. Disables: {}. ({})",
|
||||
t.program, t.needed_for, t.detail
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Every dependency must be probed with a flag that exits zero and prints
|
||||
/// a version. A typo here produces a permanent false "missing" that would
|
||||
/// train an operator to ignore the report — worse than no report at all.
|
||||
#[test]
|
||||
fn every_dependency_probe_is_a_version_query() {
|
||||
for dep in DEPENDENCIES {
|
||||
assert!(
|
||||
dep.argv.len() >= 2,
|
||||
"{} needs an argument that exits 0",
|
||||
dep.argv[0]
|
||||
);
|
||||
let flag = dep.argv[1];
|
||||
assert!(
|
||||
flag == "--version" || flag == "version",
|
||||
"{} probes with `{flag}`, which may not exit 0",
|
||||
dep.argv[0]
|
||||
);
|
||||
assert!(
|
||||
!dep.needed_for.is_empty(),
|
||||
"{} must say what breaks without it",
|
||||
dep.argv[0]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user