//! Running a command inside a container, over the Docker API. //! //! Three call sites needed this and each had shelled out to the `docker` CLI: //! the evaluator's verification sandbox, the security scanner, and the //! benchmark runner. **The server image does not ship a `docker` binary** //! (`images/server.Dockerfile` installs `git ca-certificates chromium //! fonts-liberation` and nothing else), so every one of those calls failed //! with a spawn error at runtime. //! //! The failure was invisible in the worst way. `evaluator_tools::Sandbox::run` //! turns any execution failure into evidence text rather than an error — //! deliberately, so a judge reasons about "that command did not run" instead //! of the pass collapsing. With no `docker` binary every verification command //! returned `COULD NOT RUN`, the judge correctly concluded it could not verify, //! and fail-closed returned "not met". The verdicts were right; the //! verification never happened. //! //! `bollard` was already a dependency and already reaches the daemon through //! the socket proxy (`DOCKER_HOST=tcp://socket-proxy:2375`) for every //! container operation in `mission_runtime`. This routes command execution the //! same way. //! //! The argv contract is unchanged: a command is a vector, never a shell //! string, so the allow-list in `evaluator_tools::check_argv` keeps meaning //! what it says. use bollard::exec::{CreateExecOptions, StartExecResults}; use bollard::Docker; use futures::StreamExt; use std::time::Duration; /// What a command did. Both streams are captured separately because callers /// need them for different things — the evaluator shows the judge stdout *and* /// stderr, while the scanners parse JSON from stdout alone and would choke on /// interleaved progress output. #[derive(Debug, Clone)] pub struct ExecOutput { /// `None` when the daemon reported no status (a still-running exec, which /// we treat as unknown rather than success). pub exit_code: Option, pub stdout: String, pub stderr: String, } impl ExecOutput { /// Exit status 0. An absent status is **not** success — an exec whose /// status could not be read must not be reported as a passing test run. pub fn success(&self) -> bool { self.exit_code == Some(0) } /// Both streams in the order a human reads them. Used where the consumer /// is a model rather than a parser. pub fn combined(&self) -> String { let mut out = String::new(); if !self.stdout.trim().is_empty() { out.push_str(&self.stdout); } if !self.stderr.trim().is_empty() { if !out.is_empty() { out.push('\n'); } out.push_str(&self.stderr); } out } } /// Connect to the Docker daemon the same way `mission_runtime` does: honour /// `DOCKER_HOST` when set (the socket proxy in production), else the local /// socket. pub fn connect() -> Result { if std::env::var("DOCKER_HOST").is_ok() { Docker::connect_with_defaults().map_err(|e| format!("docker connect (DOCKER_HOST): {e}")) } else { Docker::connect_with_local_defaults().map_err(|e| format!("docker connect (local): {e}")) } } /// The uid every mission artefact must belong to. /// /// The runtime container's own processes already run as this; only `docker /// exec` defaulted to root, because `CreateExecOptions::user` was never set. /// That one omission is the origin of four separate patches: root-owned /// `target/` directories appearing inside a checkout that uid 65532 then could /// not delete, `root_copy` existing at all, and a cleanup path that had to /// re-enter the container as root to undo what it had just done. pub(crate) const MISSION_UID: &str = "65532:65532"; /// Environment a non-root exec needs, because the image gives uid 65532 no /// writable `HOME` and no writable `CARGO_HOME`. /// /// Measured in the deployed image: `/zeroclaw-data` (its `HOME`) and /// `/usr/local/cargo` are both root-owned and unwritable, so switching execs to /// 65532 without this would break every `cargo` invocation — the benchmark /// runner, the judge's verification sandbox, and the delivery test gate — in a /// new and much quieter way than the problem it fixes. /// /// The missions root is bind-mounted into the runtime container at the same /// path and IS writable by 65532, so the cargo cache lives there and is shared /// across missions rather than re-downloaded per mission. Verified end to end: /// a clean `cargo build` as 65532 with these three variables produces output /// owned entirely by 65532. fn mission_env() -> Vec { let root = crate::mission_workspace::missions_root(); vec![ format!("HOME={}", root.join("_home").display()), format!("CARGO_HOME={}", root.join("_cargo").display()), "TMPDIR=/tmp".to_string(), ] } /// Whether a workdir is inside the tree missions own. /// /// The rule is positional rather than per-caller on purpose. Twelve call sites /// each remembering to pass a uid is twelve chances to forget, and the one that /// forgets leaves debris the others cannot clean up — which is exactly the /// history here. fn is_mission_path(workdir: Option<&str>) -> bool { let Some(dir) = workdir else { return false }; let root = crate::mission_workspace::missions_root(); std::path::Path::new(dir).starts_with(&root) } /// Run `argv` in `container`, optionally in `workdir`, and capture both /// streams plus the exit status. /// /// `timeout` bounds the whole exec. On expiry the error says so explicitly: /// the exec may still be running inside the container, and a caller that /// retries needs to know it is not looking at a clean slate. pub async fn exec( docker: &Docker, container: &str, workdir: Option<&str>, argv: &[String], timeout: Duration, ) -> Result { exec_with_env(docker, container, workdir, argv, &[], timeout).await } /// Run `argv` as **root**, deliberately. /// /// The one legitimate use is clearing debris that earlier root-run execs left /// behind: uid 65532 cannot delete a root-owned `target/`, so the cleanup has /// to out-rank it. Every other caller goes through [`exec`], which runs mission /// work as 65532 so no new debris is created. pub async fn exec_as_root( docker: &Docker, container: &str, workdir: Option<&str>, argv: &[String], timeout: Duration, ) -> Result { let fut = exec_inner(docker, container, workdir, argv, &[], None); match tokio::time::timeout(timeout, fut).await { Err(_) => Err(format!( "timed out after {}s (the command may still be running in {container})", timeout.as_secs() )), Ok(res) => res, } } /// As [`exec`], with extra environment for the command. pub async fn exec_with_env( docker: &Docker, container: &str, workdir: Option<&str>, argv: &[String], env: &[String], timeout: Duration, ) -> Result { // Mission work runs as 65532 with a writable HOME/CARGO_HOME; anything // outside the missions tree (runtime preflight probes, image checks) keeps // the daemon's default so this cannot break unrelated call sites. let (user, mut full_env) = if is_mission_path(workdir) { (Some(MISSION_UID), mission_env()) } else { (None, Vec::new()) }; full_env.extend_from_slice(env); let fut = exec_inner(docker, container, workdir, argv, &full_env, user); match tokio::time::timeout(timeout, fut).await { Err(_) => Err(format!( "timed out after {}s (the command may still be running in {container})", timeout.as_secs() )), Ok(res) => res, } } async fn exec_inner( docker: &Docker, container: &str, workdir: Option<&str>, argv: &[String], env: &[String], user: Option<&str>, ) -> Result { let created = docker .create_exec( container, CreateExecOptions { cmd: Some(argv.to_vec()), working_dir: workdir.map(str::to_string), env: if env.is_empty() { None } else { Some(env.to_vec()) }, user: user.map(str::to_string), attach_stdout: Some(true), attach_stderr: Some(true), ..Default::default() }, ) .await .map_err(|e| format!("create_exec on {container}: {e}"))?; let started = docker .start_exec(&created.id, None) .await .map_err(|e| format!("start_exec on {container}: {e}"))?; let StartExecResults::Attached { mut output, .. } = started else { return Err(format!("exec on {container} returned a detached result")); }; // Keep the streams apart. `LogOutput`'s Display merges them, which is what // the previous helper used and why nothing downstream could tell a JSON // payload from a progress bar. let mut stdout = String::new(); let mut stderr = String::new(); while let Some(chunk) = output.next().await { match chunk { Ok(bollard::container::LogOutput::StdOut { message }) => { stdout.push_str(&String::from_utf8_lossy(&message)); } Ok(bollard::container::LogOutput::StdErr { message }) => { stderr.push_str(&String::from_utf8_lossy(&message)); } // A container without a TTY still emits Console/StdIn frames in // some daemon versions; treat them as stdout rather than dropping // output on the floor. Ok(other) => stdout.push_str(&other.to_string()), Err(e) => return Err(format!("exec output stream on {container}: {e}")), } } // The status is only available after the stream drains. let inspected = docker .inspect_exec(&created.id) .await .map_err(|e| format!("inspect_exec on {container}: {e}"))?; Ok(ExecOutput { exit_code: inspected.exit_code, stdout, stderr, }) } #[cfg(test)] mod tests { use super::*; fn out(code: Option, stdout: &str, stderr: &str) -> ExecOutput { ExecOutput { exit_code: code, stdout: stdout.into(), stderr: stderr.into(), } } /// An exec whose status could not be read must not pass for success — /// `commit_policy = "on_green_tests"` gates on exactly this, and treating /// "unknown" as "green" would push untested work. #[test] fn an_unknown_exit_status_is_not_success() { assert!(out(Some(0), "ok", "").success()); assert!(!out(Some(1), "", "boom").success()); assert!(!out(None, "ok", "").success()); } #[test] fn combined_keeps_both_streams_and_skips_empty_ones() { assert_eq!(out(Some(0), "hello", "").combined(), "hello"); assert_eq!(out(Some(1), "", "bad").combined(), "bad"); assert_eq!(out(Some(1), "a", "b").combined(), "a\nb"); assert_eq!(out(Some(0), " ", "\n").combined(), ""); } /// Mission work is 65532; everything else keeps the daemon's default. /// /// The rule is positional so that no caller has to remember it. Twelve call /// sites each passing a uid is twelve chances to forget, and the one that /// forgets leaves debris the other eleven cannot delete — which is the /// actual history: root-owned `target/` directories inside a checkout owned /// by 65532, `root_copy` written to work around them, and a cleanup that had /// to re-enter the container as root to undo its own mess. #[test] fn only_work_inside_the_missions_tree_drops_to_the_mission_uid() { let root = crate::mission_workspace::missions_root(); let inside = root.join("019fe785-0f82-7780-8d58-da79fb4c31bc/repo"); assert!(is_mission_path(Some(&inside.display().to_string()))); assert!(is_mission_path(Some(&root.display().to_string()))); // Probes and image checks run with no workdir at all, and must not be // forced to a uid the image may not have set up for them. assert!(!is_mission_path(None)); assert!(!is_mission_path(Some("/"))); assert!(!is_mission_path(Some("/usr/local/cargo"))); // A path that merely SHARES A PREFIX is not inside the tree. // `starts_with` on `Path` compares components, so this is already true; // the assertion is here so a switch to string matching cannot pass. let sibling = format!("{}-evil/repo", root.display()); assert!(!is_mission_path(Some(&sibling))); } /// The non-root exec carries the three variables the image does not give it. /// /// Measured in the deployed image: uid 65532's `HOME` (`/zeroclaw-data`) /// and `/usr/local/cargo` are both root-owned and unwritable. Without these /// overrides, dropping execs to 65532 would break every cargo invocation — /// the benchmark runner, the judge's sandbox, the delivery test gate — far /// more quietly than the leak it fixes. #[test] fn the_mission_env_replaces_the_paths_the_image_leaves_unwritable() { let env = mission_env(); let root = crate::mission_workspace::missions_root(); assert!(env.iter().any(|v| v == &format!("HOME={}/_home", root.display()))); assert!(env.iter().any(|v| v == &format!("CARGO_HOME={}/_cargo", root.display()))); assert!(env.iter().any(|v| v == "TMPDIR=/tmp")); for v in &env { assert!( !v.contains("/usr/local/cargo") && !v.contains("/zeroclaw-data"), "{v} points back at a root-owned path" ); } } /// One place builds an exec, so one place decides its uid. /// /// The original bug was not a wrong value — it was an ABSENT one: /// `CreateExecOptions` never set `user`, so the daemon defaulted to root /// and twelve callers inherited that without any of them choosing it. A /// second construction site is how that comes back, so the guard is on the /// number of sites rather than on any particular uid. #[test] fn exactly_one_place_builds_an_exec() { let src = include_str!("container_exec.rs"); // Split so this needle does not match itself in this very file. let needle = concat!("CreateExec", "Options {"); let sites = src.matches(needle).count(); assert_eq!( sites, 1, "exec options must be built in one place; found {sites}" ); assert!( src.contains(concat!("user: ", "user.map(str::to_string)")), "that one place must set `user` — leaving it unset is the bug" ); } } /// The tail of a container's log, for putting in an error message. /// /// A turn that times out destroys the only place the reason lived: the /// per-mission runtime container is torn down after the phase, taking its logs /// with it, and the operator is left with the string "turn timed out". This /// copies the last few lines out while the container still exists. /// /// Best-effort by construction — it runs on a path that is ALREADY failing, so /// every error here degrades to a note rather than replacing the real failure /// with a docker one. pub async fn tail_logs(container: &str, lines: usize) -> String { use futures::StreamExt as _; let Ok(docker) = connect() else { return "(docker unreachable, so no container log)".into(); }; let opts = bollard::query_parameters::LogsOptionsBuilder::default() .stdout(true) .stderr(true) .tail(&lines.to_string()) .build(); let mut stream = docker.logs(container, Some(opts)); let mut out = String::new(); while let Some(chunk) = stream.next().await { match chunk { Ok(c) => out.push_str(&c.to_string()), Err(e) => { if out.is_empty() { return format!("(could not read {container} logs: {e})"); } break; } } } let out = out.trim(); if out.is_empty() { format!("({container} logged nothing)") } else { out.to_string() } }