P6: security soak, axe a11y sign-off, signed air-gapped bundle tooling
- Concurrency soak (exit criterion): 12 concurrent gated runs, every decision attempted twice concurrently, explicit resumes racing the durable sweeper — exactly one execution per approval, grants consumed at most once, every decision audited, zero stuck runs, zero unaudited executions. (Testkit pool raised to 20 connections; the 5-connection pool starved the storm.) - axe a11y sweep (exit criterion): serious+critical violations fail CI on login, shell, chat, computer home, settings app, all global pages, and the wizard. Two real violations found and fixed: aria-label on a plain div (wizard progress -> role=group) and a button directly inside a <dl> (settings -> plain bordered list). - tools/bundler (exit criterion): keygen / assemble / verify CLI — copies artifacts, writes manifest.json + sha256 checksums.txt + a detached ed25519 signature; verification is fully offline (keyless signing is internet-dependent and disqualified). Tests: round trip, tampered artifact caught by hash, tampered checksum list caught by signature, wrong key refused, missing artifact reported. 147 Rust + 63 frontend tests + 27 Playwright journeys (incl. 4 a11y). Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Fable 5
Claude Fable 5
parent
a8efada690
commit
ccf96053e6
@@ -0,0 +1,206 @@
|
||||
//! P6 security soak (spec §17 exit): under concurrent load with racing
|
||||
//! decisions and duplicated resume attempts, ZERO unaudited gated
|
||||
//! executions — every outbox row maps to exactly one approved decision,
|
||||
//! every grant is consumed at most once, and nothing executes twice.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tc_domain::{
|
||||
AccessPolicy, Agent, AgentId, AgentStatus, Role, RunState, User, UserId, Workspace, WorkspaceId,
|
||||
};
|
||||
use tc_llm::ScriptedProvider;
|
||||
use tc_runtime::{RunEventBody, Runtime, RuntimeConfig};
|
||||
use tc_safety::{approvals, Decision, ResumeReady};
|
||||
|
||||
const SCENARIOS: &str = r#"
|
||||
[[scenario]]
|
||||
marker = "[[scenario:gated-email]]"
|
||||
|
||||
[[scenario.turns]]
|
||||
events = [
|
||||
{ type = "tool_use", name = "email.send", input = { to = "[email protected]", subject = "Soak", body = "Load test." } },
|
||||
]
|
||||
|
||||
[[scenario.turns]]
|
||||
events = [
|
||||
{ type = "text", text = "Done." },
|
||||
]
|
||||
"#;
|
||||
|
||||
const RUNS: usize = 12;
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_gated_runs_never_execute_unaudited_or_twice() {
|
||||
let pool = tc_testkit::test_pool().await;
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Acme".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
tc_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
|
||||
let owner = User {
|
||||
id: UserId::new(),
|
||||
workspace_id: ws.id,
|
||||
email: format!("{}@acme.test", UserId::new()),
|
||||
role: Role::Owner,
|
||||
display_name: "Owner".into(),
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
};
|
||||
tc_db::repo::users::insert(&pool, &owner).await.unwrap();
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Scout".into(),
|
||||
job_title: "Analyst".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: owner.id,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
tc_db::repo::agents::insert(&pool, &agent, &AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
tc_db::repo::credits::add_lot(&pool, ws.id, 10_000, "soak")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let rt = Runtime::new(
|
||||
pool.clone(),
|
||||
Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()),
|
||||
RuntimeConfig::basic("scripted", 1024),
|
||||
);
|
||||
// The durable sweeper races our explicit resume calls on purpose.
|
||||
rt.spawn_resume_sweeper(Duration::from_millis(25));
|
||||
|
||||
// Launch all runs concurrently, each in its own session.
|
||||
let mut run_ids = Vec::new();
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..RUNS {
|
||||
let session = tc_db::repo::sessions::create(&pool, agent.id, ws.id, &format!("Soak {i}"))
|
||||
.await
|
||||
.unwrap();
|
||||
let rt = rt.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
let started = rt
|
||||
.send_message(session.id, "send it [[scenario:gated-email]]")
|
||||
.await
|
||||
.unwrap();
|
||||
let mut rx = started.events;
|
||||
while let Ok(envelope) = rx.recv().await {
|
||||
if matches!(
|
||||
envelope.event,
|
||||
RunEventBody::RunSuspended { .. } | RunEventBody::Error { .. }
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
started.run_id
|
||||
}));
|
||||
}
|
||||
for handle in handles {
|
||||
run_ids.push(handle.await.unwrap());
|
||||
}
|
||||
|
||||
// All suspended with pending approvals.
|
||||
let pending = approvals::list_pending(&pool, ws.id).await.unwrap();
|
||||
assert_eq!(pending.len(), RUNS);
|
||||
let outbox_before = sqlx::query_scalar::<_, i64>("SELECT count(*) FROM outbox")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(outbox_before, 0, "nothing may execute while pending");
|
||||
|
||||
// Decide concurrently: even-indexed approve, odd-indexed reject — and
|
||||
// every decision is attempted TWICE concurrently (double-click storm),
|
||||
// plus an explicit duplicate resume racing the sweeper.
|
||||
let mut deciders = Vec::new();
|
||||
for (i, approval) in pending.iter().enumerate() {
|
||||
let decision = if i % 2 == 0 {
|
||||
Decision::Approve
|
||||
} else {
|
||||
Decision::Reject
|
||||
};
|
||||
for _ in 0..2 {
|
||||
let pool = pool.clone();
|
||||
let rt = rt.clone();
|
||||
let id = approval.id;
|
||||
let run_id = approval.run_id;
|
||||
let user = owner.id;
|
||||
deciders.push(tokio::spawn(async move {
|
||||
let decided = approvals::decide(&pool, id, user, decision).await;
|
||||
// Exactly one of the two concurrent attempts wins.
|
||||
let _ = rt
|
||||
.resume_run(ResumeReady {
|
||||
run_id,
|
||||
approval_id: id,
|
||||
approved: decision == Decision::Approve,
|
||||
})
|
||||
.await;
|
||||
decided.is_ok()
|
||||
}));
|
||||
}
|
||||
}
|
||||
let mut wins = 0;
|
||||
for decider in deciders {
|
||||
if decider.await.unwrap() {
|
||||
wins += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(wins, RUNS, "each approval decided exactly once");
|
||||
|
||||
// Every run reaches Completed.
|
||||
for run_id in &run_ids {
|
||||
let mut done = false;
|
||||
for _ in 0..200 {
|
||||
let run = tc_db::repo::runs::get(&pool, *run_id).await.unwrap();
|
||||
if run.state == RunState::Completed {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
assert!(done, "run {run_id} never completed");
|
||||
}
|
||||
|
||||
// THE invariant: executions == approvals approved, exactly.
|
||||
let approved = RUNS / 2;
|
||||
let outbox = sqlx::query_scalar::<_, i64>("SELECT count(*) FROM outbox")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
outbox as usize, approved,
|
||||
"execute once per approval, never more"
|
||||
);
|
||||
|
||||
// Every approved grant consumed exactly once; rejected ones have none.
|
||||
let consumed =
|
||||
sqlx::query_scalar::<_, i64>("SELECT count(*) FROM execution_grants WHERE consumed = true")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let total_grants = sqlx::query_scalar::<_, i64>("SELECT count(*) FROM execution_grants")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(consumed as usize, approved);
|
||||
assert_eq!(total_grants as usize, approved);
|
||||
|
||||
// Every decision audited.
|
||||
let audited = sqlx::query_scalar::<_, i64>(
|
||||
"SELECT count(*) FROM audit_log WHERE event_type IN ('approval.approved', 'approval.rejected')",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(audited as usize, RUNS);
|
||||
|
||||
// No approval left pending.
|
||||
assert!(approvals::list_pending(&pool, ws.id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
}
|
||||
Reference in New Issue
Block a user