Omar SobhandClaude Opus 5
0a2bd6f868
A third kind of decision-maker between deterministic code and a full LLM call: Choice / Score / Noul questions answered as probability distributions with a confidence, behind one Decider trait, with the composition patterns (confidence gating, composite scoring, rerank) as code. Two backends: TypeSafe's Jev over HTTP, and a DeBERTa-v3 MNLI cross-encoder run in-process with candle (feature nli; metal/cuda). decide-eval measures a backend on labelled cases the way judge-eval measures the judge. eval/skill-triage.json: 20 mission tasks × 53 skills, 75 positives, hand-labelled. Measured 2026-09-21: lexical overlap AUROC 0.851 [email protected] 0.47 top-k 48/75 ECE 0.095 jev (named wording) AUROC 0.989 [email protected] 0.84 top-k 63/75 ECE 0.064 213 ms jev (plain wording) AUROC 0.970 [email protected] 0.66 top-k 52/75 nli mnli-base AUROC 0.790 [email protected] 0.28 top-k 38/75 ECE 0.263 1.5 s nli zeroshot-v2 AUROC 0.782 [email protected] 0.43 top-k 39/75 ECE 0.054 1.2 s The vendor's calibration claim survives our data; the local cross-encoder ranks below keyword overlap on either checkpoint or wording and is kept as the measured negative, not shipped. A local backend would need the logit-readout route over the fleet's 9B model — a separate spike. Shadow: one Jev call per phase launch (spawned, 10 s cap, silent without TYPESAFE_API_KEY) records a skill.triage event; the Skill-Use report carries triage_p beside each skill's Trigger verdict. It selects nothing. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
76 lines
2.2 KiB
TOML
76 lines
2.2 KiB
TOML
[package]
|
|
name = "cm-api"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
rust-version.workspace = true
|
|
license.workspace = true
|
|
publish.workspace = true
|
|
|
|
[dependencies]
|
|
getrandom = "0.2"
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
serde_yaml = "0.9"
|
|
hex = "0.4"
|
|
hmac = "0.12"
|
|
sha2 = "0.10"
|
|
base64 = "0.22"
|
|
async-stream = "0.3"
|
|
axum = { version = "0.8", features = ["ws"] }
|
|
# Docker API — matches cm-sandbox's version so the workspace resolves cleanly.
|
|
# Used by research_container to spawn/stop per-topic team runtimes via the
|
|
# same socket-proxy the server already talks to.
|
|
bollard = "0.19"
|
|
futures = "0.3"
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
sqlx = { workspace = true }
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
|
cm-auth = { path = "../cm-auth" }
|
|
cm-billing = { path = "../cm-billing" }
|
|
cm-brain = { path = "../cm-brain" }
|
|
cm-config = { path = "../cm-config" }
|
|
cm-db = { path = "../cm-db" }
|
|
cm-decide = { path = "../cm-decide" }
|
|
cm-domain = { path = "../cm-domain" }
|
|
cm-files = { path = "../cm-files" }
|
|
tar = { workspace = true }
|
|
cm-llm = { path = "../cm-llm" }
|
|
cm-orchestrator = { path = "../cm-orchestrator", features = ["provider"] }
|
|
cm-runtime = { path = "../cm-runtime" }
|
|
cm-sandbox = { path = "../cm-sandbox" }
|
|
cm-safety = { path = "../cm-safety" }
|
|
async-trait = "0.1"
|
|
cm-scheduler = { path = "../cm-scheduler" }
|
|
cm-secrets = { path = "../cm-secrets" }
|
|
cm-topology = { path = "../cm-topology" }
|
|
thiserror = { workspace = true }
|
|
tokio-tungstenite = { version = "0.26", features = ["rustls-tls-webpki-roots"] }
|
|
tower-http = { version = "0.6", features = ["trace"] }
|
|
time = { workspace = true }
|
|
tokio = { workspace = true }
|
|
urlencoding = "2"
|
|
uuid = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
axum = { version = "0.8", features = ["ws"] }
|
|
tempfile = "3"
|
|
jsonwebtoken = "9"
|
|
eventsource-stream = "0.2"
|
|
reqwest = { version = "0.12", default-features = false, features = [
|
|
"json",
|
|
"rustls-tls",
|
|
"stream",
|
|
] }
|
|
cm-llm = { path = "../cm-llm" }
|
|
cm-testkit = { path = "../cm-testkit" }
|
|
hex = "0.4"
|
|
hmac = "0.12"
|
|
base64 = "0.22"
|
|
rand_core = { version = "0.6", features = ["getrandom"] }
|
|
rsa = { version = "0.9", features = ["pem"] }
|
|
sha2 = "0.10"
|
|
|
|
[lints]
|
|
workspace = true
|