Post-1.0: OTLP tracing, broker image + compose service, install rehearsal

- tc-telemetry: fmt subscriber always; with [telemetry] otlp_endpoint
  set, spans batch-export over OTLP/HTTP. Tested against a REAL OTLP
  receiver decoding the actual protobuf (official proto types): the
  emitted span and service.name arrive on the wire. No endpoint = no
  export = no network (air-gap stance). tower-http TraceLayer gives
  every API request a span
- The broker finally has its own image (images/broker.Dockerfile,
  9.5MB from scratch) — the Helm chart referenced one that never
  existed — and the compose deployment now RUNS the broker, sharing a
  socket volume with the server (the unix-socket equivalent of the K8s
  sidecar). Compose secret flows were silently dead before this
- server.Dockerfile fixes surfaced by the rehearsal: the workspace
  build needs tools/ (bundler joined the workspace) and
  images/seccomp/ (include_str! profile) in the build context
- scripts/rehearse-install.sh (plan: clean-VM rehearsal): assembles a
  REAL signed bundle from the built images (server/frontend/broker/
  postgres/socket-proxy), runs the customer path — offline verify,
  docker load, compose up — and asserts /healthz plus the served login
  page before teardown. Passing locally; wired as a release.yml step,
  which also builds/ships the broker + socket-proxy images now

161 Rust tests + 29 journeys; clean-room rehearsal green.

Co-Authored-By: Claude Fable 5 <[email protected]>

crates/tc-telemetry/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "tc-telemetry"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+publish.workspace = true
+license.workspace = true
+
+[dependencies]
+opentelemetry = "0.32"
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client"] }
+opentelemetry_sdk = "0.32"
+thiserror = { workspace = true }
+tracing = "0.1"
+tracing-opentelemetry = "0.33"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+[dev-dependencies]
+axum = "0.8"
+opentelemetry-proto = { version = "0.32", default-features = false, features = ["trace", "gen-tonic-messages"] }
+prost = "0.14"
+tokio = { workspace = true }
+
+[lints]
+workspace = true

crates/tc-telemetry/src/lib.rs

@@ -0,0 +1,92 @@
+//! Tracing setup (§16 NFR: observability). Always installs a fmt
+//! subscriber honoring `RUST_LOG`; with an OTLP endpoint configured,
+//! spans also export over OTLP/HTTP — to a collector shipped in the
+//! air-gapped bundle or any cloud backend. No endpoint, no export, no
+//! network: telemetry never phones home uninvited.
+
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry::KeyValue;
+use opentelemetry_otlp::WithExportConfig;
+use opentelemetry_sdk::trace::SdkTracerProvider;
+use opentelemetry_sdk::Resource;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+use tracing_subscriber::{EnvFilter, Layer};
+
+/// Keeps the exporter alive; dropping it flushes pending spans.
+pub struct TelemetryGuard {
+    provider: Option<SdkTracerProvider>,
+}
+
+impl TelemetryGuard {
+    /// Forces all batched spans out — call before process exit (tests
+    /// call it to assert delivery).
+    pub fn flush(&self) {
+        if let Some(provider) = &self.provider {
+            let _ = provider.force_flush();
+        }
+    }
+}
+
+impl Drop for TelemetryGuard {
+    fn drop(&mut self) {
+        if let Some(provider) = self.provider.take() {
+            let _ = provider.force_flush();
+            let _ = provider.shutdown();
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum TelemetryError {
+    #[error("otlp exporter: {0}")]
+    Exporter(String),
+    #[error("subscriber already installed")]
+    AlreadyInstalled,
+}
+
+/// Installs the global subscriber. `otlp_endpoint` is the collector's
+/// OTLP/HTTP base (e.g. `http://otel-collector:4318`).
+pub fn init(
+    service_name: &str,
+    otlp_endpoint: Option<&str>,
+) -> Result<TelemetryGuard, TelemetryError> {
+    let fmt = tracing_subscriber::fmt::layer().with_target(false);
+    let filter = EnvFilter::try_from_default_env()
+        .unwrap_or_else(|_| EnvFilter::new("info,sqlx=warn,hyper=warn"));
+
+    match otlp_endpoint {
+        Some(endpoint) => {
+            let exporter = opentelemetry_otlp::SpanExporter::builder()
+                .with_http()
+                .with_endpoint(format!("{}/v1/traces", endpoint.trim_end_matches('/')))
+                .build()
+                .map_err(|e| TelemetryError::Exporter(e.to_string()))?;
+            let provider = SdkTracerProvider::builder()
+                .with_batch_exporter(exporter)
+                .with_resource(
+                    Resource::builder()
+                        .with_attributes([KeyValue::new("service.name", service_name.to_owned())])
+                        .build(),
+                )
+                .build();
+            let tracer = provider.tracer(service_name.to_owned());
+            let otel = tracing_opentelemetry::layer().with_tracer(tracer);
+            tracing_subscriber::registry()
+                .with(fmt.with_filter(filter))
+                .with(otel)
+                .try_init()
+                .map_err(|_| TelemetryError::AlreadyInstalled)?;
+            Ok(TelemetryGuard {
+                provider: Some(provider),
+            })
+        }
+        None => {
+            tracing_subscriber::registry()
+                .with(fmt.with_filter(filter))
+                .try_init()
+                .map_err(|_| TelemetryError::AlreadyInstalled)?;
+            Ok(TelemetryGuard { provider: None })
+        }
+    }
+}

crates/tc-telemetry/tests/otlp_export.rs

@@ -0,0 +1,81 @@
+//! Spans reach a REAL OTLP/HTTP receiver: a live server accepting
+//! protobuf at /v1/traces, decoded with the official proto types — the
+//! exact wire contract any collector speaks.
+
+use std::sync::Arc;
+
+use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
+use prost::Message;
+use tokio::sync::Mutex;
+
+type Received = Arc<Mutex<Vec<ExportTraceServiceRequest>>>;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn spans_export_to_a_real_otlp_receiver() {
+    let received: Received = Arc::new(Mutex::new(Vec::new()));
+    let state = received.clone();
+    let app = axum::Router::new()
+        .route(
+            "/v1/traces",
+            axum::routing::post(
+                |axum::extract::State(state): axum::extract::State<Received>,
+                 body: axum::body::Bytes| async move {
+                    let request =
+                        ExportTraceServiceRequest::decode(body.as_ref()).expect("otlp protobuf");
+                    state.lock().await.push(request);
+                    // An empty ExportTraceServiceResponse.
+                    (
+                        [("content-type", "application/x-protobuf")],
+                        Vec::<u8>::new(),
+                    )
+                },
+            ),
+        )
+        .with_state(state);
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let endpoint = format!("http://{}", listener.local_addr().unwrap());
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    let guard = tc_telemetry::init("teamclaw-test", Some(&endpoint)).expect("init");
+
+    {
+        let span = tracing::info_span!("approval_decision", approval_id = "ap_123");
+        let _entered = span.enter();
+        tracing::info!("decided");
+    }
+    guard.flush();
+
+    // Find our span among everything exported.
+    let requests = received.lock().await;
+    assert!(!requests.is_empty(), "no OTLP export arrived");
+    let mut span_names = Vec::new();
+    let mut service_names = Vec::new();
+    for request in requests.iter() {
+        for resource in &request.resource_spans {
+            if let Some(res) = &resource.resource {
+                for attr in &res.attributes {
+                    if attr.key == "service.name" {
+                        if let Some(value) = &attr.value {
+                            service_names.push(format!("{value:?}"));
+                        }
+                    }
+                }
+            }
+            for scope in &resource.scope_spans {
+                for span in &scope.spans {
+                    span_names.push(span.name.clone());
+                }
+            }
+        }
+    }
+    assert!(
+        span_names.iter().any(|name| name == "approval_decision"),
+        "spans: {span_names:?}"
+    );
+    assert!(
+        service_names.iter().any(|s| s.contains("teamclaw-test")),
+        "services: {service_names:?}"
+    );
+}