repos: backend — schema, /api/repos routes + GitHub sync provider

Migration 0034: two tables. repo_connections carries the workspace's
per-provider config (owner, base_url, label, last_synced_at,
last_sync_error) and points at an app_connections row for the PAT.
repos is the per-connection cache with (connection_id, external_id)
unique so upsert is idempotent across re-syncs. Cascading deletes clean
up cleanly on connection removal.

cm-secrets grows a FetchAuthorized op — GET with the stored PAT injected
as bearer, returns status + JSON body without ever exposing the
credential to cm-api. This is the least-privilege door for read-only
provider APIs (list repos), distinct from the InvokeHttp path that still
requires a single-use approval grant for outbound writes.

cm-api::routes::repos wires:
- POST /api/repos/connections (broker store_secret + insert both rows +
  initial sync + mark_synced)
- GET /api/repos/connections
- DELETE /api/repos/connections/:id
- POST /api/repos/connections/:id/sync
- GET /api/repos (500 cap, newest provider_updated first)
- GET /api/repos/:id (full detail incl. clone_url + html_url)

GitHub provider inline for v1 — paginated pull of /orgs/:owner/repos
(when owner set) or /user/repos (when absent), 100/page, capped at 20
pages (~2k repos) to keep first-sync latency bounded. Non-2xx surface
back to the caller as sync_error; parse failures are best-effort per
repo (skipped, logged, don't abort the batch).

Gitea + GitLab providers land in a follow-up — mostly URL swap +
response-shape adapter.
This commit is contained in:
Omar Sobh
2026-07-07 14:52:47 -07:00
parent 076f7724ca
commit 6d087bf537
21 changed files with 1532 additions and 5 deletions
@@ -0,0 +1,82 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, app_connection_id, provider, owner, base_url,\n label, last_synced_at, last_sync_error, created_at, updated_at\n FROM repo_connections\n WHERE workspace_id = $1\n ORDER BY created_at DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "app_connection_id",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "provider",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "owner",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "base_url",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "label",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "last_synced_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "last_sync_error",
"type_info": "Text"
},
{
"ordinal": 9,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "updated_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
true,
true,
false,
true,
true,
false,
false
]
},
"hash": "029d0c2c955ca7b66bb002ae0075e096bd5673dc01b17947f71e0fa4c2bfc6a4"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO repo_connections\n (id, workspace_id, app_connection_id, provider, owner, base_url, label)\n VALUES ($1, $2, $3, $4, $5, $6, $7)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Uuid",
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "17eddeaae42356fdbf51c9006eff470d3b72391f89be17d683c82cf1a9030605"
}
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO repos\n (id, workspace_id, connection_id, provider, external_id,\n owner, name, description, default_branch, clone_url, html_url,\n private, stars, forks, provider_updated_at, last_synced_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,\n $12, $13, $14, $15, now())\n ON CONFLICT (connection_id, external_id) DO UPDATE\n SET owner = EXCLUDED.owner,\n name = EXCLUDED.name,\n description = EXCLUDED.description,\n default_branch = EXCLUDED.default_branch,\n clone_url = EXCLUDED.clone_url,\n html_url = EXCLUDED.html_url,\n private = EXCLUDED.private,\n stars = EXCLUDED.stars,\n forks = EXCLUDED.forks,\n provider_updated_at = EXCLUDED.provider_updated_at,\n last_synced_at = now()\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Uuid",
"Text",
"Text",
"Text",
"Text",
"Text",
"Text",
"Text",
"Text",
"Bool",
"Int4",
"Int4",
"Timestamptz"
]
},
"nullable": [
false
]
},
"hash": "2e16848346fdbe69c1616cf480bb1af0b60a7b9164d971f5b553d6cc9a9dd313"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM repo_connections WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "40ddf2ca01ccd694a377d329126afbe1460d25130a5e9ed93a50ff2492f3681e"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE repo_connections\n SET last_synced_at = now(),\n last_sync_error = $2,\n updated_at = now()\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "478798e0996dbd2dc416ded6542894c7dc338a1aa4b3b2433df238177ef5565b"
}
@@ -0,0 +1,113 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, connection_id, provider, external_id,\n owner, name, description, default_branch, clone_url, html_url,\n private, stars, forks, provider_updated_at, last_synced_at\n FROM repos\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "connection_id",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "provider",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "external_id",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "owner",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 8,
"name": "default_branch",
"type_info": "Text"
},
{
"ordinal": 9,
"name": "clone_url",
"type_info": "Text"
},
{
"ordinal": 10,
"name": "html_url",
"type_info": "Text"
},
{
"ordinal": 11,
"name": "private",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "stars",
"type_info": "Int4"
},
{
"ordinal": 13,
"name": "forks",
"type_info": "Int4"
},
{
"ordinal": 14,
"name": "provider_updated_at",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "last_synced_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false,
false,
false,
true,
false
]
},
"hash": "b23e6062640be2b161ae7ebcb6d991d540bf2746ba7585b3ff4324e27f724a04"
}
@@ -0,0 +1,83 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, app_connection_id, provider, owner, base_url,\n label, last_synced_at, last_sync_error, created_at, updated_at\n FROM repo_connections\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "app_connection_id",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "provider",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "owner",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "base_url",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "label",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "last_synced_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "last_sync_error",
"type_info": "Text"
},
{
"ordinal": 9,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "updated_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
true,
true,
false,
true,
true,
false,
false
]
},
"hash": "db179364174b07cb5b58d283d9fd1c218bcb43be8d952071e7cd6bc9fb9ad6ca"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM repos WHERE connection_id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "ecff5e7cad6960b7da3daaba89f46d2b0ce1554d78b2ea34d14b5137f7b5ee67"
}
@@ -0,0 +1,113 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, connection_id, provider, external_id,\n owner, name, description, default_branch, clone_url, html_url,\n private, stars, forks, provider_updated_at, last_synced_at\n FROM repos\n WHERE workspace_id = $1\n ORDER BY provider_updated_at DESC NULLS LAST, name\n LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "connection_id",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "provider",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "external_id",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "owner",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "name",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "description",
"type_info": "Text"
},
{
"ordinal": 8,
"name": "default_branch",
"type_info": "Text"
},
{
"ordinal": 9,
"name": "clone_url",
"type_info": "Text"
},
{
"ordinal": 10,
"name": "html_url",
"type_info": "Text"
},
{
"ordinal": 11,
"name": "private",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "stars",
"type_info": "Int4"
},
{
"ordinal": 13,
"name": "forks",
"type_info": "Int4"
},
{
"ordinal": 14,
"name": "provider_updated_at",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "last_synced_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Uuid",
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false,
false,
false,
true,
false
]
},
"hash": "f3fd78a88773c6fa4cb9c3891a4d792ab569b3e939be4b104329490b8ef3662f"
}
@@ -0,0 +1,59 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, agent_id, provider, auth_type, status, secret_ref\n FROM app_connections\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Uuid"
},
{
"ordinal": 2,
"name": "agent_id",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "provider",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "auth_type",
"type_info": "Text"
},
{
"ordinal": 5,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 6,
"name": "secret_ref",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": [
false,
false,
true,
false,
false,
false,
true
]
},
"hash": "f9eb5facb16b4bd06526a71c5baba3ef77b978a58b7c4205567c92950aad72f7"
}
+15
View File
@@ -452,6 +452,21 @@ pub fn router(state: AppState) -> Router {
"/api/topology-runs/{id}/cancel", "/api/topology-runs/{id}/cancel",
post(routes::topology::cancel_run), post(routes::topology::cancel_run),
) )
// Repos tier — provider connections + cached repo list.
.route(
"/api/repos/connections",
get(routes::repos::list_connections).post(routes::repos::create_connection),
)
.route(
"/api/repos/connections/{id}",
delete(routes::repos::delete_connection),
)
.route(
"/api/repos/connections/{id}/sync",
post(routes::repos::sync_now),
)
.route("/api/repos", get(routes::repos::list_repos))
.route("/api/repos/{id}", get(routes::repos::get_repo))
.layer(tower_http::trace::TraceLayer::new_for_http()) .layer(tower_http::trace::TraceLayer::new_for_http())
.with_state(state) .with_state(state)
} }
+1
View File
@@ -17,6 +17,7 @@ pub mod nodes;
pub mod oauth; pub mod oauth;
pub mod orgs; pub mod orgs;
pub mod planner; pub mod planner;
pub mod repos;
pub mod research; pub mod research;
pub mod routines; pub mod routines;
pub mod sessions; pub mod sessions;
+458
View File
@@ -0,0 +1,458 @@
//! Repos — workspace-scoped view of connected git providers and the
//! repositories they own.
//!
//! POST /api/repos/connections create a provider connection
//! GET /api/repos/connections list connections
//! DELETE /api/repos/connections/:id remove a connection (cascades to repos)
//! POST /api/repos/connections/:id/sync refetch repos from the provider
//! GET /api/repos list every repo in the workspace
//! GET /api/repos/:id one repo detail
//!
//! Provider credentials (PATs) live in the secret broker; only their ref is
//! kept in `app_connections`. Repo fetching goes through
//! `BrokerClient::fetch_authorized`, so the PAT never leaves the broker.
//! v1 supports GitHub inline; Gitea + GitLab are next (mostly a URL swap +
//! response shape adapter).
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::Json;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use time::format_description::well_known::Rfc3339;
use uuid::Uuid;
use crate::{ApiError, AppState, Authed};
#[derive(Deserialize)]
pub struct CreateConnectionRequest {
/// `github` | `gitea` | `gitlab`
pub provider: String,
pub token: String,
/// Optional org/user login. When absent GitHub pulls `/user/repos`.
#[serde(default)]
pub owner: Option<String>,
/// Optional base URL for self-hosted providers.
#[serde(default)]
pub base_url: Option<String>,
/// Optional human label — defaults to a "<provider>/<owner>" pattern.
#[serde(default)]
pub label: Option<String>,
}
#[derive(Serialize)]
pub struct ConnectionSummary {
pub id: String,
pub provider: String,
pub owner: Option<String>,
pub base_url: Option<String>,
pub label: String,
pub status: String,
pub last_synced_at: Option<String>,
pub last_sync_error: Option<String>,
pub created_at: String,
}
fn known_provider(p: &str) -> bool {
matches!(p, "github" | "gitea" | "gitlab")
}
fn default_label(provider: &str, owner: Option<&str>) -> String {
match owner {
Some(o) => format!("{provider}/{o}"),
None => provider.to_string(),
}
}
/// `POST /api/repos/connections` — store the PAT in the broker, create the
/// app_connections + repo_connections rows, and kick off an initial sync.
pub async fn create_connection(
State(state): State<AppState>,
Authed(user): Authed,
Json(body): Json<CreateConnectionRequest>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let provider = body.provider.trim().to_lowercase();
if !known_provider(&provider) {
return Err(ApiError::BadRequest);
}
if body.token.trim().is_empty() {
return Err(ApiError::BadRequest);
}
let owner = body
.owner
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty());
let base_url = body
.base_url
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty());
let label = body
.label
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_owned)
.unwrap_or_else(|| default_label(&provider, owner));
// Stash the PAT in the broker; only the ref persists in Postgres.
let socket = state.broker_socket.as_ref().ok_or(ApiError::Internal)?;
let mut broker = cm_secrets::BrokerClient::connect(socket)
.await
.map_err(|_| ApiError::Internal)?;
let secret_ref = broker
.store_secret(
user.workspace_id,
&format!("{provider}_pat"),
body.token.trim(),
)
.await
.map_err(|_| ApiError::Internal)?;
let app_conn = cm_db::repo::connections::insert(
&state.pool,
user.workspace_id,
None,
&provider,
"keys",
secret_ref,
)
.await?;
let repo_conn_id = cm_db::repo::repo_connections::insert(
&state.pool,
cm_db::repo::repo_connections::NewRepoConnection {
workspace_id: user.workspace_id,
app_connection_id: app_conn.id,
provider: &provider,
owner,
base_url,
label: &label,
},
)
.await?;
// First sync — best-effort. Errors mark the connection with the message;
// the caller can retry via POST /:id/sync.
let sync_result = sync_connection(&state, user.workspace_id, repo_conn_id, secret_ref).await;
let (synced, sync_error) = match sync_result {
Ok(n) => (n, None),
Err(e) => (0, Some(e)),
};
let _ = cm_db::repo::repo_connections::mark_synced(
&state.pool,
repo_conn_id,
sync_error.as_deref(),
)
.await;
Ok((
StatusCode::CREATED,
Json(serde_json::json!({
"id": repo_conn_id.to_string(),
"provider": provider,
"owner": owner,
"synced": synced,
"sync_error": sync_error,
})),
))
}
/// `GET /api/repos/connections` — list every provider connection in the workspace.
pub async fn list_connections(
State(state): State<AppState>,
Authed(user): Authed,
) -> Result<Json<Vec<ConnectionSummary>>, ApiError> {
let rows =
cm_db::repo::repo_connections::list_for_workspace(&state.pool, user.workspace_id).await?;
let out = rows
.into_iter()
.map(|c| ConnectionSummary {
id: c.id.to_string(),
provider: c.provider,
owner: c.owner,
base_url: c.base_url,
label: c.label,
status: match c.last_sync_error.as_deref() {
Some(_) => "error".into(),
None if c.last_synced_at.is_some() => "connected".into(),
None => "pending".into(),
},
last_synced_at: c.last_synced_at.and_then(|t| t.format(&Rfc3339).ok()),
last_sync_error: c.last_sync_error,
created_at: c.created_at.format(&Rfc3339).unwrap_or_default(),
})
.collect();
Ok(Json(out))
}
/// `DELETE /api/repos/connections/:id` — remove the repo_connections row.
/// `ON DELETE CASCADE` cleans out its repos; the underlying app_connections
/// row + broker secret stay (the workspace may reuse the token elsewhere).
pub async fn delete_connection(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<StatusCode, ApiError> {
let removed = cm_db::repo::repo_connections::delete(&state.pool, id, user.workspace_id).await?;
if removed {
Ok(StatusCode::NO_CONTENT)
} else {
Err(ApiError::NotFound)
}
}
/// `POST /api/repos/connections/:id/sync` — refetch from the provider.
pub async fn sync_now(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<Json<Value>, ApiError> {
let conn = cm_db::repo::repo_connections::get(&state.pool, id, user.workspace_id).await?;
let app_conn =
cm_db::repo::connections::get(&state.pool, conn.app_connection_id, user.workspace_id)
.await?;
let secret_ref = app_conn.secret_ref.ok_or(ApiError::Conflict)?;
let result = sync_connection(&state, user.workspace_id, id, secret_ref).await;
let (synced, sync_error) = match result {
Ok(n) => (n, None),
Err(e) => (0, Some(e)),
};
let _ =
cm_db::repo::repo_connections::mark_synced(&state.pool, id, sync_error.as_deref()).await;
Ok(Json(serde_json::json!({
"synced": synced,
"sync_error": sync_error,
})))
}
/// `GET /api/repos` — every cached repo in the workspace, newest updated first.
pub async fn list_repos(
State(state): State<AppState>,
Authed(user): Authed,
) -> Result<Json<Value>, ApiError> {
let rows = cm_db::repo::repos::list_for_workspace(&state.pool, user.workspace_id, 500).await?;
let out: Vec<Value> = rows
.into_iter()
.map(|r| {
serde_json::json!({
"id": r.id.to_string(),
"connection_id": r.connection_id.to_string(),
"provider": r.provider,
"owner": r.owner,
"name": r.name,
"private": r.private,
"description": r.description,
"default_branch": r.default_branch,
"stars": r.stars,
"forks": r.forks,
"updated_at": r.provider_updated_at.and_then(|t| t.format(&Rfc3339).ok()),
})
})
.collect();
Ok(Json(Value::Array(out)))
}
/// `GET /api/repos/:id` — one repo detail (includes clone_url + html_url).
pub async fn get_repo(
State(state): State<AppState>,
Authed(user): Authed,
Path(id): Path<Uuid>,
) -> Result<Json<Value>, ApiError> {
let r = cm_db::repo::repos::get(&state.pool, id, user.workspace_id).await?;
Ok(Json(serde_json::json!({
"id": r.id.to_string(),
"connection_id": r.connection_id.to_string(),
"provider": r.provider,
"owner": r.owner,
"name": r.name,
"private": r.private,
"description": r.description,
"default_branch": r.default_branch,
"clone_url": r.clone_url,
"html_url": r.html_url,
"stars": r.stars,
"forks": r.forks,
"updated_at": r.provider_updated_at.and_then(|t| t.format(&Rfc3339).ok()),
"last_synced_at": r.last_synced_at.format(&Rfc3339).ok(),
})))
}
// ── provider sync ────────────────────────────────────────────────────────
/// Fetch repos from the provider (via the broker) + upsert into the cache.
/// Returns the number of upserted rows. Errors on network / provider errors —
/// the caller stashes the message on repo_connections.last_sync_error.
async fn sync_connection(
state: &AppState,
workspace_id: cm_domain::WorkspaceId,
repo_conn_id: Uuid,
secret_ref: Uuid,
) -> Result<usize, String> {
let conn = cm_db::repo::repo_connections::get(&state.pool, repo_conn_id, workspace_id)
.await
.map_err(|e| format!("load connection: {e}"))?;
match conn.provider.as_str() {
"github" => sync_github(state, workspace_id, &conn, secret_ref).await,
other => Err(format!("provider '{other}' not yet supported for sync")),
}
}
async fn sync_github(
state: &AppState,
workspace_id: cm_domain::WorkspaceId,
conn: &cm_db::repo::repo_connections::RepoConnection,
secret_ref: Uuid,
) -> Result<usize, String> {
let socket = state
.broker_socket
.as_ref()
.ok_or_else(|| "broker socket unavailable".to_string())?;
let mut broker = cm_secrets::BrokerClient::connect(socket)
.await
.map_err(|e| format!("broker: {e}"))?;
// GitHub API paginates with `per_page` + `page`. 100 is the max; stop
// when we get a short page (< per_page) — sufficient for v1.
let base = conn.base_url.as_deref().unwrap_or("https://api.github.com");
let mut page = 1u32;
let per_page = 100u32;
let mut upserted = 0usize;
loop {
let url = match conn.owner.as_deref() {
Some(owner) => format!(
"{base}/orgs/{owner}/repos?per_page={per_page}&page={page}&type=all"
),
None => format!("{base}/user/repos?per_page={per_page}&page={page}&affiliation=owner,organization_member"),
};
let (status, body) = broker
.fetch_authorized(secret_ref, &url)
.await
.map_err(|e| format!("broker fetch: {e}"))?;
if status == 404 && conn.owner.is_some() {
return Err(format!(
"org '{}' not found or PAT lacks access",
conn.owner.as_deref().unwrap_or("")
));
}
if !(200..300).contains(&status) {
return Err(format!("GitHub {status}: {body}"));
}
let arr = match body.as_array() {
Some(a) => a.clone(),
None => break,
};
if arr.is_empty() {
break;
}
let n = arr.len();
for repo in &arr {
if let Err(e) =
upsert_github_repo(&state.pool, workspace_id, conn.id, &conn.provider, repo).await
{
eprintln!(
"repos: upsert failed for {}: {e}",
repo.get("full_name")
.and_then(|v| v.as_str())
.unwrap_or("?")
);
continue;
}
upserted += 1;
}
if (n as u32) < per_page {
break;
}
page += 1;
// Cap at 20 pages (~2k repos) to keep first-sync latency bounded.
if page > 20 {
break;
}
}
Ok(upserted)
}
async fn upsert_github_repo(
pool: &sqlx::PgPool,
workspace_id: cm_domain::WorkspaceId,
connection_id: Uuid,
provider: &str,
repo: &Value,
) -> Result<(), cm_db::DbError> {
let external_id = repo
.get("id")
.and_then(|v| v.as_i64())
.map(|n| n.to_string())
.unwrap_or_default();
let owner = repo
.get("owner")
.and_then(|v| v.get("login"))
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let name = repo
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
if external_id.is_empty() || name.is_empty() {
return Ok(());
}
let description = repo
.get("description")
.and_then(|v| v.as_str())
.map(str::to_owned);
let default_branch = repo
.get("default_branch")
.and_then(|v| v.as_str())
.map(str::to_owned);
let clone_url = repo
.get("clone_url")
.and_then(|v| v.as_str())
.map(str::to_owned);
let html_url = repo
.get("html_url")
.and_then(|v| v.as_str())
.map(str::to_owned);
let private = repo
.get("private")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let stars = repo
.get("stargazers_count")
.and_then(|v| v.as_i64())
.unwrap_or(0) as i32;
let forks = repo
.get("forks_count")
.and_then(|v| v.as_i64())
.unwrap_or(0) as i32;
let provider_updated_at = repo
.get("updated_at")
.and_then(|v| v.as_str())
.and_then(|s| {
time::OffsetDateTime::parse(s, &time::format_description::well_known::Rfc3339).ok()
});
cm_db::repo::repos::upsert(
pool,
cm_db::repo::repos::RepoUpsert {
connection_id,
workspace_id,
provider,
external_id: &external_id,
owner: &owner,
name: &name,
description: description.as_deref(),
default_branch: default_branch.as_deref(),
clone_url: clone_url.as_deref(),
html_url: html_url.as_deref(),
private,
stars,
forks,
provider_updated_at,
},
)
.await?;
Ok(())
}
+22
View File
@@ -112,6 +112,28 @@ pub async fn find_provider(
Ok(row) Ok(row)
} }
/// Fetch a single connection by id, workspace-scoped so cross-tenant reads
/// return NotFound. Used when another table (e.g. repo_connections) needs
/// to resolve the secret_ref of the credential it points at.
pub async fn get(
pool: &PgPool,
id: Uuid,
workspace_id: WorkspaceId,
) -> Result<AppConnection, DbError> {
let row = sqlx::query_as!(
AppConnection,
r#"SELECT id, workspace_id, agent_id, provider, auth_type, status, secret_ref
FROM app_connections
WHERE id = $1 AND workspace_id = $2"#,
id,
workspace_id.as_uuid(),
)
.fetch_optional(pool)
.await?
.ok_or(DbError::NotFound)?;
Ok(row)
}
pub async fn disconnect(pool: &PgPool, id: Uuid) -> Result<(), DbError> { pub async fn disconnect(pool: &PgPool, id: Uuid) -> Result<(), DbError> {
let result = sqlx::query!( let result = sqlx::query!(
"UPDATE app_connections SET status = 'disconnected' WHERE id = $1", "UPDATE app_connections SET status = 'disconnected' WHERE id = $1",
+2
View File
@@ -16,6 +16,8 @@ pub mod node_tools;
pub mod nodes; pub mod nodes;
pub mod orgs; pub mod orgs;
pub mod outbox; pub mod outbox;
pub mod repo_connections;
pub mod repos;
pub mod research_publish_approvals; pub mod research_publish_approvals;
pub mod research_topics; pub mod research_topics;
pub mod routine_runs; pub mod routine_runs;
+150
View File
@@ -0,0 +1,150 @@
//! Repo connections — per-workspace binding of a repo provider (GitHub /
//! Gitea / GitLab) to the app_connections credential that holds its PAT.
//! Also stores the pull config: which owner to scope to, and (for self-
//! hosted providers) the base URL.
//!
//! The row's `id` is what shows up as `connection_id` on repos and in the
//! /api/repos/connections endpoints. The app_connections back-ref is
//! separate so tearing down the credential (revoke, rotate) doesn't
//! silently break the visible connection — deleting the app_connection
//! cascades here and blows away the repos too.
use cm_domain::WorkspaceId;
use sqlx::PgPool;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::DbError;
pub struct RepoConnection {
pub id: Uuid,
pub workspace_id: Uuid,
pub app_connection_id: Uuid,
pub provider: String,
pub owner: Option<String>,
pub base_url: Option<String>,
pub label: String,
pub last_synced_at: Option<OffsetDateTime>,
pub last_sync_error: Option<String>,
pub created_at: OffsetDateTime,
pub updated_at: OffsetDateTime,
}
pub struct NewRepoConnection<'a> {
pub workspace_id: WorkspaceId,
pub app_connection_id: Uuid,
pub provider: &'a str,
pub owner: Option<&'a str>,
pub base_url: Option<&'a str>,
pub label: &'a str,
}
pub async fn insert(pool: &PgPool, input: NewRepoConnection<'_>) -> Result<Uuid, DbError> {
let id = Uuid::now_v7();
sqlx::query!(
"INSERT INTO repo_connections
(id, workspace_id, app_connection_id, provider, owner, base_url, label)
VALUES ($1, $2, $3, $4, $5, $6, $7)",
id,
input.workspace_id.as_uuid(),
input.app_connection_id,
input.provider,
input.owner,
input.base_url,
input.label,
)
.execute(pool)
.await?;
Ok(id)
}
pub async fn list_for_workspace(
pool: &PgPool,
workspace_id: WorkspaceId,
) -> Result<Vec<RepoConnection>, DbError> {
let rows = sqlx::query!(
"SELECT id, workspace_id, app_connection_id, provider, owner, base_url,
label, last_synced_at, last_sync_error, created_at, updated_at
FROM repo_connections
WHERE workspace_id = $1
ORDER BY created_at DESC",
workspace_id.as_uuid(),
)
.fetch_all(pool)
.await?;
Ok(rows
.into_iter()
.map(|r| RepoConnection {
id: r.id,
workspace_id: r.workspace_id,
app_connection_id: r.app_connection_id,
provider: r.provider,
owner: r.owner,
base_url: r.base_url,
label: r.label,
last_synced_at: r.last_synced_at,
last_sync_error: r.last_sync_error,
created_at: r.created_at,
updated_at: r.updated_at,
})
.collect())
}
pub async fn get(
pool: &PgPool,
id: Uuid,
workspace_id: WorkspaceId,
) -> Result<RepoConnection, DbError> {
let r = sqlx::query!(
"SELECT id, workspace_id, app_connection_id, provider, owner, base_url,
label, last_synced_at, last_sync_error, created_at, updated_at
FROM repo_connections
WHERE id = $1 AND workspace_id = $2",
id,
workspace_id.as_uuid(),
)
.fetch_optional(pool)
.await?
.ok_or(DbError::NotFound)?;
Ok(RepoConnection {
id: r.id,
workspace_id: r.workspace_id,
app_connection_id: r.app_connection_id,
provider: r.provider,
owner: r.owner,
base_url: r.base_url,
label: r.label,
last_synced_at: r.last_synced_at,
last_sync_error: r.last_sync_error,
created_at: r.created_at,
updated_at: r.updated_at,
})
}
pub async fn delete(pool: &PgPool, id: Uuid, workspace_id: WorkspaceId) -> Result<bool, DbError> {
let res = sqlx::query!(
"DELETE FROM repo_connections WHERE id = $1 AND workspace_id = $2",
id,
workspace_id.as_uuid(),
)
.execute(pool)
.await?;
Ok(res.rows_affected() == 1)
}
/// Called after a sync run; timestamps the row and stores the error (or
/// clears it on success).
pub async fn mark_synced(pool: &PgPool, id: Uuid, error: Option<&str>) -> Result<(), DbError> {
sqlx::query!(
"UPDATE repo_connections
SET last_synced_at = now(),
last_sync_error = $2,
updated_at = now()
WHERE id = $1",
id,
error,
)
.execute(pool)
.await?;
Ok(())
}
+192
View File
@@ -0,0 +1,192 @@
//! Repos — workspace-scoped cache of the repositories pulled from each
//! connected provider. Rows are populated by the provider-specific sync
//! path (see cm-api::providers) and served to the REPOS tier UI.
//!
//! Ownership: `workspace_id` scopes visibility; `connection_id` groups by
//! provider account. The (`connection_id`, `external_id`) unique makes
//! upsert idempotent across re-syncs.
use cm_domain::WorkspaceId;
use sqlx::PgPool;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::DbError;
/// A repo row, as pulled from the provider.
pub struct Repo {
pub id: Uuid,
pub workspace_id: Uuid,
pub connection_id: Uuid,
pub provider: String,
pub external_id: String,
pub owner: String,
pub name: String,
pub description: Option<String>,
pub default_branch: Option<String>,
pub clone_url: Option<String>,
pub html_url: Option<String>,
pub private: bool,
pub stars: i32,
pub forks: i32,
pub provider_updated_at: Option<OffsetDateTime>,
pub last_synced_at: OffsetDateTime,
}
/// Payload for one sync-driven upsert. Fields mirror the columns 1:1 minus
/// the workspace + timestamps the DB owns.
pub struct RepoUpsert<'a> {
pub connection_id: Uuid,
pub workspace_id: WorkspaceId,
pub provider: &'a str,
pub external_id: &'a str,
pub owner: &'a str,
pub name: &'a str,
pub description: Option<&'a str>,
pub default_branch: Option<&'a str>,
pub clone_url: Option<&'a str>,
pub html_url: Option<&'a str>,
pub private: bool,
pub stars: i32,
pub forks: i32,
pub provider_updated_at: Option<OffsetDateTime>,
}
/// Insert-or-update on (connection_id, external_id). Bumps `last_synced_at`
/// on every call so the UI can show a "last synced" hint even when the
/// provider hasn't changed anything.
pub async fn upsert(pool: &PgPool, r: RepoUpsert<'_>) -> Result<Uuid, DbError> {
let id = Uuid::now_v7();
let row = sqlx::query!(
"INSERT INTO repos
(id, workspace_id, connection_id, provider, external_id,
owner, name, description, default_branch, clone_url, html_url,
private, stars, forks, provider_updated_at, last_synced_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
$12, $13, $14, $15, now())
ON CONFLICT (connection_id, external_id) DO UPDATE
SET owner = EXCLUDED.owner,
name = EXCLUDED.name,
description = EXCLUDED.description,
default_branch = EXCLUDED.default_branch,
clone_url = EXCLUDED.clone_url,
html_url = EXCLUDED.html_url,
private = EXCLUDED.private,
stars = EXCLUDED.stars,
forks = EXCLUDED.forks,
provider_updated_at = EXCLUDED.provider_updated_at,
last_synced_at = now()
RETURNING id",
id,
r.workspace_id.as_uuid(),
r.connection_id,
r.provider,
r.external_id,
r.owner,
r.name,
r.description,
r.default_branch,
r.clone_url,
r.html_url,
r.private,
r.stars,
r.forks,
r.provider_updated_at,
)
.fetch_one(pool)
.await?;
Ok(row.id)
}
/// All repos in a workspace, newest-provider-update first, cap-limited.
pub async fn list_for_workspace(
pool: &PgPool,
workspace_id: WorkspaceId,
limit: i64,
) -> Result<Vec<Repo>, DbError> {
let rows = sqlx::query!(
"SELECT id, workspace_id, connection_id, provider, external_id,
owner, name, description, default_branch, clone_url, html_url,
private, stars, forks, provider_updated_at, last_synced_at
FROM repos
WHERE workspace_id = $1
ORDER BY provider_updated_at DESC NULLS LAST, name
LIMIT $2",
workspace_id.as_uuid(),
limit,
)
.fetch_all(pool)
.await?;
Ok(rows
.into_iter()
.map(|r| Repo {
id: r.id,
workspace_id: r.workspace_id,
connection_id: r.connection_id,
provider: r.provider,
external_id: r.external_id,
owner: r.owner,
name: r.name,
description: r.description,
default_branch: r.default_branch,
clone_url: r.clone_url,
html_url: r.html_url,
private: r.private,
stars: r.stars,
forks: r.forks,
provider_updated_at: r.provider_updated_at,
last_synced_at: r.last_synced_at,
})
.collect())
}
/// A single repo, workspace-scoped so cross-tenant reads return NotFound.
pub async fn get(pool: &PgPool, id: Uuid, workspace_id: WorkspaceId) -> Result<Repo, DbError> {
let r = sqlx::query!(
"SELECT id, workspace_id, connection_id, provider, external_id,
owner, name, description, default_branch, clone_url, html_url,
private, stars, forks, provider_updated_at, last_synced_at
FROM repos
WHERE id = $1 AND workspace_id = $2",
id,
workspace_id.as_uuid(),
)
.fetch_optional(pool)
.await?
.ok_or(DbError::NotFound)?;
Ok(Repo {
id: r.id,
workspace_id: r.workspace_id,
connection_id: r.connection_id,
provider: r.provider,
external_id: r.external_id,
owner: r.owner,
name: r.name,
description: r.description,
default_branch: r.default_branch,
clone_url: r.clone_url,
html_url: r.html_url,
private: r.private,
stars: r.stars,
forks: r.forks,
provider_updated_at: r.provider_updated_at,
last_synced_at: r.last_synced_at,
})
}
/// Delete every repo cached under a connection. Called when the
/// connection itself is deleted, or as part of a full re-sync.
pub async fn delete_by_connection(
pool: &PgPool,
workspace_id: WorkspaceId,
connection_id: Uuid,
) -> Result<u64, DbError> {
let res = sqlx::query!(
"DELETE FROM repos WHERE connection_id = $1 AND workspace_id = $2",
connection_id,
workspace_id.as_uuid(),
)
.execute(pool)
.await?;
Ok(res.rows_affected())
}
+20
View File
@@ -95,4 +95,24 @@ impl BrokerClient {
other => Err(BrokerError::Io(format!("unexpected response: {other:?}"))), other => Err(BrokerError::Io(format!("unexpected response: {other:?}"))),
} }
} }
/// Read-only GET with the stored PAT injected as a bearer token. The
/// credential never leaves the broker; the caller only sees the response
/// status + parsed JSON body. Used by the repo-provider sync path.
pub async fn fetch_authorized(
&mut self,
secret_id: Uuid,
url: &str,
) -> Result<(u16, serde_json::Value), BrokerError> {
match self
.round_trip(Request::FetchAuthorized {
secret_id,
url: url.to_owned(),
})
.await?
{
Response::HttpJson { status, body } => Ok((status, body)),
other => Err(BrokerError::Io(format!("unexpected response: {other:?}"))),
}
}
} }
+32 -5
View File
@@ -37,16 +37,43 @@ pub enum Request {
#[serde(default)] #[serde(default)]
body: serde_json::Value, body: serde_json::Value,
}, },
/// Read-only HTTP GET with the secret injected as a bearer token. Returns
/// the response body as JSON without ever exposing the credential to the
/// caller. Used by low-privilege data-fetching flows (listing an org's
/// repositories on GitHub / Gitea / GitLab) that don't need the
/// single-use grant `InvokeHttp` demands. The URL scheme is still
/// restricted to http(s).
FetchAuthorized {
secret_id: Uuid,
url: String,
},
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "result", rename_all = "snake_case")] #[serde(tag = "result", rename_all = "snake_case")]
pub enum Response { pub enum Response {
SecretStored { secret_id: Uuid }, SecretStored {
SecretKind { kind: String }, secret_id: Uuid,
HttpDone { status: u16 }, },
Verified { valid: bool }, SecretKind {
Error { kind: ErrorKind, message: String }, kind: String,
},
HttpDone {
status: u16,
},
/// Response to a `FetchAuthorized`: HTTP status + parsed JSON body.
/// Body is `null` when the response wasn't JSON-parseable.
HttpJson {
status: u16,
body: serde_json::Value,
},
Verified {
valid: bool,
},
Error {
kind: ErrorKind,
message: String,
},
} }
#[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+26
View File
@@ -150,6 +150,32 @@ impl BrokerServer {
status: response.status().as_u16(), status: response.status().as_u16(),
}) })
} }
Request::FetchAuthorized { secret_id, url } => {
if !url.starts_with("https://") && !url.starts_with("http://") {
return Err(BrokerError::Invalid(format!(
"capability urls must be http(s), got {url}"
)));
}
let credential = store.reveal_internal(secret_id).await?;
// Store owns the plaintext PAT verbatim (the `store_secret`
// path stores exactly what cm-api passes in). Nothing here
// decodes it as a structured credential — it goes straight
// into the bearer header.
let response = reqwest::Client::new()
.get(&url)
.bearer_auth(credential.trim())
.header("Accept", "application/json")
.header("User-Agent", "clawmates-broker")
.send()
.await
.map_err(|e| BrokerError::Io(e.to_string()))?;
let status = response.status().as_u16();
let body = response
.json::<serde_json::Value>()
.await
.unwrap_or(serde_json::Value::Null);
Ok(Response::HttpJson { status, body })
}
} }
} }
} }
+63
View File
@@ -0,0 +1,63 @@
-- Repos: workspace-scoped cache of the repositories pulled from each
-- connected provider (GitHub / Gitea / GitLab). One row per (connection,
-- external_id). The provider PAT itself never lives here — it's already
-- in the secret broker via app_connections.secret_ref (see 0007).
--
-- Sync flow:
-- POST /api/repos/connections/:id/sync → provider API list → upsert.
-- Removed-upstream rows stay for now (v1); a future flag can prune them.
--
-- repo_connections wraps an app_connections row with the repo-specific
-- config: which owner (org/user) to pull from, provider base URL for
-- self-hosted Gitea/GitLab instances, and last-sync bookkeeping.
CREATE TABLE repo_connections (
id UUID PRIMARY KEY,
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
app_connection_id UUID NOT NULL REFERENCES app_connections (id) ON DELETE CASCADE,
provider TEXT NOT NULL, -- 'github' | 'gitea' | 'gitlab'
-- Optional owner (org or user). When NULL, the provider fetches "all
-- repos the token can see" (e.g. GitHub /user/repos).
owner TEXT,
-- Optional base URL for self-hosted providers (e.g. https://git.redclaw.dev).
base_url TEXT,
label TEXT NOT NULL DEFAULT '',
last_synced_at TIMESTAMPTZ,
last_sync_error TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX repo_connections_workspace_idx
ON repo_connections (workspace_id, created_at DESC);
CREATE TABLE repos (
id UUID PRIMARY KEY,
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
connection_id UUID NOT NULL REFERENCES repo_connections (id) ON DELETE CASCADE,
provider TEXT NOT NULL, -- 'github' | 'gitea' | 'gitlab'
-- Provider-side stable id (numeric on GitHub, integer on Gitea/GitLab).
-- Stored as TEXT so we don't care which flavor the provider uses.
external_id TEXT NOT NULL,
owner TEXT NOT NULL, -- org or user login
name TEXT NOT NULL, -- repo slug
description TEXT,
default_branch TEXT,
clone_url TEXT,
html_url TEXT,
private BOOLEAN NOT NULL DEFAULT false,
stars INTEGER NOT NULL DEFAULT 0,
forks INTEGER NOT NULL DEFAULT 0,
-- Provider's own `updated_at` (last push / edit), NOT the sync time.
provider_updated_at TIMESTAMPTZ,
last_synced_at TIMESTAMPTZ NOT NULL DEFAULT now(),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (connection_id, external_id)
);
CREATE INDEX repos_workspace_idx
ON repos (workspace_id, provider_updated_at DESC NULLS LAST);
-- Fast per-connection list (sidebar grouping).
CREATE INDEX repos_connection_idx
ON repos (connection_id, name);