research: persist bound repo + shallow-clone on start_topic
ci / gates (push) Successful in 7s
ci / frontend (push) Successful in 35s
ci / rust (push) Failing after 38s
ci / e2e (push) Has been skipped
ci / publish (push) Has been skipped

This is the minimum viable version of the "agents actually work on
a repo" architecture. Full vision (isolated ZeroClaw container per
topic, dynamic agent provisioning inside, pause/resume, commit
gate) is real weeks of work — this closes the first, most-visible
gap so the ClawHDF5 topic can actually run against its codebase.

Backend

- Migration 0037: research_topics gets repo_id UUID (nullable, FK
  to repos ON DELETE SET NULL) and repo_workspace_path TEXT for
  the on-disk checkout location. Index on repo_id when set.

- research_topics::create takes repo_id: Option<Uuid>. get + list
  select it and repo_workspace_path. set_repo_workspace_path
  persists the path once the first clone lands.

- CreateTopicRequest accepts `repo: Option<TopicRepoRef>` — the
  same denormalized shape the wizard already sends. Only repo_id
  is authoritative; other fields are ignored (dead_code-allowed
  so serde still deserializes the full body).

- start_topic branches on topic.repo_id. When set, it calls
  ensure_repo_workspace:
    · resolves repo.clone_url + repo.default_branch
    · target path = CLAWMATES_RESEARCH_WORKSPACE_ROOT
                    // <topic_id> // repo (defaults under $TMPDIR)
    · runs `git clone --depth 1 --single-branch --branch <b>` via
      tokio::process. Reuses the checkout if .git already exists.
    · persists the path so re-starts skip the clone
    · runs `git ls-files` to sample the tree (first 60 entries,
      total count reported honestly so the prompt doesn't lie
      about coverage)
  All best-effort — a clone failure logs but still starts the run
  without repo context rather than aborting.

- build_coordinator_task takes Option<&RepoContext>. When present,
  the framing gets a REPO block (slug / path / branch / file
  sample) and a USING THE REPO section instructing the coordinator
  to ground every recommendation in a concrete file reference and
  never fabricate paths. The per-topology bodies are unchanged —
  the repo guidance sits above them so it applies to every shape.

What this unblocks / doesn't unblock

Unblocks: The coordinator prompt now knows the repo exists, where
it lives on disk, and what's in it. Even without file-editing
tools wired to the checkout, the coordinator can point spokes at
concrete modules and the final artifact can reference real files.
For a spec-shaped outcome like ClawHDF5's, that's the difference
between abstract advice and a spec grounded in the actual crates.

Does NOT unblock: The agents themselves editing files, running
tests, or committing. That requires either mounting the checkout
into the ZeroClaw sandbox or exposing a new MCP tool for
repo-scoped file ops — separate follow-up.
This commit is contained in:
Omar Sobh
2026-07-08 21:38:00 -07:00
parent 1a4c5eb159
commit 3465bb7a6d
8 changed files with 322 additions and 12 deletions
@@ -1,6 +1,6 @@
{ {
"db_name": "PostgreSQL", "db_name": "PostgreSQL",
"query": "INSERT INTO research_topics\n (id, workspace_id, title, description, outcome_kind, topology_kind, status, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, 'standby', $7)", "query": "INSERT INTO research_topics\n (id, workspace_id, title, description, outcome_kind, topology_kind, repo_id, status, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, 'standby', $8)",
"describe": { "describe": {
"columns": [], "columns": [],
"parameters": { "parameters": {
@@ -11,10 +11,11 @@
"Text", "Text",
"Text", "Text",
"Text", "Text",
"Uuid",
"Uuid" "Uuid"
] ]
}, },
"nullable": [] "nullable": []
}, },
"hash": "819ff5b7f2225a932e2181eb3bd3de45b98cca6d3e88dd9f686a9eda669803b1" "hash": "15df27d0a94a927c62f0d737a11e391f5a7ff139e95d88516994ff30f782784b"
} }
@@ -1,6 +1,6 @@
{ {
"db_name": "PostgreSQL", "db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, title, description, outcome_kind, status,\n created_by, created_at, updated_at, published_at, topology_kind\n FROM research_topics\n WHERE workspace_id = $1\n ORDER BY updated_at DESC", "query": "SELECT id, workspace_id, title, description, outcome_kind, status,\n created_by, created_at, updated_at, published_at, topology_kind,\n repo_id, repo_workspace_path\n FROM research_topics\n WHERE workspace_id = $1\n ORDER BY updated_at DESC",
"describe": { "describe": {
"columns": [ "columns": [
{ {
@@ -57,6 +57,16 @@
"ordinal": 10, "ordinal": 10,
"name": "topology_kind", "name": "topology_kind",
"type_info": "Text" "type_info": "Text"
},
{
"ordinal": 11,
"name": "repo_id",
"type_info": "Uuid"
},
{
"ordinal": 12,
"name": "repo_workspace_path",
"type_info": "Text"
} }
], ],
"parameters": { "parameters": {
@@ -75,8 +85,10 @@
false, false,
false, false,
true, true,
false false,
true,
true
] ]
}, },
"hash": "7f85b5b9fa303e3f232359832a6630f3ac999a992f150d6cede6362d51eac063" "hash": "9a823772908b51bb76d5e162e9dde8265fd9cdf106828ada234b75ab106ad21e"
} }
@@ -1,6 +1,6 @@
{ {
"db_name": "PostgreSQL", "db_name": "PostgreSQL",
"query": "SELECT id, workspace_id, title, description, outcome_kind, status,\n created_by, created_at, updated_at, published_at, topology_kind\n FROM research_topics\n WHERE id = $1 AND workspace_id = $2", "query": "SELECT id, workspace_id, title, description, outcome_kind, status,\n created_by, created_at, updated_at, published_at, topology_kind,\n repo_id, repo_workspace_path\n FROM research_topics\n WHERE id = $1 AND workspace_id = $2",
"describe": { "describe": {
"columns": [ "columns": [
{ {
@@ -57,6 +57,16 @@
"ordinal": 10, "ordinal": 10,
"name": "topology_kind", "name": "topology_kind",
"type_info": "Text" "type_info": "Text"
},
{
"ordinal": 11,
"name": "repo_id",
"type_info": "Uuid"
},
{
"ordinal": 12,
"name": "repo_workspace_path",
"type_info": "Text"
} }
], ],
"parameters": { "parameters": {
@@ -76,8 +86,10 @@
false, false,
false, false,
true, true,
false false,
true,
true
] ]
}, },
"hash": "cb9c83b58db07f741510ec96e9301c5fa9c4e22f102787cbc99e488d417acf1a" "hash": "a40940823a2d1e3105140d59c513fd6c7b27430c4350bdce82ce4427c9954ee7"
} }
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE research_topics\n SET repo_workspace_path = $3, updated_at = now()\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "fd3bc406d3a36bef8c2334f5fcdbdbeed977465938a85a970bc39efc21956de3"
}
+214
View File
@@ -47,6 +47,31 @@ pub struct CreateTopicRequest {
pub topology_kind: Option<String>, pub topology_kind: Option<String>,
#[serde(default)] #[serde(default)]
pub agents: Vec<AgentSlotInput>, pub agents: Vec<AgentSlotInput>,
/// The workspace repo the wizard bound to this topic. Only `repo_id` is
/// authoritative; the rest is denormalized display data the wizard sent
/// for its own UI and is ignored here.
#[serde(default)]
pub repo: Option<TopicRepoRef>,
}
// The wizard sends a denormalized display object for its own UI. Only
// repo_id is authoritative; the rest is present so serde deserializes the
// full body cleanly (and future callers can piggyback additional
// metadata) even though start_topic ignores it.
#[derive(Deserialize)]
#[allow(dead_code)]
pub struct TopicRepoRef {
pub repo_id: Uuid,
#[serde(default)]
pub connection_id: Option<Uuid>,
#[serde(default)]
pub provider: Option<String>,
#[serde(default)]
pub owner: Option<String>,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub default_branch: Option<String>,
} }
/// Topology kinds the wizard exposes for research. Every string here must /// Topology kinds the wizard exposes for research. Every string here must
@@ -67,12 +92,43 @@ fn build_coordinator_task(
description: &str, description: &str,
topo: &cm_topology::TopologyKind, topo: &cm_topology::TopologyKind,
roster: &str, roster: &str,
repo: Option<&RepoContext>,
) -> String { ) -> String {
use cm_topology::TopologyKind::*; use cm_topology::TopologyKind::*;
let repo_block = repo
.map(|r| {
format!(
"REPO (cloned on the API host, shallow):\n\
· slug: {slug}\n\
· path: {path}\n\
· branch: {branch}\n\
· files sampled ({shown} of {total}):\n{tree}\n\n",
slug = r.slug,
path = r.path,
branch = r.branch,
shown = r.shown,
total = r.total_files,
tree = r.tree_preview,
)
})
.unwrap_or_default();
let repo_guidance = if repo.is_some() {
"USING THE REPO:\n\
The repo above is real and already checked out. Ground every claim, \
proposal, and acceptance-criterion in a concrete file or module you \
reference by path. Enumerate the files you inspected in your final \
artifact so a reviewer can walk your reasoning. If a spoke lacks \
file access, describe the module + interface you want them to \
reason about — never fabricate paths.\n\n"
} else {
""
};
let framing = format!( let framing = format!(
"RESEARCH TOPIC: {title}\n\ "RESEARCH TOPIC: {title}\n\
OUTCOME KIND: {outcome} (spec / prod_plan / roadmap / paper)\n\n\ OUTCOME KIND: {outcome} (spec / prod_plan / roadmap / paper)\n\n\
DESCRIPTION:\n{description}\n\n\ DESCRIPTION:\n{description}\n\n\
{repo_block}\
{repo_guidance}\
TEAM:\n{roster}\n\n" TEAM:\n{roster}\n\n"
); );
let body = match topo { let body = match topo {
@@ -137,6 +193,134 @@ the description."
format!("{framing}{body}") format!("{framing}{body}")
} }
/// A shallow-cloned repo attached to a research run. Populated by
/// `ensure_repo_workspace`; consumed by `build_coordinator_task` to give
/// the coordinator a concrete on-disk starting point for the team.
struct RepoContext {
/// Human-readable "owner/name".
slug: String,
/// Absolute path on the API host where the checkout lives.
path: String,
/// Branch we cloned (repo.default_branch → "main" fallback).
branch: String,
/// Line-per-entry preview of the working tree (relative paths).
tree_preview: String,
/// Files shown vs. total, so the prompt is honest about truncation.
shown: usize,
total_files: usize,
}
/// Root directory under which `start_topic` clones per-topic checkouts.
/// Overridable via `CLAWMATES_RESEARCH_WORKSPACE_ROOT` for prod deploys
/// that want a mounted volume; defaults to a subdir of the system tmpdir
/// so dev + tests just work without setup.
fn research_workspace_root() -> std::path::PathBuf {
if let Ok(root) = std::env::var("CLAWMATES_RESEARCH_WORKSPACE_ROOT") {
return std::path::PathBuf::from(root);
}
std::env::temp_dir().join("clawmates-research")
}
/// Clone the bound repo (shallow, single branch) into a per-topic
/// workspace and gather a tree preview for the coordinator prompt.
/// Persists the clone path on the topic so a re-start reuses it instead
/// of re-cloning. Best-effort — callers treat failures as "start without
/// repo context" rather than aborting the run.
async fn ensure_repo_workspace(
pool: &sqlx::PgPool,
topic_id: Uuid,
workspace_id: Uuid,
repo: &cm_db::repo::repos::Repo,
topic: &cm_db::repo::research_topics::ResearchTopic,
) -> Result<RepoContext, String> {
let clone_url = repo
.clone_url
.as_deref()
.ok_or_else(|| "repo has no clone_url".to_string())?;
let branch = repo
.default_branch
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or("main")
.to_string();
let target = topic.repo_workspace_path.clone().unwrap_or_else(|| {
research_workspace_root()
.join(topic_id.to_string())
.join("repo")
.to_string_lossy()
.into_owned()
});
let target_path = std::path::PathBuf::from(&target);
let should_clone = !target_path.join(".git").exists();
if should_clone {
if let Some(parent) = target_path.parent() {
std::fs::create_dir_all(parent).map_err(|e| format!("mkdir parent: {e}"))?;
}
let out = tokio::process::Command::new("git")
.arg("clone")
.arg("--depth")
.arg("1")
.arg("--single-branch")
.arg("--branch")
.arg(&branch)
.arg(clone_url)
.arg(&target_path)
.output()
.await
.map_err(|e| format!("spawn git clone: {e}"))?;
if !out.status.success() {
return Err(format!(
"git clone exit {:?}: {}",
out.status.code(),
String::from_utf8_lossy(&out.stderr).trim()
));
}
cm_db::repo::research_topics::set_repo_workspace_path(
pool,
topic_id,
workspace_id,
&target,
)
.await
.map_err(|e| format!("persist clone path: {e}"))?;
}
// Preview: `git ls-files` first N entries. Bounded so the prompt stays
// small even for large repos; a subsequent tool call can list more.
const MAX_TREE_LINES: usize = 60;
let ls = tokio::process::Command::new("git")
.arg("-C")
.arg(&target_path)
.arg("ls-files")
.output()
.await
.map_err(|e| format!("spawn git ls-files: {e}"))?;
let all = String::from_utf8_lossy(&ls.stdout);
let entries: Vec<&str> = all.lines().filter(|l| !l.is_empty()).collect();
let shown = entries.len().min(MAX_TREE_LINES);
let preview = entries
.iter()
.take(shown)
.map(|e| format!(" {e}"))
.collect::<Vec<_>>()
.join("\n");
Ok(RepoContext {
slug: format!("{}/{}", repo.owner, repo.name),
path: target,
branch,
tree_preview: if preview.is_empty() {
" (empty)".to_string()
} else {
preview
},
shown,
total_files: entries.len(),
})
}
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct AgentSlotInput { pub struct AgentSlotInput {
pub agent_id: Uuid, pub agent_id: Uuid,
@@ -174,6 +358,13 @@ pub async fn create_topic(
return Err(ApiError::NotFound); return Err(ApiError::NotFound);
} }
} }
// Same ownership check for the repo binding.
let repo_id = if let Some(r) = &body.repo {
cm_db::repo::repos::get(&state.pool, r.repo_id, user.workspace_id).await?;
Some(r.repo_id)
} else {
None
};
let id = cm_db::repo::research_topics::create( let id = cm_db::repo::research_topics::create(
&state.pool, &state.pool,
@@ -182,6 +373,7 @@ pub async fn create_topic(
body.description.trim(), body.description.trim(),
&body.outcome_kind, &body.outcome_kind,
topology_kind, topology_kind,
repo_id,
user.user_id.as_uuid(), user.user_id.as_uuid(),
) )
.await?; .await?;
@@ -389,6 +581,27 @@ pub async fn start_topic(
cm_db::repo::agents::get(&state.pool, cm_domain::AgentId::from(s.agent_id)).await?; cm_db::repo::agents::get(&state.pool, cm_domain::AgentId::from(s.agent_id)).await?;
roster.push((s.clone(), agent)); roster.push((s.clone(), agent));
} }
// If a repo is bound, clone (shallow) into a per-topic workspace so the
// coordinator prompt can point the team at real files. Best-effort — a
// clone failure logs but still starts the run without repo context.
let repo_context = if let Some(repo_id) = topic.repo_id {
let repo = cm_db::repo::repos::get(&state.pool, repo_id, user.workspace_id).await?;
match ensure_repo_workspace(&state.pool, id, user.workspace_id.as_uuid(), &repo, &topic)
.await
{
Ok(ctx) => Some(ctx),
Err(e) => {
eprintln!(
"research::start_topic({id}): repo clone failed for {}/{}: {e}",
repo.owner, repo.name
);
None
}
}
} else {
None
};
// Coordinator = first slot whose role_slot mentions "coordinator" (case- // Coordinator = first slot whose role_slot mentions "coordinator" (case-
// insensitive), else the first slot. The coordinator becomes the hub of // insensitive), else the first slot. The coordinator becomes the hub of
// the hub_spoke graph, so it can talk to every other agent per-turn. // the hub_spoke graph, so it can talk to every other agent per-turn.
@@ -483,6 +696,7 @@ pub async fn start_topic(
&topic.description, &topic.description,
&topo, &topo,
&roster_lines, &roster_lines,
repo_context.as_ref(),
); );
let run_id = uuid::Uuid::now_v7(); let run_id = uuid::Uuid::now_v7();
+2
View File
@@ -280,6 +280,7 @@ async fn notify_run_completed_transitions_topic_when_no_siblings_in_flight() {
"desc", "desc",
"spec", "spec",
"hub_spoke", "hub_spoke",
None,
user_id.as_uuid(), user_id.as_uuid(),
) )
.await .await
@@ -330,6 +331,7 @@ async fn notify_run_completed_leaves_topic_processing_when_siblings_in_flight()
"desc", "desc",
"spec", "spec",
"hub_spoke", "hub_spoke",
None,
user_id.as_uuid(), user_id.as_uuid(),
) )
.await .await
+38 -4
View File
@@ -27,6 +27,15 @@ pub struct ResearchTopic {
/// Topology shape start_topic builds when firing this topic. Options: /// Topology shape start_topic builds when firing this topic. Options:
/// hub_spoke, pipeline, hierarchical, star_moe. Defaults to hub_spoke. /// hub_spoke, pipeline, hierarchical, star_moe. Defaults to hub_spoke.
pub topology_kind: String, pub topology_kind: String,
/// The workspace repo the wizard bound to this topic (optional). When
/// set, `start_topic` clones it and feeds the coordinator prompt with
/// the checkout path + a file-tree overview so the agents can reason
/// about the actual code.
pub repo_id: Option<Uuid>,
/// Absolute path on the API host where `start_topic` cloned the bound
/// repo. Written once on the first successful clone; subsequent starts
/// reuse it. Null until then.
pub repo_workspace_path: Option<String>,
} }
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
@@ -43,19 +52,21 @@ pub async fn create(
description: &str, description: &str,
outcome_kind: &str, outcome_kind: &str,
topology_kind: &str, topology_kind: &str,
repo_id: Option<Uuid>,
created_by: Uuid, created_by: Uuid,
) -> Result<Uuid, DbError> { ) -> Result<Uuid, DbError> {
let id = Uuid::now_v7(); let id = Uuid::now_v7();
sqlx::query!( sqlx::query!(
"INSERT INTO research_topics "INSERT INTO research_topics
(id, workspace_id, title, description, outcome_kind, topology_kind, status, created_by) (id, workspace_id, title, description, outcome_kind, topology_kind, repo_id, status, created_by)
VALUES ($1, $2, $3, $4, $5, $6, 'standby', $7)", VALUES ($1, $2, $3, $4, $5, $6, $7, 'standby', $8)",
id, id,
workspace_id, workspace_id,
title, title,
description, description,
outcome_kind, outcome_kind,
topology_kind, topology_kind,
repo_id,
created_by, created_by,
) )
.execute(pool) .execute(pool)
@@ -63,12 +74,34 @@ pub async fn create(
Ok(id) Ok(id)
} }
/// Persist the clone path for a topic's bound repo. Set once, on the first
/// successful clone; a re-start reads it back and skips re-cloning.
pub async fn set_repo_workspace_path(
pool: &PgPool,
id: Uuid,
workspace_id: Uuid,
path: &str,
) -> Result<(), DbError> {
sqlx::query!(
"UPDATE research_topics
SET repo_workspace_path = $3, updated_at = now()
WHERE id = $1 AND workspace_id = $2",
id,
workspace_id,
path,
)
.execute(pool)
.await?;
Ok(())
}
/// Workspace's topics, newest-updated first. /// Workspace's topics, newest-updated first.
pub async fn list(pool: &PgPool, workspace_id: Uuid) -> Result<Vec<ResearchTopic>, DbError> { pub async fn list(pool: &PgPool, workspace_id: Uuid) -> Result<Vec<ResearchTopic>, DbError> {
let rows = sqlx::query_as!( let rows = sqlx::query_as!(
ResearchTopic, ResearchTopic,
"SELECT id, workspace_id, title, description, outcome_kind, status, "SELECT id, workspace_id, title, description, outcome_kind, status,
created_by, created_at, updated_at, published_at, topology_kind created_by, created_at, updated_at, published_at, topology_kind,
repo_id, repo_workspace_path
FROM research_topics FROM research_topics
WHERE workspace_id = $1 WHERE workspace_id = $1
ORDER BY updated_at DESC", ORDER BY updated_at DESC",
@@ -87,7 +120,8 @@ pub async fn get(
let row = sqlx::query_as!( let row = sqlx::query_as!(
ResearchTopic, ResearchTopic,
"SELECT id, workspace_id, title, description, outcome_kind, status, "SELECT id, workspace_id, title, description, outcome_kind, status,
created_by, created_at, updated_at, published_at, topology_kind created_by, created_at, updated_at, published_at, topology_kind,
repo_id, repo_workspace_path
FROM research_topics FROM research_topics
WHERE id = $1 AND workspace_id = $2", WHERE id = $1 AND workspace_id = $2",
id, id,
@@ -0,0 +1,19 @@
-- Bind a research topic to a workspace repo + the on-disk path where
-- `start_topic` clones a working copy. Both nullable so pre-existing
-- topics don't need backfill and topics without a repo still work.
--
-- repo_id — the repo the user picked in the wizard. ON DELETE SET
-- NULL so removing the connection doesn't cascade-nuke the topic.
--
-- repo_workspace_path — absolute path on the API host where
-- `start_topic` executed `git clone --depth 1`. Set once when the
-- clone succeeds; a subsequent start can reuse it (skip re-clone).
-- Feeds the coordinator prompt so agents know where the files live.
ALTER TABLE research_topics
ADD COLUMN repo_id UUID REFERENCES repos (id) ON DELETE SET NULL,
ADD COLUMN repo_workspace_path TEXT;
CREATE INDEX research_topics_repo_idx
ON research_topics (repo_id)
WHERE repo_id IS NOT NULL;