research: persist bound repo + shallow-clone on start_topic
This is the minimum viable version of the "agents actually work on
a repo" architecture. Full vision (isolated ZeroClaw container per
topic, dynamic agent provisioning inside, pause/resume, commit
gate) is real weeks of work — this closes the first, most-visible
gap so the ClawHDF5 topic can actually run against its codebase.
Backend
- Migration 0037: research_topics gets repo_id UUID (nullable, FK
to repos ON DELETE SET NULL) and repo_workspace_path TEXT for
the on-disk checkout location. Index on repo_id when set.
- research_topics::create takes repo_id: Option<Uuid>. get + list
select it and repo_workspace_path. set_repo_workspace_path
persists the path once the first clone lands.
- CreateTopicRequest accepts `repo: Option<TopicRepoRef>` — the
same denormalized shape the wizard already sends. Only repo_id
is authoritative; other fields are ignored (dead_code-allowed
so serde still deserializes the full body).
- start_topic branches on topic.repo_id. When set, it calls
ensure_repo_workspace:
· resolves repo.clone_url + repo.default_branch
· target path = CLAWMATES_RESEARCH_WORKSPACE_ROOT
// <topic_id> // repo (defaults under $TMPDIR)
· runs `git clone --depth 1 --single-branch --branch <b>` via
tokio::process. Reuses the checkout if .git already exists.
· persists the path so re-starts skip the clone
· runs `git ls-files` to sample the tree (first 60 entries,
total count reported honestly so the prompt doesn't lie
about coverage)
All best-effort — a clone failure logs but still starts the run
without repo context rather than aborting.
- build_coordinator_task takes Option<&RepoContext>. When present,
the framing gets a REPO block (slug / path / branch / file
sample) and a USING THE REPO section instructing the coordinator
to ground every recommendation in a concrete file reference and
never fabricate paths. The per-topology bodies are unchanged —
the repo guidance sits above them so it applies to every shape.
What this unblocks / doesn't unblock
Unblocks: The coordinator prompt now knows the repo exists, where
it lives on disk, and what's in it. Even without file-editing
tools wired to the checkout, the coordinator can point spokes at
concrete modules and the final artifact can reference real files.
For a spec-shaped outcome like ClawHDF5's, that's the difference
between abstract advice and a spec grounded in the actual crates.
Does NOT unblock: The agents themselves editing files, running
tests, or committing. That requires either mounting the checkout
into the ZeroClaw sandbox or exposing a new MCP tool for
repo-scoped file ops — separate follow-up.
This commit is contained in:
@@ -47,6 +47,31 @@ pub struct CreateTopicRequest {
|
||||
pub topology_kind: Option<String>,
|
||||
#[serde(default)]
|
||||
pub agents: Vec<AgentSlotInput>,
|
||||
/// The workspace repo the wizard bound to this topic. Only `repo_id` is
|
||||
/// authoritative; the rest is denormalized display data the wizard sent
|
||||
/// for its own UI and is ignored here.
|
||||
#[serde(default)]
|
||||
pub repo: Option<TopicRepoRef>,
|
||||
}
|
||||
|
||||
// The wizard sends a denormalized display object for its own UI. Only
|
||||
// repo_id is authoritative; the rest is present so serde deserializes the
|
||||
// full body cleanly (and future callers can piggyback additional
|
||||
// metadata) even though start_topic ignores it.
|
||||
#[derive(Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct TopicRepoRef {
|
||||
pub repo_id: Uuid,
|
||||
#[serde(default)]
|
||||
pub connection_id: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub provider: Option<String>,
|
||||
#[serde(default)]
|
||||
pub owner: Option<String>,
|
||||
#[serde(default)]
|
||||
pub name: Option<String>,
|
||||
#[serde(default)]
|
||||
pub default_branch: Option<String>,
|
||||
}
|
||||
|
||||
/// Topology kinds the wizard exposes for research. Every string here must
|
||||
@@ -67,12 +92,43 @@ fn build_coordinator_task(
|
||||
description: &str,
|
||||
topo: &cm_topology::TopologyKind,
|
||||
roster: &str,
|
||||
repo: Option<&RepoContext>,
|
||||
) -> String {
|
||||
use cm_topology::TopologyKind::*;
|
||||
let repo_block = repo
|
||||
.map(|r| {
|
||||
format!(
|
||||
"REPO (cloned on the API host, shallow):\n\
|
||||
· slug: {slug}\n\
|
||||
· path: {path}\n\
|
||||
· branch: {branch}\n\
|
||||
· files sampled ({shown} of {total}):\n{tree}\n\n",
|
||||
slug = r.slug,
|
||||
path = r.path,
|
||||
branch = r.branch,
|
||||
shown = r.shown,
|
||||
total = r.total_files,
|
||||
tree = r.tree_preview,
|
||||
)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let repo_guidance = if repo.is_some() {
|
||||
"USING THE REPO:\n\
|
||||
The repo above is real and already checked out. Ground every claim, \
|
||||
proposal, and acceptance-criterion in a concrete file or module you \
|
||||
reference by path. Enumerate the files you inspected in your final \
|
||||
artifact so a reviewer can walk your reasoning. If a spoke lacks \
|
||||
file access, describe the module + interface you want them to \
|
||||
reason about — never fabricate paths.\n\n"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let framing = format!(
|
||||
"RESEARCH TOPIC: {title}\n\
|
||||
OUTCOME KIND: {outcome} (spec / prod_plan / roadmap / paper)\n\n\
|
||||
DESCRIPTION:\n{description}\n\n\
|
||||
{repo_block}\
|
||||
{repo_guidance}\
|
||||
TEAM:\n{roster}\n\n"
|
||||
);
|
||||
let body = match topo {
|
||||
@@ -137,6 +193,134 @@ the description."
|
||||
format!("{framing}{body}")
|
||||
}
|
||||
|
||||
/// A shallow-cloned repo attached to a research run. Populated by
|
||||
/// `ensure_repo_workspace`; consumed by `build_coordinator_task` to give
|
||||
/// the coordinator a concrete on-disk starting point for the team.
|
||||
struct RepoContext {
|
||||
/// Human-readable "owner/name".
|
||||
slug: String,
|
||||
/// Absolute path on the API host where the checkout lives.
|
||||
path: String,
|
||||
/// Branch we cloned (repo.default_branch → "main" fallback).
|
||||
branch: String,
|
||||
/// Line-per-entry preview of the working tree (relative paths).
|
||||
tree_preview: String,
|
||||
/// Files shown vs. total, so the prompt is honest about truncation.
|
||||
shown: usize,
|
||||
total_files: usize,
|
||||
}
|
||||
|
||||
/// Root directory under which `start_topic` clones per-topic checkouts.
|
||||
/// Overridable via `CLAWMATES_RESEARCH_WORKSPACE_ROOT` for prod deploys
|
||||
/// that want a mounted volume; defaults to a subdir of the system tmpdir
|
||||
/// so dev + tests just work without setup.
|
||||
fn research_workspace_root() -> std::path::PathBuf {
|
||||
if let Ok(root) = std::env::var("CLAWMATES_RESEARCH_WORKSPACE_ROOT") {
|
||||
return std::path::PathBuf::from(root);
|
||||
}
|
||||
std::env::temp_dir().join("clawmates-research")
|
||||
}
|
||||
|
||||
/// Clone the bound repo (shallow, single branch) into a per-topic
|
||||
/// workspace and gather a tree preview for the coordinator prompt.
|
||||
/// Persists the clone path on the topic so a re-start reuses it instead
|
||||
/// of re-cloning. Best-effort — callers treat failures as "start without
|
||||
/// repo context" rather than aborting the run.
|
||||
async fn ensure_repo_workspace(
|
||||
pool: &sqlx::PgPool,
|
||||
topic_id: Uuid,
|
||||
workspace_id: Uuid,
|
||||
repo: &cm_db::repo::repos::Repo,
|
||||
topic: &cm_db::repo::research_topics::ResearchTopic,
|
||||
) -> Result<RepoContext, String> {
|
||||
let clone_url = repo
|
||||
.clone_url
|
||||
.as_deref()
|
||||
.ok_or_else(|| "repo has no clone_url".to_string())?;
|
||||
let branch = repo
|
||||
.default_branch
|
||||
.as_deref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or("main")
|
||||
.to_string();
|
||||
|
||||
let target = topic.repo_workspace_path.clone().unwrap_or_else(|| {
|
||||
research_workspace_root()
|
||||
.join(topic_id.to_string())
|
||||
.join("repo")
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
});
|
||||
let target_path = std::path::PathBuf::from(&target);
|
||||
|
||||
let should_clone = !target_path.join(".git").exists();
|
||||
if should_clone {
|
||||
if let Some(parent) = target_path.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("mkdir parent: {e}"))?;
|
||||
}
|
||||
let out = tokio::process::Command::new("git")
|
||||
.arg("clone")
|
||||
.arg("--depth")
|
||||
.arg("1")
|
||||
.arg("--single-branch")
|
||||
.arg("--branch")
|
||||
.arg(&branch)
|
||||
.arg(clone_url)
|
||||
.arg(&target_path)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"git clone exit {:?}: {}",
|
||||
out.status.code(),
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
));
|
||||
}
|
||||
cm_db::repo::research_topics::set_repo_workspace_path(
|
||||
pool,
|
||||
topic_id,
|
||||
workspace_id,
|
||||
&target,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("persist clone path: {e}"))?;
|
||||
}
|
||||
|
||||
// Preview: `git ls-files` first N entries. Bounded so the prompt stays
|
||||
// small even for large repos; a subsequent tool call can list more.
|
||||
const MAX_TREE_LINES: usize = 60;
|
||||
let ls = tokio::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&target_path)
|
||||
.arg("ls-files")
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("spawn git ls-files: {e}"))?;
|
||||
let all = String::from_utf8_lossy(&ls.stdout);
|
||||
let entries: Vec<&str> = all.lines().filter(|l| !l.is_empty()).collect();
|
||||
let shown = entries.len().min(MAX_TREE_LINES);
|
||||
let preview = entries
|
||||
.iter()
|
||||
.take(shown)
|
||||
.map(|e| format!(" {e}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
Ok(RepoContext {
|
||||
slug: format!("{}/{}", repo.owner, repo.name),
|
||||
path: target,
|
||||
branch,
|
||||
tree_preview: if preview.is_empty() {
|
||||
" (empty)".to_string()
|
||||
} else {
|
||||
preview
|
||||
},
|
||||
shown,
|
||||
total_files: entries.len(),
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct AgentSlotInput {
|
||||
pub agent_id: Uuid,
|
||||
@@ -174,6 +358,13 @@ pub async fn create_topic(
|
||||
return Err(ApiError::NotFound);
|
||||
}
|
||||
}
|
||||
// Same ownership check for the repo binding.
|
||||
let repo_id = if let Some(r) = &body.repo {
|
||||
cm_db::repo::repos::get(&state.pool, r.repo_id, user.workspace_id).await?;
|
||||
Some(r.repo_id)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let id = cm_db::repo::research_topics::create(
|
||||
&state.pool,
|
||||
@@ -182,6 +373,7 @@ pub async fn create_topic(
|
||||
body.description.trim(),
|
||||
&body.outcome_kind,
|
||||
topology_kind,
|
||||
repo_id,
|
||||
user.user_id.as_uuid(),
|
||||
)
|
||||
.await?;
|
||||
@@ -389,6 +581,27 @@ pub async fn start_topic(
|
||||
cm_db::repo::agents::get(&state.pool, cm_domain::AgentId::from(s.agent_id)).await?;
|
||||
roster.push((s.clone(), agent));
|
||||
}
|
||||
|
||||
// If a repo is bound, clone (shallow) into a per-topic workspace so the
|
||||
// coordinator prompt can point the team at real files. Best-effort — a
|
||||
// clone failure logs but still starts the run without repo context.
|
||||
let repo_context = if let Some(repo_id) = topic.repo_id {
|
||||
let repo = cm_db::repo::repos::get(&state.pool, repo_id, user.workspace_id).await?;
|
||||
match ensure_repo_workspace(&state.pool, id, user.workspace_id.as_uuid(), &repo, &topic)
|
||||
.await
|
||||
{
|
||||
Ok(ctx) => Some(ctx),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"research::start_topic({id}): repo clone failed for {}/{}: {e}",
|
||||
repo.owner, repo.name
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Coordinator = first slot whose role_slot mentions "coordinator" (case-
|
||||
// insensitive), else the first slot. The coordinator becomes the hub of
|
||||
// the hub_spoke graph, so it can talk to every other agent per-turn.
|
||||
@@ -483,6 +696,7 @@ pub async fn start_topic(
|
||||
&topic.description,
|
||||
&topo,
|
||||
&roster_lines,
|
||||
repo_context.as_ref(),
|
||||
);
|
||||
|
||||
let run_id = uuid::Uuid::now_v7();
|
||||
|
||||
Reference in New Issue
Block a user