clawmates/crates/cm-api/src/repo_digest.rs

//! What a repository actually contains, small enough to put in a prompt.
//!
//! The planner was given the root listing and planned "optimise the hot path"
//! for a crate whose hot path is `add(a: i64, b: i64) -> i64`. Names were not
//! enough: the mission was unachievable from the moment it was written, and
//! nothing discovered that until an agent had built a benchmark harness to
//! measure an integer addition.
//!
//! # The rule this module exists to enforce
//!
//! A digest is always partial for any repository worth planning against, and a
//! model shown a partial view without being told it is partial plans as though
//! it saw everything. So every omission is STATED — how many files were listed,
//! how many were shown, what was cut from each. That is the same distinction as
//! `Option<u32>` for the subagent probe: "we did not look" and "there is nothing
//! there" are different facts, and only one of them is about the repository.
//!
//! # Priority
//!
//! Manifests first (they say what the project IS and what it may depend on),
//! then the README, then source ascending by size — smallest-first shows the
//! most files per byte, and a planner benefits more from seeing twenty small
//! files than one large one.

/// One file in the repository tree.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileEntry {
    pub path: String,
    pub size: usize,
}

/// Total characters of file CONTENT a digest may carry.
///
/// The prompt around it is ~1.5k, and the planner is a single call per mission,
/// so this is generous by design: the cost of a too-small digest is a plan built
/// on a guess, which costs a VM boot to discover.
pub const CONTENT_BUDGET: usize = 12_000;

/// Ceiling per file, so one large file cannot spend the whole budget.
pub const PER_FILE_CAP: usize = 3_000;

/// Files worth showing before any source.
fn is_manifest(path: &str) -> bool {
    matches!(
        path,
        "Cargo.toml"
            | "package.json"
            | "pyproject.toml"
            | "setup.py"
            | "go.mod"
            | "Gemfile"
            | "pom.xml"
            | "build.gradle"
            | "Makefile"
    )
}

fn is_readme(path: &str) -> bool {
    path.eq_ignore_ascii_case("README.md") || path.eq_ignore_ascii_case("README")
}

/// Paths to fetch, in the order they earn their place.
///
/// Directories and files a planner cannot use are dropped: lockfiles are huge
/// and say nothing a manifest does not, and build output is not source.
pub fn priority(entries: &[FileEntry]) -> Vec<&FileEntry> {
    let mut useful: Vec<&FileEntry> = entries
        .iter()
        .filter(|e| {
            let p = e.path.as_str();
            !p.starts_with(".git/")
                && !p.contains("/target/")
                && !p.starts_with("target/")
                && !p.contains("node_modules/")
                && p != "Cargo.lock"
                && p != "package-lock.json"
                && p != "poetry.lock"
                && e.size > 0
        })
        .collect();
    useful.sort_by_key(|e| {
        let rank = if is_manifest(&e.path) {
            0
        } else if is_readme(&e.path) {
            1
        } else {
            2
        };
        (rank, e.size, e.path.clone())
    });
    useful
}

/// Render the digest a planner sees.
///
/// `contents` is `(path, text)` for the files that were actually fetched, in
/// priority order. Anything not fetched is still LISTED, so the model knows the
/// file exists even when it cannot read it.
pub fn render(entries: &[FileEntry], contents: &[(String, String)]) -> String {
    if entries.is_empty() {
        return "(the repository is empty, or its tree could not be read)".to_string();
    }
    let mut out = String::new();
    out.push_str(&format!("FILES ({} total):\n", entries.len()));
    // The whole tree by name is cheap and is what stops "does X exist" guessing.
    // Capped anyway: a 10k-file monorepo listing is not a prompt.
    const MAX_LISTED: usize = 300;
    for e in entries.iter().take(MAX_LISTED) {
        out.push_str(&format!("  {} ({} bytes)\n", e.path, e.size));
    }
    if entries.len() > MAX_LISTED {
        out.push_str(&format!(
            "  … and {} more files NOT listed\n",
            entries.len() - MAX_LISTED
        ));
    }

    if contents.is_empty() {
        out.push_str("\n(no file contents could be read — plan from the names alone, and say so if that is not enough)\n");
        return out;
    }

    out.push_str(&format!(
        "\nCONTENTS ({} of {} files shown; anything not shown you have NOT seen):\n",
        contents.len(),
        entries.len()
    ));
    for (path, text) in contents {
        out.push_str(&format!("\n--- {path} ---\n{text}\n"));
    }
    out
}

/// Take file texts up to the budget, truncating each at [`PER_FILE_CAP`].
///
/// Truncation is marked in the text itself rather than silently cutting: a model
/// that can see it is reading a fragment asks differently than one that believes
/// it read the file.
pub fn fit(fetched: Vec<(String, String)>) -> Vec<(String, String)> {
    let mut out = Vec::new();
    let mut spent = 0usize;
    for (path, text) in fetched {
        if spent >= CONTENT_BUDGET {
            break;
        }
        let room = (CONTENT_BUDGET - spent).min(PER_FILE_CAP);
        let text = if text.len() <= room {
            text
        } else {
            let end = (0..=room)
                .rev()
                .find(|i| text.is_char_boundary(*i))
                .unwrap_or(0);
            format!(
                "{}\n… [truncated: {} of {} bytes shown]",
                &text[..end],
                end,
                text.len()
            )
        };
        spent += text.len();
        out.push((path, text));
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    fn f(path: &str, size: usize) -> FileEntry {
        FileEntry {
            path: path.into(),
            size,
        }
    }

    /// Manifests first, then the README, then source smallest-first. A planner
    /// learns more from twenty small files than from one large one.
    #[test]
    fn the_files_that_say_what_this_is_come_first() {
        let entries = vec![
            f("src/big.rs", 9000),
            f("README.md", 400),
            f("src/lib.rs", 120),
            f("Cargo.toml", 200),
        ];
        let order: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect();
        assert_eq!(order, vec!["Cargo.toml", "README.md", "src/lib.rs", "src/big.rs"]);
    }

    /// Lockfiles and build output are dropped: enormous, and they say nothing a
    /// manifest does not.
    #[test]
    fn noise_is_not_offered_to_the_planner() {
        let entries = vec![
            f("Cargo.lock", 50_000),
            f("target/debug/thing", 900_000),
            f("node_modules/x/index.js", 400),
            f(".git/config", 100),
            f("src/lib.rs", 100),
            f("empty.rs", 0),
        ];
        let kept: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect();
        assert_eq!(kept, vec!["src/lib.rs"]);
    }

    /// THE rule. A partial view presented as complete is planned against as
    /// though it were complete — which is how "optimise the hot path" gets
    /// written for a crate that adds two integers.
    #[test]
    fn every_omission_is_stated() {
        let entries: Vec<FileEntry> = (0..400).map(|i| f(&format!("src/f{i}.rs"), 100)).collect();
        let shown = vec![("src/f0.rs".to_string(), "fn a() {}".to_string())];
        let out = render(&entries, &shown);

        assert!(out.contains("FILES (400 total)"), "{out}");
        assert!(out.contains("and 100 more files NOT listed"), "{out}");
        assert!(out.contains("1 of 400 files shown"), "{out}");
        assert!(
            out.contains("you have NOT seen"),
            "the model must be told the view is partial: {out}"
        );
    }

    /// A file cut short says so, in the text the model reads.
    #[test]
    fn a_truncated_file_says_it_was_truncated() {
        let big = "x".repeat(PER_FILE_CAP * 2);
        let out = fit(vec![("src/big.rs".into(), big.clone())]);
        assert_eq!(out.len(), 1);
        assert!(out[0].1.contains("truncated"), "{}", &out[0].1[..80]);
        assert!(out[0].1.len() < big.len());
        // And the marker names both numbers, so "how much did I miss" is
        // answerable rather than guessable.
        assert!(out[0].1.contains(&big.len().to_string()));
    }

    /// The budget is a total, not per file: one large file must not starve the
    /// rest, and the whole digest must stay promptable.
    #[test]
    fn the_budget_bounds_the_whole_digest() {
        let files: Vec<(String, String)> = (0..20)
            .map(|i| (format!("src/f{i}.rs"), "y".repeat(PER_FILE_CAP)))
            .collect();
        let out = fit(files);
        let total: usize = out.iter().map(|(_, t)| t.len()).sum();
        assert!(total <= CONTENT_BUDGET, "digest was {total} bytes");
        assert!(!out.is_empty(), "and it still shows something");
        assert!(out.len() < 20, "not everything fits, by construction");
    }

    /// An empty or unreadable tree is stated as such — never rendered as a
    /// repository that happens to contain nothing.
    #[test]
    fn an_unreadable_tree_is_not_an_empty_repository() {
        let out = render(&[], &[]);
        assert!(out.contains("could not be read"), "{out}");

        // A tree we CAN read but no contents we could fetch is a different
        // fact, and says so.
        let out = render(&[f("src/lib.rs", 100)], &[]);
        assert!(out.contains("src/lib.rs"), "{out}");
        assert!(out.contains("no file contents could be read"), "{out}");
    }
}