//! What a repository actually contains, small enough to put in a prompt. //! //! The planner was given the root listing and planned "optimise the hot path" //! for a crate whose hot path is `add(a: i64, b: i64) -> i64`. Names were not //! enough: the mission was unachievable from the moment it was written, and //! nothing discovered that until an agent had built a benchmark harness to //! measure an integer addition. //! //! # The rule this module exists to enforce //! //! A digest is always partial for any repository worth planning against, and a //! model shown a partial view without being told it is partial plans as though //! it saw everything. So every omission is STATED — how many files were listed, //! how many were shown, what was cut from each. That is the same distinction as //! `Option` for the subagent probe: "we did not look" and "there is nothing //! there" are different facts, and only one of them is about the repository. //! //! # Priority //! //! Manifests first (they say what the project IS and what it may depend on), //! then the README, then source ascending by size — smallest-first shows the //! most files per byte, and a planner benefits more from seeing twenty small //! files than one large one. /// One file in the repository tree. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileEntry { pub path: String, pub size: usize, } /// Total characters of file CONTENT a digest may carry. /// /// The prompt around it is ~1.5k, and the planner is a single call per mission, /// so this is generous by design: the cost of a too-small digest is a plan built /// on a guess, which costs a VM boot to discover. pub const CONTENT_BUDGET: usize = 12_000; /// Ceiling per file, so one large file cannot spend the whole budget. pub const PER_FILE_CAP: usize = 3_000; /// Files worth showing before any source. fn is_manifest(path: &str) -> bool { matches!( path, "Cargo.toml" | "package.json" | "pyproject.toml" | "setup.py" | "go.mod" | "Gemfile" | "pom.xml" | "build.gradle" | "Makefile" ) } fn is_readme(path: &str) -> bool { path.eq_ignore_ascii_case("README.md") || path.eq_ignore_ascii_case("README") } /// Paths to fetch, in the order they earn their place. /// /// Directories and files a planner cannot use are dropped: lockfiles are huge /// and say nothing a manifest does not, and build output is not source. pub fn priority(entries: &[FileEntry]) -> Vec<&FileEntry> { let mut useful: Vec<&FileEntry> = entries .iter() .filter(|e| { let p = e.path.as_str(); !p.starts_with(".git/") && !p.contains("/target/") && !p.starts_with("target/") && !p.contains("node_modules/") && p != "Cargo.lock" && p != "package-lock.json" && p != "poetry.lock" && e.size > 0 }) .collect(); useful.sort_by_key(|e| { let rank = if is_manifest(&e.path) { 0 } else if is_readme(&e.path) { 1 } else { 2 }; (rank, e.size, e.path.clone()) }); useful } /// Render the digest a planner sees. /// /// `contents` is `(path, text)` for the files that were actually fetched, in /// priority order. Anything not fetched is still LISTED, so the model knows the /// file exists even when it cannot read it. pub fn render(entries: &[FileEntry], contents: &[(String, String)]) -> String { if entries.is_empty() { return "(the repository is empty, or its tree could not be read)".to_string(); } let mut out = String::new(); out.push_str(&format!("FILES ({} total):\n", entries.len())); // The whole tree by name is cheap and is what stops "does X exist" guessing. // Capped anyway: a 10k-file monorepo listing is not a prompt. const MAX_LISTED: usize = 300; for e in entries.iter().take(MAX_LISTED) { out.push_str(&format!(" {} ({} bytes)\n", e.path, e.size)); } if entries.len() > MAX_LISTED { out.push_str(&format!( " … and {} more files NOT listed\n", entries.len() - MAX_LISTED )); } if contents.is_empty() { out.push_str("\n(no file contents could be read — plan from the names alone, and say so if that is not enough)\n"); return out; } out.push_str(&format!( "\nCONTENTS ({} of {} files shown; anything not shown you have NOT seen):\n", contents.len(), entries.len() )); for (path, text) in contents { out.push_str(&format!("\n--- {path} ---\n{text}\n")); } out } /// Take file texts up to the budget, truncating each at [`PER_FILE_CAP`]. /// /// Truncation is marked in the text itself rather than silently cutting: a model /// that can see it is reading a fragment asks differently than one that believes /// it read the file. pub fn fit(fetched: Vec<(String, String)>) -> Vec<(String, String)> { let mut out = Vec::new(); let mut spent = 0usize; for (path, text) in fetched { if spent >= CONTENT_BUDGET { break; } let room = (CONTENT_BUDGET - spent).min(PER_FILE_CAP); let text = if text.len() <= room { text } else { let end = (0..=room) .rev() .find(|i| text.is_char_boundary(*i)) .unwrap_or(0); format!( "{}\n… [truncated: {} of {} bytes shown]", &text[..end], end, text.len() ) }; spent += text.len(); out.push((path, text)); } out } #[cfg(test)] mod tests { use super::*; fn f(path: &str, size: usize) -> FileEntry { FileEntry { path: path.into(), size, } } /// Manifests first, then the README, then source smallest-first. A planner /// learns more from twenty small files than from one large one. #[test] fn the_files_that_say_what_this_is_come_first() { let entries = vec![ f("src/big.rs", 9000), f("README.md", 400), f("src/lib.rs", 120), f("Cargo.toml", 200), ]; let order: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect(); assert_eq!(order, vec!["Cargo.toml", "README.md", "src/lib.rs", "src/big.rs"]); } /// Lockfiles and build output are dropped: enormous, and they say nothing a /// manifest does not. #[test] fn noise_is_not_offered_to_the_planner() { let entries = vec![ f("Cargo.lock", 50_000), f("target/debug/thing", 900_000), f("node_modules/x/index.js", 400), f(".git/config", 100), f("src/lib.rs", 100), f("empty.rs", 0), ]; let kept: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect(); assert_eq!(kept, vec!["src/lib.rs"]); } /// THE rule. A partial view presented as complete is planned against as /// though it were complete — which is how "optimise the hot path" gets /// written for a crate that adds two integers. #[test] fn every_omission_is_stated() { let entries: Vec = (0..400).map(|i| f(&format!("src/f{i}.rs"), 100)).collect(); let shown = vec![("src/f0.rs".to_string(), "fn a() {}".to_string())]; let out = render(&entries, &shown); assert!(out.contains("FILES (400 total)"), "{out}"); assert!(out.contains("and 100 more files NOT listed"), "{out}"); assert!(out.contains("1 of 400 files shown"), "{out}"); assert!( out.contains("you have NOT seen"), "the model must be told the view is partial: {out}" ); } /// A file cut short says so, in the text the model reads. #[test] fn a_truncated_file_says_it_was_truncated() { let big = "x".repeat(PER_FILE_CAP * 2); let out = fit(vec![("src/big.rs".into(), big.clone())]); assert_eq!(out.len(), 1); assert!(out[0].1.contains("truncated"), "{}", &out[0].1[..80]); assert!(out[0].1.len() < big.len()); // And the marker names both numbers, so "how much did I miss" is // answerable rather than guessable. assert!(out[0].1.contains(&big.len().to_string())); } /// The budget is a total, not per file: one large file must not starve the /// rest, and the whole digest must stay promptable. #[test] fn the_budget_bounds_the_whole_digest() { let files: Vec<(String, String)> = (0..20) .map(|i| (format!("src/f{i}.rs"), "y".repeat(PER_FILE_CAP))) .collect(); let out = fit(files); let total: usize = out.iter().map(|(_, t)| t.len()).sum(); assert!(total <= CONTENT_BUDGET, "digest was {total} bytes"); assert!(!out.is_empty(), "and it still shows something"); assert!(out.len() < 20, "not everything fits, by construction"); } /// An empty or unreadable tree is stated as such — never rendered as a /// repository that happens to contain nothing. #[test] fn an_unreadable_tree_is_not_an_empty_repository() { let out = render(&[], &[]); assert!(out.contains("could not be read"), "{out}"); // A tree we CAN read but no contents we could fetch is a different // fact, and says so. let out = render(&[f("src/lib.rs", 100)], &[]); assert!(out.contains("src/lib.rs"), "{out}"); assert!(out.contains("no file contents could be read"), "{out}"); } }