wip-4a

crates/clawhdf5-bench/src/bin/longmemeval_bench.rs

@@ -11,12 +11,18 @@
 //!   field is deserialized and deliberately never read. This is **not** the official
 //!   LongMemEval metric, which is end-to-end QA accuracy (retrieve → generate → LLM
 //!   judge). Reporting retrieval recall as QA accuracy overstates by 20–30 points.
-//! - **Dataset: `longmemeval_oracle`** — evidence sessions only, not the full
+//! - **Dataset: whichever variant you point it at.** Both `longmemeval_oracle`
-//!   `longmemeval_s`/`_m` haystack. Substantially easier corpus.
+//!   (evidence sessions only — a substantially easier corpus) and the full
-//! - **Session-level metrics are degenerate here** and must not be cited: on the
+//!   `longmemeval_s` haystack are supported. The harness does not trust the
-//!   oracle variant the haystack is essentially all-evidence, so any returned
+//!   filename: [`DatasetProfile`] measures evidence-session density from the
-//!   document is a session-level hit at rank 0 by construction. Only turn-level
+//!   data and labels the run from that, so a mislabelled input cannot produce a
-//!   (`has_answer == true` on the source turn) measures the retriever.
+//!   mislabelled result.
+//! - **Session-level metrics are degenerate when evidence density is high**, and
+//!   the report says so per run rather than assuming it. On the oracle variant
+//!   the haystack is essentially all-evidence, so any returned document is a
+//!   session-level hit at rank 0 by construction; only turn-level
+//!   (`has_answer == true` on the source turn) measures the retriever there. On
+//!   the full haystack, session-level recall is meaningful.
 //! - **Not comparable to MemX's Hit@5=51.6% / MRR=0.380**, which is *fact-level*
 //!   granularity over 220,349 records from 19,195 sessions.
 //!
@@ -24,7 +30,13 @@
 //!
 //! # Usage
 //! ```
-//! cargo run --release --bin longmemeval_bench [path/to/longmemeval_oracle.json]
+//! cargo run --release --bin longmemeval_bench [PATH] [--limit N]
+//!
+//! # Usage: full haystack
+//! ```
+//! cargo run --release --bin longmemeval_bench -- \
+//!     benchmarks/longmemeval/longmemeval_s_cleaned.json --limit 50
+//! ```
 //! ```
 //!
 //! # WASM Note
@@ -302,7 +314,81 @@ fn evaluate_question(q: &Question, top_k: usize) -> EvalResult {
 // Report printing
 // ---------------------------------------------------------------------------
 
-fn print_report(overall: &Metrics, by_type: &HashMap<String, Metrics>) {
+// ---------------------------------------------------------------------------
+// Dataset profile — measured, not assumed
+// ---------------------------------------------------------------------------
+
+/// Shape of the loaded corpus, computed from the data itself.
+///
+/// The variant used to be a hardcoded `"oracle"` string in the report and the
+/// JSON summary, so pointing the harness at `longmemeval_s` would have produced
+/// full-haystack numbers labelled oracle. Everything here is derived from the
+/// questions instead, which means the label cannot drift from the corpus and a
+/// mislabelled input file cannot produce a mislabelled result.
+struct DatasetProfile {
+    n_questions: usize,
+    mean_sessions: f64,
+    mean_turns: f64,
+    /// Mean over questions of `|answer_sessions| / |haystack_sessions|`.
+    ///
+    /// This is what actually decides whether session-level recall means
+    /// anything. At ~1.0 every haystack session is an evidence session, so any
+    /// returned document is a session-level hit by construction.
+    evidence_density: f64,
+}
+
+impl DatasetProfile {
+    fn measure(questions: &[Question]) -> Self {
+        let n = questions.len().max(1) as f64;
+        let mut sessions = 0.0;
+        let mut turns = 0.0;
+        let mut density = 0.0;
+        for q in questions {
+            let n_sess = q.haystack_sessions.len();
+            sessions += n_sess as f64;
+            turns += q.haystack_sessions.iter().map(Vec::len).sum::<usize>() as f64;
+            if n_sess > 0 {
+                let evidence: HashSet<&str> =
+                    q.answer_session_ids.iter().map(String::as_str).collect();
+                let hit = q
+                    .haystack_session_ids
+                    .iter()
+                    .filter(|id| evidence.contains(id.as_str()))
+                    .count();
+                density += hit as f64 / n_sess as f64;
+            }
+        }
+        Self {
+            n_questions: questions.len(),
+            mean_sessions: sessions / n,
+            mean_turns: turns / n,
+            evidence_density: density / n,
+        }
+    }
+
+    /// Above this share of evidence sessions, session-level recall is measuring
+    /// the corpus shape rather than the retriever.
+    const DEGENERACY_THRESHOLD: f64 = 0.9;
+
+    const fn session_level_degenerate(&self) -> bool {
+        self.evidence_density > Self::DEGENERACY_THRESHOLD
+    }
+
+    /// Variant name inferred from evidence density, not from the filename.
+    const fn variant(&self) -> &'static str {
+        if self.session_level_degenerate() {
+            "oracle"
+        } else {
+            "full_haystack"
+        }
+    }
+}
+
+fn print_report(
+    overall: &Metrics,
+    by_type: &HashMap<String, Metrics>,
+    profile: &DatasetProfile,
+) {
     println!("=================================================================");
     println!("  LongMemEval Benchmark  (BM25-only retrieval, zero embeddings)");
     println!("=================================================================");
@@ -312,8 +398,24 @@ fn print_report(overall: &Metrics, by_type: &HashMap<String, Metrics>) {
     println!("Scoring target: RETRIEVAL RECALL (did the gold memory land in top-k).");
     println!("  No answer is generated or scored. This is NOT the official");
     println!("  LongMemEval metric (QA accuracy via retrieve+generate+judge).");
-    println!("Dataset: longmemeval_oracle — evidence sessions only, NOT the full");
+    println!(
-    println!("  longmemeval_s haystack. This is a substantially easier corpus.");
+        "Dataset: {} — {} questions, {:.1} sessions and {:.0} turns per question,",
+        profile.variant(),
+        profile.n_questions,
+        profile.mean_sessions,
+        profile.mean_turns,
+    );
+    println!(
+        "  {:.1}% of haystack sessions are evidence sessions.",
+        profile.evidence_density * 100.0
+    );
+    if profile.session_level_degenerate() {
+        println!("  This is the evidence-only corpus, NOT the full longmemeval_s");
+        println!("  haystack — a substantially easier retrieval problem.");
+    } else {
+        println!("  This is a full-haystack corpus: evidence sessions are a small");
+        println!("  minority, so retrieval has to actually discriminate.");
+    }
     println!();
     println!("Do NOT compare these to MemX's Hit@5=51.6% / MRR=0.380: that is");
     println!("  fact-level granularity over 220,349 records from 19,195 sessions.");
@@ -321,9 +423,20 @@ fn print_report(overall: &Metrics, by_type: &HashMap<String, Metrics>) {
     println!();
 
     println!("## Session-Level Recall  (n={})", overall.count);
-    println!("  [DEGENERATE on the oracle variant — every returned document belongs");
+    if profile.session_level_degenerate() {
-    println!("   to an answer session by construction. Reported for completeness only;");
+        println!(
-    println!("   this measures the corpus shape, not the retriever. Use turn-level.]");
+            "  [DEGENERATE — {:.1}% of haystack sessions are evidence sessions, so a",
+            profile.evidence_density * 100.0
+        );
+        println!("   returned document is a session-level hit almost by construction.");
+        println!("   This measures the corpus shape, not the retriever. Use turn-level.]");
+    } else {
+        println!(
+            "  [Meaningful on this corpus — only {:.1}% of haystack sessions are",
+            profile.evidence_density * 100.0
+        );
+        println!("   evidence sessions, so a hit reflects the retriever's discrimination.]");
+    }
     println!(
         "  Hit@1:  {:5.1}%   Hit@5:  {:5.1}%   Hit@10: {:5.1}%   MRR: {:.4}",
         overall.hit1_session_pct(),
@@ -408,10 +521,19 @@ fn print_report(overall: &Metrics, by_type: &HashMap<String, Metrics>) {
     println!("{{");
     println!("  \"benchmark\": \"longmemeval\",");
     println!("  \"mode\": \"bm25_only\",");
-    println!("  \"dataset_variant\": \"oracle\",");
+    println!("  \"dataset_variant\": \"{}\",", profile.variant());
     println!("  \"scoring_target\": \"retrieval_recall\",");
     println!("  \"k\": 10,");
-    println!("  \"session_level_degenerate\": true,");
+    println!(
+        "  \"session_level_degenerate\": {},",
+        profile.session_level_degenerate()
+    );
+    println!(
+        "  \"evidence_session_density\": {:.4},",
+        profile.evidence_density
+    );
+    println!("  \"mean_sessions_per_question\": {:.2},", profile.mean_sessions);
+    println!("  \"mean_turns_per_question\": {:.1},", profile.mean_turns);
     println!(
         "  \"total_questions\": {},",
         overall.count + overall.abstention_total
@@ -456,17 +578,59 @@ fn print_report(overall: &Metrics, by_type: &HashMap<String, Metrics>) {
 // ---------------------------------------------------------------------------
 
 fn main() {
-    let json_path = std::env::args()
+    let mut json_path: Option<String> = None;
-        .nth(1)
+    let mut limit: Option<usize> = None;
-        .unwrap_or_else(|| "benchmarks/longmemeval/longmemeval_oracle.json".to_string());
+    let mut args = std::env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--limit" => {
+                let v = args.next().expect("--limit needs a value");
+                limit = Some(v.parse().expect("--limit must be a positive integer"));
+            }
+            "--help" | "-h" => {
+                eprintln!(
+                    "usage: longmemeval_bench [PATH] [--limit N]\n\n\
+                     PATH     dataset JSON; defaults to the oracle variant.\n\
+                              longmemeval_s works too — the harness measures which\n\
+                              variant it was given rather than trusting the filename.\n\
+                     --limit  evaluate only the first N questions. The full haystack\n\
+                              is ~20x the oracle corpus per question, so this exists to\n\
+                              size a run before committing to all 500."
+                );
+                return;
+            }
+            other => json_path = Some(other.to_string()),
+        }
+    }
+    let json_path =
+        json_path.unwrap_or_else(|| "benchmarks/longmemeval/longmemeval_oracle.json".to_string());
 
     eprintln!("Loading: {json_path}");
     let data = std::fs::read_to_string(&json_path)
         .unwrap_or_else(|e| panic!("Failed to read {json_path}: {e}"));
-    let questions: Vec<Question> = serde_json::from_str(&data).expect("Failed to parse JSON");
+    let mut questions: Vec<Question> = serde_json::from_str(&data).expect("Failed to parse JSON");
+    if let Some(n) = limit
+        && n < questions.len()
+    {
+        eprintln!(
+            "Limiting to the first {n} of {} questions (--limit)",
+            questions.len()
+        );
+        questions.truncate(n);
+    }
     let total = questions.len();
     eprintln!("Loaded {total} questions");
 
+    let profile = DatasetProfile::measure(&questions);
+    eprintln!(
+        "Corpus: {} variant — {:.1} sessions / {:.0} turns per question, \
+         {:.1}% evidence-session density",
+        profile.variant(),
+        profile.mean_sessions,
+        profile.mean_turns,
+        profile.evidence_density * 100.0,
+    );
+
     let mut overall = Metrics::default();
     let mut by_type: HashMap<String, Metrics> = HashMap::new();
 
@@ -540,5 +704,5 @@ fn main() {
 
     eprintln!("\r  [{total}/{total}] done.          ");
     eprintln!();
-    print_report(&overall, &by_type);
+    print_report(&overall, &by_type, &profile);
 }