//! LongMemEval Benchmark Harness (Track 8.1)
//!
//! Evaluates BM25-based retrieval recall against the LongMemEval dataset (500 questions).
//! Since no embedding model is available at bench time, all embeddings are zero vectors
//! and `hybrid_search` operates in BM25-only mode (vector_weight=0.0, keyword_weight=1.0).
//!
//! # Scoring target (read before citing any number from this harness)
//!
//! - **Metric: retrieval recall.** A "hit" means the gold-labelled memory appeared in
//!   the top-k. No answer is generated and none is scored — the dataset's `answer`
//!   field is deserialized and deliberately never read. This is **not** the official
//!   LongMemEval metric, which is end-to-end QA accuracy (retrieve → generate → LLM
//!   judge). Reporting retrieval recall as QA accuracy overstates by 20–30 points.
//! - **Dataset: whichever variant you point it at.** Both `longmemeval_oracle`
//!   (evidence sessions only — a substantially easier corpus) and the full
//!   `longmemeval_s` haystack are supported. The harness does not trust the
//!   filename: [`DatasetProfile`] measures evidence-session density from the
//!   data and labels the run from that, so a mislabelled input cannot produce a
//!   mislabelled result.
//! - **Session-level metrics are degenerate when evidence density is high**, and
//!   the report says so per run rather than assuming it. On the oracle variant
//!   the haystack is essentially all-evidence, so any returned document is a
//!   session-level hit at rank 0 by construction; only turn-level
//!   (`has_answer == true` on the source turn) measures the retriever there. On
//!   the full haystack, session-level recall is meaningful.
//! - **Not comparable to MemX's Hit@5=51.6% / MRR=0.380**, which is *fact-level*
//!   granularity over 220,349 records from 19,195 sessions.
//!
//! See `BENCHMARKS.md` § "Retracted: session-level recall and the MemX comparison".
//!
//! # Usage
//! ```
//! cargo run --release --bin longmemeval_bench [PATH] [--limit N]
//!
//! # Usage: full haystack
//! ```
//! cargo run --release --bin longmemeval_bench -- \
//!     benchmarks/longmemeval/longmemeval_s_cleaned.json --limit 50
//! ```
//! ```
//!
//! # WASM Note
//! `#[cfg(target_arch = "wasm32")]` is not supported here. Changes required for wasm32:
//! - `std::fs::read_to_string` → fetch-based async loader (e.g. `wasm_bindgen_futures`)
//! - `TempDir` → virtual in-memory HDF5 backend (separate effort; tracked in ROADMAP)
//! - `std::time::Instant` → `web_sys::Performance::now()`
//! - HDF5 I/O layer would need a wasm32 storage backend (out of scope for this bench)

use std::collections::{HashMap, HashSet};
use std::time::{Duration, Instant};

// `#[path]` keeps the module beside its binary without Cargo autodiscovering it
// as a second bin target (which a bare `src/bin/embedder.rs` would be).
#[cfg(feature = "embeddings")]
#[path = "longmemeval_bench/embedder.rs"]
mod embedder;

use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
use serde::Deserialize;
use tempfile::TempDir;

const EMBEDDING_DIM: usize = 384;

/// A retrieval configuration: how much of the score comes from each stage.
#[derive(Clone, Copy)]
struct Mode {
    label: &'static str,
    vector_weight: f32,
    keyword_weight: f32,
}

/// The only mode available without real embeddings. Passing zero vectors with
/// `vector_weight = 0.0` is what made the vector stage inert.
const BM25_ONLY: Mode = Mode {
    label: "BM25 only (vector stage inert)",
    vector_weight: 0.0,
    keyword_weight: 1.0,
};
#[cfg(feature = "embeddings")]
const VECTOR_ONLY: Mode = Mode {
    label: "Vector only (MiniLM + HNSW)",
    vector_weight: 1.0,
    keyword_weight: 0.0,
};
/// Tuned by `--sweep` over the full haystack. The former 0.7/0.3 was a
/// documented default that had never been searched, and the sweep found it
/// strictly dominated: 0.4/0.6 is better on Hit@1, Hit@5, Hit@10 and MRR at
/// both granularities.
#[cfg(feature = "embeddings")]
const HYBRID: Mode = Mode {
    label: "Hybrid (0.4 vector / 0.6 BM25, tuned)",
    vector_weight: 0.4,
    keyword_weight: 0.6,
};

/// Every 0.1 step of vector weight, keyword weight taking the remainder.
///
/// Labels are leaked to `&'static str` because `Mode::label` is a `&'static
/// str` for the eleven named modes and a sweep is a short-lived process; the
/// alternative is threading a lifetime through the whole report path for a
/// diagnostic mode.
#[cfg(feature = "embeddings")]
fn sweep_modes() -> Vec<Mode> {
    (0..=10)
        .map(|i| {
            let v = i as f32 / 10.0;
            Mode {
                label: Box::leak(format!("sweep v={v:.1} / k={:.1}", 1.0 - v).into_boxed_str()),
                vector_weight: v,
                keyword_weight: 1.0 - v,
            }
        })
        .collect()
}

/// Text -> embedding, built once for the whole corpus.
type EmbeddingMap = HashMap<String, Vec<f32>>;

/// Look up a real embedding, falling back to zeros when running BM25-only.
fn embedding_for(map: Option<&EmbeddingMap>, text: &str) -> Vec<f32> {
    map.and_then(|m| m.get(text))
        .cloned()
        .unwrap_or_else(|| vec![0.0f32; EMBEDDING_DIM])
}

// ---------------------------------------------------------------------------
// JSON data types
// ---------------------------------------------------------------------------

/// The `answer` field in LongMemEval can be a string or a number.
fn deserialize_answer<'de, D>(deserializer: D) -> Result<String, D::Error>
where
    D: serde::Deserializer<'de>,
{
    use serde::de;
    struct AnswerVisitor;
    impl<'de> de::Visitor<'de> for AnswerVisitor {
        type Value = String;
        fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            f.write_str("a string or number")
        }
        fn visit_str<E: de::Error>(self, v: &str) -> Result<String, E> {
            Ok(v.to_string())
        }
        fn visit_string<E: de::Error>(self, v: String) -> Result<String, E> {
            Ok(v)
        }
        fn visit_i64<E: de::Error>(self, v: i64) -> Result<String, E> {
            Ok(v.to_string())
        }
        fn visit_u64<E: de::Error>(self, v: u64) -> Result<String, E> {
            Ok(v.to_string())
        }
        fn visit_f64<E: de::Error>(self, v: f64) -> Result<String, E> {
            Ok(v.to_string())
        }
    }
    deserializer.deserialize_any(AnswerVisitor)
}

#[derive(Deserialize)]
struct Turn {
    #[allow(dead_code)]
    role: String,
    content: String,
    #[serde(default)]
    has_answer: bool,
}

#[derive(Deserialize)]
struct Question {
    #[allow(dead_code)]
    question_id: String,
    question_type: String,
    question: String,
    #[allow(dead_code)]
    #[serde(deserialize_with = "deserialize_answer")]
    answer: String,
    #[allow(dead_code)]
    question_date: String,
    haystack_session_ids: Vec<String>,
    haystack_sessions: Vec<Vec<Turn>>,
    answer_session_ids: Vec<String>,
}

// ---------------------------------------------------------------------------
// Per-type metrics accumulator
// ---------------------------------------------------------------------------

#[derive(Default)]
struct Metrics {
    hit1_session: u32,
    hit5_session: u32,
    hit10_session: u32,
    rr_session: f64,
    hit1_turn: u32,
    hit5_turn: u32,
    hit10_turn: u32,
    rr_turn: f64,
    abstention_correct: u32,
    abstention_total: u32,
    latency_ns: Vec<u64>,
    count: u32,
}

impl Metrics {
    fn hit1_session_pct(&self) -> f64 {
        self.hit1_session as f64 / self.count.max(1) as f64 * 100.0
    }
    fn hit5_session_pct(&self) -> f64 {
        self.hit5_session as f64 / self.count.max(1) as f64 * 100.0
    }
    fn hit10_session_pct(&self) -> f64 {
        self.hit10_session as f64 / self.count.max(1) as f64 * 100.0
    }
    fn mrr_session(&self) -> f64 {
        self.rr_session / self.count.max(1) as f64
    }
    fn hit1_turn_pct(&self) -> f64 {
        self.hit1_turn as f64 / self.count.max(1) as f64 * 100.0
    }
    fn hit5_turn_pct(&self) -> f64 {
        self.hit5_turn as f64 / self.count.max(1) as f64 * 100.0
    }
    fn hit10_turn_pct(&self) -> f64 {
        self.hit10_turn as f64 / self.count.max(1) as f64 * 100.0
    }
    fn mrr_turn(&self) -> f64 {
        self.rr_turn / self.count.max(1) as f64
    }
    fn abstention_pct(&self) -> f64 {
        self.abstention_correct as f64 / self.abstention_total.max(1) as f64 * 100.0
    }
    fn latency_avg_us(&self) -> f64 {
        if self.latency_ns.is_empty() {
            return 0.0;
        }
        self.latency_ns.iter().sum::<u64>() as f64 / self.latency_ns.len() as f64 / 1000.0
    }
    fn latency_pct_us(&self, p: usize) -> f64 {
        if self.latency_ns.is_empty() {
            return 0.0;
        }
        let mut v = self.latency_ns.clone();
        v.sort_unstable();
        let idx = (p * v.len() / 100).min(v.len() - 1);
        v[idx] as f64 / 1000.0
    }
}

// ---------------------------------------------------------------------------
// Per-question evaluation result
// ---------------------------------------------------------------------------

struct EvalResult {
    hit1_session: bool,
    hit5_session: bool,
    hit10_session: bool,
    rr_session: Option<f64>,
    hit1_turn: bool,
    hit5_turn: bool,
    hit10_turn: bool,
    rr_turn: Option<f64>,
    latency: Duration,
}

fn evaluate_question(
    q: &Question,
    top_k: usize,
    mode: Mode,
    embeddings: Option<&EmbeddingMap>,
) -> EvalResult {
    let dir = TempDir::new().expect("failed to create temp dir");
    let mut config = MemoryConfig::new(dir.path().join("lme.h5"), "lme-bench", EMBEDDING_DIM);
    config.wal_enabled = false;
    config.compact_threshold = 0.0;

    let mut memory = HDF5Memory::create(config).expect("failed to create HDF5Memory");

    // Build MemoryEntry list from all haystack sessions
    let mut entries: Vec<MemoryEntry> = Vec::new();
    let mut turn_has_answer: Vec<bool> = Vec::new();
    let mut ts = 1_000_000.0f64;

    for (sess_idx, session) in q.haystack_sessions.iter().enumerate() {
        let sess_id = q
            .haystack_session_ids
            .get(sess_idx)
            .map(String::as_str)
            .unwrap_or("unknown");
        for turn in session {
            entries.push(MemoryEntry {
                chunk: turn.content.clone(),
                embedding: embedding_for(embeddings, &turn.content),
                source_channel: "longmemeval".to_string(),
                timestamp: ts,
                session_id: sess_id.to_string(),
                tags: if turn.has_answer {
                    "has_answer".to_string()
                } else {
                    String::new()
                },
            });
            turn_has_answer.push(turn.has_answer);
            ts += 1.0;
        }
    }

    let indices = memory.save_batch(entries).expect("failed to save entries");

    // Map memory index → has_answer
    let has_answer_indices: HashSet<usize> = indices
        .iter()
        .zip(turn_has_answer.iter())
        .filter(|(_, ha)| **ha)
        .map(|(idx, _)| *idx)
        .collect();

    // Set of session IDs that contain the answer
    let answer_sess_set: HashSet<&str> = q.answer_session_ids.iter().map(String::as_str).collect();

    let query_emb = embedding_for(embeddings, &q.question);
    let t0 = Instant::now();
    let results = memory.hybrid_search(
        &query_emb,
        &q.question,
        mode.vector_weight,
        mode.keyword_weight,
        top_k,
    );
    let latency = t0.elapsed();

    // Session-level recall
    let mut hit1_session = false;
    let mut hit5_session = false;
    let mut hit10_session = false;
    let mut rr_session: Option<f64> = None;

    for (rank, result) in results.iter().enumerate() {
        let sess_id = memory.cache.session_ids[result.index].as_str();
        if answer_sess_set.contains(sess_id) {
            hit10_session = true;
            if rank < 5 {
                hit5_session = true;
            }
            if rank == 0 {
                hit1_session = true;
            }
            if rr_session.is_none() {
                rr_session = Some(1.0 / (rank + 1) as f64);
            }
            break;
        }
    }

    // Turn-level recall
    let mut hit1_turn = false;
    let mut hit5_turn = false;
    let mut hit10_turn = false;
    let mut rr_turn: Option<f64> = None;

    for (rank, result) in results.iter().enumerate() {
        if has_answer_indices.contains(&result.index) {
            hit10_turn = true;
            if rank < 5 {
                hit5_turn = true;
            }
            if rank == 0 {
                hit1_turn = true;
            }
            if rr_turn.is_none() {
                rr_turn = Some(1.0 / (rank + 1) as f64);
            }
            break;
        }
    }

    EvalResult {
        hit1_session,
        hit5_session,
        hit10_session,
        rr_session,
        hit1_turn,
        hit5_turn,
        hit10_turn,
        rr_turn,
        latency,
    }
}

// ---------------------------------------------------------------------------
// Report printing
// ---------------------------------------------------------------------------

// ---------------------------------------------------------------------------
// Dataset profile — measured, not assumed
// ---------------------------------------------------------------------------

/// Shape of the loaded corpus, computed from the data itself.
///
/// The variant used to be a hardcoded `"oracle"` string in the report and the
/// JSON summary, so pointing the harness at `longmemeval_s` would have produced
/// full-haystack numbers labelled oracle. Everything here is derived from the
/// questions instead, which means the label cannot drift from the corpus and a
/// mislabelled input file cannot produce a mislabelled result.
struct DatasetProfile {
    n_questions: usize,
    mean_sessions: f64,
    mean_turns: f64,
    /// Mean over questions of `|answer_sessions| / |haystack_sessions|`.
    ///
    /// This is what actually decides whether session-level recall means
    /// anything. At ~1.0 every haystack session is an evidence session, so any
    /// returned document is a session-level hit by construction.
    evidence_density: f64,
}

impl DatasetProfile {
    fn measure(questions: &[Question]) -> Self {
        let n = questions.len().max(1) as f64;
        let mut sessions = 0.0;
        let mut turns = 0.0;
        let mut density = 0.0;
        for q in questions {
            let n_sess = q.haystack_sessions.len();
            sessions += n_sess as f64;
            turns += q.haystack_sessions.iter().map(Vec::len).sum::<usize>() as f64;
            if n_sess > 0 {
                let evidence: HashSet<&str> =
                    q.answer_session_ids.iter().map(String::as_str).collect();
                let hit = q
                    .haystack_session_ids
                    .iter()
                    .filter(|id| evidence.contains(id.as_str()))
                    .count();
                density += hit as f64 / n_sess as f64;
            }
        }
        Self {
            n_questions: questions.len(),
            mean_sessions: sessions / n,
            mean_turns: turns / n,
            evidence_density: density / n,
        }
    }

    /// Above this share of evidence sessions, session-level recall is measuring
    /// the corpus shape rather than the retriever.
    const DEGENERACY_THRESHOLD: f64 = 0.9;

    const fn session_level_degenerate(&self) -> bool {
        self.evidence_density > Self::DEGENERACY_THRESHOLD
    }

    /// Variant name inferred from evidence density, not from the filename.
    const fn variant(&self) -> &'static str {
        if self.session_level_degenerate() {
            "oracle"
        } else {
            "full_haystack"
        }
    }
}

fn print_report(
    overall: &Metrics,
    by_type: &HashMap<String, Metrics>,
    profile: &DatasetProfile,
    mode: Mode,
) {
    println!("=================================================================");
    println!("  LongMemEval Benchmark  —  {}", mode.label);
    println!("=================================================================");
    println!();
    println!(
        "Mode: vector_weight={:.1} / keyword_weight={:.1}",
        mode.vector_weight, mode.keyword_weight
    );
    println!();
    println!("Scoring target: RETRIEVAL RECALL (did the gold memory land in top-k).");
    println!("  No answer is generated or scored. This is NOT the official");
    println!("  LongMemEval metric (QA accuracy via retrieve+generate+judge).");
    println!(
        "Dataset: {} — {} questions, {:.1} sessions and {:.0} turns per question,",
        profile.variant(),
        profile.n_questions,
        profile.mean_sessions,
        profile.mean_turns,
    );
    println!(
        "  {:.1}% of haystack sessions are evidence sessions.",
        profile.evidence_density * 100.0
    );
    if profile.session_level_degenerate() {
        println!("  This is the evidence-only corpus, NOT the full longmemeval_s");
        println!("  haystack — a substantially easier retrieval problem.");
    } else {
        println!("  This is a full-haystack corpus: evidence sessions are a small");
        println!("  minority, so retrieval has to actually discriminate.");
    }
    println!();
    println!("Do NOT compare these to MemX's Hit@5=51.6% / MRR=0.380: that is");
    println!("  fact-level granularity over 220,349 records from 19,195 sessions.");
    println!("  Different granularity and a corpus larger by orders of magnitude.");
    println!();

    println!("## Session-Level Recall  (n={})", overall.count);
    if profile.session_level_degenerate() {
        println!(
            "  [DEGENERATE — {:.1}% of haystack sessions are evidence sessions, so a",
            profile.evidence_density * 100.0
        );
        println!("   returned document is a session-level hit almost by construction.");
        println!("   This measures the corpus shape, not the retriever. Use turn-level.]");
    } else {
        println!(
            "  [Meaningful on this corpus — only {:.1}% of haystack sessions are",
            profile.evidence_density * 100.0
        );
        println!("   evidence sessions, so a hit reflects the retriever's discrimination.]");
    }
    println!(
        "  Hit@1:  {:5.1}%   Hit@5:  {:5.1}%   Hit@10: {:5.1}%   MRR: {:.4}",
        overall.hit1_session_pct(),
        overall.hit5_session_pct(),
        overall.hit10_session_pct(),
        overall.mrr_session()
    );
    println!();

    println!("## Turn-Level Recall  (n={})", overall.count);
    println!(
        "  Hit@1:  {:5.1}%   Hit@5:  {:5.1}%   Hit@10: {:5.1}%   MRR: {:.4}",
        overall.hit1_turn_pct(),
        overall.hit5_turn_pct(),
        overall.hit10_turn_pct(),
        overall.mrr_turn()
    );
    println!();

    if overall.abstention_total > 0 {
        println!("## Abstention Accuracy");
        println!(
            "  Correct: {}/{} ({:.1}%)",
            overall.abstention_correct,
            overall.abstention_total,
            overall.abstention_pct()
        );
        println!("  (abstention = system correctly returns no matching session)");
        println!();
    }

    println!(
        "## Search Latency  (n={} queries, BM25 over variable haystack sizes)",
        overall.latency_ns.len()
    );
    println!(
        "  avg={:.1} µs   p50={:.1} µs   p95={:.1} µs   p99={:.1} µs",
        overall.latency_avg_us(),
        overall.latency_pct_us(50),
        overall.latency_pct_us(95),
        overall.latency_pct_us(99)
    );
    println!();

    println!("## Per-Type Breakdown  (session-level)");
    println!();
    println!(
        "{:<32} {:>5}  {:>7}  {:>7}  {:>7}  {:>7}",
        "Question Type", "N", "Hit@1", "Hit@5", "Hit@10", "MRR"
    );
    println!("{}", "-".repeat(72));

    let mut types: Vec<(&String, &Metrics)> = by_type.iter().collect();
    types.sort_by_key(|(t, _)| t.as_str());

    for (qtype, m) in &types {
        if m.count > 0 {
            println!(
                "{:<32} {:>5}  {:>6.1}%  {:>6.1}%  {:>6.1}%  {:>7.4}",
                qtype,
                m.count,
                m.hit1_session_pct(),
                m.hit5_session_pct(),
                m.hit10_session_pct(),
                m.mrr_session()
            );
        }
        if m.abstention_total > 0 {
            println!(
                "{:<32} {:>5}  abstention accuracy: {:>5.1}%",
                format!("{qtype}_abs"),
                m.abstention_total,
                m.abstention_pct()
            );
        }
    }

    // Machine-parseable JSON summary
    println!();
    println!("## JSON Summary");
    println!("```json");
    println!("{{");
    println!("  \"benchmark\": \"longmemeval\",");
    println!(
        "  \"mode\": \"vector_{:.1}_keyword_{:.1}\",",
        mode.vector_weight, mode.keyword_weight
    );
    println!("  \"dataset_variant\": \"{}\",", profile.variant());
    println!("  \"scoring_target\": \"retrieval_recall\",");
    println!("  \"k\": 10,");
    println!(
        "  \"session_level_degenerate\": {},",
        profile.session_level_degenerate()
    );
    println!(
        "  \"evidence_session_density\": {:.4},",
        profile.evidence_density
    );
    println!(
        "  \"mean_sessions_per_question\": {:.2},",
        profile.mean_sessions
    );
    println!("  \"mean_turns_per_question\": {:.1},", profile.mean_turns);
    println!(
        "  \"total_questions\": {},",
        overall.count + overall.abstention_total
    );
    println!("  \"session_level\": {{");
    println!(
        "    \"hit_at_1\": {:.4}, \"hit_at_5\": {:.4}, \"hit_at_10\": {:.4}, \"mrr\": {:.4}",
        overall.hit1_session_pct() / 100.0,
        overall.hit5_session_pct() / 100.0,
        overall.hit10_session_pct() / 100.0,
        overall.mrr_session()
    );
    println!("  }},");
    println!("  \"turn_level\": {{");
    println!(
        "    \"hit_at_1\": {:.4}, \"hit_at_5\": {:.4}, \"hit_at_10\": {:.4}, \"mrr\": {:.4}",
        overall.hit1_turn_pct() / 100.0,
        overall.hit5_turn_pct() / 100.0,
        overall.hit10_turn_pct() / 100.0,
        overall.mrr_turn()
    );
    println!("  }},");
    // `null`, not 0.0 — a corpus with no abstention questions has no abstention
    // accuracy, and emitting 0.0 reads as total failure at a task never posed.
    if overall.abstention_total > 0 {
        println!(
            "  \"abstention_accuracy\": {:.4},",
            overall.abstention_pct() / 100.0
        );
    } else {
        println!("  \"abstention_accuracy\": null,");
    }
    println!("  \"latency_us\": {{");
    println!(
        "    \"avg\": {:.1}, \"p50\": {:.1}, \"p95\": {:.1}, \"p99\": {:.1}",
        overall.latency_avg_us(),
        overall.latency_pct_us(50),
        overall.latency_pct_us(95),
        overall.latency_pct_us(99)
    );
    println!("  }}");
    println!("}}");
    println!("```");
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

fn main() {
    let mut json_path: Option<String> = None;
    let mut limit: Option<usize> = None;
    let mut weights_dir: Option<String> = None;
    let mut sweep = false;
    let mut args = std::env::args().skip(1);
    while let Some(arg) = args.next() {
        match arg.as_str() {
            "--limit" => {
                let v = args.next().expect("--limit needs a value");
                limit = Some(v.parse().expect("--limit must be a positive integer"));
            }
            "--sweep" => sweep = true,
            "--embeddings" => {
                weights_dir = Some(args.next().expect("--embeddings needs a directory"));
            }
            "--help" | "-h" => {
                eprintln!(
                    "usage: longmemeval_bench [PATH] [--limit N]\n\n\
                     PATH     dataset JSON; defaults to the oracle variant.\n\
                              longmemeval_s works too — the harness measures which\n\
                              variant it was given rather than trusting the filename.\n\
                     --limit  evaluate N questions, sampled evenly across the file\n\
                              rather than as a prefix — the dataset is ordered by\n\
                              question type, so a prefix samples one type only.\n\
                     --embeddings DIR\n\
                              directory holding all-MiniLM-L6-v2's model.safetensors\n\
                              and tokenizer.json. Enables the vector stage and reports\n\
                              BM25-only, vector-only, and hybrid separately. Requires\n\
                              --features embeddings; without it the vector stage is\n\
                              inert and only the BM25 row is produced.\n\
                     --sweep  instead of the three named modes, sweep vector_weight\n\
                              from 0.0 to 1.0 in 0.1 steps. The 0.7/0.3 default was\n\
                              never searched; this is what searches it."
                );
                return;
            }
            other => json_path = Some(other.to_string()),
        }
    }
    let json_path =
        json_path.unwrap_or_else(|| "benchmarks/longmemeval/longmemeval_oracle.json".to_string());

    eprintln!("Loading: {json_path}");
    let data = std::fs::read_to_string(&json_path)
        .unwrap_or_else(|e| panic!("Failed to read {json_path}: {e}"));
    let mut questions: Vec<Question> = serde_json::from_str(&data).expect("Failed to parse JSON");
    if let Some(n) = limit
        && n < questions.len()
    {
        // Stride rather than truncate. The dataset is ordered by question type,
        // so taking a prefix samples one type: `--limit 20` on longmemeval_s
        // returns 20 `single-session-user` questions and nothing else, which
        // reads as a whole-dataset result but is not one.
        let total = questions.len();
        let step = total as f64 / n as f64;
        let keep: HashSet<usize> = (0..n)
            .map(|i| ((i as f64 * step) as usize).min(total - 1))
            .collect();
        questions = questions
            .into_iter()
            .enumerate()
            .filter(|(i, _)| keep.contains(i))
            .map(|(_, q)| q)
            .collect();
        eprintln!(
            "Sampling {} of {total} questions, evenly strided (--limit)",
            questions.len()
        );
    }
    let total = questions.len();
    eprintln!("Loaded {total} questions");

    let profile = DatasetProfile::measure(&questions);
    eprintln!(
        "Corpus: {} variant — {:.1} sessions / {:.0} turns per question, \
         {:.1}% evidence-session density",
        profile.variant(),
        profile.mean_sessions,
        profile.mean_turns,
        profile.evidence_density * 100.0,
    );

    // Build the embedding table once for the whole corpus, if asked for.
    let embeddings: Option<EmbeddingMap> = weights_dir
        .as_deref()
        .map(|dir| load_embeddings(dir, &questions));
    if embeddings.is_none() && weights_dir.is_some() {
        eprintln!("warning: --embeddings ignored (build with --features embeddings)");
    }

    let modes: Vec<Mode> = if embeddings.is_some() {
        #[cfg(feature = "embeddings")]
        {
            if sweep {
                sweep_modes()
            } else {
                vec![BM25_ONLY, VECTOR_ONLY, HYBRID]
            }
        }
        #[cfg(not(feature = "embeddings"))]
        {
            vec![BM25_ONLY]
        }
    } else {
        if sweep {
            eprintln!("warning: --sweep needs --embeddings; running BM25 only");
        }
        vec![BM25_ONLY]
    };

    for (mode_idx, mode) in modes.iter().enumerate() {
        eprintln!("[{}/{}] {}", mode_idx + 1, modes.len(), mode.label);
        run_mode(&questions, *mode, embeddings.as_ref(), &profile);
    }
}

/// Load and encode the corpus. Returns `None` unless the `embeddings` feature
/// is compiled in, so the flag degrades to a warning rather than a hard error.
#[cfg(feature = "embeddings")]
fn load_embeddings(dir: &str, questions: &[Question]) -> EmbeddingMap {
    let enc = embedder::Embedder::load(std::path::Path::new(dir))
        .unwrap_or_else(|e| panic!("failed to load embedder from {dir}: {e}"));
    let texts = questions.iter().flat_map(|q| {
        q.haystack_sessions
            .iter()
            .flatten()
            .map(|t| t.content.clone())
            .chain(std::iter::once(q.question.clone()))
    });
    enc.encode_unique(texts)
        .unwrap_or_else(|e| panic!("embedding failed: {e}"))
}

#[cfg(not(feature = "embeddings"))]
fn load_embeddings(_dir: &str, _questions: &[Question]) -> EmbeddingMap {
    EmbeddingMap::new()
}

/// Evaluate every question under one retrieval mode and print its report.
fn run_mode(
    questions: &[Question],
    mode: Mode,
    embeddings: Option<&EmbeddingMap>,
    profile: &DatasetProfile,
) {
    let total = questions.len();
    let mut overall = Metrics::default();
    let mut by_type: HashMap<String, Metrics> = HashMap::new();

    for (i, q) in questions.iter().enumerate() {
        if (i + 1) % 50 == 0 || i + 1 == total {
            eprint!("\r  [{}/{}] evaluating...", i + 1, total);
        }

        let result = evaluate_question(q, 10, mode, embeddings);

        let is_abs = q.question_type.ends_with("_abs");
        let base_type = if is_abs {
            q.question_type.trim_end_matches("_abs").to_string()
        } else {
            q.question_type.clone()
        };

        let entry = by_type.entry(base_type).or_default();

        if is_abs {
            entry.abstention_total += 1;
            overall.abstention_total += 1;
            // Correct abstention: no session-level hit in top 10
            if !result.hit10_session {
                entry.abstention_correct += 1;
                overall.abstention_correct += 1;
            }
        } else {
            entry.count += 1;
            overall.count += 1;

            if result.hit1_session {
                entry.hit1_session += 1;
                overall.hit1_session += 1;
            }
            if result.hit5_session {
                entry.hit5_session += 1;
                overall.hit5_session += 1;
            }
            if result.hit10_session {
                entry.hit10_session += 1;
                overall.hit10_session += 1;
            }
            if let Some(rr) = result.rr_session {
                entry.rr_session += rr;
                overall.rr_session += rr;
            }

            if result.hit1_turn {
                entry.hit1_turn += 1;
                overall.hit1_turn += 1;
            }
            if result.hit5_turn {
                entry.hit5_turn += 1;
                overall.hit5_turn += 1;
            }
            if result.hit10_turn {
                entry.hit10_turn += 1;
                overall.hit10_turn += 1;
            }
            if let Some(rr) = result.rr_turn {
                entry.rr_turn += rr;
                overall.rr_turn += rr;
            }

            let ns = result.latency.as_nanos() as u64;
            entry.latency_ns.push(ns);
            overall.latency_ns.push(ns);
        }
    }

    eprintln!("\r  [{total}/{total}] done.          ");
    eprintln!();
    print_report(&overall, &by_type, profile, mode);
}