wip-4a2

crates/clawhdf5-bench/src/bin/longmemeval_bench.rs

@@ -593,9 +593,9 @@ fn main() {
                      PATH     dataset JSON; defaults to the oracle variant.\n\
                               longmemeval_s works too — the harness measures which\n\
                               variant it was given rather than trusting the filename.\n\
-                     --limit  evaluate only the first N questions. The full haystack\n\
+                     --limit  evaluate N questions, sampled evenly across the file\n\
-                              is ~20x the oracle corpus per question, so this exists to\n\
+                              rather than as a prefix — the dataset is ordered by\n\
-                              size a run before committing to all 500."
+                              question type, so a prefix samples one type only."
                 );
                 return;
             }
@@ -612,11 +612,25 @@ fn main() {
     if let Some(n) = limit
         && n < questions.len()
     {
+        // Stride rather than truncate. The dataset is ordered by question type,
+        // so taking a prefix samples one type: `--limit 20` on longmemeval_s
+        // returns 20 `single-session-user` questions and nothing else, which
+        // reads as a whole-dataset result but is not one.
+        let total = questions.len();
+        let step = total as f64 / n as f64;
+        let keep: HashSet<usize> = (0..n)
+            .map(|i| ((i as f64 * step) as usize).min(total - 1))
+            .collect();
+        questions = questions
+            .into_iter()
+            .enumerate()
+            .filter(|(i, _)| keep.contains(i))
+            .map(|(_, q)| q)
+            .collect();
         eprintln!(
-            "Limiting to the first {n} of {} questions (--limit)",
+            "Sampling {} of {total} questions, evenly strided (--limit)",
             questions.len()
         );
-        questions.truncate(n);
     }
     let total = questions.len();
     eprintln!("Loaded {total} questions");