Initial commit

crates/production/rtx-inference/src/speculative/metrics.rs

@@ -0,0 +1,82 @@
+//! Performance metrics for speculative decoding
+
+use std::time::Duration;
+
+/// Performance metrics for speculative decoding
+#[derive(Debug, Clone, Default)]
+pub struct PerformanceMetrics {
+    /// Total number of decoding steps
+    pub total_steps: u64,
+    /// Total tokens generated by draft model
+    pub tokens_generated: u64,
+    /// Total tokens accepted by target model
+    pub tokens_accepted: u64,
+    /// Current acceptance rate
+    pub acceptance_rate: f32,
+    /// Speedup ratio compared to sequential generation
+    pub speedup_ratio: f32,
+    /// Average time for draft generation
+    pub draft_generation_time: Duration,
+    /// Average time for verification
+    pub verification_time: Duration,
+    /// Total time spent in speculative decoding
+    pub total_time: Duration,
+}
+
+impl PerformanceMetrics {
+    pub(crate) fn new() -> Self {
+        Self {
+            total_steps: 0,
+            tokens_generated: 0,
+            tokens_accepted: 0,
+            acceptance_rate: 0.0,
+            speedup_ratio: 0.0,
+            draft_generation_time: Duration::ZERO,
+            verification_time: Duration::ZERO,
+            total_time: Duration::ZERO,
+        }
+    }
+
+    pub(crate) fn update(&mut self, step_result: &StepMetrics) {
+        self.total_steps += 1;
+        self.tokens_generated += step_result.draft_tokens as u64;
+        self.tokens_accepted += step_result.accepted_tokens as u64;
+
+        // Update running averages
+        let weighted_draft = self.draft_generation_time.as_nanos()
+            * u128::from(self.total_steps - 1)
+            + step_result.draft_time.as_nanos();
+        self.draft_generation_time =
+            Duration::from_nanos((weighted_draft / u128::from(self.total_steps)) as u64);
+
+        let weighted_verification = self.verification_time.as_nanos()
+            * u128::from(self.total_steps - 1)
+            + step_result.verification_time.as_nanos();
+        self.verification_time =
+            Duration::from_nanos((weighted_verification / u128::from(self.total_steps)) as u64);
+        self.total_time += step_result.total_time;
+
+        // Update acceptance rate
+        self.acceptance_rate = if self.tokens_generated > 0 {
+            self.tokens_accepted as f32 / self.tokens_generated as f32
+        } else {
+            0.0
+        };
+
+        // Estimate speedup ratio
+        // This is simplified - in reality would compare against baseline sequential generation
+        let draft_efficiency =
+            step_result.draft_tokens as f32 / step_result.draft_time.as_secs_f32();
+        let acceptance_benefit = self.acceptance_rate * step_result.draft_tokens as f32;
+        self.speedup_ratio = 1.0 + (acceptance_benefit / draft_efficiency).min(10.0);
+    }
+}
+
+/// Metrics for a single decoding step
+pub(crate) struct StepMetrics {
+    pub draft_tokens: usize,
+    pub accepted_tokens: usize,
+    pub draft_time: Duration,
+    pub verification_time: Duration,
+    pub total_time: Duration,
+}