Initial commit

2026-03-04 00:08:42 +00:00
commit 4d88dc0584
4449 changed files with 1556714 additions and 0 deletions
@@ -0,0 +1,119 @@
+//! Text summarization (extractive and abstractive)
+
+pub mod abstractive;
+pub mod extractive;
+
+use crate::{GeneratedOutput, ModelInterface, NlgError, Result};
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+/// Summarization configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SummarizationConfig {
+    pub summary_type: SummaryType,
+    pub max_summary_length: usize,
+    pub min_summary_length: usize,
+    pub compression_ratio: f32,
+    pub extractive_config: Option<extractive::ExtractiveConfig>,
+    pub abstractive_config: Option<abstractive::AbstractiveConfig>,
+}
+
+impl Default for SummarizationConfig {
+    fn default() -> Self {
+        Self {
+            summary_type: SummaryType::Abstractive,
+            max_summary_length: 150,
+            min_summary_length: 30,
+            compression_ratio: 0.3,
+            extractive_config: None,
+            abstractive_config: Some(abstractive::AbstractiveConfig::default()),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum SummaryType {
+    Extractive,
+    Abstractive,
+    Hybrid,
+}
+
+/// Main summarizer interface
+pub struct Summarizer {
+    model: Arc<dyn ModelInterface>,
+    config: SummarizationConfig,
+}
+
+impl Summarizer {
+    pub fn new(model: Arc<dyn ModelInterface>, config: SummarizationConfig) -> Self {
+        Self { model, config }
+    }
+
+    pub fn summarize(&self, text: &str) -> Result<GeneratedOutput> {
+        // Validate input
+        if text.trim().is_empty() {
+            return Err(NlgError::DocumentTooShort);
+        }
+
+        let word_count = text.split_whitespace().count();
+        if word_count > 10000 {
+            return Err(NlgError::document_too_long(word_count, 10000));
+        }
+
+        match self.config.summary_type {
+            SummaryType::Extractive => self.extractive_summarize(text),
+            SummaryType::Abstractive => self.abstractive_summarize(text),
+            SummaryType::Hybrid => self.hybrid_summarize(text),
+        }
+    }
+
+    fn extractive_summarize(&self, text: &str) -> Result<GeneratedOutput> {
+        let config = self
+            .config
+            .extractive_config
+            .as_ref()
+            .ok_or_else(|| NlgError::invalid_config("Missing extractive config"))?;
+
+        extractive::extract_summary(text, config)
+    }
+
+    fn abstractive_summarize(&self, text: &str) -> Result<GeneratedOutput> {
+        let config = self
+            .config
+            .abstractive_config
+            .as_ref()
+            .ok_or_else(|| NlgError::invalid_config("Missing abstractive config"))?;
+
+        abstractive::generate_summary(&*self.model, text, config)
+    }
+
+    fn hybrid_summarize(&self, text: &str) -> Result<GeneratedOutput> {
+        // Extract key sentences first, then abstractively summarize
+        let extractive_config = extractive::ExtractiveConfig::default();
+        let key_sentences = extractive::extract_key_sentences(text, &extractive_config)?;
+
+        let combined_text = key_sentences.join(" ");
+        self.abstractive_summarize(&combined_text)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MockModelInterface;
+
+    #[tokio::test]
+    async fn test_summarizer() -> Result<()> {
+        let model = Arc::new(MockModelInterface::new("/tmp/mock")?);
+        let config = SummarizationConfig::default();
+        let summarizer = Summarizer::new(model, config);
+
+        let text = "This is a long document that needs to be summarized. It contains multiple sentences with important information. The summarizer should extract or generate the key points from this text.";
+
+        let summary = summarizer.summarize(text)?;
+        assert!(!summary.text.is_empty());
+        assert!(summary.text.len() < text.len());
+
+        Ok(())
+    }
+}