//! Integration tests for communication primitives //! //! These tests validate AllReduce, Broadcast, AllGather, and other //! collective communication operations. use rtx_distributed::{ AllGatherOutput, AllReduceOp, Backend, BackendConfig, CommunicationPrimitive, ProcessGroup, ReduceOp, Result, TensorExt, TensorShapeExt, }; use rtx_tensor::{Device, Shape as TensorShape, Tensor}; /// Test AllReduce with sum operation #[tokio::test] async fn test_allreduce_sum() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config).await?; // Create test tensor let device = Device::cpu(); let shape = TensorShape::new(vec![4])?; let mut tensor = Tensor::ones(shape, &device)?; // Perform AllReduce sum pg.allreduce(&mut tensor, ReduceOp::Sum).await?; // In a 2-process setup, each element should be 2.0 after sum let data = tensor.data()?; for value in data.iter() { assert!((*value - 2.0).abs() < 1e-6); } Ok(()) } /// Test AllReduce with different reduce operations #[tokio::test] async fn test_allreduce_operations() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 4, 0, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![3])?; // Test sum let mut sum_tensor = Tensor::ones(shape.clone(), &device)?; pg.allreduce(&mut sum_tensor, ReduceOp::Sum).await?; // Test max let mut max_tensor = Tensor::ones(shape.clone(), &device)?; pg.allreduce(&mut max_tensor, ReduceOp::Max).await?; // Test min let mut min_tensor = Tensor::ones(shape, &device)?; pg.allreduce(&mut min_tensor, ReduceOp::Min).await?; Ok(()) } /// Test Broadcast operation #[tokio::test] async fn test_broadcast() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 3, 1, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![2, 2])?; let mut tensor = Tensor::zeros(shape, &device)?; // Broadcast from root (rank 0) let root = 0; pg.broadcast(&mut tensor, root).await?; // Tensor should now have values from root assert!(tensor.data()?.len() == 4); Ok(()) } /// Test AllGather operation #[tokio::test] async fn test_allgather() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![3])?; let input_tensor = Tensor::ones(shape, &device)?; // Perform AllGather let output = pg.allgather(&input_tensor).await?; // Output should contain data from all processes match output { AllGatherOutput::Tensor(gathered) => { // Should have shape [world_size * input_size] = [2 * 3] = [6] assert_eq!(gathered.shape().dims(), &[6]); } AllGatherOutput::TensorList(tensors) => { // Should have one tensor per process assert_eq!(tensors.len(), 2); for tensor in tensors { assert_eq!(tensor.shape().dims(), &[3]); } } } Ok(()) } /// Test ReduceScatter operation #[tokio::test] async fn test_reduce_scatter() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![4])?; // Should be divisible by world_size let input_tensor = Tensor::ones(shape, &device)?; // Perform ReduceScatter let output = pg.reduce_scatter(&input_tensor, ReduceOp::Sum)?; // Output should have reduced size assert_eq!(output.shape().dims(), &[2]); // 4/2 = 2 Ok(()) } /// Test point-to-point Send/Recv operations #[tokio::test] async fn test_send_recv() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![3])?; let tensor = Tensor::ones(shape, &device)?; if pg.rank() == 0 { // Send to rank 1 pg.send(&tensor, 1).await?; } else { // Receive from rank 0 let mut recv_tensor = Tensor::zeros(tensor.shape().clone(), &device)?; pg.recv(&mut recv_tensor, 0).await?; // Should match sent data for value in recv_tensor.data()?.iter() { assert!((*value - 1.0).abs() < 1e-6); } } Ok(()) } /// Test multiple simultaneous operations #[tokio::test] async fn test_concurrent_operations() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 4, 0, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![2])?; // Spawn multiple concurrent AllReduce operations let handles = (0..5) .map(|_| { let pg_clone = pg.clone(); let shape_clone = shape.clone(); let device_clone = device.clone(); tokio::spawn(async move { let mut tensor = Tensor::ones(shape_clone, &device_clone)?; pg_clone.allreduce(&mut tensor, ReduceOp::Sum).await?; Ok::<_, rtx_distributed::DistributedError>(tensor) }) }) .collect::>(); // Wait for all operations to complete for handle in handles { let tensor = handle.await.unwrap()?; // Each element should be 4.0 (world_size) for value in tensor.data()?.iter() { assert!((*value - 4.0).abs() < 1e-6); } } Ok(()) } /// Test error conditions #[tokio::test] #[ignore = "Pre-existing zero-sized storage allocation error"] async fn test_communication_errors() { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config) .await .unwrap(); let device = Device::cpu(); // Test invalid tensor shapes for AllReduce let shape = TensorShape::new(vec![0]).unwrap(); // Empty tensor let mut tensor = Tensor::zeros(shape, &device).unwrap(); let result = pg.allreduce(&mut tensor, ReduceOp::Sum).await; assert!(result.is_err()); // Test broadcast with invalid root let shape = TensorShape::new(vec![2]).unwrap(); let mut tensor = Tensor::ones(shape, &device).unwrap(); let result = pg.broadcast(&mut tensor, 999).await; // Invalid rank assert!(result.is_err()); } /// Test different tensor dtypes #[tokio::test] async fn test_different_dtypes() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config).await?; let device = Device::cpu(); let shape = TensorShape::new(vec![3])?; // Test f32 tensors let mut f32_tensor = Tensor::ones(shape.clone(), &device)?; pg.allreduce(&mut f32_tensor, ReduceOp::Sum).await?; // Test f64 tensors (if supported) // let mut f64_tensor = Tensor::ones_f64(shape.clone(), &device)?; // pg.allreduce(&mut f64_tensor, ReduceOp::Sum).await?; Ok(()) } /// Test large tensor operations #[tokio::test] async fn test_large_tensors() -> Result<()> { let config = BackendConfig::cpu(); let pg = ProcessGroup::new_with_config(Backend::Cpu, 2, 0, config).await?; let device = Device::cpu(); // Test with large tensor let shape = TensorShape::new(vec![1024, 1024])?; // 1M elements let mut tensor = Tensor::ones(shape, &device)?; let start = std::time::Instant::now(); pg.allreduce(&mut tensor, ReduceOp::Sum).await?; let elapsed = start.elapsed(); // Should complete within reasonable time assert!(elapsed < std::time::Duration::from_secs(5)); Ok(()) } /// Test operation timeout handling #[tokio::test] #[ignore = "Pre-existing timeout assertion failure"] async fn test_operation_timeout() { let mut config = BackendConfig::cpu(); config.set_timeout(std::time::Duration::from_millis(1)); // Very short timeout let pg = ProcessGroup::new_with_config(Backend::Cpu, 100, 0, config) .await .unwrap(); let device = Device::cpu(); let shape = TensorShape::new(vec![1000]).unwrap(); let mut tensor = Tensor::ones(shape, &device).unwrap(); // This should timeout let result = pg.allreduce(&mut tensor, ReduceOp::Sum).await; assert!(result.is_err()); }