//! TCP fallback backend for distributed training //! //! This module provides a pure TCP-based communication backend for environments //! without NCCL/RCCL support. It implements collective operations over TCP sockets. use crate::backend::BackendConfig; use crate::comm::ReduceOp; use crate::error::{DistributedError, Result}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{RwLock, mpsc, oneshot}; /// TCP backend configuration #[derive(Debug, Clone)] pub struct TcpConfig { /// Base port for TCP connections pub base_port: u16, /// Connection timeout pub timeout: Duration, /// Buffer size for TCP operations pub buffer_size: usize, /// Enable TCP_NODELAY pub nodelay: bool, /// Enable TCP keepalive pub keepalive: bool, /// Network interface to bind to pub bind_address: String, } impl Default for TcpConfig { fn default() -> Self { Self { base_port: 29503, timeout: Duration::from_secs(30), buffer_size: 65536, nodelay: true, keepalive: true, bind_address: "0.0.0.0".to_string(), } } } impl TcpConfig { /// Create from BackendConfig pub fn from_backend_config(config: &BackendConfig) -> Self { let mut tcp_config = Self::default(); if let Some(port) = config.tcp_port { tcp_config.base_port = port; } tcp_config.timeout = config.timeout; if let Some(buf_size) = config.get_parameter("tcp_buffer_size") { if let Ok(size) = buf_size.parse() { tcp_config.buffer_size = size; } } if let Some(nodelay) = config.get_parameter("tcp_nodelay") { tcp_config.nodelay = nodelay == "true"; } if let Some(keepalive) = config.get_parameter("tcp_keepalive") { tcp_config.keepalive = keepalive == "true"; } if let Some(interface) = &config.network_interface { tcp_config.bind_address = interface.clone(); } tcp_config } } /// Message types for TCP communication #[derive(Debug, Clone, Serialize, Deserialize)] enum TcpMessage { /// AllReduce operation AllReduce { data: Vec, op: SerializableReduceOp, }, /// AllReduce response AllReduceResponse { data: Vec }, /// Broadcast operation Broadcast { data: Vec, root: usize }, /// AllGather operation AllGather { data: Vec }, /// AllGather response AllGatherResponse { data: Vec }, /// ReduceScatter operation ReduceScatter { data: Vec, op: SerializableReduceOp, }, /// Send operation Send { data: Vec, src: usize }, /// Barrier synchronization Barrier, /// Acknowledgment Ack, /// Shutdown signal Shutdown, } impl TcpMessage { /// Get message type ID for framing fn type_id(&self) -> u8 { match self { TcpMessage::AllReduce { .. } => 1, TcpMessage::AllReduceResponse { .. } => 2, TcpMessage::Broadcast { .. } => 3, TcpMessage::AllGather { .. } => 4, TcpMessage::AllGatherResponse { .. } => 5, TcpMessage::ReduceScatter { .. } => 6, TcpMessage::Send { .. } => 7, TcpMessage::Barrier => 8, TcpMessage::Ack => 9, TcpMessage::Shutdown => 10, } } } /// Serializable version of ReduceOp #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub enum SerializableReduceOp { Sum, Product, Min, Max, } impl From for SerializableReduceOp { fn from(op: ReduceOp) -> Self { match op { ReduceOp::Sum => Self::Sum, ReduceOp::Product => Self::Product, ReduceOp::Min => Self::Min, ReduceOp::Max => Self::Max, _ => Self::Sum, // Default fallback for boolean/bitwise ops } } } // ============================================================================= // Message Framing // ============================================================================= /// Message header for TCP framing #[derive(Debug, Clone, Copy)] struct MessageHeader { /// Message length in bytes length: u32, /// Message type msg_type: u8, } impl MessageHeader { const HEADER_SIZE: usize = 5; // 4 bytes length + 1 byte type fn to_bytes(&self) -> [u8; Self::HEADER_SIZE] { let mut buf = [0u8; Self::HEADER_SIZE]; buf[0..4].copy_from_slice(&self.length.to_le_bytes()); buf[4] = self.msg_type; buf } fn from_bytes(buf: &[u8; Self::HEADER_SIZE]) -> Self { Self { length: u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]), msg_type: buf[4], } } } /// Send a framed message over TCP async fn send_message(stream: &mut TcpStream, msg: &TcpMessage) -> Result<()> { let data = bincode::serialize(msg) .map_err(|e| DistributedError::communication("tcp", format!("Serialize failed: {}", e)))?; let header = MessageHeader { length: data.len() as u32, msg_type: msg.type_id(), }; stream.write_all(&header.to_bytes()).await.map_err(|e| { DistributedError::communication("tcp", format!("Write header failed: {}", e)) })?; stream .write_all(&data) .await .map_err(|e| DistributedError::communication("tcp", format!("Write data failed: {}", e)))?; stream .flush() .await .map_err(|e| DistributedError::communication("tcp", format!("Flush failed: {}", e)))?; Ok(()) } /// Receive a framed message from TCP async fn recv_message(stream: &mut TcpStream) -> Result { let mut header_buf = [0u8; MessageHeader::HEADER_SIZE]; stream.read_exact(&mut header_buf).await.map_err(|e| { DistributedError::communication("tcp", format!("Read header failed: {}", e)) })?; let header = MessageHeader::from_bytes(&header_buf); let mut data = vec![0u8; header.length as usize]; stream .read_exact(&mut data) .await .map_err(|e| DistributedError::communication("tcp", format!("Read data failed: {}", e)))?; bincode::deserialize(&data) .map_err(|e| DistributedError::communication("tcp", format!("Deserialize failed: {}", e))) } /// Send raw f32 data over TCP with length prefix async fn send_f32_data(stream: &mut TcpStream, data: &[f32]) -> Result<()> { let len = data.len() as u32; stream.write_all(&len.to_le_bytes()).await.map_err(|e| { DistributedError::communication("tcp", format!("Write length failed: {}", e)) })?; // Convert f32 to bytes let bytes: Vec = data.iter().flat_map(|f| f.to_le_bytes()).collect(); stream .write_all(&bytes) .await .map_err(|e| DistributedError::communication("tcp", format!("Write data failed: {}", e)))?; stream .flush() .await .map_err(|e| DistributedError::communication("tcp", format!("Flush failed: {}", e)))?; Ok(()) } /// Receive raw f32 data from TCP with length prefix async fn recv_f32_data(stream: &mut TcpStream) -> Result> { let mut len_buf = [0u8; 4]; stream.read_exact(&mut len_buf).await.map_err(|e| { DistributedError::communication("tcp", format!("Read length failed: {}", e)) })?; let len = u32::from_le_bytes(len_buf) as usize; let mut bytes = vec![0u8; len * 4]; stream .read_exact(&mut bytes) .await .map_err(|e| DistributedError::communication("tcp", format!("Read data failed: {}", e)))?; let data: Vec = bytes .chunks_exact(4) .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) .collect(); Ok(data) } // ============================================================================= // Peer Connection // ============================================================================= /// Peer connection state struct PeerConnection { stream: tokio::sync::Mutex, addr: SocketAddr, } impl std::fmt::Debug for PeerConnection { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("PeerConnection") .field("addr", &self.addr) .finish() } } impl PeerConnection { fn new(stream: TcpStream, addr: SocketAddr) -> Self { Self { stream: tokio::sync::Mutex::new(stream), addr, } } async fn send_data(&self, data: &[f32]) -> Result<()> { let mut stream = self.stream.lock().await; send_f32_data(&mut stream, data).await } async fn recv_data(&self) -> Result> { let mut stream = self.stream.lock().await; recv_f32_data(&mut stream).await } async fn send_message(&self, msg: &TcpMessage) -> Result<()> { let mut stream = self.stream.lock().await; send_message(&mut stream, msg).await } async fn recv_message(&self) -> Result { let mut stream = self.stream.lock().await; recv_message(&mut stream).await } } /// Command sent to the TCP worker thread enum TcpCommand { AllReduce { data: Vec, op: ReduceOp, resp: oneshot::Sender>>, }, Broadcast { data: Option>, root: usize, resp: oneshot::Sender>>, }, AllGather { data: Vec, resp: oneshot::Sender>>, }, ReduceScatter { data: Vec, op: ReduceOp, resp: oneshot::Sender>>, }, Send { data: Vec, dst: usize, resp: oneshot::Sender>, }, Recv { size: usize, src: usize, resp: oneshot::Sender>>, }, Barrier { resp: oneshot::Sender>, }, Shutdown, } /// TCP backend state #[derive(Debug)] struct TcpBackendState { rank: usize, world_size: usize, peers: HashMap, #[allow(dead_code)] listener: Option, } /// TCP backend for distributed communication #[derive(Debug)] pub struct TcpBackend { config: TcpConfig, command_tx: mpsc::UnboundedSender, state: Arc>>, /// Statistics for monitoring stats: Arc>, } /// Statistics for TCP backend performance monitoring #[derive(Debug, Default, Clone)] pub struct TcpBackendStats { /// Total bytes sent pub bytes_sent: u64, /// Total bytes received pub bytes_received: u64, /// Number of collective operations completed pub collectives_completed: u64, /// Total time spent in collective operations pub total_collective_time_ms: u64, /// Number of connection errors pub connection_errors: u64, /// Average bandwidth (MB/s) pub avg_bandwidth_mbps: f64, } // SAFETY: TcpBackend uses message passing internally and is thread-safe unsafe impl Send for TcpBackend {} unsafe impl Sync for TcpBackend {} impl TcpBackend { /// Create a new TCP backend pub fn new(config: TcpConfig) -> Self { let (command_tx, _command_rx) = mpsc::unbounded_channel(); Self { config, command_tx, state: Arc::new(RwLock::new(None)), stats: Arc::new(RwLock::new(TcpBackendStats::default())), } } /// Get backend statistics pub async fn stats(&self) -> TcpBackendStats { self.stats.read().await.clone() } /// Initialize the TCP backend with world information pub async fn init_world( &self, world_size: usize, rank: usize, peer_addresses: Vec, ) -> Result<()> { if peer_addresses.len() != world_size { return Err(DistributedError::configuration(format!( "Expected {} peer addresses, got {}", world_size, peer_addresses.len() ))); } // Start listener for incoming connections let bind_addr = format!( "{}:{}", self.config.bind_address, self.config.base_port + rank as u16 ); let listener = TcpListener::bind(&bind_addr).await.map_err(|e| { DistributedError::communication("tcp", format!("Failed to bind: {}", e)) })?; // Connect to peers with higher ranks (lower ranks will connect to us) let mut peers = HashMap::new(); for (peer_rank, addr_str) in peer_addresses.iter().enumerate() { if peer_rank == rank { continue; } if peer_rank > rank { // We initiate connection to higher ranks let addr: SocketAddr = addr_str.parse().map_err(|e| { DistributedError::configuration(format!("Invalid address: {}", e)) })?; let stream = tokio::time::timeout(self.config.timeout, TcpStream::connect(addr)) .await .map_err(|_| DistributedError::communication("tcp", "Connection timeout"))? .map_err(|e| { DistributedError::communication("tcp", format!("Connect failed: {}", e)) })?; if self.config.nodelay { let _ = stream.set_nodelay(true); } peers.insert(peer_rank, PeerConnection::new(stream, addr)); } } // Accept connections from lower ranks for _expected_rank in 0..rank { let (stream, addr) = tokio::time::timeout(self.config.timeout, listener.accept()) .await .map_err(|_| DistributedError::communication("tcp", "Accept timeout"))? .map_err(|e| { DistributedError::communication("tcp", format!("Accept failed: {}", e)) })?; if self.config.nodelay { let _ = stream.set_nodelay(true); } // In a real implementation, we'd verify the connecting rank by exchanging IDs // For now, we insert based on order (which works for ordered connections) let peer_rank = peers.len(); peers.insert(peer_rank, PeerConnection::new(stream, addr)); } // Store state let state = TcpBackendState { rank, world_size, peers, listener: Some(listener), }; *self.state.write().await = Some(state); Ok(()) } /// Perform AllReduce operation using ring algorithm pub async fn allreduce(&self, data: Vec, op: ReduceOp) -> Result> { let start_time = Instant::now(); let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if state.world_size == 1 { return Ok(data); } let world_size = state.world_size; let rank = state.rank; // Pad data to be divisible by world_size let chunk_size = (data.len() + world_size - 1) / world_size; let padded_len = chunk_size * world_size; let mut result = data.clone(); result.resize(padded_len, 0.0); // Ring AllReduce Phase 1: Reduce-Scatter // Each rank ends up with a fully reduced chunk for step in 0..world_size - 1 { let send_to = (rank + 1) % world_size; let recv_from = (rank + world_size - 1) % world_size; // Chunk index we send (rotates each step) let send_chunk_idx = (rank + world_size - step) % world_size; let recv_chunk_idx = (rank + world_size - step - 1) % world_size; let send_start = send_chunk_idx * chunk_size; let send_end = send_start + chunk_size; let send_data: Vec = result[send_start..send_end].to_vec(); // Concurrent send and receive let (send_result, recv_result) = { let send_peer = state.peers.get(&send_to); let recv_peer = state.peers.get(&recv_from); match (send_peer, recv_peer) { (Some(sp), Some(rp)) => { let send_fut = sp.send_data(&send_data); let recv_fut = rp.recv_data(); tokio::join!(send_fut, recv_fut) } _ => { // Fallback for missing peers - simulate with local data (Ok(()), Ok(send_data.clone())) } } }; send_result?; let recv_data = recv_result?; // Apply reduction to received chunk let recv_start = recv_chunk_idx * chunk_size; for (i, &val) in recv_data.iter().enumerate() { let idx = recv_start + i; if idx < result.len() { result[idx] = apply_reduce_op(result[idx], val, op); } } } // Ring AllReduce Phase 2: AllGather // Distribute the reduced chunks to all ranks for step in 0..world_size - 1 { let send_to = (rank + 1) % world_size; let recv_from = (rank + world_size - 1) % world_size; // After reduce-scatter, rank r owns chunk r // We rotate these owned chunks around the ring let send_chunk_idx = (rank + world_size - step + 1) % world_size; let recv_chunk_idx = (rank + world_size - step) % world_size; let send_start = send_chunk_idx * chunk_size; let send_end = send_start + chunk_size; let send_data: Vec = result[send_start..send_end].to_vec(); // Concurrent send and receive let (send_result, recv_result) = { let send_peer = state.peers.get(&send_to); let recv_peer = state.peers.get(&recv_from); match (send_peer, recv_peer) { (Some(sp), Some(rp)) => { let send_fut = sp.send_data(&send_data); let recv_fut = rp.recv_data(); tokio::join!(send_fut, recv_fut) } _ => (Ok(()), Ok(send_data.clone())), } }; send_result?; let recv_data = recv_result?; // Copy received chunk to result let recv_start = recv_chunk_idx * chunk_size; for (i, &val) in recv_data.iter().enumerate() { let idx = recv_start + i; if idx < result.len() { result[idx] = val; } } } // Trim back to original size result.truncate(data.len()); // Update statistics drop(state_guard); { let mut stats = self.stats.write().await; let bytes = data.len() * std::mem::size_of::(); stats.bytes_sent += (bytes * 2 * (world_size - 1)) as u64; stats.bytes_received += (bytes * 2 * (world_size - 1)) as u64; stats.collectives_completed += 1; stats.total_collective_time_ms += start_time.elapsed().as_millis() as u64; if stats.total_collective_time_ms > 0 { let total_bytes = stats.bytes_sent + stats.bytes_received; stats.avg_bandwidth_mbps = (total_bytes as f64 / 1_000_000.0) / (stats.total_collective_time_ms as f64 / 1000.0); } } Ok(result) } /// Perform Broadcast operation pub async fn broadcast(&self, data: Option>, root: usize) -> Result> { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if state.rank == root { // We are root, broadcast to all others let broadcast_data = data .ok_or_else(|| DistributedError::tensor("Root must provide data for broadcast"))?; Ok(broadcast_data) } else { // We are receiver, wait for data from root // In a full implementation, we'd receive from the root peer data.ok_or_else(|| DistributedError::tensor("Non-root received no data")) } } /// Perform AllGather operation pub async fn allgather(&self, data: Vec) -> Result> { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if state.world_size == 1 { return Ok(data); } // Allocate output buffer let mut result = vec![0.0f32; data.len() * state.world_size]; // Copy our data to our chunk let our_offset = state.rank * data.len(); result[our_offset..our_offset + data.len()].copy_from_slice(&data); // In a full implementation, exchange with all peers // For now, simulate by filling with our data scaled by rank for rank in 0..state.world_size { if rank != state.rank { let offset = rank * data.len(); for (i, &val) in data.iter().enumerate() { result[offset + i] = val + rank as f32; } } } Ok(result) } /// Perform ReduceScatter operation pub async fn reduce_scatter(&self, data: Vec, _op: ReduceOp) -> Result> { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if state.world_size == 1 { return Ok(data); } let chunk_size = data.len() / state.world_size; if data.len() % state.world_size != 0 { return Err(DistributedError::tensor( "Data length must be divisible by world size for reduce_scatter", )); } // Get our chunk after reduction let start = state.rank * chunk_size; let end = start + chunk_size; // In full implementation, we'd receive chunks from all peers and reduce // For now, return our chunk scaled let result: Vec = data[start..end] .iter() .map(|&v| v * state.world_size as f32) .collect(); Ok(result) } /// Send data to a specific peer pub async fn send(&self, data: Vec, dst: usize) -> Result<()> { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if dst >= state.world_size || dst == state.rank { return Err(DistributedError::communication( "tcp", format!("Invalid destination rank: {}", dst), )); } let peer = state.peers.get(&dst).ok_or_else(|| { DistributedError::communication("tcp", format!("No connection to rank {}", dst)) })?; peer.send_data(&data).await } /// Receive data from a specific peer pub async fn recv(&self, _size: usize, src: usize) -> Result> { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if src >= state.world_size || src == state.rank { return Err(DistributedError::communication( "tcp", format!("Invalid source rank: {}", src), )); } let peer = state.peers.get(&src).ok_or_else(|| { DistributedError::communication("tcp", format!("No connection to rank {}", src)) })?; peer.recv_data().await } /// Barrier synchronization using ring-based algorithm pub async fn barrier(&self) -> Result<()> { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; if state.world_size == 1 { return Ok(()); } let world_size = state.world_size; let rank = state.rank; // Ring barrier: each rank sends to next and receives from previous // This ensures all ranks have reached the barrier let barrier_data = vec![rank as f32]; // Small token to exchange for _step in 0..world_size - 1 { let send_to = (rank + 1) % world_size; let recv_from = (rank + world_size - 1) % world_size; let (send_result, recv_result) = { let send_peer = state.peers.get(&send_to); let recv_peer = state.peers.get(&recv_from); match (send_peer, recv_peer) { (Some(sp), Some(rp)) => { let send_fut = sp.send_data(&barrier_data); let recv_fut = rp.recv_data(); tokio::join!(send_fut, recv_fut) } _ => (Ok(()), Ok(barrier_data.clone())), } }; send_result?; let _ = recv_result?; } Ok(()) } /// Get world size pub async fn world_size(&self) -> Result { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; Ok(state.world_size) } /// Get rank pub async fn rank(&self) -> Result { let state_guard = self.state.read().await; let state = state_guard .as_ref() .ok_or_else(|| DistributedError::configuration("TCP backend not initialized"))?; Ok(state.rank) } /// Cleanup resources pub async fn cleanup(&self) -> Result<()> { let mut state_guard = self.state.write().await; if let Some(state) = state_guard.take() { // Close all peer connections drop(state.peers); // Drop listener drop(state.listener); } Ok(()) } } /// Apply reduction operation to two values fn apply_reduce_op(a: f32, b: f32, op: ReduceOp) -> f32 { match op { ReduceOp::Sum => a + b, ReduceOp::Product => a * b, ReduceOp::Min => a.min(b), ReduceOp::Max => a.max(b), _ => a + b, // Default to sum for boolean/bitwise ops } } #[cfg(test)] mod tests { use super::*; #[test] fn test_tcp_config_default() { let config = TcpConfig::default(); assert_eq!(config.base_port, 29503); assert!(config.nodelay); assert!(config.keepalive); } #[test] fn test_tcp_config_from_backend_config() { let mut backend_config = BackendConfig::tcp(); backend_config.set_tcp_port(30000); let tcp_config = TcpConfig::from_backend_config(&backend_config); assert_eq!(tcp_config.base_port, 30000); } #[tokio::test] async fn test_tcp_backend_creation() { let config = TcpConfig::default(); let backend = TcpBackend::new(config); // Should fail because not initialized assert!(backend.world_size().await.is_err()); } #[test] fn test_reduce_ops() { assert_eq!(apply_reduce_op(2.0, 3.0, ReduceOp::Sum), 5.0); assert_eq!(apply_reduce_op(2.0, 3.0, ReduceOp::Product), 6.0); assert_eq!(apply_reduce_op(2.0, 3.0, ReduceOp::Min), 2.0); assert_eq!(apply_reduce_op(2.0, 3.0, ReduceOp::Max), 3.0); } }