osobhandClaude Sonnet 5
4aaa36a57a
Whole-workspace rustfmt pass picked up while iterating on Mamba GPU backward work. Verified formatting-only via diff sampling; no logic changed. Co-Authored-By: Claude Sonnet 5 <[email protected]>
41 lines
1.3 KiB
Rust
41 lines
1.3 KiB
Rust
//! Distributed training infrastructure for RTX Transformers
|
|
//!
|
|
//! This module provides comprehensive distributed training capabilities including:
|
|
//! - Pipeline parallelism for very large models
|
|
//! - Data parallelism for high throughput training
|
|
//! - Model parallelism for memory-constrained scenarios
|
|
//! - Hybrid parallelism strategies
|
|
|
|
use crate::Result;
|
|
|
|
pub mod pipeline_parallelism;
|
|
pub mod pipeline_verification;
|
|
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
pub mod pipeline_parallelism_tests;
|
|
|
|
// Re-export key types for convenience
|
|
pub use pipeline_parallelism::{
|
|
ActivationCheckpoint, CommunicationBuffer, DeviceMapping, GradientSynchronizer, LoadBalancer,
|
|
MicroBatch, ModelPartition, PipelineConfig, PipelineMetrics, PipelineParallelism,
|
|
PipelineSchedule, PipelineStage, StageConfig,
|
|
};
|
|
|
|
/// Initialize distributed training infrastructure
|
|
pub fn init_distributed() -> Result<()> {
|
|
tracing::info!("Initializing RTX Transformers distributed training");
|
|
Ok(())
|
|
}
|
|
|
|
/// Get distributed training capabilities
|
|
pub fn get_distributed_capabilities() -> Vec<&'static str> {
|
|
vec![
|
|
"Pipeline Parallelism",
|
|
"Micro-batch Processing",
|
|
"Gradient Synchronization",
|
|
"Activation Checkpointing",
|
|
"Load Balancing",
|
|
"Fault Tolerance",
|
|
]
|
|
}
|