101 lines
4.4 KiB
Rust
101 lines
4.4 KiB
Rust
//! L-BFGS optimizer demonstration
|
||
//!
|
||
//! This example shows how to use the L-BFGS optimizer for training transformer models.
|
||
//! L-BFGS is a quasi-Newton method that can achieve superlinear convergence on smooth problems.
|
||
//!
|
||
//! NOTE: L-BFGS optimizer is part of Phase 2 advanced optimizers and is currently disabled.
|
||
//! This example will be enabled once the L-BFGS module is implemented.
|
||
|
||
fn main() {
|
||
println!("L-BFGS Optimizer Demo");
|
||
println!("====================");
|
||
println!();
|
||
println!("⚠️ This example is currently disabled.");
|
||
println!("The L-BFGS optimizer is part of Phase 2 advanced optimizers.");
|
||
println!("It will be enabled once the L-BFGS module is implemented.");
|
||
println!();
|
||
println!("Expected features:");
|
||
println!(" • Two-loop recursion for search direction computation");
|
||
println!(" • Limited memory storage of gradient/parameter changes");
|
||
println!(" • Strong Wolfe line search for step size selection");
|
||
println!(" • Automatic initial inverse Hessian scaling");
|
||
println!(" • Memory-efficient circular buffer implementation");
|
||
println!(" • Support for L2 regularization (weight decay)");
|
||
println!(" • Convergence detection based on gradient norm");
|
||
}
|
||
|
||
#[cfg(disabled)]
|
||
mod disabled_code {
|
||
use rtx_transformers::prelude::*;
|
||
use std::collections::HashMap;
|
||
|
||
fn _demo() -> Result<()> {
|
||
// Initialize logging
|
||
tracing_subscriber::fmt::init();
|
||
|
||
println!("L-BFGS Optimizer Demo");
|
||
println!("====================");
|
||
|
||
// Create L-BFGS configuration
|
||
let lbfgs_config = LBFGSConfig {
|
||
learning_rate: 1.0, // Initial step size
|
||
history_size: 10, // Memory parameter m
|
||
max_line_search_iters: 20,
|
||
c1: 1e-4, // Wolfe condition for sufficient decrease
|
||
c2: 0.9, // Curvature condition
|
||
tolerance: 1e-5, // Convergence tolerance
|
||
weight_decay: 0.01, // L2 regularization
|
||
};
|
||
|
||
println!("Configuration:");
|
||
println!(" Learning rate: {}", lbfgs_config.learning_rate);
|
||
println!(" History size: {}", lbfgs_config.history_size);
|
||
println!(" Tolerance: {}", lbfgs_config.tolerance);
|
||
println!(" Weight decay: {}", lbfgs_config.weight_decay);
|
||
|
||
// Create optimizer
|
||
let optimizer = LBFGSOptimizer::new(lbfgs_config)?;
|
||
|
||
println!("\\nOptimizer created successfully!");
|
||
println!(" Type: {}", optimizer.optimizer_type());
|
||
println!(" History capacity: {}", optimizer.get_history_size());
|
||
|
||
// Example: Using with optimizer factory
|
||
let factory_config = OptimizerConfig::LBFGS(LBFGSConfig::default());
|
||
let factory_optimizer =
|
||
crate::optimizers::create_optimizer(factory_config, HashMap::new())?;
|
||
|
||
println!("\\nFactory integration works!");
|
||
println!(
|
||
" Created optimizer type: {}",
|
||
factory_optimizer.optimizer_type()
|
||
);
|
||
|
||
println!("\\nL-BFGS Key Features:");
|
||
println!(" ✓ Two-loop recursion for search direction computation");
|
||
println!(" ✓ Limited memory storage of gradient/parameter changes");
|
||
println!(" ✓ Strong Wolfe line search for step size selection");
|
||
println!(" ✓ Automatic initial inverse Hessian scaling");
|
||
println!(" ✓ Memory-efficient circular buffer implementation");
|
||
println!(" ✓ Support for L2 regularization (weight decay)");
|
||
println!(" ✓ Convergence detection based on gradient norm");
|
||
|
||
println!("\\nAlgorithm Overview:");
|
||
println!(" 1. Store recent {{s_k, y_k}} pairs (parameter & gradient changes)");
|
||
println!(" 2. Compute search direction using two-loop recursion");
|
||
println!(" 3. Perform line search to satisfy Wolfe conditions");
|
||
println!(" 4. Update parameters: x = x + α * search_direction");
|
||
println!(" 5. Update history with new {{s_k, y_k}} pair");
|
||
|
||
println!("\\nConvergence Properties:");
|
||
println!(" • Superlinear convergence on smooth strongly convex problems");
|
||
println!(" • Often effective on non-convex optimization landscapes");
|
||
println!(" • Memory-efficient: O(mn) storage vs O(n²) for full BFGS");
|
||
println!(" • Suitable for high-dimensional problems like neural networks");
|
||
|
||
println!("\\nL-BFGS demo completed successfully!");
|
||
|
||
Ok(())
|
||
}
|
||
}
|