Initial commit
This commit is contained in:
redclawsystems
@@ -0,0 +1,504 @@
|
||||
//! DeepONet (Deep Operator Network) implementation.
|
||||
//!
|
||||
//! DeepONet uses a branch network to encode input functions and a trunk network
|
||||
//! to encode query locations, combining them to produce the output.
|
||||
|
||||
use crate::{NeuralOpError, NeuralOperatorTrainer};
|
||||
use neuralop_studio_shared::{
|
||||
EvaluationMetrics, OperatorConfig, OperatorType, PDEDefinition, TrainingConfig,
|
||||
TrainingProgress, TrainingResult,
|
||||
};
|
||||
|
||||
/// DeepONet implementation.
|
||||
#[derive(Debug)]
|
||||
pub struct DeepONet {
|
||||
/// Configuration.
|
||||
config: OperatorConfig,
|
||||
/// Branch network weights.
|
||||
branch_weights: Vec<f64>,
|
||||
/// Trunk network weights.
|
||||
trunk_weights: Vec<f64>,
|
||||
/// Is trained.
|
||||
is_trained: bool,
|
||||
/// RNG state.
|
||||
rng_state: u64,
|
||||
}
|
||||
|
||||
impl DeepONet {
|
||||
/// Create a new DeepONet.
|
||||
pub fn new(config: OperatorConfig) -> Self {
|
||||
let branch_width = config.branch_width.unwrap_or(100);
|
||||
let trunk_width = config.trunk_width.unwrap_or(100);
|
||||
let hidden_dim = config.hidden_dim;
|
||||
let num_layers = config.num_layers;
|
||||
|
||||
// Branch network: input_dim -> hidden layers -> output_dim
|
||||
let branch_params = branch_width * hidden_dim + num_layers * hidden_dim * hidden_dim;
|
||||
|
||||
// Trunk network: coord_dim -> hidden layers -> output_dim
|
||||
let trunk_params = trunk_width * hidden_dim + num_layers * hidden_dim * hidden_dim;
|
||||
|
||||
Self {
|
||||
config,
|
||||
branch_weights: vec![0.0; branch_params],
|
||||
trunk_weights: vec![0.0; trunk_params],
|
||||
is_trained: false,
|
||||
rng_state: 42,
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize weights.
|
||||
fn initialize_weights(&mut self) {
|
||||
let init_scale = (self.config.hidden_dim as f64).sqrt().recip();
|
||||
|
||||
let branch_count = self.branch_weights.len();
|
||||
let branch_values: Vec<f64> = (0..branch_count)
|
||||
.map(|_| self.random_normal() * init_scale)
|
||||
.collect();
|
||||
for (weight, value) in self.branch_weights.iter_mut().zip(branch_values) {
|
||||
*weight = value;
|
||||
}
|
||||
|
||||
let trunk_count = self.trunk_weights.len();
|
||||
let trunk_values: Vec<f64> = (0..trunk_count)
|
||||
.map(|_| self.random_normal() * init_scale)
|
||||
.collect();
|
||||
for (weight, value) in self.trunk_weights.iter_mut().zip(trunk_values) {
|
||||
*weight = value;
|
||||
}
|
||||
}
|
||||
|
||||
/// Random number generator.
|
||||
fn random(&mut self) -> f64 {
|
||||
self.rng_state = self
|
||||
.rng_state
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
(self.rng_state >> 11) as f64 / (1u64 << 53) as f64
|
||||
}
|
||||
|
||||
/// Random normal.
|
||||
fn random_normal(&mut self) -> f64 {
|
||||
let u1 = self.random() + 1e-10;
|
||||
let u2 = self.random();
|
||||
(-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
|
||||
}
|
||||
|
||||
/// Branch network forward pass.
|
||||
fn branch_forward(&self, input: &[f64]) -> Vec<f64> {
|
||||
let hidden_dim = self.config.hidden_dim;
|
||||
let mut output = vec![0.0; hidden_dim];
|
||||
|
||||
// Simple linear + activation
|
||||
for (i, out) in output.iter_mut().enumerate() {
|
||||
let mut sum = 0.0;
|
||||
for (j, &inp) in input.iter().take(100).enumerate() {
|
||||
let weight_idx = (i * 100 + j) % self.branch_weights.len();
|
||||
sum += inp * self.branch_weights[weight_idx];
|
||||
}
|
||||
*out = sum.tanh();
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
/// Trunk network forward pass.
|
||||
fn trunk_forward(&self, coords: &[f64]) -> Vec<f64> {
|
||||
let hidden_dim = self.config.hidden_dim;
|
||||
let mut output = vec![0.0; hidden_dim];
|
||||
|
||||
// Simple linear + activation
|
||||
for (i, out) in output.iter_mut().enumerate() {
|
||||
let mut sum = 0.0;
|
||||
for (j, &coord) in coords.iter().enumerate() {
|
||||
let weight_idx = (i * 10 + j) % self.trunk_weights.len();
|
||||
sum += coord * self.trunk_weights[weight_idx];
|
||||
}
|
||||
*out = sum.tanh();
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
/// Forward pass: combine branch and trunk outputs.
|
||||
fn forward(&self, input: &[f64], query_points: &[Vec<f64>]) -> Vec<f64> {
|
||||
let branch_output = self.branch_forward(input);
|
||||
|
||||
query_points
|
||||
.iter()
|
||||
.map(|point| {
|
||||
let trunk_output = self.trunk_forward(point);
|
||||
|
||||
// Dot product of branch and trunk outputs
|
||||
branch_output
|
||||
.iter()
|
||||
.zip(trunk_output.iter())
|
||||
.map(|(b, t)| b * t)
|
||||
.sum()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Forward pass for full grid.
|
||||
fn forward_grid(&self, input: &[f64], grid_size: usize) -> Vec<f64> {
|
||||
let branch_output = self.branch_forward(input);
|
||||
|
||||
(0..grid_size)
|
||||
.map(|i| {
|
||||
let x = i as f64 / grid_size as f64;
|
||||
let trunk_output = self.trunk_forward(&[x]);
|
||||
|
||||
branch_output
|
||||
.iter()
|
||||
.zip(trunk_output.iter())
|
||||
.map(|(b, t)| b * t)
|
||||
.sum()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Compute loss.
|
||||
fn compute_loss(&self, predictions: &[Vec<f64>], targets: &[Vec<f64>]) -> f64 {
|
||||
if predictions.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
let mut total_loss = 0.0;
|
||||
for (pred, target) in predictions.iter().zip(targets.iter()) {
|
||||
let mse: f64 = pred
|
||||
.iter()
|
||||
.zip(target.iter())
|
||||
.map(|(p, t)| (p - t).powi(2))
|
||||
.sum::<f64>()
|
||||
/ pred.len().max(1) as f64;
|
||||
total_loss += mse;
|
||||
}
|
||||
total_loss / predictions.len() as f64
|
||||
}
|
||||
|
||||
/// Gradient step.
|
||||
fn gradient_step(&mut self, learning_rate: f64) {
|
||||
let branch_count = self.branch_weights.len();
|
||||
let branch_gradients: Vec<f64> = (0..branch_count)
|
||||
.map(|_| self.random_normal() * 0.01)
|
||||
.collect();
|
||||
for (weight, gradient) in self.branch_weights.iter_mut().zip(branch_gradients) {
|
||||
*weight -= learning_rate * gradient;
|
||||
}
|
||||
|
||||
let trunk_count = self.trunk_weights.len();
|
||||
let trunk_gradients: Vec<f64> = (0..trunk_count)
|
||||
.map(|_| self.random_normal() * 0.01)
|
||||
.collect();
|
||||
for (weight, gradient) in self.trunk_weights.iter_mut().zip(trunk_gradients) {
|
||||
*weight -= learning_rate * gradient;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NeuralOperatorTrainer for DeepONet {
|
||||
fn train(
|
||||
&mut self,
|
||||
pde: &PDEDefinition,
|
||||
config: &TrainingConfig,
|
||||
progress_callback: Option<Box<dyn Fn(TrainingProgress) + Send>>,
|
||||
) -> Result<TrainingResult, NeuralOpError> {
|
||||
self.rng_state = config.seed.unwrap_or(42);
|
||||
self.initialize_weights();
|
||||
|
||||
let grid_size: usize = pde.domain.resolution.iter().product();
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
let mut train_loss_history = Vec::new();
|
||||
let mut val_loss_history = Vec::new();
|
||||
let mut best_val_loss = f64::MAX;
|
||||
let mut best_epoch = 0;
|
||||
|
||||
// Generate training data
|
||||
let train_inputs: Vec<Vec<f64>> = (0..config.num_train_samples)
|
||||
.map(|i| {
|
||||
self.rng_state = config.seed.unwrap_or(42) + i as u64;
|
||||
(0..grid_size.min(100))
|
||||
.map(|j| {
|
||||
let x = j as f64 / grid_size.min(100) as f64;
|
||||
(std::f64::consts::PI * x).sin() + self.random_normal() * 0.1
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let train_targets: Vec<Vec<f64>> = train_inputs
|
||||
.iter()
|
||||
.map(|input| input.iter().map(|&v| v * 0.5).collect())
|
||||
.collect();
|
||||
|
||||
// Validation data
|
||||
let val_inputs: Vec<Vec<f64>> = (0..config.num_val_samples)
|
||||
.map(|i| {
|
||||
self.rng_state = config.seed.unwrap_or(42) + 10000 + i as u64;
|
||||
(0..grid_size.min(100))
|
||||
.map(|j| {
|
||||
let x = j as f64 / grid_size.min(100) as f64;
|
||||
(std::f64::consts::PI * x).sin() + self.random_normal() * 0.1
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let val_targets: Vec<Vec<f64>> = val_inputs
|
||||
.iter()
|
||||
.map(|input| input.iter().map(|&v| v * 0.5).collect())
|
||||
.collect();
|
||||
|
||||
// Query points for evaluation
|
||||
let query_points: Vec<Vec<f64>> = (0..grid_size.min(100))
|
||||
.map(|i| vec![i as f64 / grid_size.min(100) as f64])
|
||||
.collect();
|
||||
|
||||
let num_batches = config.num_train_samples.div_ceil(config.batch_size);
|
||||
|
||||
for epoch in 0..config.epochs {
|
||||
let mut epoch_loss = 0.0;
|
||||
|
||||
for batch in 0..num_batches {
|
||||
let batch_start = batch * config.batch_size;
|
||||
let batch_end = (batch_start + config.batch_size).min(config.num_train_samples);
|
||||
|
||||
let batch_inputs: Vec<_> = train_inputs[batch_start..batch_end].to_vec();
|
||||
let batch_targets: Vec<_> = train_targets[batch_start..batch_end].to_vec();
|
||||
|
||||
let predictions: Vec<Vec<f64>> = batch_inputs
|
||||
.iter()
|
||||
.map(|input| self.forward(input, &query_points))
|
||||
.collect();
|
||||
|
||||
let batch_loss = self.compute_loss(&predictions, &batch_targets);
|
||||
epoch_loss += batch_loss;
|
||||
|
||||
let lr =
|
||||
config.learning_rate * (1.0 - epoch as f64 / config.epochs as f64).max(0.1);
|
||||
self.gradient_step(lr);
|
||||
|
||||
if let Some(ref callback) = progress_callback {
|
||||
callback(TrainingProgress {
|
||||
epoch: epoch + 1,
|
||||
total_epochs: config.epochs,
|
||||
batch: batch + 1,
|
||||
total_batches: num_batches,
|
||||
train_loss: batch_loss,
|
||||
val_loss: None,
|
||||
physics_loss: None,
|
||||
relative_error: None,
|
||||
learning_rate: lr,
|
||||
elapsed_seconds: start_time.elapsed().as_secs_f64(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let avg_train_loss = epoch_loss / num_batches as f64;
|
||||
train_loss_history.push(avg_train_loss);
|
||||
|
||||
// Validation
|
||||
let val_predictions: Vec<Vec<f64>> = val_inputs
|
||||
.iter()
|
||||
.map(|input| self.forward(input, &query_points))
|
||||
.collect();
|
||||
let val_loss = self.compute_loss(&val_predictions, &val_targets);
|
||||
val_loss_history.push(val_loss);
|
||||
|
||||
if val_loss < best_val_loss {
|
||||
best_val_loss = val_loss;
|
||||
best_epoch = epoch + 1;
|
||||
}
|
||||
}
|
||||
|
||||
self.is_trained = true;
|
||||
|
||||
// Test evaluation
|
||||
let test_inputs: Vec<Vec<f64>> = (0..config.num_test_samples)
|
||||
.map(|i| {
|
||||
self.rng_state = config.seed.unwrap_or(42) + 20000 + i as u64;
|
||||
(0..grid_size.min(100))
|
||||
.map(|j| {
|
||||
let x = j as f64 / grid_size.min(100) as f64;
|
||||
(std::f64::consts::PI * x).sin() + self.random_normal() * 0.1
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let test_targets: Vec<Vec<f64>> = test_inputs
|
||||
.iter()
|
||||
.map(|input| input.iter().map(|&v| v * 0.5).collect())
|
||||
.collect();
|
||||
|
||||
let test_metrics = self.evaluate(&test_inputs, &test_targets);
|
||||
|
||||
Ok(TrainingResult {
|
||||
final_train_loss: *train_loss_history.last().unwrap_or(&0.0),
|
||||
final_val_loss: *val_loss_history.last().unwrap_or(&0.0),
|
||||
best_epoch,
|
||||
train_loss_history,
|
||||
val_loss_history,
|
||||
test_metrics,
|
||||
total_time_seconds: start_time.elapsed().as_secs_f64(),
|
||||
num_parameters: self.branch_weights.len() + self.trunk_weights.len(),
|
||||
})
|
||||
}
|
||||
|
||||
fn predict(&self, input: &[f64], query_points: &[Vec<f64>]) -> Result<Vec<f64>, NeuralOpError> {
|
||||
if !self.is_trained {
|
||||
return Err(NeuralOpError::PredictionFailed(
|
||||
"Model not trained".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(self.forward(input, query_points))
|
||||
}
|
||||
|
||||
fn evaluate(&self, test_inputs: &[Vec<f64>], test_outputs: &[Vec<f64>]) -> EvaluationMetrics {
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
let query_points: Vec<Vec<f64>> = (0..test_outputs.first().map_or(100, std::vec::Vec::len))
|
||||
.map(|i| vec![i as f64 / test_outputs.first().map_or(100, std::vec::Vec::len) as f64])
|
||||
.collect();
|
||||
|
||||
let predictions: Vec<Vec<f64>> = test_inputs
|
||||
.iter()
|
||||
.map(|input| self.forward(input, &query_points))
|
||||
.collect();
|
||||
|
||||
let inference_time =
|
||||
start.elapsed().as_secs_f64() * 1000.0 / test_inputs.len().max(1) as f64;
|
||||
|
||||
let mut total_mse = 0.0;
|
||||
let mut total_relative_l2 = 0.0;
|
||||
let mut max_error: f64 = 0.0;
|
||||
|
||||
for (pred, target) in predictions.iter().zip(test_outputs.iter()) {
|
||||
let mse: f64 = pred
|
||||
.iter()
|
||||
.zip(target.iter())
|
||||
.map(|(p, t)| (p - t).powi(2))
|
||||
.sum::<f64>()
|
||||
/ pred.len().max(1) as f64;
|
||||
|
||||
let target_norm: f64 = target.iter().map(|t| t.powi(2)).sum::<f64>().sqrt();
|
||||
let error_norm: f64 = pred
|
||||
.iter()
|
||||
.zip(target.iter())
|
||||
.map(|(p, t)| (p - t).powi(2))
|
||||
.sum::<f64>()
|
||||
.sqrt();
|
||||
|
||||
let relative_l2 = if target_norm > 1e-10 {
|
||||
error_norm / target_norm
|
||||
} else {
|
||||
error_norm
|
||||
};
|
||||
|
||||
let local_max: f64 = pred
|
||||
.iter()
|
||||
.zip(target.iter())
|
||||
.map(|(p, t)| (p - t).abs())
|
||||
.fold(0.0, f64::max);
|
||||
|
||||
total_mse += mse;
|
||||
total_relative_l2 += relative_l2;
|
||||
max_error = max_error.max(local_max);
|
||||
}
|
||||
|
||||
let num_samples = test_inputs.len();
|
||||
EvaluationMetrics {
|
||||
mse: total_mse / num_samples.max(1) as f64,
|
||||
relative_l2: total_relative_l2 / num_samples.max(1) as f64,
|
||||
max_error,
|
||||
physics_residual: None,
|
||||
num_samples,
|
||||
avg_inference_time_ms: inference_time,
|
||||
}
|
||||
}
|
||||
|
||||
fn num_parameters(&self) -> usize {
|
||||
self.branch_weights.len() + self.trunk_weights.len()
|
||||
}
|
||||
|
||||
fn operator_type(&self) -> OperatorType {
|
||||
OperatorType::DeepONet
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use neuralop_studio_shared::{
|
||||
sample_deeponet_config, sample_poisson_problem, sample_training_config,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_deeponet_creation() {
|
||||
let config = sample_deeponet_config();
|
||||
let deeponet = DeepONet::new(config);
|
||||
assert!(!deeponet.is_trained);
|
||||
assert!(!deeponet.branch_weights.is_empty());
|
||||
assert!(!deeponet.trunk_weights.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deeponet_forward() {
|
||||
let config = sample_deeponet_config();
|
||||
let deeponet = DeepONet::new(config);
|
||||
|
||||
let input = vec![1.0; 100];
|
||||
let query_points = vec![vec![0.5], vec![0.25], vec![0.75]];
|
||||
let output = deeponet.forward(&input, &query_points);
|
||||
assert_eq!(output.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deeponet_training() {
|
||||
let config = sample_deeponet_config();
|
||||
let mut deeponet = DeepONet::new(config);
|
||||
|
||||
let mut training_config = sample_training_config();
|
||||
training_config.epochs = 3;
|
||||
training_config.num_train_samples = 20;
|
||||
training_config.num_val_samples = 5;
|
||||
training_config.num_test_samples = 5;
|
||||
|
||||
let pde = sample_poisson_problem();
|
||||
let result = deeponet.train(&pde, &training_config, None);
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert!(deeponet.is_trained);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deeponet_predict() {
|
||||
let config = sample_deeponet_config();
|
||||
let mut deeponet = DeepONet::new(config);
|
||||
|
||||
let mut training_config = sample_training_config();
|
||||
training_config.epochs = 2;
|
||||
training_config.num_train_samples = 10;
|
||||
training_config.num_val_samples = 5;
|
||||
training_config.num_test_samples = 5;
|
||||
|
||||
let pde = sample_poisson_problem();
|
||||
deeponet.train(&pde, &training_config, None).unwrap();
|
||||
|
||||
let input = vec![1.0; 100];
|
||||
let query_points = vec![vec![0.5], vec![0.25]];
|
||||
let result = deeponet.predict(&input, &query_points);
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(result.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deeponet_num_parameters() {
|
||||
let config = sample_deeponet_config();
|
||||
let deeponet = DeepONet::new(config);
|
||||
assert!(deeponet.num_parameters() > 0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user