Initial commit
This commit is contained in:
redclawsystems
@@ -0,0 +1,675 @@
|
||||
//! `LogCoshLoss` Implementation for RTX Transformers
|
||||
//!
|
||||
//! `LogCoshLoss` (Log-Cosh Loss) provides a smooth alternative to Huber loss that's twice
|
||||
//! differentiable everywhere. It combines the smoothness of MSE near zero with the
|
||||
//! robustness of MAE for large errors.
|
||||
//!
|
||||
//! Mathematical Definition:
|
||||
//! `L(y_pred`, `y_true`) = `log(cosh(y_pred` - `y_true`))
|
||||
//!
|
||||
//! Key Properties:
|
||||
//! - Smooth everywhere (infinitely differentiable)
|
||||
//! - Approximately quadratic for small errors: log(cosh(x)) ≈ x²/2
|
||||
//! - Approximately linear for large errors: log(cosh(x)) ≈ |x| - log(2)
|
||||
//! - Twice differentiable (unlike Huber loss)
|
||||
//! - More robust to outliers than MSE, smoother than Huber
|
||||
|
||||
use super::{Loss, Reduction};
|
||||
use crate::{Result, TransformerError};
|
||||
use rtx_tensor::Tensor;
|
||||
|
||||
/// Characteristics of the `LogCosh` loss function at a specific point
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LossCharacteristics {
|
||||
/// The error value at this point
|
||||
pub error: f32,
|
||||
/// The loss value at this point
|
||||
pub loss_value: f32,
|
||||
/// The first derivative (gradient) at this point
|
||||
pub derivative: f32,
|
||||
/// The second derivative (curvature) at this point
|
||||
pub second_derivative: f32,
|
||||
/// Whether this point is in the quadratic approximation region
|
||||
pub is_quadratic_region: bool,
|
||||
/// Whether this point is in the linear approximation region
|
||||
pub is_linear_region: bool,
|
||||
/// Quadratic approximation value
|
||||
pub quadratic_approx: f32,
|
||||
/// Linear approximation value
|
||||
pub linear_approx: f32,
|
||||
/// Error of quadratic approximation
|
||||
pub quadratic_error: f32,
|
||||
/// Error of linear approximation
|
||||
pub linear_error: f32,
|
||||
}
|
||||
|
||||
/// Statistics about the `LogCosh` loss landscape
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LossStatistics {
|
||||
/// Number of samples analyzed
|
||||
pub num_samples: usize,
|
||||
/// Mean loss value across all samples
|
||||
pub mean_loss: f32,
|
||||
/// Mean gradient value across all samples
|
||||
pub mean_gradient: f32,
|
||||
/// Mean error value across all samples
|
||||
pub mean_error: f32,
|
||||
/// Variance of loss values
|
||||
pub loss_variance: f32,
|
||||
/// Fraction of samples in quadratic region
|
||||
pub quadratic_region_fraction: f32,
|
||||
/// Fraction of samples in linear region
|
||||
pub linear_region_fraction: f32,
|
||||
/// Maximum loss value
|
||||
pub max_loss: f32,
|
||||
/// Minimum loss value
|
||||
pub min_loss: f32,
|
||||
}
|
||||
|
||||
/// `LogCoshLoss` for smooth robust regression
|
||||
///
|
||||
/// This loss function provides a smooth, twice-differentiable alternative to Huber loss.
|
||||
/// It's particularly useful when smooth gradients are important for optimization.
|
||||
///
|
||||
/// # Mathematical Properties
|
||||
///
|
||||
/// For small errors |x| < 1: log(cosh(x)) ≈ x²/2 (quadratic like MSE)
|
||||
/// For large errors |x| > 3: log(cosh(x)) ≈ |x| - log(2) (linear like MAE)
|
||||
///
|
||||
/// # Usage
|
||||
///
|
||||
/// ```rust
|
||||
/// use rtx_transformers::losses::{LogCoshLoss, Loss, Reduction};
|
||||
/// use rtx_tensor::{Tensor, Device};
|
||||
///
|
||||
/// let loss = LogCoshLoss::new()
|
||||
/// .with_reduction(Reduction::Mean);
|
||||
///
|
||||
/// let device = Device::cuda(0).unwrap_or(Device::default());
|
||||
/// let predictions = Tensor::new(&[1.0f32, 2.0, 3.0], &device)?;
|
||||
/// let targets = Tensor::new(&[1.2f32, 1.8, 3.5], &device)?;
|
||||
///
|
||||
/// let loss_value = loss.forward(&predictions, &targets)?;
|
||||
/// # Ok::<(), Box<dyn std::error::Error>>(())
|
||||
/// ```
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LogCoshLoss {
|
||||
reduction: Reduction,
|
||||
}
|
||||
|
||||
impl LogCoshLoss {
|
||||
/// Create a new `LogCoshLoss` with default parameters
|
||||
///
|
||||
/// Default: Mean reduction
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
reduction: Reduction::Mean,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the reduction mode
|
||||
#[must_use]
|
||||
pub fn with_reduction(mut self, reduction: Reduction) -> Self {
|
||||
self.reduction = reduction;
|
||||
self
|
||||
}
|
||||
|
||||
/// Compute numerically stable log(cosh(x))
|
||||
///
|
||||
/// For large |x|, cosh(x) can overflow. We use the identity:
|
||||
/// log(cosh(x)) = |x| + log((1 + exp(-2|x|))/2)
|
||||
/// For large |x|, this approaches |x| - log(2)
|
||||
fn stable_logcosh(&self, x: f32) -> f32 {
|
||||
let abs_x = x.abs();
|
||||
|
||||
if abs_x < 12.0 {
|
||||
// For moderate values, compute directly
|
||||
x.cosh().ln()
|
||||
} else {
|
||||
// For large values, use stable approximation
|
||||
// log(cosh(x)) ≈ |x| - log(2) for large |x|
|
||||
abs_x - 2.0f32.ln()
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if error is in the approximately quadratic region
|
||||
#[must_use]
|
||||
pub fn is_in_quadratic_region(&self, error: f32) -> bool {
|
||||
error.abs() < 1.0
|
||||
}
|
||||
|
||||
/// Check if error is in the approximately linear region
|
||||
#[must_use]
|
||||
pub fn is_in_linear_region(&self, error: f32) -> bool {
|
||||
error.abs() > 3.0
|
||||
}
|
||||
|
||||
/// Get the quadratic approximation for small errors
|
||||
#[must_use]
|
||||
pub fn quadratic_approximation(&self, error: f32) -> f32 {
|
||||
0.5 * error * error
|
||||
}
|
||||
|
||||
/// Get the linear approximation for large errors
|
||||
#[must_use]
|
||||
pub fn linear_approximation(&self, error: f32) -> f32 {
|
||||
error.abs() - 2.0f32.ln()
|
||||
}
|
||||
|
||||
/// Compute the derivative of `LogCoshLoss`
|
||||
/// d/dx log(cosh(x)) = tanh(x)
|
||||
#[must_use]
|
||||
pub fn derivative(&self, error: f32) -> f32 {
|
||||
error.tanh()
|
||||
}
|
||||
|
||||
/// Compute the second derivative of `LogCoshLoss`
|
||||
/// d²/dx² log(cosh(x)) = sech²(x) = 1 - tanh²(x)
|
||||
#[must_use]
|
||||
pub fn second_derivative(&self, error: f32) -> f32 {
|
||||
let tanh_x = error.tanh();
|
||||
1.0 - tanh_x * tanh_x
|
||||
}
|
||||
|
||||
/// Compute loss value for a single error (useful for testing)
|
||||
#[must_use]
|
||||
pub fn compute_single_loss(&self, error: f32) -> f32 {
|
||||
self.stable_logcosh(error)
|
||||
}
|
||||
|
||||
/// Get approximation error for quadratic region
|
||||
#[must_use]
|
||||
pub fn quadratic_approximation_error(&self, error: f32) -> f32 {
|
||||
let true_loss = self.compute_single_loss(error);
|
||||
let approx_loss = self.quadratic_approximation(error);
|
||||
|
||||
if true_loss == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
((true_loss - approx_loss) / true_loss).abs()
|
||||
}
|
||||
}
|
||||
|
||||
/// Get approximation error for linear region
|
||||
#[must_use]
|
||||
pub fn linear_approximation_error(&self, error: f32) -> f32 {
|
||||
let true_loss = self.compute_single_loss(error);
|
||||
let approx_loss = self.linear_approximation(error);
|
||||
|
||||
if true_loss == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
((true_loss - approx_loss) / true_loss).abs()
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if `LogCoshLoss` is more robust than MSE for given error
|
||||
#[must_use]
|
||||
pub fn robustness_vs_mse(&self, error: f32) -> f32 {
|
||||
let logcosh_loss = self.compute_single_loss(error);
|
||||
let mse_loss = 0.5 * error * error;
|
||||
|
||||
// Return ratio - smaller values indicate better robustness
|
||||
if mse_loss == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
logcosh_loss / mse_loss
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute numerical gradient using finite differences (for testing)
|
||||
#[must_use]
|
||||
pub fn numerical_gradient(&self, error: f32, h: f32) -> f32 {
|
||||
let loss_plus = self.compute_single_loss(error + h);
|
||||
let loss_minus = self.compute_single_loss(error - h);
|
||||
(loss_plus - loss_minus) / (2.0 * h)
|
||||
}
|
||||
|
||||
/// Verify gradient correctness by comparing analytical vs numerical
|
||||
#[must_use]
|
||||
pub fn verify_gradient(&self, error: f32, tolerance: f32) -> bool {
|
||||
let analytical_grad = self.derivative(error);
|
||||
let numerical_grad = self.numerical_gradient(error, 1e-5);
|
||||
(analytical_grad - numerical_grad).abs() < tolerance
|
||||
}
|
||||
|
||||
/// Get loss function characteristics at a given point
|
||||
#[must_use]
|
||||
pub fn get_characteristics(&self, error: f32) -> LossCharacteristics {
|
||||
LossCharacteristics {
|
||||
error,
|
||||
loss_value: self.compute_single_loss(error),
|
||||
derivative: self.derivative(error),
|
||||
second_derivative: self.second_derivative(error),
|
||||
is_quadratic_region: self.is_in_quadratic_region(error),
|
||||
is_linear_region: self.is_in_linear_region(error),
|
||||
quadratic_approx: self.quadratic_approximation(error),
|
||||
linear_approx: self.linear_approximation(error),
|
||||
quadratic_error: self.quadratic_approximation_error(error),
|
||||
linear_error: self.linear_approximation_error(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the transition point where quadratic and linear approximations have equal accuracy
|
||||
#[must_use]
|
||||
pub fn transition_point() -> f32 {
|
||||
1.2
|
||||
}
|
||||
|
||||
/// Get the stability threshold above which we use the stable approximation
|
||||
#[must_use]
|
||||
pub fn stability_threshold() -> f32 {
|
||||
12.0
|
||||
}
|
||||
|
||||
/// Compute `LogCosh` loss element-wise using stable computation
|
||||
fn compute_logcosh_elementwise(&self, errors: &Tensor) -> Result<Tensor> {
|
||||
// Convert errors to CPU for computation
|
||||
let error_data = errors.to_cpu()?;
|
||||
|
||||
// Compute log(cosh(x)) for each element
|
||||
let mut result_data = Vec::new();
|
||||
for &error in &error_data {
|
||||
let logcosh_value = self.stable_logcosh(error);
|
||||
result_data.push(logcosh_value);
|
||||
}
|
||||
|
||||
// Create result tensor with same shape and device
|
||||
Ok(Tensor::from_data(
|
||||
result_data,
|
||||
errors.shape().dims(),
|
||||
errors.device(),
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Apply the specified reduction to the loss values
|
||||
fn apply_reduction(&self, losses: &Tensor) -> Result<Tensor> {
|
||||
match self.reduction {
|
||||
Reduction::None => Ok(losses.clone()),
|
||||
Reduction::Mean => {
|
||||
// Compute mean by summing all elements and dividing by count
|
||||
let sum_result = losses.sum(None)?;
|
||||
let numel = losses.numel() as f32;
|
||||
Ok(sum_result.div_scalar(numel)?)
|
||||
}
|
||||
Reduction::Sum => Ok(losses.sum(None)?),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute `LogCosh` loss with optional sample weights
|
||||
pub fn forward_weighted(
|
||||
&self,
|
||||
predictions: &Tensor,
|
||||
targets: &Tensor,
|
||||
weights: Option<&Tensor>,
|
||||
) -> Result<Tensor> {
|
||||
// Validate input shapes
|
||||
if predictions.shape() != targets.shape() {
|
||||
return Err(TransformerError::InvalidInput(format!(
|
||||
"Shape mismatch: predictions {:?} vs targets {:?}",
|
||||
predictions.shape(),
|
||||
targets.shape()
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(w) = weights
|
||||
&& w.shape() != predictions.shape() {
|
||||
return Err(TransformerError::InvalidInput(format!(
|
||||
"Weight shape {:?} doesn't match predictions shape {:?}",
|
||||
w.shape(),
|
||||
predictions.shape()
|
||||
)));
|
||||
}
|
||||
|
||||
// Compute element-wise errors
|
||||
let errors = (predictions - targets)?;
|
||||
|
||||
// Compute LogCosh loss element-wise
|
||||
let logcosh_values = self.compute_logcosh_elementwise(&errors)?;
|
||||
|
||||
// Apply weights if provided
|
||||
let weighted_losses = if let Some(weights) = weights {
|
||||
// Element-wise multiplication with weights
|
||||
self.multiply_tensors(&logcosh_values, weights)?
|
||||
} else {
|
||||
logcosh_values
|
||||
};
|
||||
|
||||
// Apply reduction
|
||||
self.apply_reduction(&weighted_losses)
|
||||
}
|
||||
|
||||
/// Helper method to multiply tensors element-wise
|
||||
fn multiply_tensors(&self, a: &Tensor, b: &Tensor) -> Result<Tensor> {
|
||||
let a_data = a.to_cpu()?;
|
||||
let b_data = b.to_cpu()?;
|
||||
|
||||
if a_data.len() != b_data.len() {
|
||||
return Err(TransformerError::InvalidInput(
|
||||
"Tensors must have same number of elements".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let result_data: Vec<f32> = a_data
|
||||
.iter()
|
||||
.zip(b_data.iter())
|
||||
.map(|(&x, &y)| x * y)
|
||||
.collect();
|
||||
|
||||
Ok(Tensor::from_data(
|
||||
result_data,
|
||||
a.shape().dims(),
|
||||
a.device(),
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Compute `LogCosh` loss with Huber-style delta parameter for comparison
|
||||
/// This allows for adaptive behavior similar to Huber loss
|
||||
pub fn forward_adaptive(
|
||||
&self,
|
||||
predictions: &Tensor,
|
||||
targets: &Tensor,
|
||||
delta: f32,
|
||||
) -> Result<Tensor> {
|
||||
// Validate inputs
|
||||
if predictions.shape() != targets.shape() {
|
||||
return Err(TransformerError::InvalidInput(format!(
|
||||
"Shape mismatch: predictions {:?} vs targets {:?}",
|
||||
predictions.shape(),
|
||||
targets.shape()
|
||||
)));
|
||||
}
|
||||
|
||||
// Compute element-wise errors
|
||||
let errors = (predictions - targets)?;
|
||||
|
||||
// Compute adaptive LogCosh loss
|
||||
let adaptive_losses = self.compute_adaptive_logcosh(&errors, delta)?;
|
||||
|
||||
// Apply reduction
|
||||
self.apply_reduction(&adaptive_losses)
|
||||
}
|
||||
|
||||
/// Compute adaptive `LogCosh` loss with scaling parameter
|
||||
fn compute_adaptive_logcosh(&self, errors: &Tensor, delta: f32) -> Result<Tensor> {
|
||||
let error_data = errors.to_cpu()?;
|
||||
|
||||
let mut result_data = Vec::new();
|
||||
for &error in &error_data {
|
||||
// Scale error by delta for adaptive behavior
|
||||
let scaled_error = error / delta;
|
||||
let logcosh_value = self.stable_logcosh(scaled_error) * delta;
|
||||
result_data.push(logcosh_value);
|
||||
}
|
||||
|
||||
Ok(Tensor::from_data(
|
||||
result_data,
|
||||
errors.shape().dims(),
|
||||
errors.device(),
|
||||
)?)
|
||||
}
|
||||
|
||||
/// Compute statistics about the loss landscape
|
||||
pub fn compute_loss_statistics(
|
||||
&self,
|
||||
predictions: &Tensor,
|
||||
targets: &Tensor,
|
||||
) -> Result<LossStatistics> {
|
||||
let errors = (predictions - targets)?;
|
||||
let error_data = errors.to_cpu()?;
|
||||
|
||||
if error_data.is_empty() {
|
||||
return Err(TransformerError::InvalidInput("Empty tensor".to_string()));
|
||||
}
|
||||
|
||||
let mut loss_values = Vec::new();
|
||||
let mut gradients = Vec::new();
|
||||
let mut quadratic_count = 0;
|
||||
let mut linear_count = 0;
|
||||
|
||||
for &error in &error_data {
|
||||
let loss = self.compute_single_loss(error);
|
||||
let grad = self.derivative(error);
|
||||
|
||||
loss_values.push(loss);
|
||||
gradients.push(grad);
|
||||
|
||||
if self.is_in_quadratic_region(error) {
|
||||
quadratic_count += 1;
|
||||
}
|
||||
if self.is_in_linear_region(error) {
|
||||
linear_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let n = error_data.len() as f32;
|
||||
let mean_loss = loss_values.iter().sum::<f32>() / n;
|
||||
let mean_gradient = gradients.iter().sum::<f32>() / n;
|
||||
let mean_error = error_data.iter().sum::<f32>() / n;
|
||||
|
||||
// Compute variance
|
||||
let loss_variance = loss_values
|
||||
.iter()
|
||||
.map(|&x| (x - mean_loss).powi(2))
|
||||
.sum::<f32>()
|
||||
/ n;
|
||||
|
||||
Ok(LossStatistics {
|
||||
num_samples: error_data.len(),
|
||||
mean_loss,
|
||||
mean_gradient,
|
||||
mean_error,
|
||||
loss_variance,
|
||||
quadratic_region_fraction: quadratic_count as f32 / n,
|
||||
linear_region_fraction: linear_count as f32 / n,
|
||||
max_loss: loss_values
|
||||
.iter()
|
||||
.copied()
|
||||
.fold(f32::NEG_INFINITY, f32::max),
|
||||
min_loss: loss_values.iter().copied().fold(f32::INFINITY, f32::min),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LogCoshLoss {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Loss for LogCoshLoss {
|
||||
fn forward(&self, predictions: &Tensor, targets: &Tensor) -> Result<Tensor> {
|
||||
// Validate input shapes
|
||||
if predictions.shape() != targets.shape() {
|
||||
return Err(TransformerError::InvalidInput(format!(
|
||||
"Shape mismatch: predictions {:?} vs targets {:?}",
|
||||
predictions.shape(),
|
||||
targets.shape()
|
||||
)));
|
||||
}
|
||||
|
||||
// Validate device compatibility
|
||||
if predictions.device() != targets.device() {
|
||||
return Err(TransformerError::InvalidInput(
|
||||
"Predictions and targets must be on the same device".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Compute element-wise errors
|
||||
let errors = (predictions - targets)?;
|
||||
|
||||
// Compute LogCosh loss element-wise
|
||||
let logcosh_values = self.compute_logcosh_elementwise(&errors)?;
|
||||
|
||||
// Apply reduction
|
||||
self.apply_reduction(&logcosh_values)
|
||||
}
|
||||
|
||||
fn reduction(&self) -> Reduction {
|
||||
self.reduction
|
||||
}
|
||||
|
||||
fn supports_backprop(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"LogCoshLoss"
|
||||
}
|
||||
}
|
||||
|
||||
/// Additional methods for autograd integration and advanced features
|
||||
impl LogCoshLoss {
|
||||
/// Create a `LogCoshLoss` builder for advanced configuration
|
||||
#[must_use]
|
||||
pub fn builder() -> LogCoshLossBuilder {
|
||||
LogCoshLossBuilder::new()
|
||||
}
|
||||
|
||||
/// Check if this loss is equivalent to another `LogCoshLoss`
|
||||
#[must_use]
|
||||
pub fn is_equivalent(&self, other: &Self) -> bool {
|
||||
self.reduction == other.reduction
|
||||
}
|
||||
|
||||
/// Clone with different reduction mode
|
||||
#[must_use]
|
||||
pub fn with_different_reduction(&self, reduction: Reduction) -> Self {
|
||||
Self { reduction }
|
||||
}
|
||||
|
||||
/// Compute loss and return both value and characteristics
|
||||
pub fn forward_with_characteristics(
|
||||
&self,
|
||||
predictions: &Tensor,
|
||||
targets: &Tensor,
|
||||
) -> Result<(Tensor, Vec<LossCharacteristics>)> {
|
||||
let loss_value = self.forward(predictions, targets)?;
|
||||
|
||||
let errors = (predictions - targets)?;
|
||||
let error_data = errors.to_cpu()?;
|
||||
|
||||
let characteristics: Vec<LossCharacteristics> = error_data
|
||||
.iter()
|
||||
.map(|&error| self.get_characteristics(error))
|
||||
.collect();
|
||||
|
||||
Ok((loss_value, characteristics))
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder pattern for `LogCoshLoss` configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LogCoshLossBuilder {
|
||||
reduction: Reduction,
|
||||
}
|
||||
|
||||
impl LogCoshLossBuilder {
|
||||
/// Create a new builder
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
reduction: Reduction::Mean,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the reduction mode
|
||||
#[must_use]
|
||||
pub fn reduction(mut self, reduction: Reduction) -> Self {
|
||||
self.reduction = reduction;
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the `LogCoshLoss`
|
||||
#[must_use]
|
||||
pub fn build(self) -> LogCoshLoss {
|
||||
LogCoshLoss {
|
||||
reduction: self.reduction,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LogCoshLossBuilder {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Comparison utilities for `LogCoshLoss` vs other losses
|
||||
pub struct LossComparison;
|
||||
|
||||
impl LossComparison {
|
||||
/// Compare `LogCosh` vs MSE loss for given errors
|
||||
#[must_use]
|
||||
pub fn logcosh_vs_mse(errors: &[f32]) -> ComparisonResult {
|
||||
let logcosh = LogCoshLoss::new();
|
||||
|
||||
let mut logcosh_losses = Vec::new();
|
||||
let mut mse_losses = Vec::new();
|
||||
|
||||
for &error in errors {
|
||||
logcosh_losses.push(logcosh.compute_single_loss(error));
|
||||
mse_losses.push(0.5 * error * error);
|
||||
}
|
||||
|
||||
let logcosh_mean = logcosh_losses.iter().sum::<f32>() / logcosh_losses.len() as f32;
|
||||
let mse_mean = mse_losses.iter().sum::<f32>() / mse_losses.len() as f32;
|
||||
|
||||
ComparisonResult {
|
||||
logcosh_mean,
|
||||
comparison_mean: mse_mean,
|
||||
robustness_ratio: logcosh_mean / mse_mean.max(1e-8),
|
||||
recommendation: if logcosh_mean < mse_mean {
|
||||
"LogCosh"
|
||||
} else {
|
||||
"MSE"
|
||||
}
|
||||
.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compare `LogCosh` vs Huber loss (simulated)
|
||||
#[must_use]
|
||||
pub fn logcosh_vs_huber(errors: &[f32], delta: f32) -> ComparisonResult {
|
||||
let logcosh = LogCoshLoss::new();
|
||||
|
||||
let mut logcosh_losses = Vec::new();
|
||||
let mut huber_losses = Vec::new();
|
||||
|
||||
for &error in errors {
|
||||
logcosh_losses.push(logcosh.compute_single_loss(error));
|
||||
|
||||
// Simulate Huber loss
|
||||
let huber_loss = if error.abs() <= delta {
|
||||
0.5 * error * error
|
||||
} else {
|
||||
delta * (error.abs() - 0.5 * delta)
|
||||
};
|
||||
huber_losses.push(huber_loss);
|
||||
}
|
||||
|
||||
let logcosh_mean = logcosh_losses.iter().sum::<f32>() / logcosh_losses.len() as f32;
|
||||
let huber_mean = huber_losses.iter().sum::<f32>() / huber_losses.len() as f32;
|
||||
|
||||
ComparisonResult {
|
||||
logcosh_mean,
|
||||
comparison_mean: huber_mean,
|
||||
robustness_ratio: logcosh_mean / huber_mean.max(1e-8),
|
||||
recommendation: "LogCosh (smoother gradients)".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of loss function comparison
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ComparisonResult {
|
||||
/// Mean `LogCosh` loss
|
||||
pub logcosh_mean: f32,
|
||||
/// Mean of comparison loss
|
||||
pub comparison_mean: f32,
|
||||
/// Ratio indicating robustness
|
||||
pub robustness_ratio: f32,
|
||||
/// Recommendation string
|
||||
pub recommendation: String,
|
||||
}
|
||||
Reference in New Issue
Block a user