503 lines
18 KiB
Rust
503 lines
18 KiB
Rust
//! Comprehensive TDD tests for LogCoshLoss implementation in RTX Transformers
|
|
//!
|
|
//! This test module follows strict TDD methodology (red-green-refactor).
|
|
//! LogCoshLoss provides a smooth alternative to Huber loss that's twice differentiable everywhere.
|
|
//!
|
|
//! Mathematical properties:
|
|
//! - L(y_pred, y_true) = mean(log(cosh(y_pred - y_true)))
|
|
//! - Approximates L2 for small errors: log(cosh(x)) ≈ x²/2
|
|
//! - Approximates L1 for large errors: log(cosh(x)) ≈ |x| - log(2)
|
|
//! - Smooth gradient transitions (no kinks like Huber loss)
|
|
//! - Twice differentiable everywhere (unlike Huber loss)
|
|
|
|
use rtx_tensor::{Tensor, Device, DType};
|
|
use rtx_autograd::TensorAutograd;
|
|
use crate::losses::{LogCoshLoss, Loss, Reduction};
|
|
use crate::{Result, TransformerError};
|
|
use approx::{assert_relative_eq, assert_abs_diff_eq};
|
|
|
|
/// Test suite for LogCoshLoss creation and configuration
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod creation_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_loss_default_creation() {
|
|
let loss = LogCoshLoss::new();
|
|
assert_eq!(loss.reduction(), Reduction::Mean);
|
|
assert!(loss.supports_backprop());
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_loss_builder_pattern() {
|
|
let loss = LogCoshLoss::new()
|
|
.with_reduction(Reduction::Sum);
|
|
assert_eq!(loss.reduction(), Reduction::Sum);
|
|
|
|
let loss2 = LogCoshLoss::new()
|
|
.with_reduction(Reduction::None);
|
|
assert_eq!(loss2.reduction(), Reduction::None);
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_loss_default_trait() {
|
|
let loss: LogCoshLoss = Default::default();
|
|
assert_eq!(loss.reduction(), Reduction::Mean);
|
|
}
|
|
}
|
|
|
|
/// Test suite for forward pass computation
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod forward_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_zero_error() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[1.0f32, 2.0, 3.0], &device).unwrap();
|
|
let targets = predictions.clone();
|
|
|
|
// Should fail because LogCoshLoss is not implemented yet
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_small_errors_quadratic_behavior() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Small errors should approximate x²/2
|
|
let predictions = Tensor::new(&[0.0f32], &device).unwrap();
|
|
let targets = Tensor::new(&[0.1f32], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_large_errors_linear_behavior() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Large errors should approximate |x| - log(2)
|
|
let predictions = Tensor::new(&[0.0f32], &device).unwrap();
|
|
let targets = Tensor::new(&[10.0f32], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_symmetry() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let pred1 = Tensor::new(&[1.0f32], &device).unwrap();
|
|
let target1 = Tensor::new(&[3.0f32], &device).unwrap();
|
|
|
|
let pred2 = Tensor::new(&[3.0f32], &device).unwrap();
|
|
let target2 = Tensor::new(&[1.0f32], &device).unwrap();
|
|
|
|
// log(cosh(x)) = log(cosh(-x)) - should be symmetric
|
|
let result1 = loss.forward(&pred1, &target1);
|
|
let result2 = loss.forward(&pred2, &target2);
|
|
|
|
assert!(result1.is_err() && result2.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_batch_processing() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[0.0f32, 1.0, 2.0, 3.0], &device).unwrap();
|
|
let targets = Tensor::new(&[0.5f32, 1.5, 1.0, 4.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_multidimensional_tensors() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::randn(&[2, 3, 4], &device).unwrap();
|
|
let targets = Tensor::randn(&[2, 3, 4], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
/// Test suite for reduction modes
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod reduction_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_reduction_none() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new().with_reduction(Reduction::None);
|
|
|
|
let predictions = Tensor::new(&[0.0f32, 1.0, 2.0], &device).unwrap();
|
|
let targets = Tensor::new(&[0.5f32, 0.5, 3.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_reduction_sum() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new().with_reduction(Reduction::Sum);
|
|
|
|
let predictions = Tensor::new(&[0.0f32, 1.0, 2.0], &device).unwrap();
|
|
let targets = Tensor::new(&[0.5f32, 0.5, 3.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_reduction_mean() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new().with_reduction(Reduction::Mean);
|
|
|
|
let predictions = Tensor::new(&[0.0f32, 1.0, 2.0], &device).unwrap();
|
|
let targets = Tensor::new(&[0.5f32, 0.5, 3.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_reduction_relationships() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
|
|
let predictions = Tensor::new(&[0.0f32, 1.0, 2.0, 3.0], &device).unwrap();
|
|
let targets = Tensor::new(&[0.5f32, 0.5, 3.0, 2.0], &device).unwrap();
|
|
|
|
let loss_none = LogCoshLoss::new().with_reduction(Reduction::None);
|
|
let loss_sum = LogCoshLoss::new().with_reduction(Reduction::Sum);
|
|
let loss_mean = LogCoshLoss::new().with_reduction(Reduction::Mean);
|
|
|
|
// All should fail in red phase
|
|
let result_none = loss_none.forward(&predictions, &targets);
|
|
let result_sum = loss_sum.forward(&predictions, &targets);
|
|
let result_mean = loss_mean.forward(&predictions, &targets);
|
|
|
|
assert!(result_none.is_err());
|
|
assert!(result_sum.is_err());
|
|
assert!(result_mean.is_err());
|
|
}
|
|
}
|
|
|
|
/// Test suite for numerical stability
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod stability_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_large_values_stability() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Test with very large values (should not overflow)
|
|
let large_predictions = Tensor::new(&[100.0f32, -100.0], &device).unwrap();
|
|
let large_targets = Tensor::new(&[110.0f32, -90.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&large_predictions, &large_targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_small_values_precision() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Test with very small values
|
|
let small_predictions = Tensor::new(&[1e-6f32, -1e-6], &device).unwrap();
|
|
let small_targets = Tensor::new(&[2e-6f32, -3e-6], &device).unwrap();
|
|
|
|
let result = loss.forward(&small_predictions, &small_targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_mixed_magnitude_values() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Mix of small and large errors
|
|
let predictions = Tensor::new(&[0.0f32, 0.0, 0.0, 0.0], &device).unwrap();
|
|
let targets = Tensor::new(&[0.001f32, 1.0, 10.0, 100.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
/// Test suite for gradient computation and backpropagation
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod gradient_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_gradient_computation() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[1.0f32, 2.0], &device).unwrap()
|
|
.requires_grad(true);
|
|
let targets = Tensor::new(&[1.5f32, 1.0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_gradient_derivative_tanh() {
|
|
// The derivative of log(cosh(x)) is tanh(x)
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[0.0f32, 1.0, -1.0, 2.0], &device).unwrap()
|
|
.requires_grad(true);
|
|
let targets = Tensor::zeros(&[4], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_second_derivative_existence() {
|
|
// LogCosh should be twice differentiable (unlike Huber)
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[1.0f32], &device).unwrap()
|
|
.requires_grad(true);
|
|
let targets = Tensor::zeros(&[1], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
/// Test suite for comparison with other losses
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod comparison_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_vs_mse_small_errors() {
|
|
// For small errors, LogCosh ≈ MSE/2
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let logcosh = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[1.0f32, 2.0, 3.0], &device).unwrap();
|
|
let targets = Tensor::new(&[1.01f32, 2.02, 2.99], &device).unwrap();
|
|
|
|
let result = logcosh.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_vs_mae_large_errors() {
|
|
// For large errors, LogCosh ≈ MAE - log(2)
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let logcosh = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[0.0f32], &device).unwrap();
|
|
let targets = Tensor::new(&[10.0f32], &device).unwrap();
|
|
|
|
let result = logcosh.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_vs_huber_smoothness() {
|
|
// LogCosh should be smoother than Huber (no kinks)
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let logcosh = LogCoshLoss::new();
|
|
|
|
// Test around the transition region where Huber has a kink
|
|
let predictions = Tensor::new(&[0.0f32; 5], &device).unwrap();
|
|
let targets = Tensor::new(&[-2.0f32, -1.0, 0.0, 1.0, 2.0], &device).unwrap();
|
|
|
|
let result = logcosh.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
/// Test suite for robustness properties
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod robustness_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_outlier_robustness() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let logcosh = LogCoshLoss::new();
|
|
|
|
// Dataset without outliers
|
|
let predictions_normal = Tensor::new(&[1.0f32, 1.0, 1.0, 1.0], &device).unwrap();
|
|
let targets_normal = Tensor::new(&[1.1f32, 0.9, 1.2, 0.8], &device).unwrap();
|
|
|
|
// Dataset with outlier
|
|
let predictions_outlier = Tensor::new(&[1.0f32, 1.0, 1.0, 1.0], &device).unwrap();
|
|
let targets_outlier = Tensor::new(&[1.1f32, 0.9, 1.2, 20.0], &device).unwrap();
|
|
|
|
let result_normal = logcosh.forward(&predictions_normal, &targets_normal);
|
|
let result_outlier = logcosh.forward(&predictions_outlier, &targets_outlier);
|
|
|
|
assert!(result_normal.is_err() && result_outlier.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_monotonicity() {
|
|
// Loss should increase monotonically with absolute error
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[0.0f32], &device).unwrap();
|
|
|
|
for &error in &[0.0, 0.5, 1.0, 2.0, 5.0] {
|
|
let targets = Tensor::new(&[error], &device).unwrap();
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Test suite for mathematical properties
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod mathematical_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_logcosh_convexity() {
|
|
// LogCosh should be convex
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let zero = Tensor::zeros(&[1], &device).unwrap();
|
|
let x1 = Tensor::new(&[2.0f32], &device).unwrap();
|
|
let x2 = Tensor::new(&[6.0f32], &device).unwrap();
|
|
let x_mid = Tensor::new(&[4.0f32], &device).unwrap();
|
|
|
|
// Test convexity: f(λx + (1-λ)y) ≤ λf(x) + (1-λ)f(y)
|
|
let result1 = loss.forward(&zero, &x1);
|
|
let result2 = loss.forward(&zero, &x2);
|
|
let result_mid = loss.forward(&zero, &x_mid);
|
|
|
|
assert!(result1.is_err() && result2.is_err() && result_mid.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_approximation_regions() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Test transition from quadratic to linear behavior
|
|
let predictions = Tensor::zeros(&[3], &device).unwrap();
|
|
|
|
// Small error (quadratic region)
|
|
let small_targets = Tensor::new(&[0.1f32], &device).unwrap();
|
|
// Medium error (transition region)
|
|
let medium_targets = Tensor::new(&[1.2f32], &device).unwrap();
|
|
// Large error (linear region)
|
|
let large_targets = Tensor::new(&[5.0f32], &device).unwrap();
|
|
|
|
let result_small = loss.forward(&predictions, &small_targets);
|
|
let result_medium = loss.forward(&predictions, &medium_targets);
|
|
let result_large = loss.forward(&predictions, &large_targets);
|
|
|
|
assert!(result_small.is_err() && result_medium.is_err() && result_large.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
/// Test suite for error conditions
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod error_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
#[should_panic(expected = "not implemented")]
|
|
fn test_logcosh_shape_mismatch() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[1.0f32, 2.0], &device).unwrap();
|
|
let targets = Tensor::new(&[1.0f32, 2.0, 3.0], &device).unwrap();
|
|
|
|
// This should panic because LogCoshLoss is not implemented
|
|
let _ = loss.forward(&predictions, &targets).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_device_mismatch() {
|
|
let cpu_device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::new(&[1.0f32, 2.0], &cpu_device).unwrap();
|
|
let targets = Tensor::new(&[1.0f32, 2.0], &cpu_device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_empty_tensors() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
let predictions = Tensor::zeros(&[0], &device).unwrap();
|
|
let targets = Tensor::zeros(&[0], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
/// Test suite for performance benchmarks
|
|
#[cfg(all(test, feature = "disabled_tests"))]
|
|
mod performance_tests {
|
|
use super::*;
|
|
use std::time::Instant;
|
|
|
|
#[test]
|
|
fn test_logcosh_performance_scaling() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Test performance with different tensor sizes
|
|
for size in [100, 1000, 10000] {
|
|
let predictions = Tensor::randn(&[size], &device).unwrap();
|
|
let targets = Tensor::randn(&[size], &device).unwrap();
|
|
|
|
let start = Instant::now();
|
|
let result = loss.forward(&predictions, &targets);
|
|
let _duration = start.elapsed();
|
|
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_logcosh_memory_efficiency() {
|
|
let device = Device::cuda(0).unwrap_or(Device::default());
|
|
let loss = LogCoshLoss::new();
|
|
|
|
// Test with large tensors to check memory usage
|
|
let predictions = Tensor::randn(&[10000], &device).unwrap();
|
|
let targets = Tensor::randn(&[10000], &device).unwrap();
|
|
|
|
let result = loss.forward(&predictions, &targets);
|
|
assert!(result.is_err()); // Should fail in red phase
|
|
}
|
|
} |