rustyverse/rustytorch
2
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
redclawsystems
2026-03-04 00:08:42 +00:00
commit 4d88dc0584
4449 changed files with 1556714 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/specialized/rtx-ml-classic/src/bayesian/gp/deep.rs
+643
View File
@@ -0,0 +1,643 @@
//! Deep Gaussian Processes
//!
//! Implements multi-layer Gaussian Processes for modeling complex,
//! hierarchical patterns in data.
use super::variational::{SVGP, SVGPConfig};
use crate::error::{MLError, Result};
use rtx_tensor::Tensor;
/// Configuration for Deep Gaussian Process
#[derive(Debug, Clone)]
pub struct DeepGPConfig {
/// Number of layers
pub num_layers: usize,
/// Hidden dimensions for each layer (excluding first and last)
pub hidden_dims: Vec<usize>,
/// Number of inducing points per layer
pub num_inducing_per_layer: usize,
/// Kernel length scale
pub length_scale: f32,
/// Output variance
pub variance: f32,
/// Noise level
pub noise: f32,
/// Jitter for numerical stability
pub jitter: f32,
}
impl Default for DeepGPConfig {
fn default() -> Self {
Self {
num_layers: 2,
hidden_dims: vec![5],
num_inducing_per_layer: 50,
length_scale: 1.0,
variance: 1.0,
noise: 0.1,
jitter: 1e-6,
}
}
}
impl DeepGPConfig {
/// Validate configuration
pub fn validate(&self) -> Result<()> {
if self.num_layers < 2 {
return Err(MLError::invalid_parameter("num_layers must be at least 2"));
}
if self.hidden_dims.len() != self.num_layers - 1 {
return Err(MLError::invalid_parameter(format!(
"hidden_dims length ({}) must equal num_layers - 1 ({})",
self.hidden_dims.len(),
self.num_layers - 1
)));
}
if self.num_inducing_per_layer == 0 {
return Err(MLError::invalid_parameter(
"num_inducing_per_layer must be positive",
));
}
for &dim in &self.hidden_dims {
if dim == 0 {
return Err(MLError::invalid_parameter(
"hidden dimensions must be positive",
));
}
}
Ok(())
}
}
/// Single GP layer in a Deep GP
#[derive(Debug, Clone)]
pub struct GPLayer {
/// SVGP model for this layer
svgp: SVGP,
/// Input dimension
input_dim: usize,
/// Output dimension
output_dim: usize,
/// Layer index
layer_idx: usize,
}
impl GPLayer {
/// Create new GP layer
pub fn new(
input_dim: usize,
output_dim: usize,
layer_idx: usize,
config: &DeepGPConfig,
) -> Result<Self> {
let svgp_config = SVGPConfig {
num_inducing: config.num_inducing_per_layer,
learn_inducing_locations: false,
jitter: config.jitter,
length_scale: config.length_scale,
variance: config.variance,
noise: config.noise,
};
let svgp = SVGP::new(svgp_config)?;
Ok(Self {
svgp,
input_dim,
output_dim,
layer_idx,
})
}
/// Initialize layer with data
pub fn initialize(&mut self, x: &Tensor, y: &Tensor) -> Result<()> {
if x.shape().dims()[1] != self.input_dim {
return Err(MLError::invalid_parameter(format!(
"Layer {} expects input dim {} but got {}",
self.layer_idx,
self.input_dim,
x.shape().dims()[1]
)));
}
if y.shape().dims()[0] != x.shape().dims()[0] {
return Err(MLError::invalid_parameter(
"x and y must have same number of samples",
));
}
self.svgp.initialize(x, y)
}
/// Forward pass: predict output for input
pub fn forward(&self, x: &Tensor) -> Result<(Tensor, Tensor)> {
self.svgp.predict(x)
}
/// Compute ELBO for this layer
pub fn elbo(&self, x: &Tensor, y: &Tensor) -> Result<f32> {
self.svgp.elbo(x, y)
}
/// Get input dimension
pub fn input_dim(&self) -> usize {
self.input_dim
}
/// Get output dimension
pub fn output_dim(&self) -> usize {
self.output_dim
}
}
/// Deep Gaussian Process
///
/// Stacks multiple GP layers to model complex hierarchical patterns.
/// Uses mean-field approximation for inference.
#[derive(Debug, Clone)]
pub struct DeepGP {
/// Configuration
config: DeepGPConfig,
/// GP layers
layers: Vec<GPLayer>,
/// Input dimension
input_dim: usize,
/// Output dimension
output_dim: usize,
}
impl DeepGP {
/// Create new Deep GP
///
/// Note: Each GP layer has scalar output (output_dim=1 for SVGP).
/// The hidden_dims control the number of parallel scalar GPs per layer.
/// For simplicity in this implementation, each layer outputs a single scalar.
pub fn new(input_dim: usize, _output_dim: usize, config: DeepGPConfig) -> Result<Self> {
config.validate()?;
let mut layers = Vec::with_capacity(config.num_layers);
// All layers have scalar output for SVGP
// First layer: input_dim -> 1 (scalar output)
layers.push(GPLayer::new(input_dim, 1, 0, &config)?);
// Hidden layers: each takes the previous output
for i in 1..config.num_layers {
// Input is output from previous layer (which is 1)
layers.push(GPLayer::new(1, 1, i, &config)?);
}
Ok(Self {
config,
layers,
input_dim,
output_dim: 1, // Scalar output
})
}
/// Initialize Deep GP with training data
pub fn initialize(&mut self, x_train: &Tensor, y_train: &Tensor) -> Result<()> {
if x_train.shape().ndim() != 2 {
return Err(MLError::invalid_parameter("x_train must be 2-dimensional"));
}
if x_train.shape().dims()[1] != self.input_dim {
return Err(MLError::invalid_parameter(format!(
"x_train has {} features but model expects {}",
x_train.shape().dims()[1],
self.input_dim
)));
}
if y_train.shape().ndim() != 1 {
return Err(MLError::invalid_parameter("y_train must be 1-dimensional"));
}
let n_samples = x_train.shape().dims()[0];
if y_train.shape().dims()[0] != n_samples {
return Err(MLError::invalid_parameter(
"x_train and y_train must have same number of samples",
));
}
// For simplicity, initialize each layer independently
// In practice, would use more sophisticated initialization
// Initialize first layer with input data
let device = x_train.device();
let mut current_x = x_train.clone();
let num_layers = self.layers.len();
for (i, layer) in self.layers.iter_mut().enumerate() {
if i == num_layers - 1 {
// Last layer: use actual targets
layer.initialize(&current_x, y_train)?;
} else {
// Intermediate layer: use random targets for initialization
// For simplicity, we treat each layer as having scalar output during initialization
let random_y_data: Vec<f32> =
(0..n_samples).map(|i| (i as f32 * 0.01).sin()).collect();
let random_y = Tensor::from_data(random_y_data, vec![n_samples], device)?;
layer.initialize(&current_x, &random_y)?;
// Propagate through layer to get input for next layer
let (mean, _) = layer.forward(&current_x)?;
// Reshape mean for next layer
// For scalar output (output_dim=1), reshape to [n_samples, output_dim]
let layer_output_dim = layer.output_dim();
current_x = mean.reshape(vec![n_samples, layer_output_dim])?;
}
}
Ok(())
}
/// Propagate input through all layers (mean-field approximation)
pub fn propagate(&self, x: &Tensor) -> Result<Vec<(Tensor, Tensor)>> {
if x.shape().dims()[1] != self.input_dim {
return Err(MLError::invalid_parameter(format!(
"Input has {} features but model expects {}",
x.shape().dims()[1],
self.input_dim
)));
}
let mut outputs = Vec::with_capacity(self.layers.len());
let mut current_x = x.clone();
for layer in &self.layers {
let (mean, var) = layer.forward(&current_x)?;
outputs.push((mean.clone(), var));
// Use mean as input to next layer (mean-field approximation)
let n_samples = mean.shape().dims()[0];
current_x = mean.reshape(vec![n_samples, 1])?;
}
Ok(outputs)
}
/// Predict mean and variance at test points
pub fn predict(&self, x_test: &Tensor) -> Result<(Tensor, Tensor)> {
let outputs = self.propagate(x_test)?;
// Return last layer output
Ok(outputs.last().unwrap().clone())
}
/// Compute total ELBO across all layers
pub fn elbo(&self, x: &Tensor, y: &Tensor) -> Result<f32> {
// Simplified ELBO: sum of individual layer ELBOs
// In practice, would need proper joint ELBO computation
let mut total_elbo = 0.0;
let mut current_x = x.clone();
for (i, layer) in self.layers.iter().enumerate() {
let layer_y = if i == self.layers.len() - 1 {
y.clone()
} else {
// For intermediate layers, use propagated values
let (mean, _) = layer.forward(&current_x)?;
mean
};
let layer_elbo = layer.elbo(&current_x, &layer_y)?;
total_elbo += layer_elbo;
if i < self.layers.len() - 1 {
let (mean, _) = layer.forward(&current_x)?;
let n_samples = mean.shape().dims()[0];
current_x = mean.reshape(vec![n_samples, 1])?;
}
}
Ok(total_elbo)
}
/// Get number of layers
pub fn num_layers(&self) -> usize {
self.layers.len()
}
/// Get configuration
pub fn config(&self) -> &DeepGPConfig {
&self.config
}
}
#[cfg(test)]
mod tests {
use super::*;
use rtx_tensor::Device;
fn create_test_data(n: usize, d: usize) -> (Tensor, Tensor) {
let device = Device::cpu();
let x_data: Vec<f32> = (0..n * d).map(|i| (i as f32) / (n * d) as f32).collect();
let y_data: Vec<f32> = (0..n).map(|i| (i as f32).sin()).collect();
let x = Tensor::from_data(x_data, vec![n, d], &device).unwrap();
let y = Tensor::from_data(y_data, vec![n], &device).unwrap();
(x, y)
}
#[test]
fn test_deep_gp_config_default() {
let config = DeepGPConfig::default();
assert_eq!(config.num_layers, 2);
assert_eq!(config.hidden_dims.len(), 1);
}
#[test]
fn test_deep_gp_config_validation() {
let config = DeepGPConfig::default();
assert!(config.validate().is_ok());
let mut bad_config = DeepGPConfig::default();
bad_config.num_layers = 1;
assert!(bad_config.validate().is_err());
let mut bad_config = DeepGPConfig::default();
bad_config.hidden_dims = vec![5, 5]; // Wrong length
assert!(bad_config.validate().is_err());
let mut bad_config = DeepGPConfig::default();
bad_config.hidden_dims = vec![0]; // Zero dimension
assert!(bad_config.validate().is_err());
}
#[test]
fn test_gp_layer_creation() {
let config = DeepGPConfig::default();
let layer = GPLayer::new(3, 5, 0, &config);
assert!(layer.is_ok());
let layer = layer.unwrap();
assert_eq!(layer.input_dim(), 3);
assert_eq!(layer.output_dim(), 5);
}
#[test]
fn test_gp_layer_initialize() {
let config = DeepGPConfig {
num_inducing_per_layer: 5,
..Default::default()
};
let mut layer = GPLayer::new(2, 1, 0, &config).unwrap();
let (x, y) = create_test_data(20, 2);
let result = layer.initialize(&x, &y);
assert!(result.is_ok());
}
#[test]
fn test_gp_layer_forward() {
let config = DeepGPConfig {
num_inducing_per_layer: 5,
..Default::default()
};
let mut layer = GPLayer::new(2, 1, 0, &config).unwrap();
let (x, y) = create_test_data(20, 2);
layer.initialize(&x, &y).unwrap();
let (x_test, _) = create_test_data(10, 2);
let result = layer.forward(&x_test);
assert!(result.is_ok());
let (mean, var) = result.unwrap();
assert_eq!(mean.shape().dims()[0], 10);
assert_eq!(var.shape().dims()[0], 10);
}
#[test]
fn test_gp_layer_elbo() {
let config = DeepGPConfig {
num_inducing_per_layer: 5,
..Default::default()
};
let mut layer = GPLayer::new(2, 1, 0, &config).unwrap();
let (x, y) = create_test_data(20, 2);
layer.initialize(&x, &y).unwrap();
let elbo = layer.elbo(&x, &y);
assert!(elbo.is_ok());
assert!(elbo.unwrap().is_finite());
}
#[test]
fn test_deep_gp_creation() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![5],
num_inducing_per_layer: 10,
..Default::default()
};
let dgp = DeepGP::new(3, 1, config);
assert!(dgp.is_ok());
let dgp = dgp.unwrap();
assert_eq!(dgp.num_layers(), 2);
}
#[test]
fn test_deep_gp_invalid_config() {
let config = DeepGPConfig {
num_layers: 1,
hidden_dims: vec![],
..Default::default()
};
assert!(DeepGP::new(3, 1, config).is_err());
}
#[test]
fn test_deep_gp_three_layers() {
let config = DeepGPConfig {
num_layers: 3,
hidden_dims: vec![5, 3],
num_inducing_per_layer: 5,
..Default::default()
};
let dgp = DeepGP::new(2, 1, config);
assert!(dgp.is_ok());
let dgp = dgp.unwrap();
assert_eq!(dgp.num_layers(), 3);
}
#[test]
fn test_deep_gp_initialize() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![3],
num_inducing_per_layer: 5,
..Default::default()
};
let mut dgp = DeepGP::new(2, 1, config).unwrap();
let (x, y) = create_test_data(20, 2);
let result = dgp.initialize(&x, &y);
assert!(result.is_ok());
}
#[test]
fn test_deep_gp_initialize_wrong_input_dim() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![3],
num_inducing_per_layer: 5,
..Default::default()
};
let mut dgp = DeepGP::new(2, 1, config).unwrap();
let (x, y) = create_test_data(20, 3); // Wrong dimension
assert!(dgp.initialize(&x, &y).is_err());
}
#[test]
fn test_deep_gp_propagate() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![3],
num_inducing_per_layer: 5,
..Default::default()
};
let mut dgp = DeepGP::new(2, 1, config).unwrap();
let (x, y) = create_test_data(20, 2);
dgp.initialize(&x, &y).unwrap();
let (x_test, _) = create_test_data(10, 2);
let result = dgp.propagate(&x_test);
assert!(result.is_ok());
let outputs = result.unwrap();
assert_eq!(outputs.len(), 2); // Two layers
// Check output dimensions
for (mean, var) in outputs {
assert_eq!(mean.shape().dims()[0], 10);
assert_eq!(var.shape().dims()[0], 10);
}
}
#[test]
fn test_deep_gp_predict() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![3],
num_inducing_per_layer: 5,
..Default::default()
};
let mut dgp = DeepGP::new(2, 1, config).unwrap();
let (x_train, y_train) = create_test_data(20, 2);
dgp.initialize(&x_train, &y_train).unwrap();
let (x_test, _) = create_test_data(10, 2);
let result = dgp.predict(&x_test);
assert!(result.is_ok());
let (mean, var) = result.unwrap();
assert_eq!(mean.shape().dims()[0], 10);
assert_eq!(var.shape().dims()[0], 10);
// Variance should be non-negative
let var_data = var.to_cpu().unwrap();
for &v in &var_data {
assert!(v >= 0.0);
}
}
#[test]
fn test_deep_gp_elbo() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![3],
num_inducing_per_layer: 5,
..Default::default()
};
let mut dgp = DeepGP::new(2, 1, config).unwrap();
let (x, y) = create_test_data(20, 2);
dgp.initialize(&x, &y).unwrap();
let elbo = dgp.elbo(&x, &y);
assert!(elbo.is_ok());
assert!(elbo.unwrap().is_finite());
}
#[test]
fn test_deep_gp_multi_layer() {
let config = DeepGPConfig {
num_layers: 4,
hidden_dims: vec![5, 4, 3],
num_inducing_per_layer: 5,
..Default::default()
};
let mut dgp = DeepGP::new(2, 1, config).unwrap();
assert_eq!(dgp.num_layers(), 4);
let (x, y) = create_test_data(20, 2);
dgp.initialize(&x, &y).unwrap();
let (x_test, _) = create_test_data(10, 2);
let outputs = dgp.propagate(&x_test).unwrap();
assert_eq!(outputs.len(), 4);
}
#[test]
fn test_deep_gp_config_access() {
let config = DeepGPConfig {
num_layers: 2,
hidden_dims: vec![3],
num_inducing_per_layer: 10,
..Default::default()
};
let dgp = DeepGP::new(2, 1, config.clone()).unwrap();
assert_eq!(dgp.config().num_layers, 2);
assert_eq!(dgp.config().num_inducing_per_layer, 10);
}
#[test]
fn test_deep_gp_layer_dimensions_propagate_correctly() {
let config = DeepGPConfig {
num_layers: 3,
hidden_dims: vec![4, 2],
num_inducing_per_layer: 5,
..Default::default()
};
let dgp = DeepGP::new(3, 1, config).unwrap();
// Check layer dimensions (all layers output scalars in simplified implementation)
assert_eq!(dgp.layers[0].input_dim(), 3);
assert_eq!(dgp.layers[0].output_dim(), 1);
assert_eq!(dgp.layers[1].input_dim(), 1);
assert_eq!(dgp.layers[1].output_dim(), 1);
assert_eq!(dgp.layers[2].input_dim(), 1);
assert_eq!(dgp.layers[2].output_dim(), 1);
}
}
crates/specialized/rtx-ml-classic/src/bayesian/gp/inducing.rs
+499
View File
@@ -0,0 +1,499 @@
//! Inducing point selection strategies for sparse GP
//!
//! Provides various strategies to select inducing points (pseudo-inputs)
//! for scalable Gaussian Process inference.
use crate::error::{MLError, Result};
use rand::seq::SliceRandom;
use rand::thread_rng;
use rtx_tensor::Tensor;
/// Inducing point selection strategy
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum InducingStrategy {
/// Random subset of training data
Random,
/// K-means cluster centers
KMeans,
/// Greedy selection maximizing determinant
Greedy,
/// User-provided fixed locations
Fixed,
}
/// Select inducing points from training data
///
/// # Arguments
/// * `x_train` - Training data [N, D]
/// * `num_inducing` - Number of inducing points to select
/// * `strategy` - Selection strategy
/// * `x_fixed` - Optional fixed inducing points (required for Fixed strategy)
///
/// # Returns
/// Tensor of shape [M, D] containing inducing point locations
pub fn select_inducing_points(
x_train: &Tensor,
num_inducing: usize,
strategy: InducingStrategy,
x_fixed: Option<&Tensor>,
) -> Result<Tensor> {
// Validate inputs
if x_train.shape().ndim() != 2 {
return Err(MLError::invalid_parameter("x_train must be 2-dimensional"));
}
let n_train = x_train.shape().dims()[0];
let n_features = x_train.shape().dims()[1];
if num_inducing == 0 {
return Err(MLError::invalid_parameter("num_inducing must be positive"));
}
if num_inducing > n_train {
return Err(MLError::invalid_parameter(format!(
"num_inducing ({}) cannot exceed training size ({})",
num_inducing, n_train
)));
}
match strategy {
InducingStrategy::Random => select_random(x_train, num_inducing),
InducingStrategy::KMeans => select_kmeans(x_train, num_inducing),
InducingStrategy::Greedy => select_greedy(x_train, num_inducing),
InducingStrategy::Fixed => {
let x_fixed = x_fixed.ok_or_else(|| {
MLError::invalid_parameter("Fixed strategy requires x_fixed parameter")
})?;
if x_fixed.shape().ndim() != 2 {
return Err(MLError::invalid_parameter("x_fixed must be 2-dimensional"));
}
if x_fixed.shape().dims()[0] != num_inducing {
return Err(MLError::invalid_parameter(format!(
"x_fixed has {} points but num_inducing is {}",
x_fixed.shape().dims()[0],
num_inducing
)));
}
if x_fixed.shape().dims()[1] != n_features {
return Err(MLError::invalid_parameter(format!(
"x_fixed has {} features but x_train has {}",
x_fixed.shape().dims()[1],
n_features
)));
}
Ok(x_fixed.clone())
}
}
}
/// Select random subset of training points
fn select_random(x_train: &Tensor, num_inducing: usize) -> Result<Tensor> {
let n_train = x_train.shape().dims()[0];
let n_features = x_train.shape().dims()[1];
let x_data = x_train.to_cpu()?;
// Randomly select indices
let mut indices: Vec<usize> = (0..n_train).collect();
indices.shuffle(&mut thread_rng());
let selected_indices = &indices[..num_inducing];
// Extract selected rows
let mut inducing_data = Vec::with_capacity(num_inducing * n_features);
for &idx in selected_indices {
let start = idx * n_features;
let end = start + n_features;
inducing_data.extend_from_slice(&x_data[start..end]);
}
Tensor::from_data(
inducing_data,
vec![num_inducing, n_features],
x_train.device(),
)
.map_err(std::convert::Into::into)
}
/// Select inducing points using k-means clustering
fn select_kmeans(x_train: &Tensor, num_inducing: usize) -> Result<Tensor> {
let n_features = x_train.shape().dims()[1];
let x_data = x_train.to_cpu()?;
// Simple k-means implementation
let max_iters = 100;
let tolerance = 1e-4;
// Initialize centers randomly
let mut rng = thread_rng();
let all_indices: Vec<usize> = (0..x_train.shape().dims()[0]).collect();
let init_indices: Vec<usize> = all_indices
.choose_multiple(&mut rng, num_inducing)
.copied()
.collect();
let mut centers = vec![0.0f32; num_inducing * n_features];
for (i, &idx) in init_indices.iter().enumerate() {
let start_src = idx * n_features;
let start_dst = i * n_features;
centers[start_dst..start_dst + n_features]
.copy_from_slice(&x_data[start_src..start_src + n_features]);
}
let n_train = x_train.shape().dims()[0];
let mut assignments = vec![0usize; n_train];
for _iter in 0..max_iters {
let old_centers = centers.clone();
// Assignment step
for i in 0..n_train {
let point = &x_data[i * n_features..(i + 1) * n_features];
let mut min_dist = f32::INFINITY;
let mut best_cluster = 0;
for j in 0..num_inducing {
let center = &centers[j * n_features..(j + 1) * n_features];
let dist: f32 = point
.iter()
.zip(center.iter())
.map(|(a, b)| (a - b).powi(2))
.sum();
if dist < min_dist {
min_dist = dist;
best_cluster = j;
}
}
assignments[i] = best_cluster;
}
// Update step
centers.fill(0.0);
let mut counts = vec![0usize; num_inducing];
for i in 0..n_train {
let cluster = assignments[i];
let point = &x_data[i * n_features..(i + 1) * n_features];
for d in 0..n_features {
centers[cluster * n_features + d] += point[d];
}
counts[cluster] += 1;
}
for j in 0..num_inducing {
if counts[j] > 0 {
for d in 0..n_features {
centers[j * n_features + d] /= counts[j] as f32;
}
}
}
// Check convergence
let change: f32 = centers
.iter()
.zip(old_centers.iter())
.map(|(a, b)| (a - b).abs())
.sum();
if change < tolerance {
break;
}
}
Tensor::from_data(centers, vec![num_inducing, n_features], x_train.device())
.map_err(std::convert::Into::into)
}
/// Select inducing points greedily to maximize determinant
fn select_greedy(x_train: &Tensor, num_inducing: usize) -> Result<Tensor> {
let n_train = x_train.shape().dims()[0];
let n_features = x_train.shape().dims()[1];
let x_data = x_train.to_cpu()?;
// Start with the point that has maximum variance
let mut selected_indices = Vec::with_capacity(num_inducing);
// Select first point (center of data)
let mut mean = vec![0.0f32; n_features];
for i in 0..n_train {
for d in 0..n_features {
mean[d] += x_data[i * n_features + d];
}
}
for d in 0..n_features {
mean[d] /= n_train as f32;
}
// Find point closest to mean
let mut min_dist = f32::INFINITY;
let mut first_idx = 0;
for i in 0..n_train {
let point = &x_data[i * n_features..(i + 1) * n_features];
let dist: f32 = point
.iter()
.zip(mean.iter())
.map(|(a, b)| (a - b).powi(2))
.sum();
if dist < min_dist {
min_dist = dist;
first_idx = i;
}
}
selected_indices.push(first_idx);
// Greedily select remaining points to maximize minimum distance
for _ in 1..num_inducing {
let mut max_min_dist = 0.0f32;
let mut best_idx = 0;
for i in 0..n_train {
if selected_indices.contains(&i) {
continue;
}
let point = &x_data[i * n_features..(i + 1) * n_features];
// Find minimum distance to selected points
let mut min_dist = f32::INFINITY;
for &sel_idx in &selected_indices {
let sel_point = &x_data[sel_idx * n_features..(sel_idx + 1) * n_features];
let dist: f32 = point
.iter()
.zip(sel_point.iter())
.map(|(a, b)| (a - b).powi(2))
.sum::<f32>()
.sqrt();
if dist < min_dist {
min_dist = dist;
}
}
if min_dist > max_min_dist {
max_min_dist = min_dist;
best_idx = i;
}
}
selected_indices.push(best_idx);
}
// Extract selected points
let mut inducing_data = Vec::with_capacity(num_inducing * n_features);
for &idx in &selected_indices {
let start = idx * n_features;
let end = start + n_features;
inducing_data.extend_from_slice(&x_data[start..end]);
}
Tensor::from_data(
inducing_data,
vec![num_inducing, n_features],
x_train.device(),
)
.map_err(std::convert::Into::into)
}
#[cfg(test)]
mod tests {
use super::*;
use rtx_tensor::Device;
fn create_test_data(n: usize, d: usize) -> Tensor {
let device = Device::cpu();
let data: Vec<f32> = (0..n * d).map(|i| (i as f32) / 10.0).collect();
Tensor::from_data(data, vec![n, d], &device).unwrap()
}
#[test]
fn test_select_random_basic() {
let x_train = create_test_data(100, 3);
let result = select_inducing_points(&x_train, 10, InducingStrategy::Random, None);
assert!(result.is_ok());
let inducing = result.unwrap();
assert_eq!(inducing.shape().dims(), &[10, 3]);
}
#[test]
fn test_select_random_full_size() {
let x_train = create_test_data(50, 2);
let result = select_inducing_points(&x_train, 50, InducingStrategy::Random, None);
assert!(result.is_ok());
let inducing = result.unwrap();
assert_eq!(inducing.shape().dims(), &[50, 2]);
}
#[test]
fn test_select_invalid_size() {
let x_train = create_test_data(50, 2);
// Too many inducing points
let result = select_inducing_points(&x_train, 100, InducingStrategy::Random, None);
assert!(result.is_err());
// Zero inducing points
let result = select_inducing_points(&x_train, 0, InducingStrategy::Random, None);
assert!(result.is_err());
}
#[test]
fn test_select_invalid_dims() {
let device = Device::cpu();
let data: Vec<f32> = (0..10).map(|i| i as f32).collect();
let x_train = Tensor::from_data(data, vec![10], &device).unwrap();
let result = select_inducing_points(&x_train, 5, InducingStrategy::Random, None);
assert!(result.is_err());
}
#[test]
fn test_select_kmeans_basic() {
let x_train = create_test_data(100, 3);
let result = select_inducing_points(&x_train, 10, InducingStrategy::KMeans, None);
assert!(result.is_ok());
let inducing = result.unwrap();
assert_eq!(inducing.shape().dims(), &[10, 3]);
}
#[test]
fn test_select_kmeans_clusters() {
let device = Device::cpu();
// Create two distinct clusters
let mut data = Vec::new();
for i in 0..50 {
data.push((i as f32) / 50.0); // Cluster 1: [0, 1]
data.push(0.0);
}
for i in 0..50 {
data.push(10.0 + (i as f32) / 50.0); // Cluster 2: [10, 11]
data.push(0.0);
}
let x_train = Tensor::from_data(data, vec![100, 2], &device).unwrap();
let inducing = select_inducing_points(&x_train, 2, InducingStrategy::KMeans, None).unwrap();
let inducing_data = inducing.to_cpu().unwrap();
// Should have centers near 0.5 and 10.5
let center1 = inducing_data[0];
let center2 = inducing_data[2];
assert!(center1 < 2.0 || center2 < 2.0); // One center in first cluster
assert!(center1 > 9.0 || center2 > 9.0); // One center in second cluster
}
#[test]
fn test_select_greedy_basic() {
let x_train = create_test_data(100, 3);
let result = select_inducing_points(&x_train, 10, InducingStrategy::Greedy, None);
assert!(result.is_ok());
let inducing = result.unwrap();
assert_eq!(inducing.shape().dims(), &[10, 3]);
}
#[test]
fn test_select_greedy_spacing() {
let device = Device::cpu();
// Create evenly spaced 1D points
let data: Vec<f32> = (0..100).map(|i| i as f32).collect();
let x_train = Tensor::from_data(data, vec![100, 1], &device).unwrap();
let inducing = select_inducing_points(&x_train, 5, InducingStrategy::Greedy, None).unwrap();
let inducing_data = inducing.to_cpu().unwrap();
// Check that points are reasonably spaced
for i in 0..4 {
let dist = (inducing_data[i + 1] - inducing_data[i]).abs();
assert!(dist > 5.0); // Should be well-separated
}
}
#[test]
fn test_select_fixed_basic() {
let x_train = create_test_data(100, 3);
let device = Device::cpu();
let fixed_data: Vec<f32> = (0..30).map(|i| i as f32).collect();
let x_fixed = Tensor::from_data(fixed_data, vec![10, 3], &device).unwrap();
let result = select_inducing_points(&x_train, 10, InducingStrategy::Fixed, Some(&x_fixed));
assert!(result.is_ok());
let inducing = result.unwrap();
assert_eq!(inducing.shape().dims(), &[10, 3]);
// Should be identical to x_fixed
let inducing_data = inducing.to_cpu().unwrap();
let fixed_data_cpu = x_fixed.to_cpu().unwrap();
assert_eq!(inducing_data, fixed_data_cpu);
}
#[test]
fn test_select_fixed_missing_param() {
let x_train = create_test_data(100, 3);
let result = select_inducing_points(&x_train, 10, InducingStrategy::Fixed, None);
assert!(result.is_err());
}
#[test]
fn test_select_fixed_wrong_size() {
let x_train = create_test_data(100, 3);
let device = Device::cpu();
// Wrong number of points
let fixed_data: Vec<f32> = (0..15).map(|i| i as f32).collect();
let x_fixed = Tensor::from_data(fixed_data, vec![5, 3], &device).unwrap();
let result = select_inducing_points(&x_train, 10, InducingStrategy::Fixed, Some(&x_fixed));
assert!(result.is_err());
}
#[test]
fn test_select_fixed_wrong_features() {
let x_train = create_test_data(100, 3);
let device = Device::cpu();
// Wrong number of features
let fixed_data: Vec<f32> = (0..20).map(|i| i as f32).collect();
let x_fixed = Tensor::from_data(fixed_data, vec![10, 2], &device).unwrap();
let result = select_inducing_points(&x_train, 10, InducingStrategy::Fixed, Some(&x_fixed));
assert!(result.is_err());
}
#[test]
fn test_different_strategies_produce_different_results() {
let x_train = create_test_data(100, 3);
let random1 = select_inducing_points(&x_train, 10, InducingStrategy::Random, None).unwrap();
let random2 = select_inducing_points(&x_train, 10, InducingStrategy::Random, None).unwrap();
// Random selections should differ (with high probability)
let data1 = random1.to_cpu().unwrap();
let data2 = random2.to_cpu().unwrap();
assert_ne!(data1, data2);
let kmeans = select_inducing_points(&x_train, 10, InducingStrategy::KMeans, None).unwrap();
let greedy = select_inducing_points(&x_train, 10, InducingStrategy::Greedy, None).unwrap();
let kmeans_data = kmeans.to_cpu().unwrap();
let greedy_data = greedy.to_cpu().unwrap();
assert_ne!(kmeans_data, greedy_data);
}
#[test]
fn test_inducing_strategy_enum() {
assert_eq!(InducingStrategy::Random, InducingStrategy::Random);
assert_ne!(InducingStrategy::Random, InducingStrategy::KMeans);
}
}
crates/specialized/rtx-ml-classic/src/bayesian/gp/kernels.rs
+485
View File
@@ -0,0 +1,485 @@
//! Advanced kernel functions for Gaussian Processes
//!
//! Provides sophisticated kernels for modeling complex patterns:
//! - SpectralMixtureKernel: Learn spectral structure
//! - PeriodicKernel: Model periodic patterns
//! - RationalQuadraticKernel: Infinite mixture of RBF kernels
//! - CompositeKernel: Combine kernels via addition/multiplication
//! - ScaledKernel: Apply variance scaling
use crate::error::Result;
use std::f32::consts::PI;
/// Kernel operation for composition
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum KernelOp {
/// Add two kernels
Add,
/// Multiply two kernels
Multiply,
}
/// Advanced kernel types
#[derive(Debug, Clone)]
pub enum AdvancedKernel {
/// Spectral Mixture Kernel: sum of Gaussians in frequency domain
SpectralMixture(SpectralMixtureKernel),
/// Periodic kernel for periodic patterns
Periodic(PeriodicKernel),
/// Rational Quadratic kernel (infinite mixture of RBF)
RationalQuadratic(RationalQuadraticKernel),
/// Composite kernel (addition or multiplication)
Composite(CompositeKernel),
/// Scaled kernel
Scaled(ScaledKernel),
}
/// Spectral Mixture Kernel
///
/// Represents a mixture of Q Gaussians in the frequency domain:
/// k(x, x') = Σ_q w_q * exp(-2π²τ²v_q) * cos(2πτμ_q)
/// where τ = x - x'
#[derive(Debug, Clone)]
pub struct SpectralMixtureKernel {
/// Mixture weights (Q components)
pub weights: Vec<f32>,
/// Mean frequencies (Q components x D dimensions)
pub means: Vec<Vec<f32>>,
/// Variance scales (Q components x D dimensions)
pub variances: Vec<Vec<f32>>,
}
impl SpectralMixtureKernel {
/// Create new spectral mixture kernel
pub fn new(weights: Vec<f32>, means: Vec<Vec<f32>>, variances: Vec<Vec<f32>>) -> Result<Self> {
let q = weights.len();
if means.len() != q || variances.len() != q {
return Err(crate::error::MLError::invalid_parameter(
"Weights, means, and variances must have same number of components",
));
}
if q == 0 {
return Err(crate::error::MLError::invalid_parameter(
"Must have at least one mixture component",
));
}
let dim = means[0].len();
for (mean, var) in means.iter().zip(variances.iter()) {
if mean.len() != dim || var.len() != dim {
return Err(crate::error::MLError::invalid_parameter(
"All means and variances must have same dimensionality",
));
}
}
Ok(Self {
weights,
means,
variances,
})
}
/// Compute kernel value between two points
pub fn compute(&self, x1: &[f32], x2: &[f32]) -> f32 {
let mut result = 0.0;
for q in 0..self.weights.len() {
let mut exp_term = 0.0;
let mut cos_term = 0.0;
for d in 0..x1.len() {
let tau_d = x1[d] - x2[d];
exp_term += -2.0 * PI * PI * tau_d * tau_d * self.variances[q][d];
cos_term += 2.0 * PI * tau_d * self.means[q][d];
}
result += self.weights[q] * exp_term.exp() * cos_term.cos();
}
result
}
/// Number of mixture components
pub fn num_components(&self) -> usize {
self.weights.len()
}
}
/// Periodic Kernel
///
/// Models periodic patterns: k(x, x') = σ² exp(-2 sin²(π|x-x'|/p) / l²)
#[derive(Debug, Clone)]
pub struct PeriodicKernel {
/// Output variance
pub variance: f32,
/// Period length
pub period: f32,
/// Length scale
pub length_scale: f32,
}
impl PeriodicKernel {
/// Create new periodic kernel
pub fn new(variance: f32, period: f32, length_scale: f32) -> Result<Self> {
if variance <= 0.0 || period <= 0.0 || length_scale <= 0.0 {
return Err(crate::error::MLError::invalid_parameter(
"Variance, period, and length_scale must be positive",
));
}
Ok(Self {
variance,
period,
length_scale,
})
}
/// Compute kernel value
pub fn compute(&self, x1: &[f32], x2: &[f32]) -> f32 {
let mut dist_sq = 0.0;
for (a, b) in x1.iter().zip(x2.iter()) {
dist_sq += (a - b).powi(2);
}
let dist = dist_sq.sqrt();
let sin_term = (PI * dist / self.period).sin();
let exp_arg = -2.0 * sin_term * sin_term / (self.length_scale * self.length_scale);
self.variance * exp_arg.exp()
}
}
/// Rational Quadratic Kernel
///
/// Equivalent to infinite mixture of RBF kernels:
/// k(x, x') = σ² (1 + ||x-x'||²/(2αl²))^(-α)
#[derive(Debug, Clone)]
pub struct RationalQuadraticKernel {
/// Output variance
pub variance: f32,
/// Length scale
pub length_scale: f32,
/// Relative weighting (α)
pub alpha: f32,
}
impl RationalQuadraticKernel {
/// Create new rational quadratic kernel
pub fn new(variance: f32, length_scale: f32, alpha: f32) -> Result<Self> {
if variance <= 0.0 || length_scale <= 0.0 || alpha <= 0.0 {
return Err(crate::error::MLError::invalid_parameter(
"All parameters must be positive",
));
}
Ok(Self {
variance,
length_scale,
alpha,
})
}
/// Compute kernel value
pub fn compute(&self, x1: &[f32], x2: &[f32]) -> f32 {
let mut dist_sq = 0.0;
for (a, b) in x1.iter().zip(x2.iter()) {
dist_sq += (a - b).powi(2);
}
let base = 1.0 + dist_sq / (2.0 * self.alpha * self.length_scale * self.length_scale);
self.variance * base.powf(-self.alpha)
}
}
/// Composite Kernel
///
/// Combines two kernels via addition or multiplication
#[derive(Debug, Clone)]
pub struct CompositeKernel {
/// Left kernel
pub left: Box<AdvancedKernel>,
/// Right kernel
pub right: Box<AdvancedKernel>,
/// Operation (add or multiply)
pub op: KernelOp,
}
impl CompositeKernel {
/// Create new composite kernel
pub fn new(left: AdvancedKernel, right: AdvancedKernel, op: KernelOp) -> Self {
Self {
left: Box::new(left),
right: Box::new(right),
op,
}
}
/// Compute kernel value
pub fn compute(&self, x1: &[f32], x2: &[f32]) -> f32 {
let left_val = self.left.compute(x1, x2);
let right_val = self.right.compute(x1, x2);
match self.op {
KernelOp::Add => left_val + right_val,
KernelOp::Multiply => left_val * right_val,
}
}
}
/// Scaled Kernel
///
/// Applies variance scaling to another kernel: k_scaled(x, x') = σ² k(x, x')
#[derive(Debug, Clone)]
pub struct ScaledKernel {
/// Base kernel
pub base: Box<AdvancedKernel>,
/// Output scale (variance)
pub scale: f32,
}
impl ScaledKernel {
/// Create new scaled kernel
pub fn new(base: AdvancedKernel, scale: f32) -> Result<Self> {
if scale <= 0.0 {
return Err(crate::error::MLError::invalid_parameter(
"Scale must be positive",
));
}
Ok(Self {
base: Box::new(base),
scale,
})
}
/// Compute kernel value
pub fn compute(&self, x1: &[f32], x2: &[f32]) -> f32 {
self.scale * self.base.compute(x1, x2)
}
}
impl AdvancedKernel {
/// Compute kernel value between two points
pub fn compute(&self, x1: &[f32], x2: &[f32]) -> f32 {
match self {
Self::SpectralMixture(k) => k.compute(x1, x2),
Self::Periodic(k) => k.compute(x1, x2),
Self::RationalQuadratic(k) => k.compute(x1, x2),
Self::Composite(k) => k.compute(x1, x2),
Self::Scaled(k) => k.compute(x1, x2),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use approx::assert_relative_eq;
#[test]
fn test_spectral_mixture_kernel_creation() {
let weights = vec![0.5, 0.5];
let means = vec![vec![1.0], vec![2.0]];
let variances = vec![vec![0.1], vec![0.2]];
let kernel = SpectralMixtureKernel::new(weights, means, variances).unwrap();
assert_eq!(kernel.num_components(), 2);
}
#[test]
fn test_spectral_mixture_kernel_invalid_params() {
let weights = vec![0.5, 0.5];
let means = vec![vec![1.0]]; // Wrong size
let variances = vec![vec![0.1], vec![0.2]];
assert!(SpectralMixtureKernel::new(weights, means, variances).is_err());
}
#[test]
fn test_spectral_mixture_kernel_compute() {
let weights = vec![1.0];
let means = vec![vec![0.0]];
let variances = vec![vec![1.0]];
let kernel = SpectralMixtureKernel::new(weights, means, variances).unwrap();
// Same point should give value close to weight
let val = kernel.compute(&[0.0], &[0.0]);
assert_relative_eq!(val, 1.0, epsilon = 1e-5);
// Kernel should be symmetric
let val1 = kernel.compute(&[1.0], &[2.0]);
let val2 = kernel.compute(&[2.0], &[1.0]);
assert_relative_eq!(val1, val2, epsilon = 1e-5);
}
#[test]
fn test_periodic_kernel_creation() {
let kernel = PeriodicKernel::new(1.0, 2.0, 0.5).unwrap();
assert_eq!(kernel.variance, 1.0);
assert_eq!(kernel.period, 2.0);
assert_eq!(kernel.length_scale, 0.5);
}
#[test]
fn test_periodic_kernel_invalid_params() {
assert!(PeriodicKernel::new(-1.0, 2.0, 0.5).is_err());
assert!(PeriodicKernel::new(1.0, 0.0, 0.5).is_err());
assert!(PeriodicKernel::new(1.0, 2.0, 0.0).is_err());
}
#[test]
fn test_periodic_kernel_compute() {
let kernel = PeriodicKernel::new(1.0, 1.0, 1.0).unwrap();
// Same point should give variance
let val = kernel.compute(&[0.0], &[0.0]);
assert_relative_eq!(val, 1.0, epsilon = 1e-5);
// Symmetric
let val1 = kernel.compute(&[0.0], &[0.5]);
let val2 = kernel.compute(&[0.5], &[0.0]);
assert_relative_eq!(val1, val2, epsilon = 1e-5);
// Periodic property: k(x, x+period) should be close to k(x, x)
let val_0 = kernel.compute(&[0.0], &[0.0]);
let val_period = kernel.compute(&[0.0], &[1.0]);
// Should be close due to periodicity
assert!(val_period > 0.5 * val_0);
}
#[test]
fn test_rational_quadratic_kernel_creation() {
let kernel = RationalQuadraticKernel::new(1.0, 1.0, 1.0).unwrap();
assert_eq!(kernel.variance, 1.0);
assert_eq!(kernel.length_scale, 1.0);
assert_eq!(kernel.alpha, 1.0);
}
#[test]
fn test_rational_quadratic_kernel_invalid_params() {
assert!(RationalQuadraticKernel::new(0.0, 1.0, 1.0).is_err());
assert!(RationalQuadraticKernel::new(1.0, -1.0, 1.0).is_err());
assert!(RationalQuadraticKernel::new(1.0, 1.0, 0.0).is_err());
}
#[test]
fn test_rational_quadratic_kernel_compute() {
let kernel = RationalQuadraticKernel::new(1.0, 1.0, 1.0).unwrap();
// Same point should give variance
let val = kernel.compute(&[0.0], &[0.0]);
assert_relative_eq!(val, 1.0, epsilon = 1e-5);
// Symmetric
let val1 = kernel.compute(&[0.0], &[1.0]);
let val2 = kernel.compute(&[1.0], &[0.0]);
assert_relative_eq!(val1, val2, epsilon = 1e-5);
// Should decrease with distance
let val_near = kernel.compute(&[0.0], &[0.1]);
let val_far = kernel.compute(&[0.0], &[1.0]);
assert!(val_near > val_far);
}
#[test]
fn test_composite_kernel_addition() {
let k1 = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
let k2 =
AdvancedKernel::RationalQuadratic(RationalQuadraticKernel::new(1.0, 1.0, 1.0).unwrap());
let composite = CompositeKernel::new(k1.clone(), k2.clone(), KernelOp::Add);
let x1 = &[0.0];
let x2 = &[0.5];
let val_composite = composite.compute(x1, x2);
let val_k1 = k1.compute(x1, x2);
let val_k2 = k2.compute(x1, x2);
assert_relative_eq!(val_composite, val_k1 + val_k2, epsilon = 1e-5);
}
#[test]
fn test_composite_kernel_multiplication() {
let k1 = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
let k2 =
AdvancedKernel::RationalQuadratic(RationalQuadraticKernel::new(1.0, 1.0, 1.0).unwrap());
let composite = CompositeKernel::new(k1.clone(), k2.clone(), KernelOp::Multiply);
let x1 = &[0.0];
let x2 = &[0.5];
let val_composite = composite.compute(x1, x2);
let val_k1 = k1.compute(x1, x2);
let val_k2 = k2.compute(x1, x2);
assert_relative_eq!(val_composite, val_k1 * val_k2, epsilon = 1e-5);
}
#[test]
fn test_scaled_kernel() {
let base = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
let scale = 2.5;
let scaled = ScaledKernel::new(base.clone(), scale).unwrap();
let x1 = &[0.0];
let x2 = &[0.5];
let val_scaled = scaled.compute(x1, x2);
let val_base = base.compute(x1, x2);
assert_relative_eq!(val_scaled, scale * val_base, epsilon = 1e-5);
}
#[test]
fn test_scaled_kernel_invalid_scale() {
let base = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
assert!(ScaledKernel::new(base, 0.0).is_err());
let base = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
assert!(ScaledKernel::new(base, -1.0).is_err());
}
#[test]
fn test_advanced_kernel_enum() {
let kernel = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
let val = kernel.compute(&[0.0], &[0.0]);
assert_relative_eq!(val, 1.0, epsilon = 1e-5);
}
#[test]
fn test_spectral_mixture_multivariate() {
let weights = vec![0.6, 0.4];
let means = vec![vec![1.0, 2.0], vec![0.5, 1.5]];
let variances = vec![vec![0.1, 0.2], vec![0.15, 0.25]];
let kernel = SpectralMixtureKernel::new(weights, means, variances).unwrap();
let x1 = &[0.0, 0.0];
let x2 = &[0.1, 0.1];
let val = kernel.compute(x1, x2);
// Spectral mixture kernel can exceed 1.0 due to sum of weighted components
assert!(val.is_finite() && !val.is_nan());
// Same point should give sum of weights
let val_same = kernel.compute(x1, x1);
assert!((val_same - 1.0).abs() < 0.1); // Should be close to sum of weights
}
#[test]
fn test_nested_composite_kernels() {
let k1 = AdvancedKernel::Periodic(PeriodicKernel::new(1.0, 1.0, 1.0).unwrap());
let k2 =
AdvancedKernel::RationalQuadratic(RationalQuadraticKernel::new(1.0, 1.0, 1.0).unwrap());
let k3 = AdvancedKernel::Periodic(PeriodicKernel::new(0.5, 2.0, 0.5).unwrap());
// (k1 + k2) * k3
let inner = CompositeKernel::new(k1, k2, KernelOp::Add);
let outer = CompositeKernel::new(AdvancedKernel::Composite(inner), k3, KernelOp::Multiply);
let val = outer.compute(&[0.0], &[0.5]);
assert!(val > 0.0);
}
}
crates/specialized/rtx-ml-classic/src/bayesian/gp/mod.rs
+35
View File
@@ -0,0 +1,35 @@
//! Gaussian Process module with deep GP support
//!
//! This module provides:
//! - Basic and advanced kernel functions
//! - Sparse Variational Gaussian Processes (SVGP)
//! - Deep Gaussian Processes (DGP)
//! - Inducing point selection strategies
mod deep;
mod inducing;
mod kernels;
mod variational;
// Re-export core GP from parent module
pub use super::gaussian_process::{GaussianProcess, Kernel};
// Re-export new components
pub use deep::{DeepGP, DeepGPConfig, GPLayer};
pub use inducing::{InducingStrategy, select_inducing_points};
pub use kernels::{
AdvancedKernel, CompositeKernel, KernelOp, PeriodicKernel, RationalQuadraticKernel,
ScaledKernel, SpectralMixtureKernel,
};
pub use variational::{SVGP, SVGPConfig};
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_module_exports() {
// Verify all exports are accessible
let _kernel = Kernel::RBF { gamma: 1.0 };
}
}
crates/specialized/rtx-ml-classic/src/bayesian/gp/variational.rs
+812
View File
@@ -0,0 +1,812 @@
//! Sparse Variational Gaussian Process (SVGP)
//!
//! Implements scalable GP inference using variational approximations
//! with inducing points for handling large datasets (10K+ samples).
use super::inducing::{InducingStrategy, select_inducing_points};
use crate::error::{MLError, Result};
use rtx_tensor::Tensor;
/// Configuration for Sparse Variational GP
#[derive(Debug, Clone)]
pub struct SVGPConfig {
/// Number of inducing points
pub num_inducing: usize,
/// Whether to learn inducing point locations
pub learn_inducing_locations: bool,
/// Jitter for numerical stability
pub jitter: f32,
/// Kernel length scale
pub length_scale: f32,
/// Output variance
pub variance: f32,
/// Noise level
pub noise: f32,
}
impl Default for SVGPConfig {
fn default() -> Self {
Self {
num_inducing: 100,
learn_inducing_locations: false,
jitter: 1e-6,
length_scale: 1.0,
variance: 1.0,
noise: 0.1,
}
}
}
/// Sparse Variational Gaussian Process
///
/// Uses inducing points to approximate the full GP for scalability.
/// Implements the SVGP algorithm with variational inference.
#[derive(Debug, Clone)]
pub struct SVGP {
/// Configuration
config: SVGPConfig,
/// Inducing point locations [M, D]
inducing_points: Option<Tensor>,
/// Variational mean m [M]
variational_mean: Option<Tensor>,
/// Variational variance (diagonal) S [M]
variational_variance: Option<Tensor>,
/// Number of features
n_features: usize,
/// Training data size
n_train: usize,
}
impl SVGP {
/// Create new SVGP with configuration
pub fn new(config: SVGPConfig) -> Result<Self> {
if config.num_inducing == 0 {
return Err(MLError::invalid_parameter("num_inducing must be positive"));
}
if config.jitter <= 0.0 {
return Err(MLError::invalid_parameter("jitter must be positive"));
}
if config.length_scale <= 0.0 {
return Err(MLError::invalid_parameter("length_scale must be positive"));
}
if config.variance <= 0.0 {
return Err(MLError::invalid_parameter("variance must be positive"));
}
if config.noise <= 0.0 {
return Err(MLError::invalid_parameter("noise must be positive"));
}
Ok(Self {
config,
inducing_points: None,
variational_mean: None,
variational_variance: None,
n_features: 0,
n_train: 0,
})
}
/// Initialize with training data
pub fn initialize(&mut self, x_train: &Tensor, y_train: &Tensor) -> Result<()> {
if x_train.shape().ndim() != 2 {
return Err(MLError::invalid_parameter("x_train must be 2-dimensional"));
}
if y_train.shape().ndim() != 1 {
return Err(MLError::invalid_parameter("y_train must be 1-dimensional"));
}
self.n_train = x_train.shape().dims()[0];
self.n_features = x_train.shape().dims()[1];
if y_train.shape().dims()[0] != self.n_train {
return Err(MLError::invalid_parameter(
"x_train and y_train must have same number of samples",
));
}
if self.config.num_inducing > self.n_train {
return Err(MLError::invalid_parameter(format!(
"num_inducing ({}) cannot exceed training size ({})",
self.config.num_inducing, self.n_train
)));
}
// Select inducing points
let inducing = select_inducing_points(
x_train,
self.config.num_inducing,
InducingStrategy::KMeans,
None,
)?;
self.inducing_points = Some(inducing);
// Initialize variational parameters
let device = x_train.device();
let m = self.config.num_inducing;
// Initialize mean to zero
let var_mean = Tensor::zeros(vec![m], device)?;
self.variational_mean = Some(var_mean);
// Initialize variance to prior
let var_variance = Tensor::ones(vec![m], device)?.mul_scalar(self.config.variance)?;
self.variational_variance = Some(var_variance);
Ok(())
}
/// Compute RBF kernel between two point sets
fn compute_kernel(&self, x1: &Tensor, x2: &Tensor) -> Result<Tensor> {
let n1 = x1.shape().dims()[0];
let n2 = x2.shape().dims()[0];
let d = x1.shape().dims()[1];
let x1_data = x1.to_cpu()?;
let x2_data = x2.to_cpu()?;
let mut k_data = vec![0.0f32; n1 * n2];
let gamma = 1.0 / (2.0 * self.config.length_scale * self.config.length_scale);
for i in 0..n1 {
for j in 0..n2 {
let mut dist_sq = 0.0f32;
for k in 0..d {
let diff = x1_data[i * d + k] - x2_data[j * d + k];
dist_sq += diff * diff;
}
k_data[i * n2 + j] = self.config.variance * (-gamma * dist_sq).exp();
}
}
Tensor::from_data(k_data, vec![n1, n2], x1.device()).map_err(std::convert::Into::into)
}
/// Compute Evidence Lower Bound (ELBO)
///
/// ELBO = log p(y|f) - KL(q(u) || p(u))
pub fn elbo(&self, x_batch: &Tensor, y_batch: &Tensor) -> Result<f32> {
let xu = self
.inducing_points
.as_ref()
.ok_or_else(|| MLError::not_fitted("SVGP not initialized"))?;
let m = self.variational_mean.as_ref().unwrap();
let s = self.variational_variance.as_ref().unwrap();
let n_batch = x_batch.shape().dims()[0];
// Compute kernels
let kuu = self.compute_kernel(xu, xu)?;
let kuf = self.compute_kernel(xu, x_batch)?;
// Add jitter to Kuu for stability
let mut kuu_data = kuu.to_cpu()?;
let num_inducing = xu.shape().dims()[0];
for i in 0..num_inducing {
kuu_data[i * num_inducing + i] += self.config.jitter;
}
let kuu_stable =
Tensor::from_data(kuu_data, vec![num_inducing, num_inducing], xu.device())?;
// Cholesky decomposition of Kuu
let l_uu = self.cholesky(&kuu_stable)?;
// Solve L_uu^(-1) Kuf
let a = self.solve_triangular(&l_uu, &kuf, true)?;
// Mean prediction: Kfu Kuu^(-1) m
let l_inv_m = self.solve_triangular(&l_uu, m, true)?;
let kuu_inv_m = self.solve_triangular_transpose(&l_uu, &l_inv_m)?;
// Reshape to column vector for matrix multiplication
let kuu_inv_m_reshaped = kuu_inv_m.reshape(vec![num_inducing, 1])?;
let mu = kuf.transpose(0, 1)?.matmul(&kuu_inv_m_reshaped)?;
let mu = mu.reshape(vec![n_batch])?;
// Variance prediction (diagonal only for efficiency)
let a_data = a.to_cpu()?;
let s_data = s.to_cpu()?;
let m_inducing = xu.shape().dims()[0];
let mut var_diag = vec![0.0f32; n_batch];
for i in 0..n_batch {
let mut var_i = self.config.variance; // Kff diagonal
// Subtract A^T A (predictive variance reduction)
for j in 0..m_inducing {
var_i -= a_data[j * n_batch + i] * a_data[j * n_batch + i];
}
// Add uncertainty from variational posterior
for j in 0..m_inducing {
let a_ji = a_data[j * n_batch + i];
var_i += a_ji * a_ji * s_data[j];
}
var_diag[i] = var_i + self.config.noise;
}
// Log likelihood: log N(y | mu, var + noise)
let mu_data = mu.to_cpu()?;
let y_data = y_batch.to_cpu()?;
let mut log_lik = 0.0f32;
for i in 0..n_batch {
let residual = y_data[i] - mu_data[i];
log_lik += -0.5
* (residual * residual / var_diag[i]
+ var_diag[i].ln()
+ (2.0 * std::f32::consts::PI).ln());
}
// KL divergence: KL(q(u) || p(u))
let kl = self.kl_divergence()?;
// ELBO = log_lik - KL
let elbo = log_lik - kl;
Ok(elbo)
}
/// Compute KL divergence: KL(q(u) || p(u))
///
/// KL = 0.5 * [tr(Kuu^(-1) S) + m^T Kuu^(-1) m - M + log|Kuu| - log|S|]
pub fn kl_divergence(&self) -> Result<f32> {
let xu = self
.inducing_points
.as_ref()
.ok_or_else(|| MLError::not_fitted("SVGP not initialized"))?;
let m = self.variational_mean.as_ref().unwrap();
let s = self.variational_variance.as_ref().unwrap();
let num_inducing = xu.shape().dims()[0];
// Compute Kuu
let kuu = self.compute_kernel(xu, xu)?;
// Add jitter
let mut kuu_data = kuu.to_cpu()?;
for i in 0..num_inducing {
kuu_data[i * num_inducing + i] += self.config.jitter;
}
let kuu_stable =
Tensor::from_data(kuu_data, vec![num_inducing, num_inducing], xu.device())?;
// Cholesky: Kuu = L L^T
let l = self.cholesky(&kuu_stable)?;
// Solve L^(-1) m
let l_inv_m = self.solve_triangular(&l, m, true)?;
let l_inv_m_data = l_inv_m.to_cpu()?;
// m^T Kuu^(-1) m = ||L^(-1) m||^2
let m_kuu_inv_m: f32 = l_inv_m_data.iter().map(|&x| x * x).sum();
// tr(Kuu^(-1) S) = Σ (L^(-1))_ii^2 * S_ii (for diagonal S)
let l_data = l.to_cpu()?;
let s_data = s.to_cpu()?;
let mut trace_term = 0.0f32;
for i in 0..num_inducing {
// Compute (L^(-1))_ii using forward substitution
let mut l_inv_ii = 0.0f32;
let mut e_i = vec![0.0f32; num_inducing];
e_i[i] = 1.0;
for j in 0..num_inducing {
let mut sum = 0.0f32;
for k in 0..j {
sum += l_data[j * num_inducing + k] * e_i[k];
}
e_i[j] = (e_i[j] - sum) / l_data[j * num_inducing + j];
if j == i {
l_inv_ii = e_i[j];
}
}
trace_term += l_inv_ii * l_inv_ii * s_data[i];
}
// log|Kuu| = 2 * Σ log(L_ii)
let mut log_det_kuu = 0.0f32;
for i in 0..num_inducing {
log_det_kuu += 2.0 * l_data[i * num_inducing + i].ln();
}
// log|S| = Σ log(S_ii)
let log_det_s: f32 = s_data.iter().map(|&x| x.ln()).sum();
// KL = 0.5 * [trace + m_term - M + log_det_kuu - log_det_s]
let kl = 0.5 * (trace_term + m_kuu_inv_m - num_inducing as f32 + log_det_kuu - log_det_s);
Ok(kl)
}
/// Predict mean and variance for test points
pub fn predict(&self, x_test: &Tensor) -> Result<(Tensor, Tensor)> {
let xu = self
.inducing_points
.as_ref()
.ok_or_else(|| MLError::not_fitted("SVGP not initialized"))?;
let m = self.variational_mean.as_ref().unwrap();
let s = self.variational_variance.as_ref().unwrap();
if x_test.shape().ndim() != 2 {
return Err(MLError::invalid_parameter("x_test must be 2-dimensional"));
}
if x_test.shape().dims()[1] != self.n_features {
return Err(MLError::invalid_parameter(format!(
"x_test has {} features but model expects {}",
x_test.shape().dims()[1],
self.n_features
)));
}
let n_test = x_test.shape().dims()[0];
let m_inducing = xu.shape().dims()[0];
// Compute kernels
let kuu = self.compute_kernel(xu, xu)?;
let kuf = self.compute_kernel(xu, x_test)?;
// Add jitter
let mut kuu_data = kuu.to_cpu()?;
for i in 0..m_inducing {
kuu_data[i * m_inducing + i] += self.config.jitter;
}
let kuu_stable = Tensor::from_data(kuu_data, vec![m_inducing, m_inducing], xu.device())?;
// Cholesky
let l_uu = self.cholesky(&kuu_stable)?;
// Solve L^(-1) Kuf
let a = self.solve_triangular(&l_uu, &kuf, true)?;
// Mean: Kfu Kuu^(-1) m
let l_inv_m = self.solve_triangular(&l_uu, m, true)?;
let kuu_inv_m = self.solve_triangular_transpose(&l_uu, &l_inv_m)?;
let kuu_inv_m_reshaped = kuu_inv_m.reshape(vec![m_inducing, 1])?;
let mean = kuf.transpose(0, 1)?.matmul(&kuu_inv_m_reshaped)?;
let mean = mean.reshape(vec![n_test])?;
// Variance (diagonal)
let a_data = a.to_cpu()?;
let s_data = s.to_cpu()?;
let mut var_data = vec![0.0f32; n_test];
for i in 0..n_test {
let mut var_i = self.config.variance;
// Predictive variance reduction
for j in 0..m_inducing {
var_i -= a_data[j * n_test + i] * a_data[j * n_test + i];
}
// Variational uncertainty
for j in 0..m_inducing {
let a_ji = a_data[j * n_test + i];
var_i += a_ji * a_ji * s_data[j];
}
var_data[i] = var_i.max(0.0);
}
let variance = Tensor::from_data(var_data, vec![n_test], x_test.device())?;
Ok((mean, variance))
}
/// Update variational parameters (simplified gradient step)
pub fn update_variational_params(
&mut self,
_x_batch: &Tensor,
_y_batch: &Tensor,
_learning_rate: f32,
) -> Result<()> {
// Placeholder for gradient-based update
// In practice, would compute gradients of ELBO w.r.t. m and S
// and update using optimizer
Ok(())
}
/// Cholesky decomposition
fn cholesky(&self, matrix: &Tensor) -> Result<Tensor> {
let n = matrix.shape().dims()[0];
let data = matrix.to_cpu()?;
let mut l_data = vec![0.0f32; n * n];
for i in 0..n {
for j in 0..=i {
if i == j {
let mut sum = 0.0;
for k in 0..j {
sum += l_data[j * n + k] * l_data[j * n + k];
}
let val = data[j * n + j] - sum;
if val <= 0.0 {
return Err(MLError::invalid_parameter(
"Matrix not positive definite in Cholesky decomposition",
));
}
l_data[j * n + j] = val.sqrt();
} else {
let mut sum = 0.0;
for k in 0..j {
sum += l_data[i * n + k] * l_data[j * n + k];
}
l_data[i * n + j] = (data[i * n + j] - sum) / l_data[j * n + j];
}
}
}
Tensor::from_data(l_data, vec![n, n], matrix.device()).map_err(std::convert::Into::into)
}
/// Solve triangular system L x = b (lower=true) or U x = b (lower=false)
fn solve_triangular(&self, l: &Tensor, b: &Tensor, lower: bool) -> Result<Tensor> {
let n = l.shape().dims()[0];
let l_data = l.to_cpu()?;
let b_data = b.to_cpu()?;
let b_is_matrix = b.shape().ndim() == 2;
let n_cols = if b_is_matrix { b.shape().dims()[1] } else { 1 };
let mut x_data = vec![0.0f32; n * n_cols];
for col in 0..n_cols {
if lower {
// Forward substitution
for i in 0..n {
let mut sum = 0.0;
for j in 0..i {
sum += l_data[i * n + j] * x_data[j * n_cols + col];
}
let b_val = if b_is_matrix {
b_data[i * n_cols + col]
} else {
b_data[i]
};
x_data[i * n_cols + col] = (b_val - sum) / l_data[i * n + i];
}
} else {
// Backward substitution
for i in (0..n).rev() {
let mut sum = 0.0;
for j in (i + 1)..n {
sum += l_data[i * n + j] * x_data[j * n_cols + col];
}
let b_val = if b_is_matrix {
b_data[i * n_cols + col]
} else {
b_data[i]
};
x_data[i * n_cols + col] = (b_val - sum) / l_data[i * n + i];
}
}
}
let shape = if b_is_matrix {
vec![n, n_cols]
} else {
vec![n]
};
Tensor::from_data(x_data, shape, l.device()).map_err(std::convert::Into::into)
}
/// Solve L^T x = b
fn solve_triangular_transpose(&self, l: &Tensor, b: &Tensor) -> Result<Tensor> {
let n = l.shape().dims()[0];
let l_data = l.to_cpu()?;
let b_data = b.to_cpu()?;
let mut x_data = vec![0.0f32; n];
// Backward substitution with transpose
for i in (0..n).rev() {
let mut sum = 0.0;
for j in (i + 1)..n {
sum += l_data[j * n + i] * x_data[j]; // Note: j,i for transpose
}
x_data[i] = (b_data[i] - sum) / l_data[i * n + i];
}
Tensor::from_data(x_data, vec![n], l.device()).map_err(std::convert::Into::into)
}
/// Get configuration
pub fn config(&self) -> &SVGPConfig {
&self.config
}
/// Check if initialized
pub fn is_initialized(&self) -> bool {
self.inducing_points.is_some()
}
}
#[cfg(test)]
mod tests {
use super::*;
use rtx_tensor::Device;
fn create_test_data(n: usize, d: usize) -> (Tensor, Tensor) {
let device = Device::cpu();
let x_data: Vec<f32> = (0..n * d).map(|i| (i as f32) / (n * d) as f32).collect();
let y_data: Vec<f32> = (0..n).map(|i| (i as f32).sin()).collect();
let x = Tensor::from_data(x_data, vec![n, d], &device).unwrap();
let y = Tensor::from_data(y_data, vec![n], &device).unwrap();
(x, y)
}
#[test]
fn test_svgp_config_default() {
let config = SVGPConfig::default();
assert_eq!(config.num_inducing, 100);
assert!(!config.learn_inducing_locations);
assert!(config.jitter > 0.0);
}
#[test]
fn test_svgp_creation() {
let config = SVGPConfig::default();
let svgp = SVGP::new(config);
assert!(svgp.is_ok());
}
#[test]
fn test_svgp_invalid_config() {
let mut config = SVGPConfig::default();
config.num_inducing = 0;
assert!(SVGP::new(config).is_err());
let mut config = SVGPConfig::default();
config.jitter = 0.0;
assert!(SVGP::new(config).is_err());
let mut config = SVGPConfig::default();
config.length_scale = -1.0;
assert!(SVGP::new(config).is_err());
}
#[test]
fn test_svgp_initialize() {
let config = SVGPConfig {
num_inducing: 10,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x, y) = create_test_data(50, 3);
let result = svgp.initialize(&x, &y);
assert!(result.is_ok());
assert!(svgp.is_initialized());
}
#[test]
fn test_svgp_initialize_invalid_dims() {
let config = SVGPConfig::default();
let mut svgp = SVGP::new(config).unwrap();
let device = Device::cpu();
let x = Tensor::from_data(vec![1.0, 2.0, 3.0], vec![3], &device).unwrap();
let y = Tensor::from_data(vec![1.0, 2.0, 3.0], vec![3], &device).unwrap();
assert!(svgp.initialize(&x, &y).is_err());
}
#[test]
fn test_svgp_initialize_mismatched_sizes() {
let config = SVGPConfig::default();
let mut svgp = SVGP::new(config).unwrap();
let device = Device::cpu();
let x = Tensor::from_data(vec![1.0, 2.0, 3.0, 4.0], vec![2, 2], &device).unwrap();
let y = Tensor::from_data(vec![1.0, 2.0, 3.0], vec![3], &device).unwrap();
assert!(svgp.initialize(&x, &y).is_err());
}
#[test]
fn test_svgp_initialize_too_many_inducing() {
let config = SVGPConfig {
num_inducing: 100,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x, y) = create_test_data(50, 3);
assert!(svgp.initialize(&x, &y).is_err());
}
#[test]
fn test_svgp_kl_divergence() {
let config = SVGPConfig {
num_inducing: 5,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x, y) = create_test_data(20, 2);
svgp.initialize(&x, &y).unwrap();
let kl = svgp.kl_divergence();
assert!(kl.is_ok());
let kl_val = kl.unwrap();
assert!(kl_val >= 0.0); // KL divergence is always non-negative
}
#[test]
fn test_svgp_elbo() {
let config = SVGPConfig {
num_inducing: 5,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x, y) = create_test_data(20, 2);
svgp.initialize(&x, &y).unwrap();
let elbo = svgp.elbo(&x, &y);
assert!(elbo.is_ok());
let elbo_val = elbo.unwrap();
assert!(elbo_val.is_finite());
}
#[test]
fn test_svgp_elbo_not_initialized() {
let config = SVGPConfig::default();
let svgp = SVGP::new(config).unwrap();
let (x, y) = create_test_data(20, 2);
assert!(svgp.elbo(&x, &y).is_err());
}
#[test]
fn test_svgp_predict() {
let config = SVGPConfig {
num_inducing: 5,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x_train, y_train) = create_test_data(20, 2);
svgp.initialize(&x_train, &y_train).unwrap();
let (x_test, _) = create_test_data(10, 2);
let result = svgp.predict(&x_test);
assert!(result.is_ok());
let (mean, var) = result.unwrap();
assert_eq!(mean.shape().dims(), &[10]);
assert_eq!(var.shape().dims(), &[10]);
// Variance should be non-negative
let var_data = var.to_cpu().unwrap();
for &v in &var_data {
assert!(v >= 0.0);
}
}
#[test]
fn test_svgp_predict_wrong_features() {
let config = SVGPConfig {
num_inducing: 5,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x_train, y_train) = create_test_data(20, 2);
svgp.initialize(&x_train, &y_train).unwrap();
let (x_test, _) = create_test_data(10, 3); // Wrong number of features
assert!(svgp.predict(&x_test).is_err());
}
#[test]
fn test_svgp_predict_not_initialized() {
let config = SVGPConfig::default();
let svgp = SVGP::new(config).unwrap();
let (x_test, _) = create_test_data(10, 2);
assert!(svgp.predict(&x_test).is_err());
}
#[test]
fn test_svgp_update_params() {
let config = SVGPConfig {
num_inducing: 5,
..Default::default()
};
let mut svgp = SVGP::new(config).unwrap();
let (x, y) = create_test_data(20, 2);
svgp.initialize(&x, &y).unwrap();
let result = svgp.update_variational_params(&x, &y, 0.01);
assert!(result.is_ok());
}
#[test]
fn test_svgp_kernel_computation() {
let config = SVGPConfig {
num_inducing: 5,
length_scale: 1.0,
variance: 2.0,
..Default::default()
};
let svgp = SVGP::new(config).unwrap();
let device = Device::cpu();
let x1 = Tensor::from_data(vec![0.0, 0.0, 1.0, 1.0], vec![2, 2], &device).unwrap();
let x2 = Tensor::from_data(vec![0.0, 0.0, 2.0, 2.0], vec![2, 2], &device).unwrap();
let k = svgp.compute_kernel(&x1, &x2).unwrap();
assert_eq!(k.shape().dims(), &[2, 2]);
let k_data = k.to_cpu().unwrap();
// k(x, x) should be variance
assert!((k_data[0] - 2.0).abs() < 1e-4);
// For symmetry, compute K(x2, x1) and compare
let k_sym = svgp.compute_kernel(&x2, &x1).unwrap();
let k_sym_data = k_sym.to_cpu().unwrap();
// K(x1[i], x2[j]) should equal K(x2[j], x1[i])
assert!((k_data[1] - k_sym_data[2]).abs() < 1e-4);
}
#[test]
fn test_svgp_cholesky() {
let config = SVGPConfig::default();
let svgp = SVGP::new(config).unwrap();
let device = Device::cpu();
// Simple 2x2 positive definite matrix
let a = Tensor::from_data(vec![4.0, 2.0, 2.0, 3.0], vec![2, 2], &device).unwrap();
let l = svgp.cholesky(&a);
assert!(l.is_ok());
let l_data = l.unwrap().to_cpu().unwrap();
// L should be lower triangular with L L^T = A
assert!(l_data[0] > 0.0); // L[0,0]
assert!(l_data[1] < 1e-6); // L[0,1] ≈ 0
assert!(l_data[2] > 0.0); // L[1,0]
assert!(l_data[3] > 0.0); // L[1,1]
}
#[test]
fn test_svgp_solve_triangular() {
let config = SVGPConfig::default();
let svgp = SVGP::new(config).unwrap();
let device = Device::cpu();
// Lower triangular matrix
let l = Tensor::from_data(vec![2.0, 0.0, 1.0, 3.0], vec![2, 2], &device).unwrap();
let b = Tensor::from_data(vec![4.0, 7.0], vec![2], &device).unwrap();
let x = svgp.solve_triangular(&l, &b, true);
assert!(x.is_ok());
let x_data = x.unwrap().to_cpu().unwrap();
// Verify L x = b
// [2 0][x0] [4]
// [1 3][x1] = [7]
// x0 = 2, x1 = (7-1*2)/3 = 5/3
assert!((x_data[0] - 2.0).abs() < 1e-4);
assert!((x_data[1] - 5.0 / 3.0).abs() < 1e-4);
}
}