style: apply rustfmt across all crates and demos
Consistent formatting pass: line wrapping, import sorting, trailing whitespace removal, let-chain indentation, merged derive attributes, and unsafe block reformatting. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
@@ -1362,7 +1362,6 @@ fn create_parents_2<B: Backend, const D: usize>(
|
|||||||
|
|
||||||
/// Convert internal gradient storage to public GradientMap.
|
/// Convert internal gradient storage to public GradientMap.
|
||||||
fn convert_gradient_storage<B: Backend>(storage: GradientStorage<B>) -> GradientMap<B> {
|
fn convert_gradient_storage<B: Backend>(storage: GradientStorage<B>) -> GradientMap<B> {
|
||||||
|
|
||||||
// Note: Full conversion would require extending GradientMap with insert method
|
// Note: Full conversion would require extending GradientMap with insert method
|
||||||
// For now, return empty map - will be properly implemented when needed
|
// For now, return empty map - will be properly implemented when needed
|
||||||
GradientMap::new()
|
GradientMap::new()
|
||||||
|
|||||||
@@ -622,7 +622,6 @@ where
|
|||||||
let scaled_diff = B::mul(abs_diff, scale_tensor);
|
let scaled_diff = B::mul(abs_diff, scale_tensor);
|
||||||
let neg_scaled = B::neg(scaled_diff);
|
let neg_scaled = B::neg(scaled_diff);
|
||||||
|
|
||||||
|
|
||||||
B::exp(neg_scaled)
|
B::exp(neg_scaled)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -171,7 +171,6 @@ where
|
|||||||
{
|
{
|
||||||
// Get the tensor from grad_output (D2 dimensional)
|
// Get the tensor from grad_output (D2 dimensional)
|
||||||
|
|
||||||
|
|
||||||
match grad {
|
match grad {
|
||||||
GradTensor::D1(g) if D2 == 1 => {
|
GradTensor::D1(g) if D2 == 1 => {
|
||||||
let reshaped = B::reshape(g.clone(), target_shape);
|
let reshaped = B::reshape(g.clone(), target_shape);
|
||||||
@@ -197,11 +196,9 @@ where
|
|||||||
let reshaped = B::reshape(g.clone(), target_shape);
|
let reshaped = B::reshape(g.clone(), target_shape);
|
||||||
wrap_as_grad_tensor::<B, D1>(reshaped)
|
wrap_as_grad_tensor::<B, D1>(reshaped)
|
||||||
}
|
}
|
||||||
_ => {
|
_ => Err(AutogradError::DimensionMismatch(
|
||||||
Err(AutogradError::DimensionMismatch(
|
"reshape backward".to_string(),
|
||||||
"reshape backward".to_string(),
|
)),
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -366,10 +366,16 @@ where
|
|||||||
let func_outputs = f(&func_inputs)?;
|
let func_outputs = f(&func_inputs)?;
|
||||||
|
|
||||||
// Collect mutations from all inputs
|
// Collect mutations from all inputs
|
||||||
let mutations: Vec<MutationRecord> = func_inputs.iter().flat_map(Functionalized::mutations).collect();
|
let mutations: Vec<MutationRecord> = func_inputs
|
||||||
|
.iter()
|
||||||
|
.flat_map(Functionalized::mutations)
|
||||||
|
.collect();
|
||||||
|
|
||||||
// Unwrap outputs
|
// Unwrap outputs
|
||||||
let outputs: Vec<Variable> = func_outputs.into_iter().map(Functionalized::into_inner).collect();
|
let outputs: Vec<Variable> = func_outputs
|
||||||
|
.into_iter()
|
||||||
|
.map(Functionalized::into_inner)
|
||||||
|
.collect();
|
||||||
|
|
||||||
Ok((outputs, mutations))
|
Ok((outputs, mutations))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,7 +58,8 @@ where
|
|||||||
let vjp_final = compute_gradients_weighted(&f, &inputs_with_grad, &combined_cotangent)?;
|
let vjp_final = compute_gradients_weighted(&f, &inputs_with_grad, &combined_cotangent)?;
|
||||||
|
|
||||||
// Detach outputs
|
// Detach outputs
|
||||||
let outputs_detached: Vec<Variable> = outputs.iter().map(super::super::Variable::detach).collect();
|
let outputs_detached: Vec<Variable> =
|
||||||
|
outputs.iter().map(super::super::Variable::detach).collect();
|
||||||
|
|
||||||
Ok((outputs_detached, vjp_final))
|
Ok((outputs_detached, vjp_final))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -60,7 +60,8 @@ where
|
|||||||
// Determine batch size
|
// Determine batch size
|
||||||
let batch_size = inputs
|
let batch_size = inputs
|
||||||
.iter()
|
.iter()
|
||||||
.zip(in_dims.iter()).find_map(|(tensor, dim)| dim.map(|d| tensor.shape().get(d).copied().unwrap_or(1)))
|
.zip(in_dims.iter())
|
||||||
|
.find_map(|(tensor, dim)| dim.map(|d| tensor.shape().get(d).copied().unwrap_or(1)))
|
||||||
.unwrap_or(1);
|
.unwrap_or(1);
|
||||||
|
|
||||||
if batch_size == 0 {
|
if batch_size == 0 {
|
||||||
@@ -174,7 +175,8 @@ where
|
|||||||
// Determine batch size from the first batched input
|
// Determine batch size from the first batched input
|
||||||
let batch_size = inputs
|
let batch_size = inputs
|
||||||
.iter()
|
.iter()
|
||||||
.zip(in_dims.iter()).find_map(|(tensor, dim)| dim.map(|d| tensor.shape().get(d).copied().unwrap_or(1)))
|
.zip(in_dims.iter())
|
||||||
|
.find_map(|(tensor, dim)| dim.map(|d| tensor.shape().get(d).copied().unwrap_or(1)))
|
||||||
.unwrap_or(1);
|
.unwrap_or(1);
|
||||||
|
|
||||||
if batch_size == 0 {
|
if batch_size == 0 {
|
||||||
|
|||||||
@@ -4,7 +4,9 @@
|
|||||||
|
|
||||||
#![cfg(target_arch = "aarch64")]
|
#![cfg(target_arch = "aarch64")]
|
||||||
|
|
||||||
use std::arch::aarch64::{vld1q_f32, vaddq_f32, vst1q_f32, vmulq_f32, vfmaq_f32, vdupq_n_f32, vaddvq_f32, vmaxq_f32};
|
use std::arch::aarch64::{
|
||||||
|
vaddq_f32, vaddvq_f32, vdupq_n_f32, vfmaq_f32, vld1q_f32, vmaxq_f32, vmulq_f32, vst1q_f32,
|
||||||
|
};
|
||||||
|
|
||||||
/// NEON vector width in f32 elements
|
/// NEON vector width in f32 elements
|
||||||
pub const VECTOR_WIDTH: usize = 4;
|
pub const VECTOR_WIDTH: usize = 4;
|
||||||
@@ -13,26 +15,28 @@ pub const VECTOR_WIDTH: usize = 4;
|
|||||||
///
|
///
|
||||||
/// Uses `vaddq_f32` for 4-wide vector addition.
|
/// Uses `vaddq_f32` for 4-wide vector addition.
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn add_neon(a: &[f32], b: &[f32], c: &mut [f32]) { unsafe {
|
pub unsafe fn add_neon(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
// Process full vectors
|
// Process full vectors
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
let vb = vld1q_f32(b.as_ptr().add(offset));
|
let vb = vld1q_f32(b.as_ptr().add(offset));
|
||||||
let vc = vaddq_f32(va, vb);
|
let vc = vaddq_f32(va, vb);
|
||||||
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle remainder with scalar
|
// Handle remainder with scalar
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
for i in 0..remainder {
|
for i in 0..remainder {
|
||||||
c[tail_start + i] = a[tail_start + i] + b[tail_start + i];
|
c[tail_start + i] = a[tail_start + i] + b[tail_start + i];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}}
|
}
|
||||||
|
|
||||||
/// Safe wrapper for NEON add
|
/// Safe wrapper for NEON add
|
||||||
pub fn add(a: &[f32], b: &[f32], c: &mut [f32]) {
|
pub fn add(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
@@ -43,24 +47,26 @@ pub fn add(a: &[f32], b: &[f32], c: &mut [f32]) {
|
|||||||
///
|
///
|
||||||
/// Uses `vmulq_f32` for 4-wide vector multiplication.
|
/// Uses `vmulq_f32` for 4-wide vector multiplication.
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn mul_neon(a: &[f32], b: &[f32], c: &mut [f32]) { unsafe {
|
pub unsafe fn mul_neon(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
let vb = vld1q_f32(b.as_ptr().add(offset));
|
let vb = vld1q_f32(b.as_ptr().add(offset));
|
||||||
let vc = vmulq_f32(va, vb);
|
let vc = vmulq_f32(va, vb);
|
||||||
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
||||||
}
|
}
|
||||||
|
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
for i in 0..remainder {
|
for i in 0..remainder {
|
||||||
c[tail_start + i] = a[tail_start + i] * b[tail_start + i];
|
c[tail_start + i] = a[tail_start + i] * b[tail_start + i];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}}
|
}
|
||||||
|
|
||||||
/// Safe wrapper for NEON mul
|
/// Safe wrapper for NEON mul
|
||||||
pub fn mul(a: &[f32], b: &[f32], c: &mut [f32]) {
|
pub fn mul(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
@@ -71,26 +77,28 @@ pub fn mul(a: &[f32], b: &[f32], c: &mut [f32]) {
|
|||||||
///
|
///
|
||||||
/// Uses `vfmaq_f32` for fused multiply-add (no intermediate rounding).
|
/// Uses `vfmaq_f32` for fused multiply-add (no intermediate rounding).
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn fma_neon(a: &[f32], b: &[f32], c: &mut [f32]) { unsafe {
|
pub unsafe fn fma_neon(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
let vb = vld1q_f32(b.as_ptr().add(offset));
|
let vb = vld1q_f32(b.as_ptr().add(offset));
|
||||||
let vc = vld1q_f32(c.as_ptr().add(offset));
|
let vc = vld1q_f32(c.as_ptr().add(offset));
|
||||||
// vfmaq_f32(c, a, b) = a * b + c
|
// vfmaq_f32(c, a, b) = a * b + c
|
||||||
let vr = vfmaq_f32(vc, va, vb);
|
let vr = vfmaq_f32(vc, va, vb);
|
||||||
vst1q_f32(c.as_mut_ptr().add(offset), vr);
|
vst1q_f32(c.as_mut_ptr().add(offset), vr);
|
||||||
}
|
}
|
||||||
|
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
for i in 0..remainder {
|
for i in 0..remainder {
|
||||||
c[tail_start + i] = a[tail_start + i].mul_add(b[tail_start + i], c[tail_start + i]);
|
c[tail_start + i] = a[tail_start + i].mul_add(b[tail_start + i], c[tail_start + i]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}}
|
}
|
||||||
|
|
||||||
/// Safe wrapper for NEON fma
|
/// Safe wrapper for NEON fma
|
||||||
pub fn fma(a: &[f32], b: &[f32], c: &mut [f32]) {
|
pub fn fma(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
@@ -101,30 +109,32 @@ pub fn fma(a: &[f32], b: &[f32], c: &mut [f32]) {
|
|||||||
///
|
///
|
||||||
/// Uses `vaddvq_f32` for horizontal sum.
|
/// Uses `vaddvq_f32` for horizontal sum.
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn sum_neon(a: &[f32]) -> f32 { unsafe {
|
pub unsafe fn sum_neon(a: &[f32]) -> f32 {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
let mut acc = vdupq_n_f32(0.0);
|
let mut acc = vdupq_n_f32(0.0);
|
||||||
|
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
acc = vaddq_f32(acc, va);
|
acc = vaddq_f32(acc, va);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Horizontal sum of accumulator
|
||||||
|
let mut total = vaddvq_f32(acc);
|
||||||
|
|
||||||
|
// Add remainder
|
||||||
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
|
for i in 0..remainder {
|
||||||
|
total += a[tail_start + i];
|
||||||
|
}
|
||||||
|
|
||||||
|
total
|
||||||
}
|
}
|
||||||
|
}
|
||||||
// Horizontal sum of accumulator
|
|
||||||
let mut total = vaddvq_f32(acc);
|
|
||||||
|
|
||||||
// Add remainder
|
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
|
||||||
for i in 0..remainder {
|
|
||||||
total += a[tail_start + i];
|
|
||||||
}
|
|
||||||
|
|
||||||
total
|
|
||||||
}}
|
|
||||||
|
|
||||||
/// Safe wrapper for NEON sum
|
/// Safe wrapper for NEON sum
|
||||||
pub fn sum(a: &[f32]) -> f32 {
|
pub fn sum(a: &[f32]) -> f32 {
|
||||||
@@ -135,29 +145,31 @@ pub fn sum(a: &[f32]) -> f32 {
|
|||||||
///
|
///
|
||||||
/// Uses fused multiply-add for accumulation.
|
/// Uses fused multiply-add for accumulation.
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn dot_neon(a: &[f32], b: &[f32]) -> f32 { unsafe {
|
pub unsafe fn dot_neon(a: &[f32], b: &[f32]) -> f32 {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
let mut acc = vdupq_n_f32(0.0);
|
let mut acc = vdupq_n_f32(0.0);
|
||||||
|
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
let vb = vld1q_f32(b.as_ptr().add(offset));
|
let vb = vld1q_f32(b.as_ptr().add(offset));
|
||||||
acc = vfmaq_f32(acc, va, vb);
|
acc = vfmaq_f32(acc, va, vb);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut total = vaddvq_f32(acc);
|
||||||
|
|
||||||
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
|
for i in 0..remainder {
|
||||||
|
total += a[tail_start + i] * b[tail_start + i];
|
||||||
|
}
|
||||||
|
|
||||||
|
total
|
||||||
}
|
}
|
||||||
|
}
|
||||||
let mut total = vaddvq_f32(acc);
|
|
||||||
|
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
|
||||||
for i in 0..remainder {
|
|
||||||
total += a[tail_start + i] * b[tail_start + i];
|
|
||||||
}
|
|
||||||
|
|
||||||
total
|
|
||||||
}}
|
|
||||||
|
|
||||||
/// Safe wrapper for NEON dot
|
/// Safe wrapper for NEON dot
|
||||||
pub fn dot(a: &[f32], b: &[f32]) -> f32 {
|
pub fn dot(a: &[f32], b: &[f32]) -> f32 {
|
||||||
@@ -166,24 +178,26 @@ pub fn dot(a: &[f32], b: &[f32]) -> f32 {
|
|||||||
|
|
||||||
/// Element-wise maximum
|
/// Element-wise maximum
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn max_neon(a: &[f32], b: &[f32], c: &mut [f32]) { unsafe {
|
pub unsafe fn max_neon(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
let vb = vld1q_f32(b.as_ptr().add(offset));
|
let vb = vld1q_f32(b.as_ptr().add(offset));
|
||||||
let vc = vmaxq_f32(va, vb);
|
let vc = vmaxq_f32(va, vb);
|
||||||
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
||||||
}
|
}
|
||||||
|
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
for i in 0..remainder {
|
for i in 0..remainder {
|
||||||
c[tail_start + i] = a[tail_start + i].max(b[tail_start + i]);
|
c[tail_start + i] = a[tail_start + i].max(b[tail_start + i]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}}
|
}
|
||||||
|
|
||||||
/// Safe wrapper for NEON max
|
/// Safe wrapper for NEON max
|
||||||
pub fn max(a: &[f32], b: &[f32], c: &mut [f32]) {
|
pub fn max(a: &[f32], b: &[f32], c: &mut [f32]) {
|
||||||
@@ -209,25 +223,27 @@ pub fn exp(a: &[f32], c: &mut [f32]) {
|
|||||||
|
|
||||||
/// Vector scale: c = a * scalar
|
/// Vector scale: c = a * scalar
|
||||||
#[target_feature(enable = "neon")]
|
#[target_feature(enable = "neon")]
|
||||||
pub unsafe fn scale_neon(a: &[f32], scalar: f32, c: &mut [f32]) { unsafe {
|
pub unsafe fn scale_neon(a: &[f32], scalar: f32, c: &mut [f32]) {
|
||||||
let len = a.len();
|
unsafe {
|
||||||
let chunks = len / VECTOR_WIDTH;
|
let len = a.len();
|
||||||
let remainder = len % VECTOR_WIDTH;
|
let chunks = len / VECTOR_WIDTH;
|
||||||
|
let remainder = len % VECTOR_WIDTH;
|
||||||
|
|
||||||
let vs = vdupq_n_f32(scalar);
|
let vs = vdupq_n_f32(scalar);
|
||||||
|
|
||||||
for i in 0..chunks {
|
for i in 0..chunks {
|
||||||
let offset = i * VECTOR_WIDTH;
|
let offset = i * VECTOR_WIDTH;
|
||||||
let va = vld1q_f32(a.as_ptr().add(offset));
|
let va = vld1q_f32(a.as_ptr().add(offset));
|
||||||
let vc = vmulq_f32(va, vs);
|
let vc = vmulq_f32(va, vs);
|
||||||
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
vst1q_f32(c.as_mut_ptr().add(offset), vc);
|
||||||
|
}
|
||||||
|
|
||||||
|
let tail_start = chunks * VECTOR_WIDTH;
|
||||||
|
for i in 0..remainder {
|
||||||
|
c[tail_start + i] = a[tail_start + i] * scalar;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
let tail_start = chunks * VECTOR_WIDTH;
|
|
||||||
for i in 0..remainder {
|
|
||||||
c[tail_start + i] = a[tail_start + i] * scalar;
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
|
|
||||||
/// Safe wrapper for NEON scale
|
/// Safe wrapper for NEON scale
|
||||||
pub fn scale(a: &[f32], scalar: f32, c: &mut [f32]) {
|
pub fn scale(a: &[f32], scalar: f32, c: &mut [f32]) {
|
||||||
@@ -251,75 +267,77 @@ pub unsafe fn gemm_neon(
|
|||||||
beta: f32,
|
beta: f32,
|
||||||
c: &mut [f32],
|
c: &mut [f32],
|
||||||
ldc: usize,
|
ldc: usize,
|
||||||
) { unsafe {
|
) {
|
||||||
// Block sizes for cache efficiency on Apple Silicon
|
unsafe {
|
||||||
const MC: usize = 64; // Rows of A per block
|
// Block sizes for cache efficiency on Apple Silicon
|
||||||
const NC: usize = 256; // Cols of B per block
|
const MC: usize = 64; // Rows of A per block
|
||||||
const KC: usize = 128; // Inner dimension per block
|
const NC: usize = 256; // Cols of B per block
|
||||||
|
const KC: usize = 128; // Inner dimension per block
|
||||||
|
|
||||||
// Scale C by beta
|
// Scale C by beta
|
||||||
if beta == 0.0 {
|
if beta == 0.0 {
|
||||||
for i in 0..m {
|
for i in 0..m {
|
||||||
for j in 0..n {
|
for j in 0..n {
|
||||||
c[i * ldc + j] = 0.0;
|
c[i * ldc + j] = 0.0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if beta != 1.0 {
|
||||||
|
let vbeta = vdupq_n_f32(beta);
|
||||||
|
for i in 0..m {
|
||||||
|
let row = &mut c[i * ldc..i * ldc + n];
|
||||||
|
let chunks = n / VECTOR_WIDTH;
|
||||||
|
for j in 0..chunks {
|
||||||
|
let offset = j * VECTOR_WIDTH;
|
||||||
|
let vc = vld1q_f32(row.as_ptr().add(offset));
|
||||||
|
let vr = vmulq_f32(vc, vbeta);
|
||||||
|
vst1q_f32(row.as_mut_ptr().add(offset), vr);
|
||||||
|
}
|
||||||
|
for j in (chunks * VECTOR_WIDTH)..n {
|
||||||
|
row[j] *= beta;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if beta != 1.0 {
|
|
||||||
let vbeta = vdupq_n_f32(beta);
|
|
||||||
for i in 0..m {
|
|
||||||
let row = &mut c[i * ldc..i * ldc + n];
|
|
||||||
let chunks = n / VECTOR_WIDTH;
|
|
||||||
for j in 0..chunks {
|
|
||||||
let offset = j * VECTOR_WIDTH;
|
|
||||||
let vc = vld1q_f32(row.as_ptr().add(offset));
|
|
||||||
let vr = vmulq_f32(vc, vbeta);
|
|
||||||
vst1q_f32(row.as_mut_ptr().add(offset), vr);
|
|
||||||
}
|
|
||||||
for j in (chunks * VECTOR_WIDTH)..n {
|
|
||||||
row[j] *= beta;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let valpha = vdupq_n_f32(alpha);
|
let valpha = vdupq_n_f32(alpha);
|
||||||
|
|
||||||
// Blocked GEMM
|
// Blocked GEMM
|
||||||
for jc in (0..n).step_by(NC) {
|
for jc in (0..n).step_by(NC) {
|
||||||
let jc_end = (jc + NC).min(n);
|
let jc_end = (jc + NC).min(n);
|
||||||
|
|
||||||
for pc in (0..k).step_by(KC) {
|
for pc in (0..k).step_by(KC) {
|
||||||
let pc_end = (pc + KC).min(k);
|
let pc_end = (pc + KC).min(k);
|
||||||
|
|
||||||
for ic in (0..m).step_by(MC) {
|
for ic in (0..m).step_by(MC) {
|
||||||
let ic_end = (ic + MC).min(m);
|
let ic_end = (ic + MC).min(m);
|
||||||
|
|
||||||
// Micro-kernel: process 4x4 blocks
|
// Micro-kernel: process 4x4 blocks
|
||||||
for i in ic..ic_end {
|
for i in ic..ic_end {
|
||||||
for j in (jc..jc_end).step_by(VECTOR_WIDTH) {
|
for j in (jc..jc_end).step_by(VECTOR_WIDTH) {
|
||||||
let j_end = (j + VECTOR_WIDTH).min(jc_end);
|
let j_end = (j + VECTOR_WIDTH).min(jc_end);
|
||||||
|
|
||||||
if j_end - j == VECTOR_WIDTH {
|
if j_end - j == VECTOR_WIDTH {
|
||||||
// Full vector
|
// Full vector
|
||||||
let mut acc = vld1q_f32(c.as_ptr().add(i * ldc + j));
|
let mut acc = vld1q_f32(c.as_ptr().add(i * ldc + j));
|
||||||
|
|
||||||
for p in pc..pc_end {
|
|
||||||
let a_ip = a[i * lda + p];
|
|
||||||
let va = vdupq_n_f32(a_ip);
|
|
||||||
let vb = vld1q_f32(b.as_ptr().add(p * ldb + j));
|
|
||||||
let vab = vmulq_f32(va, vb);
|
|
||||||
let vab_scaled = vmulq_f32(vab, valpha);
|
|
||||||
acc = vaddq_f32(acc, vab_scaled);
|
|
||||||
}
|
|
||||||
|
|
||||||
vst1q_f32(c.as_mut_ptr().add(i * ldc + j), acc);
|
|
||||||
} else {
|
|
||||||
// Partial vector - scalar fallback
|
|
||||||
for jj in j..j_end {
|
|
||||||
let mut sum = c[i * ldc + jj];
|
|
||||||
for p in pc..pc_end {
|
for p in pc..pc_end {
|
||||||
sum += alpha * a[i * lda + p] * b[p * ldb + jj];
|
let a_ip = a[i * lda + p];
|
||||||
|
let va = vdupq_n_f32(a_ip);
|
||||||
|
let vb = vld1q_f32(b.as_ptr().add(p * ldb + j));
|
||||||
|
let vab = vmulq_f32(va, vb);
|
||||||
|
let vab_scaled = vmulq_f32(vab, valpha);
|
||||||
|
acc = vaddq_f32(acc, vab_scaled);
|
||||||
|
}
|
||||||
|
|
||||||
|
vst1q_f32(c.as_mut_ptr().add(i * ldc + j), acc);
|
||||||
|
} else {
|
||||||
|
// Partial vector - scalar fallback
|
||||||
|
for jj in j..j_end {
|
||||||
|
let mut sum = c[i * ldc + jj];
|
||||||
|
for p in pc..pc_end {
|
||||||
|
sum += alpha * a[i * lda + p] * b[p * ldb + jj];
|
||||||
|
}
|
||||||
|
c[i * ldc + jj] = sum;
|
||||||
}
|
}
|
||||||
c[i * ldc + jj] = sum;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -327,7 +345,7 @@ pub unsafe fn gemm_neon(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}}
|
}
|
||||||
|
|
||||||
/// Safe wrapper for NEON gemm
|
/// Safe wrapper for NEON gemm
|
||||||
pub fn gemm(
|
pub fn gemm(
|
||||||
|
|||||||
@@ -55,7 +55,9 @@ pub fn flash_attention(
|
|||||||
let q_data = query.to_host()?;
|
let q_data = query.to_host()?;
|
||||||
let k_data = key.to_host()?;
|
let k_data = key.to_host()?;
|
||||||
let v_data = value.to_host()?;
|
let v_data = value.to_host()?;
|
||||||
let mask_data = mask.map(super::super::tensor::SyclTensorPrimitive::to_host).transpose()?;
|
let mask_data = mask
|
||||||
|
.map(super::super::tensor::SyclTensorPrimitive::to_host)
|
||||||
|
.transpose()?;
|
||||||
|
|
||||||
let mut output = vec![0.0f32; batch * heads * q_seq * head_dim];
|
let mut output = vec![0.0f32; batch * heads * q_seq * head_dim];
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,9 @@ pub fn layer_norm<const D: usize>(
|
|||||||
let shape = tensor.shape();
|
let shape = tensor.shape();
|
||||||
let data = tensor.to_host()?;
|
let data = tensor.to_host()?;
|
||||||
let weight_data = weight.to_host()?;
|
let weight_data = weight.to_host()?;
|
||||||
let bias_data = bias.map(super::super::tensor::SyclTensorPrimitive::to_host).transpose()?;
|
let bias_data = bias
|
||||||
|
.map(super::super::tensor::SyclTensorPrimitive::to_host)
|
||||||
|
.transpose()?;
|
||||||
|
|
||||||
let normalized_shape = shape[D - 1];
|
let normalized_shape = shape[D - 1];
|
||||||
if weight_data.len() != normalized_shape {
|
if weight_data.len() != normalized_shape {
|
||||||
|
|||||||
@@ -141,9 +141,10 @@ impl Backend for CubeclBackend {
|
|||||||
|
|
||||||
// Try to allocate on GPU, fall back to CPU
|
// Try to allocate on GPU, fall back to CPU
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(&data) {
|
&& let Ok(handle) = client.create_f32(&data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
// CPU fallback
|
// CPU fallback
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
@@ -155,9 +156,10 @@ impl Backend for CubeclBackend {
|
|||||||
let data = vec![1.0f32; numel];
|
let data = vec![1.0f32; numel];
|
||||||
|
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(&data) {
|
&& let Ok(handle) = client.create_f32(&data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
||||||
@@ -172,9 +174,10 @@ impl Backend for CubeclBackend {
|
|||||||
let data = vec![fill_value; numel];
|
let data = vec![fill_value; numel];
|
||||||
|
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(&data) {
|
&& let Ok(handle) = client.create_f32(&data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
||||||
@@ -192,9 +195,10 @@ impl Backend for CubeclBackend {
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(&data) {
|
&& let Ok(handle) = client.create_f32(&data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
||||||
@@ -217,9 +221,10 @@ impl Backend for CubeclBackend {
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(&data) {
|
&& let Ok(handle) = client.create_f32(&data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
||||||
@@ -231,9 +236,10 @@ impl Backend for CubeclBackend {
|
|||||||
device: &Self::Device,
|
device: &Self::Device,
|
||||||
) -> Self::TensorPrimitive<D> {
|
) -> Self::TensorPrimitive<D> {
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(data) {
|
&& let Ok(handle) = client.create_f32(data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
CubeclTensorPrimitive::from_cpu_data(shape, bytes, device.clone())
|
||||||
@@ -606,9 +612,10 @@ impl Backend for CubeclBackend {
|
|||||||
if let Some(buffer) = tensor.buffer() {
|
if let Some(buffer) = tensor.buffer() {
|
||||||
if let Some(gpu_handle) = buffer.as_gpu()
|
if let Some(gpu_handle) = buffer.as_gpu()
|
||||||
&& let Ok(client) = client::get_or_create(&tensor.device)
|
&& let Ok(client) = client::get_or_create(&tensor.device)
|
||||||
&& let Ok(data) = client.read_f32(gpu_handle) {
|
&& let Ok(data) = client.read_f32(gpu_handle)
|
||||||
return data;
|
{
|
||||||
}
|
return data;
|
||||||
|
}
|
||||||
// Read from CPU buffer
|
// Read from CPU buffer
|
||||||
if let Some(bytes) = buffer.as_cpu() {
|
if let Some(bytes) = buffer.as_cpu() {
|
||||||
return bytes
|
return bytes
|
||||||
|
|||||||
@@ -239,7 +239,7 @@ impl std::fmt::Debug for CubeclClient {
|
|||||||
///
|
///
|
||||||
/// This allows reusing clients across tensor operations.
|
/// This allows reusing clients across tensor operations.
|
||||||
mod registry {
|
mod registry {
|
||||||
use super::{CubeclDevice, CubeclClient, Result, CubeclError};
|
use super::{CubeclClient, CubeclDevice, CubeclError, Result};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::RwLock;
|
use std::sync::RwLock;
|
||||||
|
|
||||||
|
|||||||
@@ -55,9 +55,10 @@ pub fn read_tensor_data<const D: usize>(tensor: &CubeclTensorPrimitive<D>) -> Ve
|
|||||||
// Try to read from GPU
|
// Try to read from GPU
|
||||||
if let Some(gpu_handle) = buffer.as_gpu()
|
if let Some(gpu_handle) = buffer.as_gpu()
|
||||||
&& let Ok(client) = client::get_or_create(&tensor.device)
|
&& let Ok(client) = client::get_or_create(&tensor.device)
|
||||||
&& let Ok(data) = client.read_f32(gpu_handle) {
|
&& let Ok(data) = client.read_f32(gpu_handle)
|
||||||
return data;
|
{
|
||||||
}
|
return data;
|
||||||
|
}
|
||||||
// Read from CPU buffer
|
// Read from CPU buffer
|
||||||
if let Some(bytes) = buffer.as_cpu() {
|
if let Some(bytes) = buffer.as_cpu() {
|
||||||
return bytes
|
return bytes
|
||||||
@@ -78,9 +79,10 @@ pub fn write_tensor_data<const D: usize>(
|
|||||||
) -> CubeclTensorPrimitive<D> {
|
) -> CubeclTensorPrimitive<D> {
|
||||||
// Try to write to GPU
|
// Try to write to GPU
|
||||||
if let Ok(client) = client::get_or_create(device)
|
if let Ok(client) = client::get_or_create(device)
|
||||||
&& let Ok(handle) = client.create_f32(&data) {
|
&& let Ok(handle) = client.create_f32(&data)
|
||||||
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
{
|
||||||
}
|
return CubeclTensorPrimitive::from_handle(shape, handle, device.clone());
|
||||||
|
}
|
||||||
|
|
||||||
// CPU fallback
|
// CPU fallback
|
||||||
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
let bytes: Vec<u8> = data.iter().flat_map(|f| f.to_le_bytes()).collect();
|
||||||
|
|||||||
@@ -461,8 +461,10 @@ impl SelfOptimizer {
|
|||||||
0.02 // Significant
|
0.02 // Significant
|
||||||
};
|
};
|
||||||
|
|
||||||
let effect_size =
|
let effect_size = f64::midpoint(
|
||||||
f64::midpoint(execution_time_improvement.abs(), throughput_improvement.abs()) / 100.0;
|
execution_time_improvement.abs(),
|
||||||
|
throughput_improvement.abs(),
|
||||||
|
) / 100.0;
|
||||||
let confidence_interval = (effect_size - 0.1, effect_size + 0.1);
|
let confidence_interval = (effect_size - 0.1, effect_size + 0.1);
|
||||||
|
|
||||||
Ok(AbTestResults {
|
Ok(AbTestResults {
|
||||||
|
|||||||
@@ -153,9 +153,7 @@ fn test_cost_benefit_analysis() {
|
|||||||
// Should create a valid optimization decision
|
// Should create a valid optimization decision
|
||||||
assert!(decision.confidence >= 0.0 && decision.confidence <= 1.0);
|
assert!(decision.confidence >= 0.0 && decision.confidence <= 1.0);
|
||||||
assert!(decision.expected_benefit.execution_time_reduction >= 0.0);
|
assert!(decision.expected_benefit.execution_time_reduction >= 0.0);
|
||||||
assert!(
|
assert!(decision.cost_estimate.risk_score >= 0.0 && decision.cost_estimate.risk_score <= 1.0);
|
||||||
decision.cost_estimate.risk_score >= 0.0 && decision.cost_estimate.risk_score <= 1.0
|
|
||||||
);
|
|
||||||
assert_eq!(decision.status, DecisionStatus::Pending);
|
assert_eq!(decision.status, DecisionStatus::Pending);
|
||||||
assert_eq!(decision.target_pattern, WorkloadPatternType::ComputeBound);
|
assert_eq!(decision.target_pattern, WorkloadPatternType::ComputeBound);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,10 +9,7 @@ async fn create_test_manager() -> ZeroCopyManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Helper to create test GPU memory region
|
/// Helper to create test GPU memory region
|
||||||
async fn create_test_gpu_region(
|
async fn create_test_gpu_region(manager: &ZeroCopyManager, size: usize) -> Arc<GpuMemoryRegion> {
|
||||||
manager: &ZeroCopyManager,
|
|
||||||
size: usize,
|
|
||||||
) -> Arc<GpuMemoryRegion> {
|
|
||||||
manager
|
manager
|
||||||
.allocate_gpu_memory(0, size, 4096)
|
.allocate_gpu_memory(0, size, 4096)
|
||||||
.expect("Failed to allocate test GPU memory")
|
.expect("Failed to allocate test GPU memory")
|
||||||
|
|||||||
@@ -332,9 +332,8 @@ impl TCAV {
|
|||||||
gradient_data.push(grad_i);
|
gradient_data.push(grad_i);
|
||||||
}
|
}
|
||||||
|
|
||||||
Tensor::from_slice(&gradient_data, input.shape().dims(), device).map_err(|e| {
|
Tensor::from_slice(&gradient_data, input.shape().dims(), device)
|
||||||
InterpretError::tensor(format!("Failed to create gradient tensor: {}", e))
|
.map_err(|e| InterpretError::tensor(format!("Failed to create gradient tensor: {}", e)))
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract class output from model output tensor
|
/// Extract class output from model output tensor
|
||||||
|
|||||||
@@ -452,9 +452,7 @@ impl FeatureAnalyzer {
|
|||||||
|
|
||||||
/// Get total samples analyzed
|
/// Get total samples analyzed
|
||||||
pub fn total_samples(&self) -> usize {
|
pub fn total_samples(&self) -> usize {
|
||||||
self.feature_stats
|
self.feature_stats.first().map_or(0, |s| s.samples_seen)
|
||||||
.first()
|
|
||||||
.map_or(0, |s| s.samples_seen)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -323,7 +323,10 @@ impl CombinedLoss {
|
|||||||
|
|
||||||
/// Get total effective weight (sum of enabled component weights)
|
/// Get total effective weight (sum of enabled component weights)
|
||||||
pub fn total_effective_weight(&self) -> f32 {
|
pub fn total_effective_weight(&self) -> f32 {
|
||||||
self.components.iter().map(LossComponent::effective_weight).sum()
|
self.components
|
||||||
|
.iter()
|
||||||
|
.map(LossComponent::effective_weight)
|
||||||
|
.sum()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Validate configuration
|
/// Validate configuration
|
||||||
|
|||||||
@@ -156,7 +156,10 @@ impl HingeLoss {
|
|||||||
/// # Panics
|
/// # Panics
|
||||||
/// Panics if smoothing is not in [0, 1)
|
/// Panics if smoothing is not in [0, 1)
|
||||||
pub fn with_label_smoothing(mut self, smoothing: f32) -> Self {
|
pub fn with_label_smoothing(mut self, smoothing: f32) -> Self {
|
||||||
assert!((0.0..1.0).contains(&smoothing), "Label smoothing must be in [0, 1), got {smoothing}");
|
assert!(
|
||||||
|
(0.0..1.0).contains(&smoothing),
|
||||||
|
"Label smoothing must be in [0, 1), got {smoothing}"
|
||||||
|
);
|
||||||
self.label_smoothing = smoothing;
|
self.label_smoothing = smoothing;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,7 +93,10 @@ impl QuantileLoss {
|
|||||||
/// - tau = 0.5: 50th percentile / median (equivalent to MAE)
|
/// - tau = 0.5: 50th percentile / median (equivalent to MAE)
|
||||||
/// - tau = 0.9: 90th percentile (penalizes under-prediction more)
|
/// - tau = 0.9: 90th percentile (penalizes under-prediction more)
|
||||||
pub fn with_tau(mut self, tau: f32) -> Self {
|
pub fn with_tau(mut self, tau: f32) -> Self {
|
||||||
assert!(!(tau <= 0.0 || tau >= 1.0), "Tau must be between 0 and 1 (exclusive)");
|
assert!(
|
||||||
|
!(tau <= 0.0 || tau >= 1.0),
|
||||||
|
"Tau must be between 0 and 1 (exclusive)"
|
||||||
|
);
|
||||||
self.tau = tau;
|
self.tau = tau;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,7 +62,8 @@ impl SwAVMinimal {
|
|||||||
|
|
||||||
/// Set custom prototypes
|
/// Set custom prototypes
|
||||||
pub fn with_prototypes(mut self, prototypes: MinimalTensor) -> Self {
|
pub fn with_prototypes(mut self, prototypes: MinimalTensor) -> Self {
|
||||||
assert!(prototypes.shape()[0] == self.num_prototypes,
|
assert!(
|
||||||
|
prototypes.shape()[0] == self.num_prototypes,
|
||||||
"Prototype tensor must have {} prototypes",
|
"Prototype tensor must have {} prototypes",
|
||||||
self.num_prototypes
|
self.num_prototypes
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -133,7 +133,10 @@ impl WassersteinLoss {
|
|||||||
|
|
||||||
/// Set number of random projections for sliced Wasserstein
|
/// Set number of random projections for sliced Wasserstein
|
||||||
pub fn with_num_projections(mut self, num_projections: usize) -> Self {
|
pub fn with_num_projections(mut self, num_projections: usize) -> Self {
|
||||||
assert!(num_projections != 0, "Number of projections must be positive");
|
assert!(
|
||||||
|
num_projections != 0,
|
||||||
|
"Number of projections must be positive"
|
||||||
|
);
|
||||||
self.num_projections = num_projections;
|
self.num_projections = num_projections;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,14 +50,14 @@ impl ConfigCodegen {
|
|||||||
return Err(syn::Error::new_spanned(
|
return Err(syn::Error::new_spanned(
|
||||||
ast,
|
ast,
|
||||||
"Config derive only supports named fields",
|
"Config derive only supports named fields",
|
||||||
))
|
));
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
_ => {
|
_ => {
|
||||||
return Err(syn::Error::new_spanned(
|
return Err(syn::Error::new_spanned(
|
||||||
ast,
|
ast,
|
||||||
"Config derive only supports structs",
|
"Config derive only supports structs",
|
||||||
))
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -236,9 +236,10 @@ impl ConfigCodegen {
|
|||||||
/// Check if a type is Option<T>.
|
/// Check if a type is Option<T>.
|
||||||
fn is_option_type(ty: &Type) -> bool {
|
fn is_option_type(ty: &Type) -> bool {
|
||||||
if let Type::Path(type_path) = ty
|
if let Type::Path(type_path) = ty
|
||||||
&& let Some(segment) = type_path.path.segments.last() {
|
&& let Some(segment) = type_path.path.segments.last()
|
||||||
return segment.ident == "Option";
|
{
|
||||||
}
|
return segment.ident == "Option";
|
||||||
|
}
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -92,7 +92,10 @@ mod shared;
|
|||||||
/// including parameter management, device handling, and training mode control.
|
/// including parameter management, device handling, and training mode control.
|
||||||
///
|
///
|
||||||
/// See the [crate-level documentation](crate) for detailed usage examples.
|
/// See the [crate-level documentation](crate) for detailed usage examples.
|
||||||
#[proc_macro_derive(Module, attributes(param, module, constant, device_field, training_field))]
|
#[proc_macro_derive(
|
||||||
|
Module,
|
||||||
|
attributes(param, module, constant, device_field, training_field)
|
||||||
|
)]
|
||||||
pub fn module_derive(input: TokenStream) -> TokenStream {
|
pub fn module_derive(input: TokenStream) -> TokenStream {
|
||||||
let input = syn::parse_macro_input!(input as syn::DeriveInput);
|
let input = syn::parse_macro_input!(input as syn::DeriveInput);
|
||||||
module::derive_impl(&input)
|
module::derive_impl(&input)
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
//! Code generation for the Module derive macro.
|
//! Code generation for the Module derive macro.
|
||||||
|
|
||||||
use crate::shared::field::{
|
use crate::shared::field::{
|
||||||
find_device_field, find_training_field, get_param_fields, parse_fields, AnalyzedField,
|
AnalyzedField, FieldKind, find_device_field, find_training_field, get_param_fields,
|
||||||
FieldKind,
|
parse_fields,
|
||||||
};
|
};
|
||||||
use proc_macro2::{Ident, TokenStream};
|
use proc_macro2::{Ident, TokenStream};
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
@@ -67,7 +67,8 @@ impl ModuleCodegen {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let param_code: Vec<TokenStream> = param_fields.iter().map(|f| f.gen_parameters()).collect();
|
let param_code: Vec<TokenStream> =
|
||||||
|
param_fields.iter().map(|f| f.gen_parameters()).collect();
|
||||||
|
|
||||||
quote! {
|
quote! {
|
||||||
fn parameters(&self) -> Vec<&crate::Tensor> {
|
fn parameters(&self) -> Vec<&crate::Tensor> {
|
||||||
@@ -90,8 +91,10 @@ impl ModuleCodegen {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let param_code: Vec<TokenStream> =
|
let param_code: Vec<TokenStream> = param_fields
|
||||||
param_fields.iter().map(|f| f.gen_parameters_mut()).collect();
|
.iter()
|
||||||
|
.map(|f| f.gen_parameters_mut())
|
||||||
|
.collect();
|
||||||
|
|
||||||
quote! {
|
quote! {
|
||||||
fn parameters_mut(&mut self) -> Vec<&mut crate::Tensor> {
|
fn parameters_mut(&mut self) -> Vec<&mut crate::Tensor> {
|
||||||
@@ -107,7 +110,12 @@ impl ModuleCodegen {
|
|||||||
let to_device_code: Vec<TokenStream> = self
|
let to_device_code: Vec<TokenStream> = self
|
||||||
.fields
|
.fields
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|f| matches!(f.kind, FieldKind::Param | FieldKind::Module | FieldKind::Device))
|
.filter(|f| {
|
||||||
|
matches!(
|
||||||
|
f.kind,
|
||||||
|
FieldKind::Param | FieldKind::Module | FieldKind::Device
|
||||||
|
)
|
||||||
|
})
|
||||||
.map(super::super::shared::field::AnalyzedField::gen_to_device)
|
.map(super::super::shared::field::AnalyzedField::gen_to_device)
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
@@ -53,9 +53,9 @@ pub fn extract_doc_comments(attrs: &[Attribute]) -> Vec<String> {
|
|||||||
&& let syn::Expr::Lit(syn::ExprLit {
|
&& let syn::Expr::Lit(syn::ExprLit {
|
||||||
lit: Lit::Str(s), ..
|
lit: Lit::Str(s), ..
|
||||||
}) = &nv.value
|
}) = &nv.value
|
||||||
{
|
{
|
||||||
return Some(s.value().trim().to_string());
|
return Some(s.value().trim().to_string());
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
|
|||||||
@@ -42,9 +42,10 @@ impl AnalyzedField {
|
|||||||
/// Check if this is an Option type.
|
/// Check if this is an Option type.
|
||||||
pub fn is_option_type(ty: &Type) -> bool {
|
pub fn is_option_type(ty: &Type) -> bool {
|
||||||
if let Type::Path(type_path) = ty
|
if let Type::Path(type_path) = ty
|
||||||
&& let Some(segment) = type_path.path.segments.last() {
|
&& let Some(segment) = type_path.path.segments.last()
|
||||||
return segment.ident == "Option";
|
{
|
||||||
}
|
return segment.ident == "Option";
|
||||||
|
}
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,11 +54,12 @@ impl AnalyzedField {
|
|||||||
pub fn option_inner_type(ty: &Type) -> Option<&Type> {
|
pub fn option_inner_type(ty: &Type) -> Option<&Type> {
|
||||||
if let Type::Path(type_path) = ty
|
if let Type::Path(type_path) = ty
|
||||||
&& let Some(segment) = type_path.path.segments.last()
|
&& let Some(segment) = type_path.path.segments.last()
|
||||||
&& segment.ident == "Option"
|
&& segment.ident == "Option"
|
||||||
&& let syn::PathArguments::AngleBracketed(args) = &segment.arguments
|
&& let syn::PathArguments::AngleBracketed(args) = &segment.arguments
|
||||||
&& let Some(syn::GenericArgument::Type(inner)) = args.args.first() {
|
&& let Some(syn::GenericArgument::Type(inner)) = args.args.first()
|
||||||
return Some(inner);
|
{
|
||||||
}
|
return Some(inner);
|
||||||
|
}
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,14 +210,14 @@ pub fn parse_fields(ast: &syn::DeriveInput) -> syn::Result<Vec<AnalyzedField>> {
|
|||||||
return Err(syn::Error::new_spanned(
|
return Err(syn::Error::new_spanned(
|
||||||
ast,
|
ast,
|
||||||
"Module derive only supports named fields",
|
"Module derive only supports named fields",
|
||||||
))
|
));
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
_ => {
|
_ => {
|
||||||
return Err(syn::Error::new_spanned(
|
return Err(syn::Error::new_spanned(
|
||||||
ast,
|
ast,
|
||||||
"Module derive only supports structs",
|
"Module derive only supports structs",
|
||||||
))
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Shared utilities for the derive macros.
|
//! Shared utilities for the derive macros.
|
||||||
|
|
||||||
pub mod field;
|
|
||||||
pub mod attribute;
|
pub mod attribute;
|
||||||
|
pub mod field;
|
||||||
|
|||||||
@@ -220,8 +220,8 @@ mod config_tests {
|
|||||||
Self {
|
Self {
|
||||||
in_features,
|
in_features,
|
||||||
out_features,
|
out_features,
|
||||||
bias: true, // default
|
bias: true, // default
|
||||||
dropout: 0.0, // default
|
dropout: 0.0, // default
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -247,7 +247,11 @@ mod config_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Conv2dConfig {
|
impl Conv2dConfig {
|
||||||
pub fn new(in_channels: usize, out_channels: usize, kernel_size: (usize, usize)) -> Self {
|
pub fn new(
|
||||||
|
in_channels: usize,
|
||||||
|
out_channels: usize,
|
||||||
|
kernel_size: (usize, usize),
|
||||||
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
in_channels,
|
in_channels,
|
||||||
out_channels,
|
out_channels,
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
//! Arena-based GPU memory allocator implementation
|
//! Arena-based GPU memory allocator implementation
|
||||||
|
|
||||||
use super::types::{ArenaBlock, CompactionStats, DeviceId, GpuMemoryBlock, GpuMemoryStats, GpuMemoryType};
|
use super::types::{
|
||||||
|
ArenaBlock, CompactionStats, DeviceId, GpuMemoryBlock, GpuMemoryStats, GpuMemoryType,
|
||||||
|
};
|
||||||
use crate::{MemoryError, Result};
|
use crate::{MemoryError, Result};
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::{BTreeMap, HashMap, VecDeque};
|
use std::collections::{BTreeMap, HashMap, VecDeque};
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ use crate::gpu_allocator::{DeviceId, GpuMemoryBlock, GpuMemoryStats};
|
|||||||
use crate::{MemoryError, Result};
|
use crate::{MemoryError, Result};
|
||||||
use parking_lot::{Mutex, RwLock};
|
use parking_lot::{Mutex, RwLock};
|
||||||
use std::collections::{BTreeMap, HashMap, VecDeque};
|
use std::collections::{BTreeMap, HashMap, VecDeque};
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
|
||||||
use std::sync::Weak;
|
use std::sync::Weak;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
/// OOM recovery strategy
|
/// OOM recovery strategy
|
||||||
|
|||||||
@@ -507,7 +507,10 @@ async fn test_metrics_accuracy() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mid_metrics = pool_manager.metrics().await;
|
let mid_metrics = pool_manager.metrics().await;
|
||||||
assert_eq!(mid_metrics.allocation_count() - initial_alloc_count, NUM_ALLOCS);
|
assert_eq!(
|
||||||
|
mid_metrics.allocation_count() - initial_alloc_count,
|
||||||
|
NUM_ALLOCS
|
||||||
|
);
|
||||||
// Note: active_allocations tracking may be affected by pool warmup
|
// Note: active_allocations tracking may be affected by pool warmup
|
||||||
// Just verify allocations were counted correctly
|
// Just verify allocations were counted correctly
|
||||||
assert!(mid_metrics.peak_memory_usage() >= NUM_ALLOCS * BLOCK_SIZE as u64);
|
assert!(mid_metrics.peak_memory_usage() >= NUM_ALLOCS * BLOCK_SIZE as u64);
|
||||||
@@ -519,8 +522,14 @@ async fn test_metrics_accuracy() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let final_metrics = pool_manager.metrics().await;
|
let final_metrics = pool_manager.metrics().await;
|
||||||
assert_eq!(final_metrics.allocation_count() - initial_alloc_count, NUM_ALLOCS);
|
assert_eq!(
|
||||||
assert_eq!(final_metrics.deallocation_count() - initial_dealloc_count, NUM_ALLOCS / 2);
|
final_metrics.allocation_count() - initial_alloc_count,
|
||||||
|
NUM_ALLOCS
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
final_metrics.deallocation_count() - initial_dealloc_count,
|
||||||
|
NUM_ALLOCS / 2
|
||||||
|
);
|
||||||
|
|
||||||
// Cleanup remaining blocks
|
// Cleanup remaining blocks
|
||||||
for block in blocks {
|
for block in blocks {
|
||||||
|
|||||||
@@ -41,8 +41,7 @@ use crate::gpu_simple::MemoryPressure;
|
|||||||
use crate::paged_attention::{BlockTable, PageTable, PagedAttentionConfig, SequenceId};
|
use crate::paged_attention::{BlockTable, PageTable, PagedAttentionConfig, SequenceId};
|
||||||
|
|
||||||
/// Data type for KV cache storage
|
/// Data type for KV cache storage
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
#[derive(Default)]
|
|
||||||
pub enum KvDataType {
|
pub enum KvDataType {
|
||||||
/// 32-bit floating point
|
/// 32-bit floating point
|
||||||
Float32,
|
Float32,
|
||||||
@@ -69,10 +68,8 @@ impl KvDataType {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Memory tier for KV cache storage
|
/// Memory tier for KV cache storage
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
#[derive(Default)]
|
|
||||||
pub enum KvMemoryTier {
|
pub enum KvMemoryTier {
|
||||||
/// GPU memory (fastest)
|
/// GPU memory (fastest)
|
||||||
#[default]
|
#[default]
|
||||||
@@ -83,7 +80,6 @@ pub enum KvMemoryTier {
|
|||||||
Disk,
|
Disk,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Configuration for KV cache allocator
|
/// Configuration for KV cache allocator
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct KvCacheConfig {
|
pub struct KvCacheConfig {
|
||||||
@@ -387,8 +383,7 @@ impl KvCacheAllocator {
|
|||||||
num_tokens: usize,
|
num_tokens: usize,
|
||||||
) -> Result<KvCacheHandle> {
|
) -> Result<KvCacheHandle> {
|
||||||
// Check if we need to evict first
|
// Check if we need to evict first
|
||||||
let pages_needed =
|
let pages_needed = num_tokens.div_ceil(self.config.page_size_tokens);
|
||||||
num_tokens.div_ceil(self.config.page_size_tokens);
|
|
||||||
let free_pages = self.page_table.free_page_count();
|
let free_pages = self.page_table.free_page_count();
|
||||||
|
|
||||||
if pages_needed > free_pages {
|
if pages_needed > free_pages {
|
||||||
@@ -665,9 +660,10 @@ impl KvCacheAllocator {
|
|||||||
let sequences = self.sequences.read();
|
let sequences = self.sequences.read();
|
||||||
|
|
||||||
if let Some(cache) = sequences.get(&handle.sequence_id)
|
if let Some(cache) = sequences.get(&handle.sequence_id)
|
||||||
&& cache.handle_id == handle.handle_id {
|
&& cache.handle_id == handle.handle_id
|
||||||
return Ok(());
|
{
|
||||||
}
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
Err(MemoryError::invalid_block_id(handle.sequence_id))
|
Err(MemoryError::invalid_block_id(handle.sequence_id))
|
||||||
}
|
}
|
||||||
@@ -724,7 +720,9 @@ impl KvCacheAllocator {
|
|||||||
/// Get entropy tracker stats (if enabled)
|
/// Get entropy tracker stats (if enabled)
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn entropy_stats(&self) -> Option<crate::entropy_cache::EntropyStats> {
|
pub fn entropy_stats(&self) -> Option<crate::entropy_cache::EntropyStats> {
|
||||||
self.entropy_tracker.as_ref().map(super::entropy_cache::EntropyTracker::stats)
|
self.entropy_tracker
|
||||||
|
.as_ref()
|
||||||
|
.map(super::entropy_cache::EntropyTracker::stats)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Apply temporal decay to entropy scores
|
/// Apply temporal decay to entropy scores
|
||||||
|
|||||||
@@ -5,8 +5,8 @@
|
|||||||
|
|
||||||
#![cfg(all(test, feature = "loom"))]
|
#![cfg(all(test, feature = "loom"))]
|
||||||
|
|
||||||
use loom::sync::{Arc, Mutex};
|
|
||||||
use loom::sync::atomic::{AtomicU64, Ordering};
|
use loom::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
use loom::sync::{Arc, Mutex};
|
||||||
use loom::thread;
|
use loom::thread;
|
||||||
|
|
||||||
/// Mock memory block for loom testing.
|
/// Mock memory block for loom testing.
|
||||||
@@ -84,13 +84,9 @@ fn test_atomic_block_id_uniqueness() {
|
|||||||
let g1 = Arc::clone(&id_gen);
|
let g1 = Arc::clone(&id_gen);
|
||||||
let g2 = Arc::clone(&id_gen);
|
let g2 = Arc::clone(&id_gen);
|
||||||
|
|
||||||
let t1 = thread::spawn(move || {
|
let t1 = thread::spawn(move || g1.fetch_add(1, Ordering::SeqCst));
|
||||||
g1.fetch_add(1, Ordering::SeqCst)
|
|
||||||
});
|
|
||||||
|
|
||||||
let t2 = thread::spawn(move || {
|
let t2 = thread::spawn(move || g2.fetch_add(1, Ordering::SeqCst));
|
||||||
g2.fetch_add(1, Ordering::SeqCst)
|
|
||||||
});
|
|
||||||
|
|
||||||
let r1 = t1.join().unwrap();
|
let r1 = t1.join().unwrap();
|
||||||
let r2 = t2.join().unwrap();
|
let r2 = t2.join().unwrap();
|
||||||
@@ -111,13 +107,9 @@ fn test_pool_concurrent_allocate() {
|
|||||||
let p1 = Arc::clone(&pool);
|
let p1 = Arc::clone(&pool);
|
||||||
let p2 = Arc::clone(&pool);
|
let p2 = Arc::clone(&pool);
|
||||||
|
|
||||||
let t1 = thread::spawn(move || {
|
let t1 = thread::spawn(move || p1.allocate(512));
|
||||||
p1.allocate(512)
|
|
||||||
});
|
|
||||||
|
|
||||||
let t2 = thread::spawn(move || {
|
let t2 = thread::spawn(move || p2.allocate(512));
|
||||||
p2.allocate(512)
|
|
||||||
});
|
|
||||||
|
|
||||||
let id1 = t1.join().unwrap();
|
let id1 = t1.join().unwrap();
|
||||||
let id2 = t2.join().unwrap();
|
let id2 = t2.join().unwrap();
|
||||||
@@ -143,13 +135,9 @@ fn test_pool_allocate_free_interleaving() {
|
|||||||
let p1 = Arc::clone(&pool);
|
let p1 = Arc::clone(&pool);
|
||||||
let p2 = Arc::clone(&pool);
|
let p2 = Arc::clone(&pool);
|
||||||
|
|
||||||
let t1 = thread::spawn(move || {
|
let t1 = thread::spawn(move || p1.free(block_id));
|
||||||
p1.free(block_id)
|
|
||||||
});
|
|
||||||
|
|
||||||
let t2 = thread::spawn(move || {
|
let t2 = thread::spawn(move || p2.allocate(256));
|
||||||
p2.allocate(256)
|
|
||||||
});
|
|
||||||
|
|
||||||
let freed = t1.join().unwrap();
|
let freed = t1.join().unwrap();
|
||||||
let new_id = t2.join().unwrap();
|
let new_id = t2.join().unwrap();
|
||||||
@@ -169,19 +157,18 @@ fn test_pool_arena_exhaustion() {
|
|||||||
let p1 = Arc::clone(&pool);
|
let p1 = Arc::clone(&pool);
|
||||||
let p2 = Arc::clone(&pool);
|
let p2 = Arc::clone(&pool);
|
||||||
|
|
||||||
let t1 = thread::spawn(move || {
|
let t1 = thread::spawn(move || p1.allocate(512));
|
||||||
p1.allocate(512)
|
|
||||||
});
|
|
||||||
|
|
||||||
let t2 = thread::spawn(move || {
|
let t2 = thread::spawn(move || p2.allocate(512));
|
||||||
p2.allocate(512)
|
|
||||||
});
|
|
||||||
|
|
||||||
let id1 = t1.join().unwrap();
|
let id1 = t1.join().unwrap();
|
||||||
let id2 = t2.join().unwrap();
|
let id2 = t2.join().unwrap();
|
||||||
|
|
||||||
// Exactly one should succeed, one should fail
|
// Exactly one should succeed, one should fail
|
||||||
let successes = [id1.is_some(), id2.is_some()].iter().filter(|&&x| x).count();
|
let successes = [id1.is_some(), id2.is_some()]
|
||||||
|
.iter()
|
||||||
|
.filter(|&&x| x)
|
||||||
|
.count();
|
||||||
assert_eq!(successes, 1);
|
assert_eq!(successes, 1);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -407,8 +407,7 @@ impl PageTable {
|
|||||||
|
|
||||||
let current_pages = seq.num_pages();
|
let current_pages = seq.num_pages();
|
||||||
let new_total_tokens = seq.num_tokens + additional_tokens;
|
let new_total_tokens = seq.num_tokens + additional_tokens;
|
||||||
let new_total_pages =
|
let new_total_pages = new_total_tokens.div_ceil(self.config.block_size);
|
||||||
new_total_tokens.div_ceil(self.config.block_size);
|
|
||||||
let pages_needed = new_total_pages - current_pages;
|
let pages_needed = new_total_pages - current_pages;
|
||||||
|
|
||||||
if pages_needed == 0 {
|
if pages_needed == 0 {
|
||||||
@@ -697,11 +696,12 @@ impl PageTable {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if let Some(seq_id) = sequence_to_evict
|
if let Some(seq_id) = sequence_to_evict
|
||||||
&& let Ok(pages_freed) = self.free_sequence(seq_id) {
|
&& let Ok(pages_freed) = self.free_sequence(seq_id)
|
||||||
evicted += pages_freed;
|
{
|
||||||
let mut stats = self.stats.write();
|
evicted += pages_freed;
|
||||||
stats.evictions += 1;
|
let mut stats = self.stats.write();
|
||||||
}
|
stats.evictions += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(evicted)
|
Ok(evicted)
|
||||||
|
|||||||
@@ -112,22 +112,23 @@ impl AccessPattern {
|
|||||||
|
|
||||||
// Calculate stride from previous access
|
// Calculate stride from previous access
|
||||||
if let Some(last_access) = self.access_times.last()
|
if let Some(last_access) = self.access_times.last()
|
||||||
&& let Ok(duration) = now.duration_since(*last_access) {
|
&& let Ok(duration) = now.duration_since(*last_access)
|
||||||
self.access_stride.push(duration);
|
{
|
||||||
|
self.access_stride.push(duration);
|
||||||
|
|
||||||
// Keep only recent access strides (sliding window)
|
// Keep only recent access strides (sliding window)
|
||||||
if self.access_stride.len() > 10 {
|
if self.access_stride.len() > 10 {
|
||||||
self.access_stride.remove(0);
|
self.access_stride.remove(0);
|
||||||
}
|
|
||||||
|
|
||||||
// Predict next access based on average stride
|
|
||||||
if self.access_stride.len() >= 2 {
|
|
||||||
let total_duration: Duration = self.access_stride.iter().sum();
|
|
||||||
let avg_stride = total_duration / self.access_stride.len() as u32;
|
|
||||||
self.predicted_next_access = Some(now + avg_stride);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Predict next access based on average stride
|
||||||
|
if self.access_stride.len() >= 2 {
|
||||||
|
let total_duration: Duration = self.access_stride.iter().sum();
|
||||||
|
let avg_stride = total_duration / self.access_stride.len() as u32;
|
||||||
|
self.predicted_next_access = Some(now + avg_stride);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Record the access time
|
// Record the access time
|
||||||
self.access_times.push(now);
|
self.access_times.push(now);
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
//! Tests for activation layers (ReLU, GELU, LeakyReLU, ELU)
|
//! Tests for activation layers (ReLU, GELU, LeakyReLU, ELU)
|
||||||
|
|
||||||
use crate::generic::{GenericELU, GenericGELU, GenericLeakyReLU, GenericModule, GenericModule1D, GenericReLU};
|
use crate::generic::{
|
||||||
|
GenericELU, GenericGELU, GenericLeakyReLU, GenericModule, GenericModule1D, GenericReLU,
|
||||||
|
};
|
||||||
use rtx_backend_cpu::{CpuBackend, CpuDevice};
|
use rtx_backend_cpu::{CpuBackend, CpuDevice};
|
||||||
use rtx_tensor::generic::GenericTensor;
|
use rtx_tensor::generic::GenericTensor;
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
//! Tests for convolution layers (Conv1d, Conv2d)
|
//! Tests for convolution layers (Conv1d, Conv2d)
|
||||||
|
|
||||||
use crate::generic::{GenericConv1d, GenericConv2d, GenericModule, GenericModule3D, GenericModule4D};
|
use crate::generic::{
|
||||||
|
GenericConv1d, GenericConv2d, GenericModule, GenericModule3D, GenericModule4D,
|
||||||
|
};
|
||||||
use rtx_backend_cpu::{CpuBackend, CpuDevice};
|
use rtx_backend_cpu::{CpuBackend, CpuDevice};
|
||||||
use rtx_tensor::generic::GenericTensor;
|
use rtx_tensor::generic::GenericTensor;
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
//! Tests for dropout layers (Dropout, Dropout2d, Dropout3d)
|
//! Tests for dropout layers (Dropout, Dropout2d, Dropout3d)
|
||||||
|
|
||||||
use crate::generic::{GenericDropout, GenericDropout2d, GenericDropout3d, GenericModule, GenericModule4D};
|
use crate::generic::{
|
||||||
|
GenericDropout, GenericDropout2d, GenericDropout3d, GenericModule, GenericModule4D,
|
||||||
|
};
|
||||||
use rtx_backend_cpu::{CpuBackend, CpuDevice};
|
use rtx_backend_cpu::{CpuBackend, CpuDevice};
|
||||||
use rtx_tensor::generic::GenericTensor;
|
use rtx_tensor::generic::GenericTensor;
|
||||||
|
|
||||||
|
|||||||
@@ -12,8 +12,7 @@ use rtx_tensor::generic::GenericTensor;
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_layer_norm_creation() {
|
fn test_layer_norm_creation() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let layer_norm: GenericLayerNorm<CpuBackend> =
|
let layer_norm: GenericLayerNorm<CpuBackend> = GenericLayerNorm::new(64, 1e-5, true, &device);
|
||||||
GenericLayerNorm::new(64, 1e-5, true, &device);
|
|
||||||
|
|
||||||
assert_eq!(layer_norm.normalized_shape(), 64);
|
assert_eq!(layer_norm.normalized_shape(), 64);
|
||||||
assert_eq!(layer_norm.eps(), 1e-5);
|
assert_eq!(layer_norm.eps(), 1e-5);
|
||||||
@@ -23,8 +22,7 @@ fn test_layer_norm_creation() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_layer_norm_forward_shape() {
|
fn test_layer_norm_forward_shape() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let layer_norm: GenericLayerNorm<CpuBackend> =
|
let layer_norm: GenericLayerNorm<CpuBackend> = GenericLayerNorm::new(8, 1e-5, true, &device);
|
||||||
GenericLayerNorm::new(8, 1e-5, true, &device);
|
|
||||||
|
|
||||||
let input: GenericTensor<CpuBackend, 2> = GenericTensor::randn([4, 8], &device);
|
let input: GenericTensor<CpuBackend, 2> = GenericTensor::randn([4, 8], &device);
|
||||||
let output = layer_norm.forward(&input);
|
let output = layer_norm.forward(&input);
|
||||||
@@ -35,8 +33,7 @@ fn test_layer_norm_forward_shape() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_layer_norm_normalizes() {
|
fn test_layer_norm_normalizes() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let layer_norm: GenericLayerNorm<CpuBackend> =
|
let layer_norm: GenericLayerNorm<CpuBackend> = GenericLayerNorm::new(4, 1e-5, false, &device);
|
||||||
GenericLayerNorm::new(4, 1e-5, false, &device);
|
|
||||||
|
|
||||||
// Input with known values
|
// Input with known values
|
||||||
let input: GenericTensor<CpuBackend, 2> =
|
let input: GenericTensor<CpuBackend, 2> =
|
||||||
@@ -54,8 +51,7 @@ fn test_layer_norm_normalizes() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_layer_norm_parameters() {
|
fn test_layer_norm_parameters() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let layer_norm: GenericLayerNorm<CpuBackend> =
|
let layer_norm: GenericLayerNorm<CpuBackend> = GenericLayerNorm::new(64, 1e-5, true, &device);
|
||||||
GenericLayerNorm::new(64, 1e-5, true, &device);
|
|
||||||
|
|
||||||
// weight: 64, bias: 64
|
// weight: 64, bias: 64
|
||||||
assert_eq!(layer_norm.num_parameters(), 128);
|
assert_eq!(layer_norm.num_parameters(), 128);
|
||||||
@@ -64,8 +60,7 @@ fn test_layer_norm_parameters() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_layer_norm_no_affine() {
|
fn test_layer_norm_no_affine() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let layer_norm: GenericLayerNorm<CpuBackend> =
|
let layer_norm: GenericLayerNorm<CpuBackend> = GenericLayerNorm::new(64, 1e-5, false, &device);
|
||||||
GenericLayerNorm::new(64, 1e-5, false, &device);
|
|
||||||
|
|
||||||
// With elementwise_affine=false, only weight (no bias)
|
// With elementwise_affine=false, only weight (no bias)
|
||||||
assert_eq!(layer_norm.num_parameters(), 64);
|
assert_eq!(layer_norm.num_parameters(), 64);
|
||||||
|
|||||||
@@ -117,8 +117,7 @@ fn test_avg_pool2d_no_parameters() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_avg_pool2d_creation() {
|
fn test_adaptive_avg_pool2d_creation() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let pool: GenericAdaptiveAvgPool2d<CpuBackend> =
|
let pool: GenericAdaptiveAvgPool2d<CpuBackend> = GenericAdaptiveAvgPool2d::new([1, 1], &device);
|
||||||
GenericAdaptiveAvgPool2d::new([1, 1], &device);
|
|
||||||
assert_eq!(pool.output_size(), [1, 1]);
|
assert_eq!(pool.output_size(), [1, 1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,8 +125,7 @@ fn test_adaptive_avg_pool2d_creation() {
|
|||||||
fn test_adaptive_avg_pool2d_forward_shape() {
|
fn test_adaptive_avg_pool2d_forward_shape() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
// Global average pooling (output 1x1)
|
// Global average pooling (output 1x1)
|
||||||
let pool: GenericAdaptiveAvgPool2d<CpuBackend> =
|
let pool: GenericAdaptiveAvgPool2d<CpuBackend> = GenericAdaptiveAvgPool2d::new([1, 1], &device);
|
||||||
GenericAdaptiveAvgPool2d::new([1, 1], &device);
|
|
||||||
|
|
||||||
let input: GenericTensor<CpuBackend, 4> = GenericTensor::randn([2, 16, 32, 32], &device);
|
let input: GenericTensor<CpuBackend, 4> = GenericTensor::randn([2, 16, 32, 32], &device);
|
||||||
let output = pool.forward_4d(&input);
|
let output = pool.forward_4d(&input);
|
||||||
@@ -139,8 +137,7 @@ fn test_adaptive_avg_pool2d_forward_shape() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_avg_pool2d_forward_non_square() {
|
fn test_adaptive_avg_pool2d_forward_non_square() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let pool: GenericAdaptiveAvgPool2d<CpuBackend> =
|
let pool: GenericAdaptiveAvgPool2d<CpuBackend> = GenericAdaptiveAvgPool2d::new([4, 4], &device);
|
||||||
GenericAdaptiveAvgPool2d::new([4, 4], &device);
|
|
||||||
|
|
||||||
let input: GenericTensor<CpuBackend, 4> = GenericTensor::randn([1, 8, 32, 32], &device);
|
let input: GenericTensor<CpuBackend, 4> = GenericTensor::randn([1, 8, 32, 32], &device);
|
||||||
let output = pool.forward_4d(&input);
|
let output = pool.forward_4d(&input);
|
||||||
@@ -151,8 +148,7 @@ fn test_adaptive_avg_pool2d_forward_non_square() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_adaptive_avg_pool2d_no_parameters() {
|
fn test_adaptive_avg_pool2d_no_parameters() {
|
||||||
let device = CpuDevice::default();
|
let device = CpuDevice::default();
|
||||||
let pool: GenericAdaptiveAvgPool2d<CpuBackend> =
|
let pool: GenericAdaptiveAvgPool2d<CpuBackend> = GenericAdaptiveAvgPool2d::new([1, 1], &device);
|
||||||
GenericAdaptiveAvgPool2d::new([1, 1], &device);
|
|
||||||
assert_eq!(pool.num_parameters(), 0);
|
assert_eq!(pool.num_parameters(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -76,8 +76,7 @@ fn test_sequential_training_mode() {
|
|||||||
let linear: GenericLinear<CpuBackend> = GenericLinear::new(4, 3, true, &device);
|
let linear: GenericLinear<CpuBackend> = GenericLinear::new(4, 3, true, &device);
|
||||||
let relu: GenericReLU<CpuBackend> = GenericReLU::new(&device);
|
let relu: GenericReLU<CpuBackend> = GenericReLU::new(&device);
|
||||||
|
|
||||||
let mut seq =
|
let mut seq = GenericSequential::new(vec![Box::new(linear), Box::new(relu)], device.clone());
|
||||||
GenericSequential::new(vec![Box::new(linear), Box::new(relu)], device.clone());
|
|
||||||
|
|
||||||
assert!(seq.training());
|
assert!(seq.training());
|
||||||
|
|
||||||
|
|||||||
@@ -396,7 +396,11 @@ impl<B: Backend<FloatElem = f32>> GenericModule<B> for GenericTransformerEncoder
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn num_parameters(&self) -> usize {
|
fn num_parameters(&self) -> usize {
|
||||||
let block_params: usize = self.blocks.iter().map(super::module::GenericModule::num_parameters).sum();
|
let block_params: usize = self
|
||||||
|
.blocks
|
||||||
|
.iter()
|
||||||
|
.map(super::module::GenericModule::num_parameters)
|
||||||
|
.sum();
|
||||||
block_params + self.final_norm.num_parameters()
|
block_params + self.final_norm.num_parameters()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ pub fn calculate_fan_out(out_channels: usize, kernel_size: &[usize], groups: usi
|
|||||||
|
|
||||||
/// Padding utilities
|
/// Padding utilities
|
||||||
pub mod padding {
|
pub mod padding {
|
||||||
use super::{Tensor, Device, Result, NNError};
|
use super::{Device, NNError, Result, Tensor};
|
||||||
|
|
||||||
/// Apply zero padding to a 2D tensor
|
/// Apply zero padding to a 2D tensor
|
||||||
pub fn pad_2d(input: &Tensor, padding: (usize, usize), device: &Device) -> Result<Tensor> {
|
pub fn pad_2d(input: &Tensor, padding: (usize, usize), device: &Device) -> Result<Tensor> {
|
||||||
@@ -156,7 +156,7 @@ pub mod padding {
|
|||||||
|
|
||||||
/// Im2col utilities for efficient convolution
|
/// Im2col utilities for efficient convolution
|
||||||
pub mod im2col {
|
pub mod im2col {
|
||||||
use super::{Tensor, Result};
|
use super::{Result, Tensor};
|
||||||
|
|
||||||
/// Im2col for 2D convolution
|
/// Im2col for 2D convolution
|
||||||
pub fn im2col_2d(
|
pub fn im2col_2d(
|
||||||
|
|||||||
@@ -43,7 +43,10 @@ impl BatchNormConfig {
|
|||||||
|
|
||||||
/// Set momentum for running statistics updates
|
/// Set momentum for running statistics updates
|
||||||
pub fn momentum(mut self, momentum: f32) -> Self {
|
pub fn momentum(mut self, momentum: f32) -> Self {
|
||||||
assert!(!(momentum <= 0.0 || momentum >= 1.0), "Momentum must be in (0, 1), got {momentum}");
|
assert!(
|
||||||
|
!(momentum <= 0.0 || momentum >= 1.0),
|
||||||
|
"Momentum must be in (0, 1), got {momentum}"
|
||||||
|
);
|
||||||
self.momentum = momentum;
|
self.momentum = momentum;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -193,8 +193,14 @@ impl crate::layers::Module for LayerNorm {
|
|||||||
.map_err(|e| NNError::Tensor(e))?;
|
.map_err(|e| NNError::Tensor(e))?;
|
||||||
|
|
||||||
// Propagate requires_grad from input or parameters
|
// Propagate requires_grad from input or parameters
|
||||||
let weight_requires_grad = self.weight.as_ref().is_some_and(rtx_tensor::Tensor::requires_grad);
|
let weight_requires_grad = self
|
||||||
let bias_requires_grad = self.bias.as_ref().is_some_and(rtx_tensor::Tensor::requires_grad);
|
.weight
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(rtx_tensor::Tensor::requires_grad);
|
||||||
|
let bias_requires_grad = self
|
||||||
|
.bias
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(rtx_tensor::Tensor::requires_grad);
|
||||||
if input.requires_grad() || weight_requires_grad || bias_requires_grad {
|
if input.requires_grad() || weight_requires_grad || bias_requires_grad {
|
||||||
result.set_requires_grad(true);
|
result.set_requires_grad(true);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -589,10 +589,12 @@ impl JITCompiler {
|
|||||||
} else {
|
} else {
|
||||||
16
|
16
|
||||||
};
|
};
|
||||||
let shuffle = i32::from(profile
|
let shuffle = i32::from(
|
||||||
.tensor_shapes
|
profile
|
||||||
.iter()
|
.tensor_shapes
|
||||||
.any(|s| s.iter().product::<usize>() > 1024));
|
.iter()
|
||||||
|
.any(|s| s.iter().product::<usize>() > 1024),
|
||||||
|
);
|
||||||
Ok(format!(
|
Ok(format!(
|
||||||
"#define WARP_SPECIALIZED 1\n#define WARP_SIZE {warp_size}\n#define SHUFFLE_OPERATIONS {shuffle}\n{code}\n"
|
"#define WARP_SPECIALIZED 1\n#define WARP_SIZE {warp_size}\n#define SHUFFLE_OPERATIONS {shuffle}\n{code}\n"
|
||||||
))
|
))
|
||||||
|
|||||||
@@ -153,7 +153,11 @@ impl FusionAnalyzer {
|
|||||||
|
|
||||||
/// Check if two operation types can be fused together
|
/// Check if two operation types can be fused together
|
||||||
fn are_operations_fuseable(&self, op1: &FusionOpType, op2: &FusionOpType) -> bool {
|
fn are_operations_fuseable(&self, op1: &FusionOpType, op2: &FusionOpType) -> bool {
|
||||||
use FusionOpType::{Add, Mul, Div, Sub, ReLU, Sigmoid, GELU, Sum, Max, Mean, Min, BroadcastScalar, Linear, LayerNorm, Dropout, AttentionCompute, AttentionApply, QuantumEncode, QuantumSimulate, QuantumDecode, SpikeEncode, SpikingCompute, SpikeToDense};
|
use FusionOpType::{
|
||||||
|
Add, AttentionApply, AttentionCompute, BroadcastScalar, Div, Dropout, GELU, LayerNorm,
|
||||||
|
Linear, Max, Mean, Min, Mul, QuantumDecode, QuantumEncode, QuantumSimulate, ReLU,
|
||||||
|
Sigmoid, SpikeEncode, SpikeToDense, SpikingCompute, Sub, Sum,
|
||||||
|
};
|
||||||
|
|
||||||
match (op1, op2) {
|
match (op1, op2) {
|
||||||
// Elementwise operations can always be fused with other elementwise
|
// Elementwise operations can always be fused with other elementwise
|
||||||
@@ -256,7 +260,10 @@ impl FusionAnalyzer {
|
|||||||
|
|
||||||
/// Estimate additional speedup from specific fusion patterns
|
/// Estimate additional speedup from specific fusion patterns
|
||||||
fn estimate_pattern_specific_speedup(&self, operations: &[FusionOp]) -> f64 {
|
fn estimate_pattern_specific_speedup(&self, operations: &[FusionOp]) -> f64 {
|
||||||
use FusionOpType::{Linear, AttentionCompute, AttentionApply, GELU, ReLU, QuantumEncode, QuantumSimulate, QuantumDecode, SpikeEncode, SpikingCompute, SpikeToDense};
|
use FusionOpType::{
|
||||||
|
AttentionApply, AttentionCompute, GELU, Linear, QuantumDecode, QuantumEncode,
|
||||||
|
QuantumSimulate, ReLU, SpikeEncode, SpikeToDense, SpikingCompute,
|
||||||
|
};
|
||||||
|
|
||||||
// Look for high-value patterns
|
// Look for high-value patterns
|
||||||
let op_types: Vec<_> = operations.iter().map(|op| &op.op_type).collect();
|
let op_types: Vec<_> = operations.iter().map(|op| &op.op_type).collect();
|
||||||
|
|||||||
@@ -102,7 +102,11 @@ impl PerformanceEstimator {
|
|||||||
|
|
||||||
/// Estimate execution time for a single operation
|
/// Estimate execution time for a single operation
|
||||||
fn estimate_operation_time(&self, op: &FusionOp) -> Result<f64> {
|
fn estimate_operation_time(&self, op: &FusionOp) -> Result<f64> {
|
||||||
use FusionOpType::{Add, Sub, Copy, BroadcastScalar, Mul, Div, ReLU, Sigmoid, GELU, Sum, Max, Mean, Min, Linear, LayerNorm, Softmax, AttentionCompute, AttentionApply, QuantumEncode, QuantumSimulate, QuantumDecode, SpikeEncode, SpikingCompute, SpikeToDense, Dropout};
|
use FusionOpType::{
|
||||||
|
Add, AttentionApply, AttentionCompute, BroadcastScalar, Copy, Div, Dropout, GELU,
|
||||||
|
LayerNorm, Linear, Max, Mean, Min, Mul, QuantumDecode, QuantumEncode, QuantumSimulate,
|
||||||
|
ReLU, Sigmoid, Softmax, SpikeEncode, SpikeToDense, SpikingCompute, Sub, Sum,
|
||||||
|
};
|
||||||
|
|
||||||
let element_count = op.dimensions.iter().product::<usize>();
|
let element_count = op.dimensions.iter().product::<usize>();
|
||||||
let memory_size = element_count * op.element_size;
|
let memory_size = element_count * op.element_size;
|
||||||
@@ -238,7 +242,11 @@ impl PerformanceEstimator {
|
|||||||
|
|
||||||
/// Get computational complexity factor for operation type
|
/// Get computational complexity factor for operation type
|
||||||
fn get_op_complexity(&self, op_type: &FusionOpType) -> usize {
|
fn get_op_complexity(&self, op_type: &FusionOpType) -> usize {
|
||||||
use FusionOpType::{Add, Sub, Copy, BroadcastScalar, Mul, Div, ReLU, Sigmoid, GELU, Sum, Max, Mean, Min, Linear, LayerNorm, Softmax, AttentionCompute, AttentionApply, QuantumEncode, QuantumSimulate, QuantumDecode, SpikeEncode, SpikingCompute, SpikeToDense, Dropout};
|
use FusionOpType::{
|
||||||
|
Add, AttentionApply, AttentionCompute, BroadcastScalar, Copy, Div, Dropout, GELU,
|
||||||
|
LayerNorm, Linear, Max, Mean, Min, Mul, QuantumDecode, QuantumEncode, QuantumSimulate,
|
||||||
|
ReLU, Sigmoid, Softmax, SpikeEncode, SpikeToDense, SpikingCompute, Sub, Sum,
|
||||||
|
};
|
||||||
match op_type {
|
match op_type {
|
||||||
Add | Sub | Copy | BroadcastScalar => 1,
|
Add | Sub | Copy | BroadcastScalar => 1,
|
||||||
Mul | Div => 2,
|
Mul | Div => 2,
|
||||||
@@ -346,7 +354,11 @@ impl PerformanceEstimator {
|
|||||||
|
|
||||||
/// Estimate pattern-specific performance improvement
|
/// Estimate pattern-specific performance improvement
|
||||||
pub fn estimate_pattern_performance(&self, pattern: &AdvancedFusionPattern) -> Result<f64> {
|
pub fn estimate_pattern_performance(&self, pattern: &AdvancedFusionPattern) -> Result<f64> {
|
||||||
use AdvancedFusionPattern::{AttentionLayerNormFusion, FeedForwardFusion, TransformerBlockFusion, QuantumClassicalFusion, NeuromorphicClassicalFusion, ElementwiseReductionFusion, BroadcastElementwiseFusion};
|
use AdvancedFusionPattern::{
|
||||||
|
AttentionLayerNormFusion, BroadcastElementwiseFusion, ElementwiseReductionFusion,
|
||||||
|
FeedForwardFusion, NeuromorphicClassicalFusion, QuantumClassicalFusion,
|
||||||
|
TransformerBlockFusion,
|
||||||
|
};
|
||||||
|
|
||||||
let speedup = match pattern {
|
let speedup = match pattern {
|
||||||
AttentionLayerNormFusion {
|
AttentionLayerNormFusion {
|
||||||
|
|||||||
@@ -230,9 +230,10 @@ impl LazyExecutor {
|
|||||||
// Add remaining non-fused operations
|
// Add remaining non-fused operations
|
||||||
for node_id in needed {
|
for node_id in needed {
|
||||||
if !fused_nodes.contains(&node_id)
|
if !fused_nodes.contains(&node_id)
|
||||||
&& let Some(node) = graph.get_node(node_id) {
|
&& let Some(node) = graph.get_node(node_id)
|
||||||
plan.add_single(node);
|
{
|
||||||
}
|
plan.add_single(node);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
plan
|
plan
|
||||||
|
|||||||
@@ -162,8 +162,7 @@ impl MetalFusionExecutor {
|
|||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 2);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 2);
|
||||||
|
|
||||||
let count_u32 = count as u32;
|
let count_u32 = count as u32;
|
||||||
let count_ptr =
|
let count_ptr = NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 3);
|
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -225,8 +224,7 @@ impl MetalFusionExecutor {
|
|||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 1);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 1);
|
||||||
|
|
||||||
let count_u32 = count as u32;
|
let count_u32 = count as u32;
|
||||||
let count_ptr =
|
let count_ptr = NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 2);
|
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -289,8 +287,7 @@ impl MetalFusionExecutor {
|
|||||||
let scalar_ptr = NonNull::new_unchecked(&raw const scalar as *mut std::ffi::c_void);
|
let scalar_ptr = NonNull::new_unchecked(&raw const scalar as *mut std::ffi::c_void);
|
||||||
encoder.setBytes_length_atIndex(scalar_ptr, std::mem::size_of::<f32>(), 2);
|
encoder.setBytes_length_atIndex(scalar_ptr, std::mem::size_of::<f32>(), 2);
|
||||||
let count_u32 = count as u32;
|
let count_u32 = count as u32;
|
||||||
let count_ptr =
|
let count_ptr = NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 3);
|
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -354,8 +351,7 @@ impl MetalFusionExecutor {
|
|||||||
unsafe {
|
unsafe {
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_in), 0, 0);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_in), 0, 0);
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 1);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 1);
|
||||||
let params_ptr =
|
let params_ptr = NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<ReductionParams>(), 2);
|
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<ReductionParams>(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -424,8 +420,7 @@ impl MetalFusionExecutor {
|
|||||||
unsafe {
|
unsafe {
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_in), 0, 0);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_in), 0, 0);
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 1);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 1);
|
||||||
let params_ptr =
|
let params_ptr = NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<SoftmaxParams>(), 2);
|
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<SoftmaxParams>(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -494,8 +489,7 @@ impl MetalFusionExecutor {
|
|||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_gamma), 0, 1);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_gamma), 0, 1);
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_beta), 0, 2);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_beta), 0, 2);
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 3);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 3);
|
||||||
let params_ptr =
|
let params_ptr = NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<LayerNormParams>(), 4);
|
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<LayerNormParams>(), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -546,8 +540,7 @@ impl MetalFusionExecutor {
|
|||||||
let scale_ptr = NonNull::new_unchecked(&raw const scale as *mut std::ffi::c_void);
|
let scale_ptr = NonNull::new_unchecked(&raw const scale as *mut std::ffi::c_void);
|
||||||
encoder.setBytes_length_atIndex(scale_ptr, std::mem::size_of::<f32>(), 3);
|
encoder.setBytes_length_atIndex(scale_ptr, std::mem::size_of::<f32>(), 3);
|
||||||
let count_u32 = count as u32;
|
let count_u32 = count as u32;
|
||||||
let count_ptr =
|
let count_ptr = NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const count_u32 as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 4);
|
encoder.setBytes_length_atIndex(count_ptr, std::mem::size_of::<u32>(), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -619,8 +612,7 @@ impl MetalFusionExecutor {
|
|||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_weight), 0, 1);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_weight), 0, 1);
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_bias), 0, 2);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_bias), 0, 2);
|
||||||
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 3);
|
encoder.setBuffer_offset_atIndex(Some(&buffer_out), 0, 3);
|
||||||
let params_ptr =
|
let params_ptr = NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
||||||
NonNull::new_unchecked(&raw const params as *mut std::ffi::c_void);
|
|
||||||
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<MatmulParams>(), 4);
|
encoder.setBytes_length_atIndex(params_ptr, std::mem::size_of::<MatmulParams>(), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -272,7 +272,9 @@ pub fn is_gpu_available() -> bool {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if rtx_tensor::Device::cuda(config.device_id).is_ok() { true } else {
|
if rtx_tensor::Device::cuda(config.device_id).is_ok() {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
warn!(
|
warn!(
|
||||||
"GPU device {} not available, falling back to CPU",
|
"GPU device {} not available, falling back to CPU",
|
||||||
config.device_id
|
config.device_id
|
||||||
@@ -335,7 +337,9 @@ pub fn version_info() -> HashMap<String, String> {
|
|||||||
pub fn get_device() -> rtx_tensor::Device {
|
pub fn get_device() -> rtx_tensor::Device {
|
||||||
let config = get_gpu_config();
|
let config = get_gpu_config();
|
||||||
if config.enabled {
|
if config.enabled {
|
||||||
if let Ok(device) = rtx_tensor::Device::cuda(config.device_id) { device } else {
|
if let Ok(device) = rtx_tensor::Device::cuda(config.device_id) {
|
||||||
|
device
|
||||||
|
} else {
|
||||||
warn!("Failed to create CUDA device, using CPU");
|
warn!("Failed to create CUDA device, using CPU");
|
||||||
rtx_tensor::Device::cuda(0).unwrap_or_default()
|
rtx_tensor::Device::cuda(0).unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ use tokio::time::interval;
|
|||||||
use tracing::{debug, info, instrument};
|
use tracing::{debug, info, instrument};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
DataRecord, DataValue, Result, monitoring::EtlMetrics,
|
DataRecord, DataValue, Result, monitoring::EtlMetrics, state::StateManager,
|
||||||
state::StateManager, transform::Transformation,
|
transform::Transformation,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Configuration for stream processing
|
/// Configuration for stream processing
|
||||||
|
|||||||
@@ -168,10 +168,11 @@ impl BurnSession {
|
|||||||
if self.config.cache_models {
|
if self.config.cache_models {
|
||||||
// Evict oldest if at capacity
|
// Evict oldest if at capacity
|
||||||
if self.models.len() >= self.config.max_cached_models
|
if self.models.len() >= self.config.max_cached_models
|
||||||
&& let Some(key) = self.models.keys().next().cloned() {
|
&& let Some(key) = self.models.keys().next().cloned()
|
||||||
debug!("Evicting cached model: {}", key);
|
{
|
||||||
self.models.remove(&key);
|
debug!("Evicting cached model: {}", key);
|
||||||
}
|
self.models.remove(&key);
|
||||||
|
}
|
||||||
self.models.insert(path_str.clone(), model);
|
self.models.insert(path_str.clone(), model);
|
||||||
self.stats.cached_models = self.models.len();
|
self.stats.cached_models = self.models.len();
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -55,8 +55,7 @@ impl CandleBackend {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Device wrapper for Candle
|
/// Device wrapper for Candle
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||||
#[derive(Default)]
|
|
||||||
pub struct CandleDevice {
|
pub struct CandleDevice {
|
||||||
/// Backend type
|
/// Backend type
|
||||||
pub backend: CandleBackend,
|
pub backend: CandleBackend,
|
||||||
@@ -64,7 +63,6 @@ pub struct CandleDevice {
|
|||||||
pub ordinal: usize,
|
pub ordinal: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
impl CandleDevice {
|
impl CandleDevice {
|
||||||
/// Create a CPU device
|
/// Create a CPU device
|
||||||
pub fn cpu() -> Self {
|
pub fn cpu() -> Self {
|
||||||
|
|||||||
@@ -160,9 +160,10 @@ impl CandleSession {
|
|||||||
// Cache if enabled
|
// Cache if enabled
|
||||||
if self.config.cache_models {
|
if self.config.cache_models {
|
||||||
if self.models.len() >= self.config.max_cached_models
|
if self.models.len() >= self.config.max_cached_models
|
||||||
&& let Some(key) = self.models.keys().next().cloned() {
|
&& let Some(key) = self.models.keys().next().cloned()
|
||||||
self.models.remove(&key);
|
{
|
||||||
}
|
self.models.remove(&key);
|
||||||
|
}
|
||||||
self.models.insert(path_str.clone(), model);
|
self.models.insert(path_str.clone(), model);
|
||||||
self.stats.cached_models = self.models.len();
|
self.stats.cached_models = self.models.len();
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ use tracing::debug;
|
|||||||
/// Convert an rtx-tensor Tensor to Candle tensor
|
/// Convert an rtx-tensor Tensor to Candle tensor
|
||||||
pub fn rtx_to_candle(tensor: &Tensor, device: &CandleDevice) -> Result<CandleTensor> {
|
pub fn rtx_to_candle(tensor: &Tensor, device: &CandleDevice) -> Result<CandleTensor> {
|
||||||
let shape: Vec<usize> = tensor.shape().to_vec();
|
let shape: Vec<usize> = tensor.shape().to_vec();
|
||||||
let data = tensor.to_vec_f32().map_err(|e| {
|
let data = tensor
|
||||||
CandleError::tensor_conversion(format!("Failed to extract f32 data: {e}"))
|
.to_vec_f32()
|
||||||
})?;
|
.map_err(|e| CandleError::tensor_conversion(format!("Failed to extract f32 data: {e}")))?;
|
||||||
|
|
||||||
debug!("Converting rtx tensor {:?} to Candle", shape);
|
debug!("Converting rtx tensor {:?} to Candle", shape);
|
||||||
|
|
||||||
|
|||||||
@@ -507,7 +507,7 @@ impl TextGenerator {
|
|||||||
|
|
||||||
/// Utility functions for generation
|
/// Utility functions for generation
|
||||||
pub mod utils {
|
pub mod utils {
|
||||||
use super::{Tensor, Result, GenerationConfig};
|
use super::{GenerationConfig, Result, Tensor};
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
|
|
||||||
/// Apply temperature scaling to logits
|
/// Apply temperature scaling to logits
|
||||||
|
|||||||
@@ -284,7 +284,9 @@ pub fn is_gpu_available() -> bool {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if rtx_tensor::Device::cuda(config.device_id).is_ok() { true } else {
|
if rtx_tensor::Device::cuda(config.device_id).is_ok() {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
warn!(
|
warn!(
|
||||||
"GPU device {} not available, falling back to CPU",
|
"GPU device {} not available, falling back to CPU",
|
||||||
config.device_id
|
config.device_id
|
||||||
@@ -347,7 +349,9 @@ pub fn version_info() -> HashMap<String, String> {
|
|||||||
pub fn get_device() -> rtx_tensor::Device {
|
pub fn get_device() -> rtx_tensor::Device {
|
||||||
let config = get_gpu_config();
|
let config = get_gpu_config();
|
||||||
if config.enabled {
|
if config.enabled {
|
||||||
if let Ok(device) = rtx_tensor::Device::cuda(config.device_id) { device } else {
|
if let Ok(device) = rtx_tensor::Device::cuda(config.device_id) {
|
||||||
|
device
|
||||||
|
} else {
|
||||||
warn!("Failed to create CUDA device, using CPU");
|
warn!("Failed to create CUDA device, using CPU");
|
||||||
rtx_tensor::Device::Cuda(0)
|
rtx_tensor::Device::Cuda(0)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,7 +67,9 @@ impl PositionalEncoding {
|
|||||||
|
|
||||||
/// Forward pass - add positional encoding to input
|
/// Forward pass - add positional encoding to input
|
||||||
pub fn forward(&self, input: &Tensor) -> Result<Tensor> {
|
pub fn forward(&self, input: &Tensor) -> Result<Tensor> {
|
||||||
if let PositionalEncodingType::None = self.encoding_type { Ok(input.clone()) } else {
|
if let PositionalEncodingType::None = self.encoding_type {
|
||||||
|
Ok(input.clone())
|
||||||
|
} else {
|
||||||
let Some(ref pos_emb) = self.position_embeddings else {
|
let Some(ref pos_emb) = self.position_embeddings else {
|
||||||
return Ok(input.clone());
|
return Ok(input.clone());
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
use crate::Tensor;
|
use crate::Tensor;
|
||||||
use crate::error::{Result, VisionError};
|
use crate::error::{Result, VisionError};
|
||||||
use crate::preprocessing::{ImageProcessor, ImageTensor};
|
use crate::preprocessing::{ImageProcessor, ImageTensor};
|
||||||
use rand::{thread_rng, Rng};
|
use rand::{Rng, thread_rng};
|
||||||
|
|
||||||
/// Augmentation operations for data augmentation during training
|
/// Augmentation operations for data augmentation during training
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
|
|||||||
@@ -428,7 +428,12 @@ pub fn convert_hf_config(hf_config: &HFModelConfig) -> HubResult<RTXModelConfig>
|
|||||||
let model_type = hf_config
|
let model_type = hf_config
|
||||||
.model_type
|
.model_type
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.or_else(|| hf_config.architectures.first().map(std::string::String::as_str))
|
.or_else(|| {
|
||||||
|
hf_config
|
||||||
|
.architectures
|
||||||
|
.first()
|
||||||
|
.map(std::string::String::as_str)
|
||||||
|
})
|
||||||
.unwrap_or("unknown");
|
.unwrap_or("unknown");
|
||||||
|
|
||||||
let architecture = RTXArchitecture::from_hf_model_type(model_type);
|
let architecture = RTXArchitecture::from_hf_model_type(model_type);
|
||||||
@@ -496,7 +501,8 @@ pub fn convert_hf_config(hf_config: &HFModelConfig) -> HubResult<RTXModelConfig>
|
|||||||
let bos_token_id = hf_config.bos_token_id.unwrap_or(1);
|
let bos_token_id = hf_config.bos_token_id.unwrap_or(1);
|
||||||
let eos_token_ids = hf_config
|
let eos_token_ids = hf_config
|
||||||
.eos_token_id
|
.eos_token_id
|
||||||
.as_ref().map_or_else(|| vec![2], EosTokenId::all);
|
.as_ref()
|
||||||
|
.map_or_else(|| vec![2], EosTokenId::all);
|
||||||
let pad_token_id = hf_config.pad_token_id;
|
let pad_token_id = hf_config.pad_token_id;
|
||||||
|
|
||||||
Ok(RTXModelConfig {
|
Ok(RTXModelConfig {
|
||||||
|
|||||||
@@ -163,7 +163,11 @@ impl SafeTensors {
|
|||||||
|
|
||||||
/// Get the names of all tensors in the file.
|
/// Get the names of all tensors in the file.
|
||||||
pub fn tensor_names(&self) -> Vec<&str> {
|
pub fn tensor_names(&self) -> Vec<&str> {
|
||||||
self.header.tensors.keys().map(std::string::String::as_str).collect()
|
self.header
|
||||||
|
.tensors
|
||||||
|
.keys()
|
||||||
|
.map(std::string::String::as_str)
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get information about a specific tensor.
|
/// Get information about a specific tensor.
|
||||||
|
|||||||
@@ -19,7 +19,8 @@ fn main() {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}).map_or_else(|| "unknown".to_string(), |s| s.trim().to_string());
|
})
|
||||||
|
.map_or_else(|| "unknown".to_string(), |s| s.trim().to_string());
|
||||||
|
|
||||||
println!("cargo:rustc-env=RUSTYTORCH_GIT_COMMIT={git_commit}");
|
println!("cargo:rustc-env=RUSTYTORCH_GIT_COMMIT={git_commit}");
|
||||||
|
|
||||||
@@ -35,7 +36,8 @@ fn main() {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}).map_or_else(|| "unknown".to_string(), |s| s.trim().to_string())
|
})
|
||||||
|
.map_or_else(|| "unknown".to_string(), |s| s.trim().to_string())
|
||||||
});
|
});
|
||||||
|
|
||||||
println!("cargo:rustc-env=RUSTYTORCH_RUST_VERSION={rustc_version}");
|
println!("cargo:rustc-env=RUSTYTORCH_RUST_VERSION={rustc_version}");
|
||||||
|
|||||||
@@ -399,7 +399,10 @@ impl PagedKvCache {
|
|||||||
let result_keys = if key_tensors.len() == 1 {
|
let result_keys = if key_tensors.len() == 1 {
|
||||||
// SAFETY: We just checked len() == 1, so next() will succeed
|
// SAFETY: We just checked len() == 1, so next() will succeed
|
||||||
let Some(key) = key_tensors.into_iter().next() else {
|
let Some(key) = key_tensors.into_iter().next() else {
|
||||||
return Err(InferenceError::kv_cache_error("get", "Key tensor disappeared"));
|
return Err(InferenceError::kv_cache_error(
|
||||||
|
"get",
|
||||||
|
"Key tensor disappeared",
|
||||||
|
));
|
||||||
};
|
};
|
||||||
key
|
key
|
||||||
} else if !key_tensors.is_empty() {
|
} else if !key_tensors.is_empty() {
|
||||||
@@ -416,7 +419,10 @@ impl PagedKvCache {
|
|||||||
let result_values = if value_tensors.len() == 1 {
|
let result_values = if value_tensors.len() == 1 {
|
||||||
// SAFETY: We just checked len() == 1, so next() will succeed
|
// SAFETY: We just checked len() == 1, so next() will succeed
|
||||||
let Some(value) = value_tensors.into_iter().next() else {
|
let Some(value) = value_tensors.into_iter().next() else {
|
||||||
return Err(InferenceError::kv_cache_error("get", "Value tensor disappeared"));
|
return Err(InferenceError::kv_cache_error(
|
||||||
|
"get",
|
||||||
|
"Value tensor disappeared",
|
||||||
|
));
|
||||||
};
|
};
|
||||||
value
|
value
|
||||||
} else if !value_tensors.is_empty() {
|
} else if !value_tensors.is_empty() {
|
||||||
|
|||||||
@@ -487,7 +487,7 @@ impl AlertManager {
|
|||||||
|
|
||||||
/// Pre-defined alert rules for common ML scenarios
|
/// Pre-defined alert rules for common ML scenarios
|
||||||
pub mod presets {
|
pub mod presets {
|
||||||
use super::{AlertRule, AlertCondition, AlertSeverity};
|
use super::{AlertCondition, AlertRule, AlertSeverity};
|
||||||
|
|
||||||
/// High GPU memory usage alert
|
/// High GPU memory usage alert
|
||||||
pub fn high_gpu_memory(threshold_percent: f64) -> AlertRule {
|
pub fn high_gpu_memory(threshold_percent: f64) -> AlertRule {
|
||||||
|
|||||||
@@ -4,9 +4,9 @@ use proc_macro2::TokenStream;
|
|||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::codegen::CodegenConfig;
|
use crate::codegen::CodegenConfig;
|
||||||
|
use crate::error::Result;
|
||||||
use crate::ir::OnnxGraph;
|
use crate::ir::OnnxGraph;
|
||||||
use crate::ops::{generate_node_code, sanitize_name};
|
use crate::ops::{generate_node_code, sanitize_name};
|
||||||
use crate::error::Result;
|
|
||||||
|
|
||||||
/// Generator for the forward function.
|
/// Generator for the forward function.
|
||||||
pub struct ForwardGenerator<'a> {
|
pub struct ForwardGenerator<'a> {
|
||||||
@@ -43,7 +43,9 @@ impl<'a> ForwardGenerator<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn generate_input_params(&self, graph: &OnnxGraph) -> Result<TokenStream> {
|
fn generate_input_params(&self, graph: &OnnxGraph) -> Result<TokenStream> {
|
||||||
let params: Vec<TokenStream> = graph.inputs.iter()
|
let params: Vec<TokenStream> = graph
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
.filter(|input| !graph.is_initializer(&input.name))
|
.filter(|input| !graph.is_initializer(&input.name))
|
||||||
.map(|input| {
|
.map(|input| {
|
||||||
let name = quote::format_ident!("{}", sanitize_name(&input.name));
|
let name = quote::format_ident!("{}", sanitize_name(&input.name));
|
||||||
@@ -91,7 +93,9 @@ impl<'a> ForwardGenerator<'a> {
|
|||||||
let output_name = quote::format_ident!("{}", sanitize_name(&graph.outputs[0].name));
|
let output_name = quote::format_ident!("{}", sanitize_name(&graph.outputs[0].name));
|
||||||
Ok(quote! { Ok(#output_name) })
|
Ok(quote! { Ok(#output_name) })
|
||||||
} else {
|
} else {
|
||||||
let output_names: Vec<_> = graph.outputs.iter()
|
let output_names: Vec<_> = graph
|
||||||
|
.outputs
|
||||||
|
.iter()
|
||||||
.map(|o| quote::format_ident!("{}", sanitize_name(&o.name)))
|
.map(|o| quote::format_ident!("{}", sanitize_name(&o.name)))
|
||||||
.collect();
|
.collect();
|
||||||
Ok(quote! { Ok((#(#output_names),*)) })
|
Ok(quote! { Ok((#(#output_names),*)) })
|
||||||
|
|||||||
@@ -5,19 +5,19 @@
|
|||||||
//! - A constructor that loads weights from SafeTensors
|
//! - A constructor that loads weights from SafeTensors
|
||||||
//! - A forward method that implements the model computation
|
//! - A forward method that implements the model computation
|
||||||
|
|
||||||
mod module_gen;
|
|
||||||
mod forward_gen;
|
mod forward_gen;
|
||||||
|
mod module_gen;
|
||||||
mod weight_gen;
|
mod weight_gen;
|
||||||
|
|
||||||
pub use module_gen::ModuleGenerator;
|
|
||||||
pub use forward_gen::ForwardGenerator;
|
pub use forward_gen::ForwardGenerator;
|
||||||
|
pub use module_gen::ModuleGenerator;
|
||||||
pub use weight_gen::WeightGenerator;
|
pub use weight_gen::WeightGenerator;
|
||||||
|
|
||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::OnnxGraph;
|
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
|
use crate::ir::OnnxGraph;
|
||||||
|
|
||||||
/// Configuration for code generation.
|
/// Configuration for code generation.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|||||||
@@ -4,9 +4,9 @@ use proc_macro2::TokenStream;
|
|||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::codegen::CodegenConfig;
|
use crate::codegen::CodegenConfig;
|
||||||
|
use crate::error::Result;
|
||||||
use crate::ir::OnnxGraph;
|
use crate::ir::OnnxGraph;
|
||||||
use crate::ops::sanitize_name;
|
use crate::ops::sanitize_name;
|
||||||
use crate::error::Result;
|
|
||||||
|
|
||||||
/// Generator for the model struct definition.
|
/// Generator for the model struct definition.
|
||||||
pub struct ModuleGenerator<'a> {
|
pub struct ModuleGenerator<'a> {
|
||||||
|
|||||||
@@ -4,9 +4,9 @@ use proc_macro2::TokenStream;
|
|||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::codegen::CodegenConfig;
|
use crate::codegen::CodegenConfig;
|
||||||
|
use crate::error::Result;
|
||||||
use crate::ir::OnnxGraph;
|
use crate::ir::OnnxGraph;
|
||||||
use crate::ops::sanitize_name;
|
use crate::ops::sanitize_name;
|
||||||
use crate::error::Result;
|
|
||||||
|
|
||||||
/// Generator for weight loading code.
|
/// Generator for weight loading code.
|
||||||
pub struct WeightGenerator<'a> {
|
pub struct WeightGenerator<'a> {
|
||||||
@@ -106,7 +106,9 @@ impl<'a> WeightGenerator<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn generate_field_initializers(&self, graph: &OnnxGraph) -> Result<TokenStream> {
|
fn generate_field_initializers(&self, graph: &OnnxGraph) -> Result<TokenStream> {
|
||||||
let inits: Vec<TokenStream> = graph.initializers.keys()
|
let inits: Vec<TokenStream> = graph
|
||||||
|
.initializers
|
||||||
|
.keys()
|
||||||
.map(|name| {
|
.map(|name| {
|
||||||
let field_name = quote::format_ident!("{}", sanitize_name(name));
|
let field_name = quote::format_ident!("{}", sanitize_name(name));
|
||||||
quote! { #field_name, }
|
quote! { #field_name, }
|
||||||
|
|||||||
@@ -74,8 +74,14 @@ impl OnnxGraph {
|
|||||||
value_info.insert(info.name.clone(), info);
|
value_info.insert(info.name.clone(), info);
|
||||||
}
|
}
|
||||||
|
|
||||||
log::debug!("Graph '{}': {} inputs, {} outputs, {} nodes, {} initializers",
|
log::debug!(
|
||||||
name, inputs.len(), outputs.len(), nodes.len(), initializers.len());
|
"Graph '{}': {} inputs, {} outputs, {} nodes, {} initializers",
|
||||||
|
name,
|
||||||
|
inputs.len(),
|
||||||
|
outputs.len(),
|
||||||
|
nodes.len(),
|
||||||
|
initializers.len()
|
||||||
|
);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
name,
|
name,
|
||||||
@@ -111,11 +117,8 @@ impl OnnxGraph {
|
|||||||
/// Validate the graph structure.
|
/// Validate the graph structure.
|
||||||
pub fn validate(&self) -> Result<()> {
|
pub fn validate(&self) -> Result<()> {
|
||||||
// Check all inputs are defined
|
// Check all inputs are defined
|
||||||
let mut defined: std::collections::HashSet<&str> = self
|
let mut defined: std::collections::HashSet<&str> =
|
||||||
.inputs
|
self.inputs.iter().map(|i| i.name.as_str()).collect();
|
||||||
.iter()
|
|
||||||
.map(|i| i.name.as_str())
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Add initializers
|
// Add initializers
|
||||||
for name in self.initializers.keys() {
|
for name in self.initializers.keys() {
|
||||||
|
|||||||
@@ -150,9 +150,7 @@ impl Tensor {
|
|||||||
.collect();
|
.collect();
|
||||||
Ok(TensorData::Int64(data))
|
Ok(TensorData::Int64(data))
|
||||||
}
|
}
|
||||||
DataType::UInt8 | DataType::Bool => {
|
DataType::UInt8 | DataType::Bool => Ok(TensorData::UInt8(raw.to_vec())),
|
||||||
Ok(TensorData::UInt8(raw.to_vec()))
|
|
||||||
}
|
|
||||||
_ => {
|
_ => {
|
||||||
// Store as raw bytes for unsupported types
|
// Store as raw bytes for unsupported types
|
||||||
Ok(TensorData::Raw(raw.to_vec()))
|
Ok(TensorData::Raw(raw.to_vec()))
|
||||||
|
|||||||
@@ -173,7 +173,10 @@ impl Shape {
|
|||||||
|
|
||||||
/// Get static dimensions, panics if dynamic.
|
/// Get static dimensions, panics if dynamic.
|
||||||
pub fn static_dims(&self) -> Vec<i64> {
|
pub fn static_dims(&self) -> Vec<i64> {
|
||||||
self.dims.iter().map(|d| d.expect("Dynamic dimension")).collect()
|
self.dims
|
||||||
|
.iter()
|
||||||
|
.map(|d| d.expect("Dynamic dimension"))
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Total number of elements (for static shapes).
|
/// Total number of elements (for static shapes).
|
||||||
@@ -218,7 +221,9 @@ impl ValueInfo {
|
|||||||
pub fn from_proto(proto: &ValueInfoProto) -> Result<Self> {
|
pub fn from_proto(proto: &ValueInfoProto) -> Result<Self> {
|
||||||
let name = proto.name.clone();
|
let name = proto.name.clone();
|
||||||
|
|
||||||
let type_proto = proto.r#type.as_ref()
|
let type_proto = proto
|
||||||
|
.r#type
|
||||||
|
.as_ref()
|
||||||
.ok_or_else(|| Error::InvalidModel(format!("Missing type for {}", name)))?;
|
.ok_or_else(|| Error::InvalidModel(format!("Missing type for {}", name)))?;
|
||||||
|
|
||||||
let (dtype, shape) = Self::extract_tensor_type(type_proto)?;
|
let (dtype, shape) = Self::extract_tensor_type(type_proto)?;
|
||||||
@@ -252,7 +257,9 @@ impl ValueInfo {
|
|||||||
};
|
};
|
||||||
Ok((dtype, shape))
|
Ok((dtype, shape))
|
||||||
}
|
}
|
||||||
_ => Err(Error::InvalidModel("Non-tensor types not supported".to_string())),
|
_ => Err(Error::InvalidModel(
|
||||||
|
"Non-tensor types not supported".to_string(),
|
||||||
|
)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,12 +2,12 @@
|
|||||||
//!
|
//!
|
||||||
//! Provides a fluent API for generating Rust code from ONNX models.
|
//! Provides a fluent API for generating Rust code from ONNX models.
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use crate::codegen::{CodeGenerator, CodegenConfig, Visibility};
|
use crate::codegen::{CodeGenerator, CodegenConfig, Visibility};
|
||||||
use crate::parser::OnnxParser;
|
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
|
use crate::parser::OnnxParser;
|
||||||
|
|
||||||
/// Builder for generating Rust code from ONNX models.
|
/// Builder for generating Rust code from ONNX models.
|
||||||
///
|
///
|
||||||
@@ -136,20 +136,23 @@ impl ModelGen {
|
|||||||
/// - Code generation fails
|
/// - Code generation fails
|
||||||
/// - The output file can't be written
|
/// - The output file can't be written
|
||||||
pub fn run(&self) -> Result<PathBuf> {
|
pub fn run(&self) -> Result<PathBuf> {
|
||||||
let input_path = self.input_path.as_ref()
|
let input_path = self
|
||||||
|
.input_path
|
||||||
|
.as_ref()
|
||||||
.ok_or_else(|| Error::CodeGen("No input path specified".into()))?;
|
.ok_or_else(|| Error::CodeGen("No input path specified".into()))?;
|
||||||
|
|
||||||
// Determine output path
|
// Determine output path
|
||||||
let out_dir = self.out_dir.clone()
|
let out_dir = self.out_dir.clone().unwrap_or_else(|| {
|
||||||
.unwrap_or_else(|| std::env::var("OUT_DIR").map_or_else(|_| PathBuf::from("."), PathBuf::from));
|
std::env::var("OUT_DIR").map_or_else(|_| PathBuf::from("."), PathBuf::from)
|
||||||
|
});
|
||||||
|
|
||||||
let out_file = self.out_file.clone()
|
let out_file = self.out_file.clone().unwrap_or_else(|| {
|
||||||
.unwrap_or_else(|| {
|
input_path
|
||||||
input_path.file_stem()
|
.file_stem()
|
||||||
.and_then(|s| s.to_str())
|
.and_then(|s| s.to_str())
|
||||||
.unwrap_or("model")
|
.unwrap_or("model")
|
||||||
.to_string()
|
.to_string()
|
||||||
});
|
});
|
||||||
|
|
||||||
let output_path = out_dir.join(format!("{}.rs", out_file));
|
let output_path = out_dir.join(format!("{}.rs", out_file));
|
||||||
|
|
||||||
@@ -210,7 +213,9 @@ impl ModelGen {
|
|||||||
///
|
///
|
||||||
/// Useful for testing or generating code programmatically.
|
/// Useful for testing or generating code programmatically.
|
||||||
pub fn generate_string(&self) -> Result<String> {
|
pub fn generate_string(&self) -> Result<String> {
|
||||||
let input_path = self.input_path.as_ref()
|
let input_path = self
|
||||||
|
.input_path
|
||||||
|
.as_ref()
|
||||||
.ok_or_else(|| Error::CodeGen("No input path specified".into()))?;
|
.ok_or_else(|| Error::CodeGen("No input path specified".into()))?;
|
||||||
|
|
||||||
let graph = OnnxParser::parse_file(input_path)?;
|
let graph = OnnxParser::parse_file(input_path)?;
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::Node;
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::Node;
|
||||||
|
|
||||||
/// Generate softmax operation.
|
/// Generate softmax operation.
|
||||||
pub fn generate_softmax(node: &Node) -> Result<TokenStream> {
|
pub fn generate_softmax(node: &Node) -> Result<TokenStream> {
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::Node;
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::Node;
|
||||||
|
|
||||||
/// Generate binary arithmetic operation.
|
/// Generate binary arithmetic operation.
|
||||||
pub fn generate_binary_op(node: &Node, op: &str) -> Result<TokenStream> {
|
pub fn generate_binary_op(node: &Node, op: &str) -> Result<TokenStream> {
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::{Node, OnnxGraph};
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::{Node, OnnxGraph};
|
||||||
|
|
||||||
/// Default strides for 2D convolution.
|
/// Default strides for 2D convolution.
|
||||||
const DEFAULT_STRIDES: &[i64] = &[1, 1];
|
const DEFAULT_STRIDES: &[i64] = &[1, 1];
|
||||||
@@ -78,7 +78,9 @@ pub fn generate_conv_transpose(node: &Node, _graph: &OnnxGraph) -> Result<TokenS
|
|||||||
// Get attributes
|
// Get attributes
|
||||||
let strides = node.get_ints("strides").unwrap_or(DEFAULT_STRIDES);
|
let strides = node.get_ints("strides").unwrap_or(DEFAULT_STRIDES);
|
||||||
let pads = node.get_ints("pads").unwrap_or(DEFAULT_PADS);
|
let pads = node.get_ints("pads").unwrap_or(DEFAULT_PADS);
|
||||||
let output_padding = node.get_ints("output_padding").unwrap_or(DEFAULT_OUTPUT_PADDING);
|
let output_padding = node
|
||||||
|
.get_ints("output_padding")
|
||||||
|
.unwrap_or(DEFAULT_OUTPUT_PADDING);
|
||||||
let dilations = node.get_ints("dilations").unwrap_or(DEFAULT_DILATIONS);
|
let dilations = node.get_ints("dilations").unwrap_or(DEFAULT_DILATIONS);
|
||||||
let group = node.get_int("group").unwrap_or(1) as usize;
|
let group = node.get_int("group").unwrap_or(1) as usize;
|
||||||
|
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::{Node, OnnxGraph};
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::{Node, OnnxGraph};
|
||||||
|
|
||||||
/// Generate MatMul operation.
|
/// Generate MatMul operation.
|
||||||
pub fn generate_matmul(node: &Node) -> Result<TokenStream> {
|
pub fn generate_matmul(node: &Node) -> Result<TokenStream> {
|
||||||
|
|||||||
@@ -2,16 +2,16 @@
|
|||||||
//!
|
//!
|
||||||
//! This module contains code generators for individual ONNX operators.
|
//! This module contains code generators for individual ONNX operators.
|
||||||
|
|
||||||
mod arithmetic;
|
|
||||||
mod activation;
|
mod activation;
|
||||||
|
mod arithmetic;
|
||||||
mod conv;
|
mod conv;
|
||||||
mod matmul;
|
mod matmul;
|
||||||
mod normalization;
|
mod normalization;
|
||||||
mod pooling;
|
mod pooling;
|
||||||
mod reshape;
|
mod reshape;
|
||||||
|
|
||||||
pub use arithmetic::*;
|
|
||||||
pub use activation::*;
|
pub use activation::*;
|
||||||
|
pub use arithmetic::*;
|
||||||
pub use conv::*;
|
pub use conv::*;
|
||||||
pub use matmul::*;
|
pub use matmul::*;
|
||||||
pub use normalization::*;
|
pub use normalization::*;
|
||||||
@@ -21,8 +21,8 @@ pub use reshape::*;
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::{Node, NodeKind, OnnxGraph};
|
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
|
use crate::ir::{Node, NodeKind, OnnxGraph};
|
||||||
|
|
||||||
/// Generate code for a single node.
|
/// Generate code for a single node.
|
||||||
pub fn generate_node_code(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
pub fn generate_node_code(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
||||||
@@ -183,12 +183,12 @@ pub fn sanitize_name(name: &str) -> String {
|
|||||||
|
|
||||||
// Handle Rust keywords
|
// Handle Rust keywords
|
||||||
match result.as_str() {
|
match result.as_str() {
|
||||||
"type" | "fn" | "let" | "mut" | "ref" | "self" | "Self" | "mod" | "pub" | "use" |
|
"type" | "fn" | "let" | "mut" | "ref" | "self" | "Self" | "mod" | "pub" | "use"
|
||||||
"struct" | "enum" | "trait" | "impl" | "for" | "while" | "loop" | "if" | "else" |
|
| "struct" | "enum" | "trait" | "impl" | "for" | "while" | "loop" | "if" | "else"
|
||||||
"match" | "return" | "break" | "continue" | "move" | "box" | "where" | "async" |
|
| "match" | "return" | "break" | "continue" | "move" | "box" | "where" | "async"
|
||||||
"await" | "dyn" | "abstract" | "become" | "const" | "crate" | "do" | "extern" |
|
| "await" | "dyn" | "abstract" | "become" | "const" | "crate" | "do" | "extern"
|
||||||
"final" | "in" | "macro" | "override" | "priv" | "static" | "super" | "try" |
|
| "final" | "in" | "macro" | "override" | "priv" | "static" | "super" | "try"
|
||||||
"typeof" | "unsafe" | "unsized" | "virtual" | "yield" => {
|
| "typeof" | "unsafe" | "unsized" | "virtual" | "yield" => {
|
||||||
result.push('_');
|
result.push('_');
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
@@ -223,7 +223,9 @@ fn generate_min_max(node: &Node, op: &str) -> Result<TokenStream> {
|
|||||||
let output = &node.outputs[0];
|
let output = &node.outputs[0];
|
||||||
let out_ident = quote::format_ident!("{}", sanitize_name(output));
|
let out_ident = quote::format_ident!("{}", sanitize_name(output));
|
||||||
|
|
||||||
let input_idents: Vec<_> = node.inputs.iter()
|
let input_idents: Vec<_> = node
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
.filter(|s| !s.is_empty())
|
.filter(|s| !s.is_empty())
|
||||||
.map(|s| quote::format_ident!("{}", sanitize_name(s)))
|
.map(|s| quote::format_ident!("{}", sanitize_name(s)))
|
||||||
.collect();
|
.collect();
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::{Node, OnnxGraph};
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::{Node, OnnxGraph};
|
||||||
|
|
||||||
/// Generate BatchNormalization operation.
|
/// Generate BatchNormalization operation.
|
||||||
pub fn generate_batch_norm(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
pub fn generate_batch_norm(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::Node;
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::Node;
|
||||||
|
|
||||||
/// Default kernel shape for pooling.
|
/// Default kernel shape for pooling.
|
||||||
const DEFAULT_KERNEL: &[i64] = &[2, 2];
|
const DEFAULT_KERNEL: &[i64] = &[2, 2];
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
use proc_macro2::TokenStream;
|
use proc_macro2::TokenStream;
|
||||||
use quote::quote;
|
use quote::quote;
|
||||||
|
|
||||||
use crate::ir::{Node, OnnxGraph};
|
|
||||||
use crate::error::Result;
|
|
||||||
use super::sanitize_name;
|
use super::sanitize_name;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::ir::{Node, OnnxGraph};
|
||||||
|
|
||||||
/// Generate Reshape operation.
|
/// Generate Reshape operation.
|
||||||
pub fn generate_reshape(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
pub fn generate_reshape(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
||||||
@@ -87,7 +87,9 @@ pub fn generate_concat(node: &Node) -> Result<TokenStream> {
|
|||||||
let axis = node.get_int("axis").unwrap_or(0);
|
let axis = node.get_int("axis").unwrap_or(0);
|
||||||
|
|
||||||
// Collect all input identifiers
|
// Collect all input identifiers
|
||||||
let input_idents: Vec<_> = node.inputs.iter()
|
let input_idents: Vec<_> = node
|
||||||
|
.inputs
|
||||||
|
.iter()
|
||||||
.filter(|s| !s.is_empty())
|
.filter(|s| !s.is_empty())
|
||||||
.map(|s| {
|
.map(|s| {
|
||||||
let ident = quote::format_ident!("{}", sanitize_name(s));
|
let ident = quote::format_ident!("{}", sanitize_name(s));
|
||||||
@@ -165,7 +167,9 @@ pub fn generate_split(node: &Node) -> Result<TokenStream> {
|
|||||||
let axis = node.get_int("axis").unwrap_or(0);
|
let axis = node.get_int("axis").unwrap_or(0);
|
||||||
|
|
||||||
// Output identifiers
|
// Output identifiers
|
||||||
let output_idents: Vec<_> = node.outputs.iter()
|
let output_idents: Vec<_> = node
|
||||||
|
.outputs
|
||||||
|
.iter()
|
||||||
.filter(|s| !s.is_empty())
|
.filter(|s| !s.is_empty())
|
||||||
.map(|s| quote::format_ident!("{}", sanitize_name(s)))
|
.map(|s| quote::format_ident!("{}", sanitize_name(s)))
|
||||||
.collect();
|
.collect();
|
||||||
@@ -203,22 +207,25 @@ pub fn generate_slice(node: &Node, graph: &OnnxGraph) -> Result<TokenStream> {
|
|||||||
let starts_name = &node.inputs[1];
|
let starts_name = &node.inputs[1];
|
||||||
let ends_name = &node.inputs[2];
|
let ends_name = &node.inputs[2];
|
||||||
|
|
||||||
if let (Some(starts_tensor), Some(ends_tensor)) =
|
if let (Some(starts_tensor), Some(ends_tensor)) = (
|
||||||
(graph.get_initializer(starts_name), graph.get_initializer(ends_name))
|
graph.get_initializer(starts_name),
|
||||||
{
|
graph.get_initializer(ends_name),
|
||||||
|
) {
|
||||||
let starts = starts_tensor.shape.static_dims();
|
let starts = starts_tensor.shape.static_dims();
|
||||||
let ends = ends_tensor.shape.static_dims();
|
let ends = ends_tensor.shape.static_dims();
|
||||||
|
|
||||||
// Optional axes and steps
|
// Optional axes and steps
|
||||||
let axes = if node.inputs.len() > 3 && !node.inputs[3].is_empty() {
|
let axes = if node.inputs.len() > 3 && !node.inputs[3].is_empty() {
|
||||||
graph.get_initializer(&node.inputs[3])
|
graph
|
||||||
|
.get_initializer(&node.inputs[3])
|
||||||
.map(|t| t.shape.static_dims())
|
.map(|t| t.shape.static_dims())
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let steps = if node.inputs.len() > 4 && !node.inputs[4].is_empty() {
|
let steps = if node.inputs.len() > 4 && !node.inputs[4].is_empty() {
|
||||||
graph.get_initializer(&node.inputs[4])
|
graph
|
||||||
|
.get_initializer(&node.inputs[4])
|
||||||
.map(|t| t.shape.static_dims())
|
.map(|t| t.shape.static_dims())
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
|||||||
@@ -41,11 +41,13 @@ impl OnnxParser {
|
|||||||
.graph
|
.graph
|
||||||
.ok_or_else(|| Error::InvalidModel("Model has no graph".to_string()))?;
|
.ok_or_else(|| Error::InvalidModel("Model has no graph".to_string()))?;
|
||||||
|
|
||||||
log::info!("Model: {} (IR version {})",
|
log::info!(
|
||||||
|
"Model: {} (IR version {})",
|
||||||
model.producer_name,
|
model.producer_name,
|
||||||
model.ir_version
|
model.ir_version
|
||||||
);
|
);
|
||||||
log::info!("Graph: {} nodes, {} initializers",
|
log::info!(
|
||||||
|
"Graph: {} nodes, {} initializers",
|
||||||
graph.node.len(),
|
graph.node.len(),
|
||||||
graph.initializer.len()
|
graph.initializer.len()
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -121,8 +121,8 @@ impl OnnxSession {
|
|||||||
model_bytes.len()
|
model_bytes.len()
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut builder = Session::builder()
|
let mut builder =
|
||||||
.map_err(|e| OnnxError::SessionCreation(e.to_string()))?;
|
Session::builder().map_err(|e| OnnxError::SessionCreation(e.to_string()))?;
|
||||||
|
|
||||||
builder = builder
|
builder = builder
|
||||||
.with_optimization_level(config.optimization_level.into())
|
.with_optimization_level(config.optimization_level.into())
|
||||||
@@ -137,8 +137,8 @@ impl OnnxSession {
|
|||||||
|
|
||||||
/// Create session from file with configuration
|
/// Create session from file with configuration
|
||||||
fn create_session(path: &Path, config: &OnnxSessionConfig) -> Result<Session> {
|
fn create_session(path: &Path, config: &OnnxSessionConfig) -> Result<Session> {
|
||||||
let mut builder = Session::builder()
|
let mut builder =
|
||||||
.map_err(|e| OnnxError::SessionCreation(e.to_string()))?;
|
Session::builder().map_err(|e| OnnxError::SessionCreation(e.to_string()))?;
|
||||||
|
|
||||||
// Set optimization level
|
// Set optimization level
|
||||||
builder = builder
|
builder = builder
|
||||||
|
|||||||
@@ -170,7 +170,9 @@ impl ContinuousBatchingController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Safe: front() returned Some, so pop_front() will succeed
|
// Safe: front() returned Some, so pop_front() will succeed
|
||||||
let Some(mut request) = queue.pop_front() else { break };
|
let Some(mut request) = queue.pop_front() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
total_tokens += request.total_tokens();
|
total_tokens += request.total_tokens();
|
||||||
total_memory += request.estimated_memory;
|
total_memory += request.estimated_memory;
|
||||||
request.state = RequestState::Processing;
|
request.state = RequestState::Processing;
|
||||||
|
|||||||
@@ -169,7 +169,9 @@ impl ContextFreeGrammar {
|
|||||||
|
|
||||||
for rule in &self.rules {
|
for rule in &self.rules {
|
||||||
let lhs = &rule.left_hand_side;
|
let lhs = &rule.left_hand_side;
|
||||||
let old_size = first_sets.get(lhs).map_or(0, std::collections::HashSet::len);
|
let old_size = first_sets
|
||||||
|
.get(lhs)
|
||||||
|
.map_or(0, std::collections::HashSet::len);
|
||||||
|
|
||||||
if rule.is_epsilon() {
|
if rule.is_epsilon() {
|
||||||
first_sets
|
first_sets
|
||||||
@@ -216,7 +218,9 @@ impl ContextFreeGrammar {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let new_size = first_sets.get(lhs).map_or(0, std::collections::HashSet::len);
|
let new_size = first_sets
|
||||||
|
.get(lhs)
|
||||||
|
.map_or(0, std::collections::HashSet::len);
|
||||||
if new_size > old_size {
|
if new_size > old_size {
|
||||||
changed = true;
|
changed = true;
|
||||||
}
|
}
|
||||||
@@ -247,13 +251,14 @@ impl ContextFreeGrammar {
|
|||||||
for rule in &self.rules {
|
for rule in &self.rules {
|
||||||
for symbol in &rule.right_hand_side {
|
for symbol in &rule.right_hand_side {
|
||||||
if let GrammarSymbol::NonTerminal(nt) = symbol
|
if let GrammarSymbol::NonTerminal(nt) = symbol
|
||||||
&& !self.non_terminals.contains(nt) {
|
&& !self.non_terminals.contains(nt)
|
||||||
errors.push(GrammarError {
|
{
|
||||||
message: format!("Undefined non-terminal '{nt}' in rule"),
|
errors.push(GrammarError {
|
||||||
error_type: GrammarErrorType::UndefinedSymbol,
|
message: format!("Undefined non-terminal '{nt}' in rule"),
|
||||||
location: Some(rule.left_hand_side.clone()),
|
error_type: GrammarErrorType::UndefinedSymbol,
|
||||||
});
|
location: Some(rule.left_hand_side.clone()),
|
||||||
}
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,10 +303,11 @@ impl ContextFreeGrammar {
|
|||||||
for rule in self.get_rules(&symbol) {
|
for rule in self.get_rules(&symbol) {
|
||||||
for rhs_symbol in &rule.right_hand_side {
|
for rhs_symbol in &rule.right_hand_side {
|
||||||
if let GrammarSymbol::NonTerminal(nt) = rhs_symbol
|
if let GrammarSymbol::NonTerminal(nt) = rhs_symbol
|
||||||
&& !reachable.contains(nt) {
|
&& !reachable.contains(nt)
|
||||||
reachable.insert(nt.clone());
|
{
|
||||||
queue.push_back(nt.clone());
|
reachable.insert(nt.clone());
|
||||||
}
|
queue.push_back(nt.clone());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -424,9 +430,10 @@ impl GrammarGuidedSampler {
|
|||||||
// Filter token probabilities by expected terminals
|
// Filter token probabilities by expected terminals
|
||||||
for (token, prob) in token_probabilities {
|
for (token, prob) in token_probabilities {
|
||||||
if expected_terminals.contains(token)
|
if expected_terminals.contains(token)
|
||||||
&& let Some(new_state) = self.advance_state(state, token)? {
|
&& let Some(new_state) = self.advance_state(state, token)?
|
||||||
candidates.push((token.clone(), *prob, new_state));
|
{
|
||||||
}
|
candidates.push((token.clone(), *prob, new_state));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sort by probability and take top candidates
|
// Sort by probability and take top candidates
|
||||||
|
|||||||
@@ -14,12 +14,7 @@ use dashmap::DashMap;
|
|||||||
use parking_lot::{Mutex, RwLock};
|
use parking_lot::{Mutex, RwLock};
|
||||||
use rand::{Rng, thread_rng};
|
use rand::{Rng, thread_rng};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::{
|
use std::{collections::HashMap, hash::Hash, sync::Arc, time::Duration};
|
||||||
collections::HashMap,
|
|
||||||
hash::Hash,
|
|
||||||
sync::Arc,
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
use tokio::time::{Instant, sleep};
|
use tokio::time::{Instant, sleep};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -457,8 +452,7 @@ impl MultiModelManager {
|
|||||||
|
|
||||||
// Check reservation result
|
// Check reservation result
|
||||||
if let Err(e) = reserve_result {
|
if let Err(e) = reserve_result {
|
||||||
instance.state =
|
instance.state = ModelState::Error(format!("Resource reservation failed: {e}"));
|
||||||
ModelState::Error(format!("Resource reservation failed: {e}"));
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -511,9 +505,10 @@ impl MultiModelManager {
|
|||||||
|
|
||||||
// Check circuit breaker
|
// Check circuit breaker
|
||||||
if let Some(cb) = self.circuit_breakers.get(&instance_id)
|
if let Some(cb) = self.circuit_breakers.get(&instance_id)
|
||||||
&& !cb.can_execute() {
|
&& !cb.can_execute()
|
||||||
return Err(anyhow!("Circuit breaker open for model {instance_id}"));
|
{
|
||||||
}
|
return Err(anyhow!("Circuit breaker open for model {instance_id}"));
|
||||||
|
}
|
||||||
|
|
||||||
// Execute with retries
|
// Execute with retries
|
||||||
let retry_policy = &self.routing_config.read().retry_policy;
|
let retry_policy = &self.routing_config.read().retry_policy;
|
||||||
|
|||||||
@@ -9,9 +9,9 @@
|
|||||||
//! - Custom sampling strategies with pluggable algorithms
|
//! - Custom sampling strategies with pluggable algorithms
|
||||||
|
|
||||||
use anyhow::{Result, anyhow};
|
use anyhow::{Result, anyhow};
|
||||||
|
use rand::SeedableRng;
|
||||||
use rand::distributions::{Distribution, WeightedIndex};
|
use rand::distributions::{Distribution, WeightedIndex};
|
||||||
use rand::rngs::StdRng;
|
use rand::rngs::StdRng;
|
||||||
use rand::SeedableRng;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::{cmp::Ordering, collections::HashMap};
|
use std::{cmp::Ordering, collections::HashMap};
|
||||||
|
|
||||||
|
|||||||
@@ -84,7 +84,9 @@ impl DynamicBatchController {
|
|||||||
pub fn get_optimal_batch_size(&self, stream_id: &str) -> usize {
|
pub fn get_optimal_batch_size(&self, stream_id: &str) -> usize {
|
||||||
self.current_batch_sizes
|
self.current_batch_sizes
|
||||||
.get(stream_id)
|
.get(stream_id)
|
||||||
.map_or(self.config.initial_batch_size, |state| state.current_size.load(Ordering::Relaxed))
|
.map_or(self.config.initial_batch_size, |state| {
|
||||||
|
state.current_size.load(Ordering::Relaxed)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update performance metrics
|
/// Update performance metrics
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ pub mod state_handle;
|
|||||||
// External Imports
|
// External Imports
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
|
use crate::StreamingResult;
|
||||||
use crate::types_final::FailureDetector;
|
use crate::types_final::FailureDetector;
|
||||||
use crate::types_processing::StateCoordinator;
|
use crate::types_processing::StateCoordinator;
|
||||||
use crate::StreamingResult;
|
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Re-exports from config module
|
// Re-exports from config module
|
||||||
|
|||||||
@@ -50,13 +50,14 @@ pub use tokenizer::*;
|
|||||||
pub use webgpu::*;
|
pub use webgpu::*;
|
||||||
|
|
||||||
// Re-export core tensor types (no_std compatible)
|
// Re-export core tensor types (no_std compatible)
|
||||||
pub use tensor_core::{DType, TensorCore, TensorError};
|
|
||||||
pub use tensor_core::simd::{SIMD_AVAILABLE, SIMD_WIDTH};
|
|
||||||
pub use tensor_core::backend::{
|
pub use tensor_core::backend::{
|
||||||
BackendLimits, BackendPreference, BackendType, UnifiedBackend,
|
BackendLimits, BackendPreference, BackendType, UnifiedBackend, available_backends,
|
||||||
available_backends,
|
|
||||||
};
|
};
|
||||||
pub use tensor_core::wasm_api::{WasmTensor, WasmBackend, get_backend_info, is_simd_available, get_simd_width};
|
pub use tensor_core::simd::{SIMD_AVAILABLE, SIMD_WIDTH};
|
||||||
|
pub use tensor_core::wasm_api::{
|
||||||
|
WasmBackend, WasmTensor, get_backend_info, get_simd_width, is_simd_available,
|
||||||
|
};
|
||||||
|
pub use tensor_core::{DType, TensorCore, TensorError};
|
||||||
|
|
||||||
/// Initialize panic hook for better error messages in browser console
|
/// Initialize panic hook for better error messages in browser console
|
||||||
#[wasm_bindgen(start)]
|
#[wasm_bindgen(start)]
|
||||||
@@ -556,7 +557,10 @@ impl WasmInferenceEngine {
|
|||||||
/// Get memory usage estimate
|
/// Get memory usage estimate
|
||||||
#[wasm_bindgen]
|
#[wasm_bindgen]
|
||||||
pub fn memory_usage(&self) -> usize {
|
pub fn memory_usage(&self) -> usize {
|
||||||
let model_mem = self.model.as_ref().map_or(0, model::WasmModel::memory_usage);
|
let model_mem = self
|
||||||
|
.model
|
||||||
|
.as_ref()
|
||||||
|
.map_or(0, model::WasmModel::memory_usage);
|
||||||
let cache_mem = self
|
let cache_mem = self
|
||||||
.kv_cache
|
.kv_cache
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
|||||||
@@ -344,11 +344,7 @@ impl UnifiedBackend {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Tensor matrix multiplication
|
/// Tensor matrix multiplication
|
||||||
pub fn tensor_matmul(
|
pub fn tensor_matmul(&self, a: &TensorCore, b: &TensorCore) -> Result<TensorCore, TensorError> {
|
||||||
&self,
|
|
||||||
a: &TensorCore,
|
|
||||||
b: &TensorCore,
|
|
||||||
) -> Result<TensorCore, TensorError> {
|
|
||||||
a.matmul(b)
|
a.matmul(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -439,7 +435,10 @@ mod tests {
|
|||||||
fn test_backend_creation() {
|
fn test_backend_creation() {
|
||||||
let backend = UnifiedBackend::new();
|
let backend = UnifiedBackend::new();
|
||||||
// Should always succeed with CPU fallback
|
// Should always succeed with CPU fallback
|
||||||
assert!(backend.backend_type() == BackendType::Cpu || backend.backend_type() == BackendType::WebGpu);
|
assert!(
|
||||||
|
backend.backend_type() == BackendType::Cpu
|
||||||
|
|| backend.backend_type() == BackendType::WebGpu
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -262,7 +262,10 @@ impl WasmTensor {
|
|||||||
/// Matrix multiplication (2D tensors only)
|
/// Matrix multiplication (2D tensors only)
|
||||||
#[wasm_bindgen]
|
#[wasm_bindgen]
|
||||||
pub fn matmul(&self, other: &WasmTensor) -> Result<WasmTensor, JsError> {
|
pub fn matmul(&self, other: &WasmTensor) -> Result<WasmTensor, JsError> {
|
||||||
let inner = self.inner.matmul(&other.inner).map_err(tensor_error_to_js)?;
|
let inner = self
|
||||||
|
.inner
|
||||||
|
.matmul(&other.inner)
|
||||||
|
.map_err(tensor_error_to_js)?;
|
||||||
Ok(WasmTensor { inner })
|
Ok(WasmTensor { inner })
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -350,21 +353,30 @@ impl WasmBackend {
|
|||||||
/// Tensor addition using this backend
|
/// Tensor addition using this backend
|
||||||
#[wasm_bindgen(js_name = tensorAdd)]
|
#[wasm_bindgen(js_name = tensorAdd)]
|
||||||
pub fn tensor_add(&self, a: &WasmTensor, b: &WasmTensor) -> Result<WasmTensor, JsError> {
|
pub fn tensor_add(&self, a: &WasmTensor, b: &WasmTensor) -> Result<WasmTensor, JsError> {
|
||||||
let inner = self.inner.tensor_add(&a.inner, &b.inner).map_err(tensor_error_to_js)?;
|
let inner = self
|
||||||
|
.inner
|
||||||
|
.tensor_add(&a.inner, &b.inner)
|
||||||
|
.map_err(tensor_error_to_js)?;
|
||||||
Ok(WasmTensor { inner })
|
Ok(WasmTensor { inner })
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tensor multiplication using this backend
|
/// Tensor multiplication using this backend
|
||||||
#[wasm_bindgen(js_name = tensorMul)]
|
#[wasm_bindgen(js_name = tensorMul)]
|
||||||
pub fn tensor_mul(&self, a: &WasmTensor, b: &WasmTensor) -> Result<WasmTensor, JsError> {
|
pub fn tensor_mul(&self, a: &WasmTensor, b: &WasmTensor) -> Result<WasmTensor, JsError> {
|
||||||
let inner = self.inner.tensor_mul(&a.inner, &b.inner).map_err(tensor_error_to_js)?;
|
let inner = self
|
||||||
|
.inner
|
||||||
|
.tensor_mul(&a.inner, &b.inner)
|
||||||
|
.map_err(tensor_error_to_js)?;
|
||||||
Ok(WasmTensor { inner })
|
Ok(WasmTensor { inner })
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tensor matmul using this backend
|
/// Tensor matmul using this backend
|
||||||
#[wasm_bindgen(js_name = tensorMatmul)]
|
#[wasm_bindgen(js_name = tensorMatmul)]
|
||||||
pub fn tensor_matmul(&self, a: &WasmTensor, b: &WasmTensor) -> Result<WasmTensor, JsError> {
|
pub fn tensor_matmul(&self, a: &WasmTensor, b: &WasmTensor) -> Result<WasmTensor, JsError> {
|
||||||
let inner = self.inner.tensor_matmul(&a.inner, &b.inner).map_err(tensor_error_to_js)?;
|
let inner = self
|
||||||
|
.inner
|
||||||
|
.tensor_matmul(&a.inner, &b.inner)
|
||||||
|
.map_err(tensor_error_to_js)?;
|
||||||
Ok(WasmTensor { inner })
|
Ok(WasmTensor { inner })
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -380,7 +392,9 @@ impl WasmBackend {
|
|||||||
#[wasm_bindgen(js_name = tensorLayerNorm)]
|
#[wasm_bindgen(js_name = tensorLayerNorm)]
|
||||||
pub fn tensor_layer_norm(&self, input: &WasmTensor, eps: Option<f32>) -> WasmTensor {
|
pub fn tensor_layer_norm(&self, input: &WasmTensor, eps: Option<f32>) -> WasmTensor {
|
||||||
WasmTensor {
|
WasmTensor {
|
||||||
inner: self.inner.tensor_layer_norm(&input.inner, eps.unwrap_or(1e-5)),
|
inner: self
|
||||||
|
.inner
|
||||||
|
.tensor_layer_norm(&input.inner, eps.unwrap_or(1e-5)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -284,7 +284,9 @@ impl AdvancedDofNumbering {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|&&neighbor| !visited.contains(&neighbor))
|
.filter(|&&neighbor| !visited.contains(&neighbor))
|
||||||
.map(|&neighbor| {
|
.map(|&neighbor| {
|
||||||
let degree = adjacency.get(&neighbor).map_or(0, std::collections::HashSet::len);
|
let degree = adjacency
|
||||||
|
.get(&neighbor)
|
||||||
|
.map_or(0, std::collections::HashSet::len);
|
||||||
(neighbor, degree)
|
(neighbor, degree)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|||||||
@@ -585,7 +585,10 @@ impl std::ops::Add<&SparseMatrix> for &SparseMatrix {
|
|||||||
type Output = SparseMatrix;
|
type Output = SparseMatrix;
|
||||||
|
|
||||||
fn add(self, rhs: &SparseMatrix) -> Self::Output {
|
fn add(self, rhs: &SparseMatrix) -> Self::Output {
|
||||||
assert!(!(self.nrows != rhs.nrows || self.ncols != rhs.ncols), "Matrix dimensions must match for addition");
|
assert!(
|
||||||
|
!(self.nrows != rhs.nrows || self.ncols != rhs.ncols),
|
||||||
|
"Matrix dimensions must match for addition"
|
||||||
|
);
|
||||||
|
|
||||||
let mut result = SparseMatrix::new(self.nrows, self.ncols);
|
let mut result = SparseMatrix::new(self.nrows, self.ncols);
|
||||||
|
|
||||||
@@ -938,7 +941,10 @@ impl std::ops::Add<&Self> for SparseMatrix {
|
|||||||
type Output = Self;
|
type Output = Self;
|
||||||
|
|
||||||
fn add(self, rhs: &Self) -> Self::Output {
|
fn add(self, rhs: &Self) -> Self::Output {
|
||||||
assert!(!(self.nrows != rhs.nrows || self.ncols != rhs.ncols), "Matrix dimensions must match for addition");
|
assert!(
|
||||||
|
!(self.nrows != rhs.nrows || self.ncols != rhs.ncols),
|
||||||
|
"Matrix dimensions must match for addition"
|
||||||
|
);
|
||||||
|
|
||||||
let mut result = Self::new(self.nrows, self.ncols);
|
let mut result = Self::new(self.nrows, self.ncols);
|
||||||
|
|
||||||
|
|||||||
@@ -402,7 +402,10 @@ impl JacobianQuality {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let determinants: Vec<f64> = jacobians.iter().map(|j| j.determinant).collect();
|
let determinants: Vec<f64> = jacobians.iter().map(|j| j.determinant).collect();
|
||||||
let conditions: Vec<f64> = jacobians.iter().map(JacobianEval::condition_number).collect();
|
let conditions: Vec<f64> = jacobians
|
||||||
|
.iter()
|
||||||
|
.map(JacobianEval::condition_number)
|
||||||
|
.collect();
|
||||||
|
|
||||||
let min_determinant = determinants.iter().fold(f64::INFINITY, |a, &b| a.min(b));
|
let min_determinant = determinants.iter().fold(f64::INFINITY, |a, &b| a.min(b));
|
||||||
let max_determinant = determinants
|
let max_determinant = determinants
|
||||||
|
|||||||
@@ -59,7 +59,6 @@
|
|||||||
//! - `cantilever_beam`: Classical beam bending problem
|
//! - `cantilever_beam`: Classical beam bending problem
|
||||||
//! - Advanced examples with nonlinear materials and contact
|
//! - Advanced examples with nonlinear materials and contact
|
||||||
|
|
||||||
|
|
||||||
pub mod analysis;
|
pub mod analysis;
|
||||||
pub mod assembly;
|
pub mod assembly;
|
||||||
pub mod boundary;
|
pub mod boundary;
|
||||||
|
|||||||
@@ -585,7 +585,7 @@ impl GpuMaterialOps {
|
|||||||
|
|
||||||
/// Utility functions for material calculations.
|
/// Utility functions for material calculations.
|
||||||
pub mod utils {
|
pub mod utils {
|
||||||
use super::{Vector6, Vector3};
|
use super::{Vector3, Vector6};
|
||||||
|
|
||||||
/// Convert engineering strain to Voigt notation.
|
/// Convert engineering strain to Voigt notation.
|
||||||
pub fn engineering_to_voigt(engineering_strain: &Vector6<f64>) -> Vector6<f64> {
|
pub fn engineering_to_voigt(engineering_strain: &Vector6<f64>) -> Vector6<f64> {
|
||||||
|
|||||||
@@ -186,7 +186,10 @@ impl PartitioningStats {
|
|||||||
}
|
}
|
||||||
let total_nodes = all_nodes.len();
|
let total_nodes = all_nodes.len();
|
||||||
|
|
||||||
let element_counts: Vec<usize> = partitions.iter().map(MeshPartition::element_count).collect();
|
let element_counts: Vec<usize> = partitions
|
||||||
|
.iter()
|
||||||
|
.map(MeshPartition::element_count)
|
||||||
|
.collect();
|
||||||
let max_elements = element_counts.iter().max().copied().unwrap_or(0);
|
let max_elements = element_counts.iter().max().copied().unwrap_or(0);
|
||||||
let min_elements = element_counts.iter().min().copied().unwrap_or(0);
|
let min_elements = element_counts.iter().min().copied().unwrap_or(0);
|
||||||
let avg_elements = if num_partitions > 0 {
|
let avg_elements = if num_partitions > 0 {
|
||||||
@@ -203,7 +206,10 @@ impl PartitioningStats {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Count total interface nodes
|
// Count total interface nodes
|
||||||
let total_interface_nodes: usize = partitions.iter().map(MeshPartition::boundary_node_count).sum();
|
let total_interface_nodes: usize = partitions
|
||||||
|
.iter()
|
||||||
|
.map(MeshPartition::boundary_node_count)
|
||||||
|
.sum();
|
||||||
|
|
||||||
// Estimate communication volume
|
// Estimate communication volume
|
||||||
let communication_volume: usize = partitions
|
let communication_volume: usize = partitions
|
||||||
|
|||||||
@@ -550,9 +550,8 @@ impl MeshRefinement {
|
|||||||
let key = if ni < nj { (ni, nj) } else { (nj, ni) };
|
let key = if ni < nj { (ni, nj) } else { (nj, ni) };
|
||||||
|
|
||||||
*edge_midpoints.entry(key).or_insert_with(|| {
|
*edge_midpoints.entry(key).or_insert_with(|| {
|
||||||
let mid_coords = (&mesh.nodes[&ni].coordinates
|
let mid_coords =
|
||||||
+ &mesh.nodes[&nj].coordinates)
|
(&mesh.nodes[&ni].coordinates + &mesh.nodes[&nj].coordinates) * 0.5;
|
||||||
* 0.5;
|
|
||||||
let mid_node = Node {
|
let mid_node = Node {
|
||||||
coordinates: mid_coords,
|
coordinates: mid_coords,
|
||||||
dofs: mesh.nodes[&ni].dofs.clone(),
|
dofs: mesh.nodes[&ni].dofs.clone(),
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user