Initial commit

This commit is contained in:
redclawsystems
2026-03-04 00:08:42 +00:00
commit 4d88dc0584
4449 changed files with 1556714 additions and 0 deletions
@@ -0,0 +1,293 @@
//! ResNet backbone and related building blocks for R-CNN models.
use crate::{VisionError, VisionResult};
use rtx_tensor::{Device, Tensor};
use tracing::{debug, info};
use super::config::BackboneType;
/// ResNet backbone for R-CNN models
pub struct ResNetBackbone {
backbone_type: BackboneType,
stem: ConvStem,
layer1: ResLayer,
layer2: ResLayer,
layer3: ResLayer,
layer4: ResLayer,
out_channels: Vec<usize>,
}
impl ResNetBackbone {
pub fn new(backbone_type: BackboneType) -> VisionResult<Self> {
let (depths, channels) = match backbone_type {
BackboneType::ResNet50 => (vec![3, 4, 6, 3], vec![64, 128, 256, 512]),
BackboneType::ResNet101 => (vec![3, 4, 23, 3], vec![64, 128, 256, 512]),
BackboneType::ResNeXt50 => (vec![3, 4, 6, 3], vec![128, 256, 512, 1024]),
BackboneType::ResNeXt101 => (vec![3, 4, 23, 3], vec![128, 256, 512, 1024]),
BackboneType::EfficientNet => (vec![2, 2, 2, 3], vec![32, 64, 128, 256]),
};
let stem = ConvStem::new(3, 64)?;
let layer1 = ResLayer::new(64, channels[0], depths[0], 1)?;
let layer2 = ResLayer::new(channels[0] * 4, channels[1], depths[1], 2)?;
let layer3 = ResLayer::new(channels[1] * 4, channels[2], depths[2], 2)?;
let layer4 = ResLayer::new(channels[2] * 4, channels[3], depths[3], 2)?;
let out_channels = channels.iter().map(|&c| c * 4).collect();
info!("Created ResNet backbone: {:?}", backbone_type);
Ok(Self {
backbone_type,
stem,
layer1,
layer2,
layer3,
layer4,
out_channels,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Vec<Tensor>> {
let mut features = Vec::new();
// Stem
let x = self.stem.forward(x)?;
// ResNet layers
let c1 = self.layer1.forward(&x)?;
features.push(c1.clone());
let c2 = self.layer2.forward(&c1)?;
features.push(c2.clone());
let c3 = self.layer3.forward(&c2)?;
features.push(c3.clone());
let c4 = self.layer4.forward(&c3)?;
features.push(c4);
debug!("ResNet backbone produced {} feature maps", features.len());
Ok(features)
}
pub fn out_channels(&self) -> &[usize] {
&self.out_channels
}
}
/// Convolutional stem
pub struct ConvStem {
conv: Tensor,
bn_weight: Tensor,
bn_bias: Tensor,
maxpool: MaxPoolLayer,
}
impl ConvStem {
pub fn new(in_channels: usize, out_channels: usize) -> VisionResult<Self> {
let conv_weight = Tensor::randn(&[out_channels, in_channels, 7, 7], &Device::default())?;
let bn_weight = Tensor::ones([out_channels], &Device::default())?;
let bn_bias = Tensor::zeros([out_channels], &Device::default())?;
let maxpool = MaxPoolLayer::new(3, 2, 1)?;
Ok(Self {
conv: conv_weight,
bn_weight,
bn_bias,
maxpool,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
let conv_out = x.conv2d(&self.conv, None, 3, 2, 1, 1)?;
let norm_out = conv_out
.mul(&self.bn_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
.add(&self.bn_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
let activated = norm_out.relu()?;
let pooled = self.maxpool.forward(&activated)?;
Ok(pooled)
}
}
/// ResNet layer containing multiple residual blocks
pub struct ResLayer {
blocks: Vec<ResidualBlock>,
}
impl ResLayer {
pub fn new(
in_channels: usize,
out_channels: usize,
num_blocks: usize,
stride: usize,
) -> VisionResult<Self> {
let mut blocks = Vec::new();
// First block with potential downsampling
blocks.push(ResidualBlock::new(in_channels, out_channels, stride, true)?);
// Remaining blocks
for _ in 1..num_blocks {
blocks.push(ResidualBlock::new(
out_channels * 4,
out_channels,
1,
false,
)?);
}
Ok(Self { blocks })
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
let mut out = x.clone();
for block in &self.blocks {
out = block.forward(&out)?;
}
Ok(out)
}
}
/// Residual block
pub struct ResidualBlock {
conv1: Tensor,
bn1_weight: Tensor,
bn1_bias: Tensor,
conv2: Tensor,
bn2_weight: Tensor,
bn2_bias: Tensor,
conv3: Tensor,
bn3_weight: Tensor,
bn3_bias: Tensor,
downsample: Option<DownsampleLayer>,
stride: usize,
}
impl ResidualBlock {
pub fn new(
in_channels: usize,
out_channels: usize,
stride: usize,
downsample: bool,
) -> VisionResult<Self> {
let conv1 = Tensor::randn(&[out_channels, in_channels, 1, 1], &Device::default())?;
let bn1_weight = Tensor::ones([out_channels], &Device::default())?;
let bn1_bias = Tensor::zeros([out_channels], &Device::default())?;
let conv2 = Tensor::randn(&[out_channels, out_channels, 3, 3], &Device::default())?;
let bn2_weight = Tensor::ones([out_channels], &Device::default())?;
let bn2_bias = Tensor::zeros([out_channels], &Device::default())?;
let conv3 = Tensor::randn(&[out_channels * 4, out_channels, 1, 1], &Device::default())?;
let bn3_weight = Tensor::ones([out_channels * 4], &Device::default())?;
let bn3_bias = Tensor::zeros([out_channels * 4], &Device::default())?;
let downsample_layer = if downsample {
Some(DownsampleLayer::new(in_channels, out_channels * 4, stride)?)
} else {
None
};
Ok(Self {
conv1,
bn1_weight,
bn1_bias,
conv2,
bn2_weight,
bn2_bias,
conv3,
bn3_weight,
bn3_bias,
downsample: downsample_layer,
stride,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
let identity = if let Some(downsample) = &self.downsample {
downsample.forward(x)?
} else {
x.clone()
};
// First conv + bn + relu
let out = x.conv2d(&self.conv1, None, 0, 1, 1, 1)?;
let out = out
.mul(&self.bn1_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
.add(&self.bn1_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
let out = out.relu()?;
// Second conv + bn + relu
let out = out.conv2d(&self.conv2, None, 1, self.stride, 1, 1)?;
let out = out
.mul(&self.bn2_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
.add(&self.bn2_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
let out = out.relu()?;
// Third conv + bn
let out = out.conv2d(&self.conv3, None, 0, 1, 1, 1)?;
let out = out
.mul(&self.bn3_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
.add(&self.bn3_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
// Add residual connection and apply ReLU
let out = out.add(&identity)?;
let out = out.relu()?;
Ok(out)
}
}
/// Downsample layer for residual connections
pub struct DownsampleLayer {
conv: Tensor,
bn_weight: Tensor,
bn_bias: Tensor,
stride: usize,
}
impl DownsampleLayer {
pub fn new(in_channels: usize, out_channels: usize, stride: usize) -> VisionResult<Self> {
let conv = Tensor::randn(&[out_channels, in_channels, 1, 1], &Device::default())?;
let bn_weight = Tensor::ones([out_channels], &Device::default())?;
let bn_bias = Tensor::zeros([out_channels], &Device::default())?;
Ok(Self {
conv,
bn_weight,
bn_bias,
stride,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
let conv_out = x.conv2d(&self.conv, None, 0, self.stride, 1, 1)?;
let norm_out = conv_out
.mul(&self.bn_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
.add(&self.bn_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
Ok(norm_out)
}
}
/// Max pooling layer
pub struct MaxPoolLayer {
kernel_size: usize,
stride: usize,
padding: usize,
}
impl MaxPoolLayer {
pub fn new(kernel_size: usize, stride: usize, padding: usize) -> VisionResult<Self> {
Ok(Self {
kernel_size,
stride,
padding,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
x.max_pool2d(self.kernel_size, self.stride, self.padding)
.map_err(|e| VisionError::tensor_error_with_source("Max pooling failed", e))
}
}
@@ -0,0 +1,129 @@
//! R-CNN configuration types and variants.
/// R-CNN model variants
#[derive(Debug, Clone, Copy)]
pub enum RcnnVariant {
/// Fast R-CNN with external proposals
FastRcnn,
/// Faster R-CNN with integrated RPN
FasterRcnn,
/// Mask R-CNN with instance segmentation
MaskRcnn,
/// RetinaNet with focal loss
RetinaNet,
/// FCOS (Fully Convolutional One-Stage)
Fcos,
}
impl RcnnVariant {
/// Get model configuration
pub fn config(&self) -> RcnnConfig {
match self {
Self::FastRcnn => RcnnConfig {
backbone_type: BackboneType::ResNet50,
neck_type: Some(NeckType::FPN),
rpn_enabled: false,
mask_head_enabled: false,
roi_head_type: RoiHeadType::StandardRoiHead,
anchor_scales: vec![8.0, 16.0, 32.0],
anchor_ratios: vec![0.5, 1.0, 2.0],
roi_pool_size: 7,
mask_pool_size: 14,
nms_threshold: 0.7,
score_threshold: 0.05,
},
Self::FasterRcnn => RcnnConfig {
backbone_type: BackboneType::ResNet50,
neck_type: Some(NeckType::FPN),
rpn_enabled: true,
mask_head_enabled: false,
roi_head_type: RoiHeadType::StandardRoiHead,
anchor_scales: vec![8.0, 16.0, 32.0],
anchor_ratios: vec![0.5, 1.0, 2.0],
roi_pool_size: 7,
mask_pool_size: 14,
nms_threshold: 0.7,
score_threshold: 0.05,
},
Self::MaskRcnn => RcnnConfig {
backbone_type: BackboneType::ResNet50,
neck_type: Some(NeckType::FPN),
rpn_enabled: true,
mask_head_enabled: true,
roi_head_type: RoiHeadType::StandardRoiHead,
anchor_scales: vec![8.0, 16.0, 32.0],
anchor_ratios: vec![0.5, 1.0, 2.0],
roi_pool_size: 7,
mask_pool_size: 14,
nms_threshold: 0.7,
score_threshold: 0.05,
},
Self::RetinaNet => RcnnConfig {
backbone_type: BackboneType::ResNet50,
neck_type: Some(NeckType::FPN),
rpn_enabled: false,
mask_head_enabled: false,
roi_head_type: RoiHeadType::RetinaHead,
anchor_scales: vec![4.0, 5.04, 6.35],
anchor_ratios: vec![0.5, 1.0, 2.0],
roi_pool_size: 7,
mask_pool_size: 14,
nms_threshold: 0.5,
score_threshold: 0.05,
},
Self::Fcos => RcnnConfig {
backbone_type: BackboneType::ResNet50,
neck_type: Some(NeckType::FPN),
rpn_enabled: false,
mask_head_enabled: false,
roi_head_type: RoiHeadType::FcosHead,
anchor_scales: vec![],
anchor_ratios: vec![],
roi_pool_size: 7,
mask_pool_size: 14,
nms_threshold: 0.6,
score_threshold: 0.05,
},
}
}
}
/// R-CNN configuration
#[derive(Debug, Clone)]
pub struct RcnnConfig {
pub backbone_type: BackboneType,
pub neck_type: Option<NeckType>,
pub rpn_enabled: bool,
pub mask_head_enabled: bool,
pub roi_head_type: RoiHeadType,
pub anchor_scales: Vec<f32>,
pub anchor_ratios: Vec<f32>,
pub roi_pool_size: usize,
pub mask_pool_size: usize,
pub nms_threshold: f32,
pub score_threshold: f32,
}
#[derive(Debug, Clone)]
pub enum BackboneType {
ResNet50,
ResNet101,
ResNeXt50,
ResNeXt101,
EfficientNet,
}
#[derive(Debug, Clone)]
pub enum NeckType {
FPN,
PAFPN,
BiFPN,
}
#[derive(Debug, Clone)]
pub enum RoiHeadType {
StandardRoiHead,
RetinaHead,
FcosHead,
CascadeRoiHead,
}
@@ -0,0 +1,393 @@
//! Complete R-CNN detector implementation.
use crate::detection::{Detector, DetectorStats};
use crate::utils::{nms, preprocessing};
use crate::{BoundingBox, DetectionResult, VisionConfig, VisionError, VisionResult};
use rtx_tensor::Tensor;
use tracing::{debug, info};
use super::backbone::ResNetBackbone;
use super::config::{NeckType, RcnnConfig, RcnnVariant};
use super::fpn::FeaturePyramidNetwork;
use super::roi_head::RoiHead;
use super::rpn::RegionProposalNetwork;
/// Complete R-CNN detector implementation
pub struct RcnnDetector {
variant: RcnnVariant,
backbone: ResNetBackbone,
neck: Option<FeaturePyramidNetwork>,
rpn: Option<RegionProposalNetwork>,
roi_head: RoiHead,
config: RcnnConfig,
input_size: (usize, usize),
num_classes: usize,
}
impl RcnnDetector {
pub fn new(
variant: RcnnVariant,
weights_path: Option<&str>,
vision_config: &VisionConfig,
) -> VisionResult<Self> {
info!("Initializing R-CNN detector: {:?}", variant);
let config = variant.config();
let input_size = vision_config.input_size;
let num_classes = vision_config.num_classes;
let backbone = ResNetBackbone::new(config.backbone_type.clone())?;
let neck = if let Some(neck_type) = &config.neck_type {
match neck_type {
NeckType::FPN => Some(FeaturePyramidNetwork::new(backbone.out_channels(), 256)?),
_ => None, // Other neck types not implemented yet
}
} else {
None
};
let rpn = if config.rpn_enabled {
let num_anchors = config.anchor_scales.len() * config.anchor_ratios.len();
Some(RegionProposalNetwork::new(256, num_anchors)?)
} else {
None
};
let roi_head = RoiHead::new(
256,
num_classes,
config.roi_pool_size,
config.mask_head_enabled,
config.mask_pool_size,
)?;
let mut detector = Self {
variant,
backbone,
neck,
rpn,
roi_head,
config,
input_size,
num_classes,
};
// Load pre-trained weights if provided
if let Some(path) = weights_path {
detector.load_weights(path)?;
}
info!("R-CNN detector initialized successfully");
Ok(detector)
}
pub fn load_weights(&mut self, path: &str) -> VisionResult<()> {
info!("Loading R-CNN weights from: {}", path);
// Load and apply weights (simplified)
let _weights = std::fs::read(path)
.map_err(|e| VisionError::model_load_error(format!("Failed to read weights: {e}")))?;
// Apply weights to model components
// Implementation would depend on weight format
info!("Successfully loaded R-CNN weights");
Ok(())
}
fn generate_proposals_from_rpn(
&self,
objectness_logits: &[Tensor],
bbox_deltas: &[Tensor],
anchors: &[Tensor],
) -> VisionResult<Vec<BoundingBox>> {
let mut proposals = Vec::new();
// Process each feature level
for ((obj_logits, bbox_delta), anchor_tensor) in objectness_logits
.iter()
.zip(bbox_deltas.iter())
.zip(anchors.iter())
{
// Convert tensors to proposals
let obj_scores = obj_logits.sigmoid()?.to_vec()?;
let deltas = bbox_delta.to_vec()?;
let anchor_coords = anchor_tensor.to_vec()?;
let num_anchors = obj_scores.len();
for i in 0..num_anchors {
let score = obj_scores[i];
if score > 0.5 {
// Objectness threshold
let delta_idx = i * 4;
let anchor_idx = i * 4;
if delta_idx + 3 < deltas.len() && anchor_idx + 3 < anchor_coords.len() {
// Apply bbox regression
let dx = deltas[delta_idx];
let dy = deltas[delta_idx + 1];
let dw = deltas[delta_idx + 2];
let dh = deltas[delta_idx + 3];
let anchor_x = anchor_coords[anchor_idx];
let anchor_y = anchor_coords[anchor_idx + 1];
let anchor_w = anchor_coords[anchor_idx + 2] - anchor_coords[anchor_idx];
let anchor_h =
anchor_coords[anchor_idx + 3] - anchor_coords[anchor_idx + 1];
let pred_x = anchor_x + dx * anchor_w;
let pred_y = anchor_y + dy * anchor_h;
let pred_w = anchor_w * dw.exp();
let pred_h = anchor_h * dh.exp();
proposals.push(BoundingBox::new(
pred_x - pred_w / 2.0,
pred_y - pred_h / 2.0,
pred_w,
pred_h,
score,
0, // RPN doesn't predict class
));
}
}
}
}
// Filter and sort proposals
if let Some(rpn) = &self.rpn {
let filtered_proposals =
rpn.filter_proposals(proposals, &objectness_logits[0], self.input_size, false)?;
Ok(filtered_proposals)
} else {
Ok(proposals)
}
}
fn generate_sliding_window_proposals(&self) -> VisionResult<Vec<BoundingBox>> {
// Generate sliding window proposals for Fast R-CNN
let mut proposals = Vec::new();
let (img_h, img_w) = self.input_size;
let scales = vec![0.25, 0.5, 1.0, 2.0];
let ratios: Vec<f32> = vec![0.5, 1.0, 2.0];
let stride = 16;
for y in (0..img_h).step_by(stride) {
for x in (0..img_w).step_by(stride) {
for &scale in &scales {
for &ratio in &ratios {
let w = 32.0f32 * scale * ratio.sqrt();
let h = 32.0f32 * scale / ratio.sqrt();
let x1 = (x as f32 - w / 2.0).max(0.0);
let y1 = (y as f32 - h / 2.0).max(0.0);
let x2 = (x as f32 + w / 2.0).min(img_w as f32);
let y2 = (y as f32 + h / 2.0).min(img_h as f32);
if x2 > x1 && y2 > y1 {
proposals.push(BoundingBox::new(
x1,
y1,
x2 - x1,
y2 - y1,
1.0, // Default confidence for sliding window
0, // No class prediction yet
));
}
}
}
}
}
// Limit number of proposals
proposals.truncate(2000);
Ok(proposals)
}
fn postprocess_detections(
&self,
class_logits: &Tensor,
bbox_deltas: &Tensor,
proposals: &[BoundingBox],
config: &VisionConfig,
) -> VisionResult<Vec<BoundingBox>> {
let num_proposals = proposals.len();
if num_proposals == 0 {
return Ok(Vec::new());
}
// Get class probabilities
let class_probs = class_logits.softmax(-1)?;
let prob_data = class_probs.to_vec()?;
let delta_data = bbox_deltas.to_vec()?;
// Assume shape is [num_proposals, num_classes] for class_probs
// and [num_proposals, num_classes * 4] for bbox_deltas
let num_classes = class_logits.shape()[class_logits.shape().len() - 1];
let mut detections = Vec::new();
for (i, proposal) in proposals.iter().enumerate() {
let prob_start = i * num_classes;
let delta_start_base = i * num_classes * 4;
if prob_start + num_classes > prob_data.len() {
break;
}
// Skip background class (index 0)
for class_id in 1..num_classes {
let confidence = prob_data[prob_start + class_id];
if confidence > config.confidence_threshold {
// Apply bbox regression
let delta_start = delta_start_base + class_id * 4;
if delta_start + 3 < delta_data.len() {
let dx = delta_data[delta_start];
let dy = delta_data[delta_start + 1];
let dw = delta_data[delta_start + 2];
let dh = delta_data[delta_start + 3];
let pred_x = proposal.x + dx * proposal.width;
let pred_y = proposal.y + dy * proposal.height;
let pred_w = proposal.width * dw.exp();
let pred_h = proposal.height * dh.exp();
detections.push(BoundingBox::new(
pred_x,
pred_y,
pred_w,
pred_h,
confidence,
class_id - 1, // Adjust for background class
));
}
}
}
}
// Apply NMS
let final_detections = nms::apply_nms_per_class(detections, config.nms_threshold)?;
Ok(final_detections)
}
}
impl Detector for RcnnDetector {
fn detect(
&mut self,
image: &Tensor,
vision_config: &VisionConfig,
) -> VisionResult<DetectionResult> {
let start_time = std::time::Instant::now();
// Preprocess image
let processed = preprocessing::resize_tensor(image, self.input_size, true)?;
let normalized = preprocessing::imagenet_normalize(&processed)?;
let batched = normalized.unsqueeze(0)?;
// Backbone forward pass
let backbone_features = self.backbone.forward(&batched)?;
// Neck forward pass (if present)
let features = if let Some(neck) = &self.neck {
neck.forward(&backbone_features)?
} else {
backbone_features
};
// RPN forward pass (if present)
let proposals = if let Some(rpn) = &self.rpn {
let image_sizes = vec![self.input_size];
let (objectness_logits, bbox_deltas, anchors) = rpn.forward(&features, &image_sizes)?;
// Generate proposals from RPN predictions
self.generate_proposals_from_rpn(&objectness_logits, &bbox_deltas, &anchors)?
} else {
// Use external proposals or sliding window
self.generate_sliding_window_proposals()?
};
// RoI head forward pass
let (class_logits, bbox_deltas, _masks) = self.roi_head.forward(&features, &proposals)?;
// Post-process predictions
let detections =
self.postprocess_detections(&class_logits, &bbox_deltas, &proposals, vision_config)?;
let processing_time = start_time.elapsed().as_millis() as f32;
debug!(
"R-CNN detection completed: {} boxes in {:.1}ms",
detections.len(),
processing_time
);
Ok(DetectionResult::new(
detections,
(image.shape()[2], image.shape()[3]),
processing_time,
format!("{:?}", self.variant),
))
}
fn detect_batch(
&mut self,
images: &Tensor,
config: &VisionConfig,
) -> VisionResult<Vec<DetectionResult>> {
let batch_size = images.shape()[0];
let mut results = Vec::with_capacity(batch_size);
// Process each image individually for simplicity
// In practice, you'd implement proper batch processing
for i in 0..batch_size {
let image = images.narrow(0, i, 1)?.squeeze(Some(0))?;
let result = self.detect(&image, config)?;
results.push(result);
}
Ok(results)
}
fn name(&self) -> &str {
match self.variant {
RcnnVariant::FastRcnn => "Fast R-CNN",
RcnnVariant::FasterRcnn => "Faster R-CNN",
RcnnVariant::MaskRcnn => "Mask R-CNN",
RcnnVariant::RetinaNet => "RetinaNet",
RcnnVariant::Fcos => "FCOS",
}
}
fn input_size(&self) -> (usize, usize) {
self.input_size
}
fn num_classes(&self) -> usize {
self.num_classes
}
fn get_stats(&self) -> DetectorStats {
let params = match self.variant {
RcnnVariant::FastRcnn => 25_600_000,
RcnnVariant::FasterRcnn => 41_800_000,
RcnnVariant::MaskRcnn => 44_400_000,
RcnnVariant::RetinaNet => 36_300_000,
RcnnVariant::Fcos => 32_100_000,
};
DetectorStats {
total_params: params,
flops: params as f64 * 5.0, // Rough estimate
model_size_mb: params as f32 * 4.0 / (1024.0 * 1024.0),
avg_inference_ms: 0.0,
peak_memory_mb: 0.0,
features: vec![
"Two-stage detection".to_string(),
"Feature Pyramid Network".to_string(),
"Region Proposal Network".to_string(),
],
}
}
}
@@ -0,0 +1,76 @@
//! Feature Pyramid Network (FPN) for multi-scale feature extraction.
use crate::VisionResult;
use rtx_tensor::Tensor;
use super::layers::ConvBlock;
/// Feature Pyramid Network (FPN)
pub struct FeaturePyramidNetwork {
lateral_convs: Vec<ConvBlock>,
fpn_convs: Vec<ConvBlock>,
out_channels: usize,
}
impl FeaturePyramidNetwork {
pub fn new(in_channels: &[usize], out_channels: usize) -> VisionResult<Self> {
let mut lateral_convs = Vec::new();
let mut fpn_convs = Vec::new();
for &in_ch in in_channels {
lateral_convs.push(ConvBlock::new(in_ch, out_channels, 1, 1, 0)?);
fpn_convs.push(ConvBlock::new(out_channels, out_channels, 3, 1, 1)?);
}
Ok(Self {
lateral_convs,
fpn_convs,
out_channels,
})
}
pub fn forward(&self, features: &[Tensor]) -> VisionResult<Vec<Tensor>> {
let mut fpn_features = Vec::new();
// Top-down pathway
let mut top_down =
self.lateral_convs[features.len() - 1].forward(&features[features.len() - 1])?;
fpn_features.push(self.fpn_convs[features.len() - 1].forward(&top_down)?);
for i in (0..features.len() - 1).rev() {
// Upsample
let shape = features[i].shape();
let upsampled = self.upsample(&top_down, (shape[2], shape[3]))?;
// Lateral connection
let lateral = self.lateral_convs[i].forward(&features[i])?;
// Add
top_down = upsampled.add(&lateral)?;
fpn_features.insert(0, self.fpn_convs[i].forward(&top_down)?);
}
Ok(fpn_features)
}
fn upsample(&self, x: &Tensor, target_size: (usize, usize)) -> VisionResult<Tensor> {
// Simple nearest neighbor upsampling
let shape = x.shape();
let scale_h = target_size.0 as f32 / shape[2] as f32;
let scale_w = target_size.1 as f32 / shape[3] as f32;
// For simplicity, we'll create a basic upsampling implementation
// In production, you'd use proper interpolation
if scale_h.round() as usize == 2 && scale_w.round() as usize == 2 {
let upsampled = x
.unsqueeze(4)?
.unsqueeze(4)?
.repeat(&[1, 1, 1, 1, 2, 2])?
.view([shape[0], shape[1], target_size.0, target_size.1])?;
Ok(upsampled)
} else {
// Fallback to identity for other scales
Ok(x.clone())
}
}
}
@@ -0,0 +1,81 @@
//! Common neural network layers used in R-CNN models.
use crate::{VisionError, VisionResult};
use rtx_tensor::{Device, Tensor};
/// Linear (fully connected) layer
pub struct LinearLayer {
weight: Tensor,
bias: Option<Tensor>,
}
impl LinearLayer {
pub fn new(in_features: usize, out_features: usize) -> VisionResult<Self> {
let weight = Tensor::randn(&[out_features, in_features], &Device::default())?;
let bias = Some(Tensor::zeros([out_features], &Device::default())?);
Ok(Self { weight, bias })
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
let linear_out = x.matmul(&self.weight.transpose(-2, -1)?)?;
if let Some(bias) = &self.bias {
linear_out.add(bias).map_err(|e| {
VisionError::tensor_error_with_source("Linear layer bias addition failed", e)
})
} else {
Ok(linear_out)
}
}
}
/// Conv block used in various parts
pub struct ConvBlock {
conv: Tensor,
bn_weight: Option<Tensor>,
bn_bias: Option<Tensor>,
kernel_size: usize,
stride: usize,
padding: usize,
}
impl ConvBlock {
pub fn new(
in_channels: usize,
out_channels: usize,
kernel_size: usize,
stride: usize,
padding: usize,
) -> VisionResult<Self> {
let conv = Tensor::randn(
&[out_channels, in_channels, kernel_size, kernel_size],
&Device::default(),
)?;
let bn_weight = Some(Tensor::ones([out_channels], &Device::default())?);
let bn_bias = Some(Tensor::zeros([out_channels], &Device::default())?);
Ok(Self {
conv,
bn_weight,
bn_bias,
kernel_size,
stride,
padding,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
let conv_out = x.conv2d(&self.conv, None, self.padding, self.stride, 1, 1)?;
let normalized = if let (Some(weight), Some(bias)) = (&self.bn_weight, &self.bn_bias) {
conv_out
.mul(&weight.unsqueeze(-1)?.unsqueeze(-1)?)?
.add(&bias.unsqueeze(-1)?.unsqueeze(-1)?)?
} else {
conv_out
};
Ok(normalized)
}
}
@@ -0,0 +1,55 @@
//! R-CNN family detection implementation
//!
//! Implements:
//! - Fast R-CNN: Region-based CNN with RoI pooling
//! - Faster R-CNN: End-to-end detection with RPN
//! - Mask R-CNN: Instance segmentation with mask heads
//! - Feature Pyramid Networks (FPN) for multi-scale detection
mod backbone;
mod config;
mod detector;
mod fpn;
mod layers;
mod roi_head;
mod rpn;
pub use backbone::{
ConvStem, DownsampleLayer, MaxPoolLayer, ResLayer, ResNetBackbone, ResidualBlock,
};
pub use config::{BackboneType, NeckType, RcnnConfig, RcnnVariant, RoiHeadType};
pub use detector::RcnnDetector;
pub use fpn::FeaturePyramidNetwork;
pub use layers::{ConvBlock, LinearLayer};
pub use roi_head::{BboxHead, MaskHead, RoiExtractor, RoiHead};
pub use rpn::{AnchorGenerator, AnchorMatcher, RegionProposalNetwork};
#[cfg(test)]
mod tests {
use super::*;
use crate::VisionConfig;
#[test]
fn test_rcnn_config() {
let config = RcnnVariant::FasterRcnn.config();
assert!(config.rpn_enabled);
assert!(!config.mask_head_enabled);
assert_eq!(config.roi_pool_size, 7);
}
#[test]
fn test_anchor_generator() {
let generator = AnchorGenerator::new(vec![8.0, 16.0, 32.0], vec![0.5, 1.0, 2.0]).unwrap();
assert_eq!(generator.base_anchors.len(), 9); // 3 scales * 3 ratios
}
#[tokio::test]
async fn test_rcnn_detector_creation() {
let config = VisionConfig::default();
let detector = RcnnDetector::new(RcnnVariant::FasterRcnn, None, &config);
assert!(detector.is_ok());
let detector = detector.unwrap();
assert_eq!(detector.name(), "Faster R-CNN");
}
}
@@ -0,0 +1,259 @@
//! RoI head components for R-CNN detection.
use crate::{BoundingBox, SegmentationMask, VisionError, VisionResult};
use rtx_tensor::Tensor;
use super::layers::{ConvBlock, LinearLayer};
/// RoI head for final classification and regression
pub struct RoiHead {
bbox_head: BboxHead,
mask_head: Option<MaskHead>,
roi_extractor: RoiExtractor,
}
impl RoiHead {
pub fn new(
in_channels: usize,
num_classes: usize,
roi_pool_size: usize,
enable_mask: bool,
mask_pool_size: usize,
) -> VisionResult<Self> {
let bbox_head = BboxHead::new(in_channels, num_classes, roi_pool_size)?;
let mask_head = if enable_mask {
Some(MaskHead::new(in_channels, num_classes, mask_pool_size)?)
} else {
None
};
let roi_extractor = RoiExtractor::new(roi_pool_size)?;
Ok(Self {
bbox_head,
mask_head,
roi_extractor,
})
}
pub fn forward(
&self,
features: &[Tensor],
proposals: &[BoundingBox],
) -> VisionResult<(Tensor, Tensor, Option<Vec<SegmentationMask>>)> {
// Extract RoI features
let roi_features = self.roi_extractor.forward(features, proposals)?;
// Bbox head forward
let (class_logits, bbox_deltas) = self.bbox_head.forward(&roi_features)?;
// Mask head forward (if enabled)
let masks = if let Some(mask_head) = &self.mask_head {
Some(mask_head.forward(&roi_features, proposals)?)
} else {
None
};
Ok((class_logits, bbox_deltas, masks))
}
}
/// Bounding box head
pub struct BboxHead {
fc1: LinearLayer,
fc2: LinearLayer,
cls_score: LinearLayer,
bbox_pred: LinearLayer,
pool_size: usize,
}
impl BboxHead {
pub fn new(in_channels: usize, num_classes: usize, pool_size: usize) -> VisionResult<Self> {
let flattened_size = in_channels * pool_size * pool_size;
let hidden_size = 1024;
let fc1 = LinearLayer::new(flattened_size, hidden_size)?;
let fc2 = LinearLayer::new(hidden_size, hidden_size)?;
let cls_score = LinearLayer::new(hidden_size, num_classes + 1)?; // +1 for background
let bbox_pred = LinearLayer::new(hidden_size, (num_classes + 1) * 4)?;
Ok(Self {
fc1,
fc2,
cls_score,
bbox_pred,
pool_size,
})
}
pub fn forward(&self, x: &Tensor) -> VisionResult<(Tensor, Tensor)> {
// Flatten
let batch_size = x.shape()[0];
let total_elements = x.numel();
let flattened_size = total_elements / batch_size;
let flattened = x.view([batch_size, flattened_size])?;
// FC layers
let h1 = self.fc1.forward(&flattened)?.relu()?;
let h2 = self.fc2.forward(&h1)?.relu()?;
// Output heads
let cls_score = self.cls_score.forward(&h2)?;
let bbox_pred = self.bbox_pred.forward(&h2)?;
Ok((cls_score, bbox_pred))
}
}
/// Mask head for instance segmentation
pub struct MaskHead {
convs: Vec<ConvBlock>,
mask_fcn_logits: ConvBlock,
num_classes: usize,
pool_size: usize,
}
impl MaskHead {
pub fn new(in_channels: usize, num_classes: usize, pool_size: usize) -> VisionResult<Self> {
let mut convs = Vec::new();
let hidden_channels = 256;
// 4 conv layers
convs.push(ConvBlock::new(in_channels, hidden_channels, 3, 1, 1)?);
for _ in 1..4 {
convs.push(ConvBlock::new(hidden_channels, hidden_channels, 3, 1, 1)?);
}
let mask_fcn_logits = ConvBlock::new(hidden_channels, num_classes, 1, 1, 0)?;
Ok(Self {
convs,
mask_fcn_logits,
num_classes,
pool_size,
})
}
pub fn forward(
&self,
x: &Tensor,
proposals: &[BoundingBox],
) -> VisionResult<Vec<SegmentationMask>> {
let mut mask_features = x.clone();
// Apply conv layers
for conv in &self.convs {
mask_features = conv.forward(&mask_features)?.relu()?;
}
// Final mask prediction
let mask_logits = self.mask_fcn_logits.forward(&mask_features)?;
// Convert to individual masks
let mut masks = Vec::new();
let batch_size = mask_logits.shape()[0];
for i in 0..batch_size.min(proposals.len()) {
let mask_logit = mask_logits.narrow(0, i, 1)?.squeeze(Some(0))?;
// Apply sigmoid to get probabilities
let mask_prob = mask_logit.sigmoid()?;
// Create segmentation mask
let mask =
SegmentationMask::new(mask_prob, proposals[i].class_id, proposals[i].confidence);
masks.push(mask);
}
Ok(masks)
}
}
/// RoI feature extractor
pub struct RoiExtractor {
pool_size: usize,
}
impl RoiExtractor {
pub fn new(pool_size: usize) -> VisionResult<Self> {
Ok(Self { pool_size })
}
pub fn forward(&self, features: &[Tensor], proposals: &[BoundingBox]) -> VisionResult<Tensor> {
if proposals.is_empty() {
return Err(VisionError::invalid_input(
"No proposals provided to RoI extractor",
));
}
let feature_channels = features[0].shape()[1];
let output_shape = vec![
proposals.len(),
feature_channels,
self.pool_size,
self.pool_size,
];
// Simplified RoI pooling implementation
// In practice, you'd implement proper RoI align
let pooled_features =
Tensor::zeros_typed(&output_shape, features[0].dtype(), features[0].device())?;
for proposal in proposals.iter() {
// Find appropriate feature level based on RoI size
let roi_area = proposal.width * proposal.height;
let feature_level = self.select_feature_level(roi_area, features.len());
let feature = &features[feature_level.min(features.len() - 1)];
// Extract and pool RoI
let _roi_feature = self.roi_align(feature, proposal)?;
// Copy to output tensor (simplified)
// In practice, you'd use proper tensor assignment
}
Ok(pooled_features)
}
fn select_feature_level(&self, roi_area: f32, num_levels: usize) -> usize {
// Heuristic: larger RoIs use lower resolution features
let base_area = 224.0f32 * 224.0f32; // Base image size
let level = (roi_area.log2() - base_area.log2()) / 2.0f32;
(level as usize).min(num_levels - 1)
}
fn roi_align(&self, feature: &Tensor, proposal: &BoundingBox) -> VisionResult<Tensor> {
// Simplified RoI align - in practice you'd implement bilinear sampling
let feat_shape = feature.shape();
let feat_h = feat_shape[2] as f32;
let feat_w = feat_shape[3] as f32;
// Map proposal coordinates to feature map coordinates
let x1 = (proposal.x / feat_w * feat_w).max(0.0) as usize;
let y1 = (proposal.y / feat_h * feat_h).max(0.0) as usize;
let x2 = ((proposal.x + proposal.width) / feat_w * feat_w).min(feat_w) as usize;
let y2 = ((proposal.y + proposal.height) / feat_h * feat_h).min(feat_h) as usize;
// Extract region and resize to pool_size
let roi_region = feature
.narrow(2, y1, y2.saturating_sub(y1).max(1))?
.narrow(3, x1, x2.saturating_sub(x1).max(1))?;
// Simple average pooling to target size
self.adaptive_avg_pool(&roi_region, self.pool_size)
}
fn adaptive_avg_pool(&self, x: &Tensor, target_size: usize) -> VisionResult<Tensor> {
let shape = x.shape();
if shape[2] == target_size && shape[3] == target_size {
return Ok(x.clone());
}
// Simplified adaptive pooling - in practice you'd use proper implementation
let stride_h = shape[2] / target_size;
let _stride_w = shape[3] / target_size;
x.avg_pool2d(stride_h.max(1), stride_h.max(1), 0)
.map_err(|e| VisionError::tensor_error_with_source("Adaptive pooling failed", e))
}
}
@@ -0,0 +1,220 @@
//! Region Proposal Network (RPN) and anchor generation.
use crate::utils::nms;
use crate::{BoundingBox, VisionError, VisionResult};
use rtx_tensor::{Device, Tensor};
use tracing::debug;
use super::layers::ConvBlock;
/// Region Proposal Network (RPN)
pub struct RegionProposalNetwork {
conv: ConvBlock,
cls_logits: ConvBlock,
bbox_pred: ConvBlock,
anchor_generator: AnchorGenerator,
anchor_matcher: AnchorMatcher,
}
impl RegionProposalNetwork {
pub fn new(in_channels: usize, num_anchors_per_location: usize) -> VisionResult<Self> {
let conv = ConvBlock::new(in_channels, in_channels, 3, 1, 1)?;
let cls_logits = ConvBlock::new(in_channels, num_anchors_per_location, 1, 1, 0)?;
let bbox_pred = ConvBlock::new(in_channels, num_anchors_per_location * 4, 1, 1, 0)?;
let anchor_generator = AnchorGenerator::new(vec![8.0, 16.0, 32.0], vec![0.5, 1.0, 2.0])?;
let anchor_matcher = AnchorMatcher::new(0.3, 0.7)?;
Ok(Self {
conv,
cls_logits,
bbox_pred,
anchor_generator,
anchor_matcher,
})
}
pub fn forward(
&self,
features: &[Tensor],
image_sizes: &[(usize, usize)],
) -> VisionResult<(Vec<Tensor>, Vec<Tensor>, Vec<Tensor>)> {
let mut objectness_logits = Vec::new();
let mut pred_bbox_deltas = Vec::new();
let mut anchors = Vec::new();
for (feature, &image_size) in features.iter().zip(image_sizes.iter()) {
// RPN head
let conv_out = self.conv.forward(feature)?;
// Classification (objectness)
let cls_logits = self.cls_logits.forward(&conv_out)?;
objectness_logits.push(cls_logits);
// Regression (bbox deltas)
let bbox_deltas = self.bbox_pred.forward(&conv_out)?;
pred_bbox_deltas.push(bbox_deltas);
// Generate anchors
let feature_shape = (feature.shape()[2], feature.shape()[3]);
let level_anchors = self
.anchor_generator
.generate_anchors(feature_shape, image_size)?;
anchors.push(level_anchors);
}
Ok((objectness_logits, pred_bbox_deltas, anchors))
}
pub fn filter_proposals(
&self,
proposals: Vec<BoundingBox>,
_objectness_logits: &Tensor,
_image_size: (usize, usize),
training: bool,
) -> VisionResult<Vec<BoundingBox>> {
let num_proposals_train = 2000;
let num_proposals_test = 1000;
let nms_threshold = 0.7;
let max_proposals = if training {
num_proposals_train
} else {
num_proposals_test
};
// Apply NMS to proposals
let mut filtered_proposals = proposals;
filtered_proposals.retain(|prop| prop.confidence > 0.5);
if filtered_proposals.len() > max_proposals {
filtered_proposals.sort_by(|a, b| b.confidence.total_cmp(&a.confidence));
filtered_proposals.truncate(max_proposals);
}
// Apply NMS
let final_proposals = nms::apply_nms(filtered_proposals, nms_threshold)?;
debug!("RPN generated {} proposals", final_proposals.len());
Ok(final_proposals)
}
}
/// Anchor generator for RPN
pub struct AnchorGenerator {
scales: Vec<f32>,
aspect_ratios: Vec<f32>,
pub base_anchors: Vec<BoundingBox>,
}
impl AnchorGenerator {
pub fn new(scales: Vec<f32>, aspect_ratios: Vec<f32>) -> VisionResult<Self> {
let mut base_anchors = Vec::new();
for &scale in &scales {
for &ratio in &aspect_ratios {
let width = scale * ratio.sqrt();
let height = scale / ratio.sqrt();
base_anchors.push(BoundingBox::new(
-width / 2.0,
-height / 2.0,
width,
height,
1.0, // Default confidence
0, // Default class
));
}
}
Ok(Self {
scales,
aspect_ratios,
base_anchors,
})
}
pub fn generate_anchors(
&self,
feature_shape: (usize, usize),
image_size: (usize, usize),
) -> VisionResult<Tensor> {
let (feat_h, feat_w) = feature_shape;
let (img_h, img_w) = image_size;
let stride_h = img_h as f32 / feat_h as f32;
let stride_w = img_w as f32 / feat_w as f32;
let mut all_anchors = Vec::new();
for y in 0..feat_h {
for x in 0..feat_w {
let center_x = (x as f32 + 0.5) * stride_w;
let center_y = (y as f32 + 0.5) * stride_h;
for base_anchor in &self.base_anchors {
let anchor_x = center_x + base_anchor.x;
let anchor_y = center_y + base_anchor.y;
all_anchors.push([
anchor_x,
anchor_y,
anchor_x + base_anchor.width,
anchor_y + base_anchor.height,
]);
}
}
}
// Convert Vec<[f32; 4]> to Tensor
let flattened: Vec<f32> = all_anchors.iter().flat_map(|a| a.iter().copied()).collect();
Tensor::from_vec(flattened, &[all_anchors.len(), 4], &Device::default())
.map_err(|e| VisionError::tensor_error_with_source("Failed to create anchor tensor", e))
}
}
/// Anchor matcher for training
pub struct AnchorMatcher {
low_threshold: f32,
high_threshold: f32,
}
impl AnchorMatcher {
pub fn new(low_threshold: f32, high_threshold: f32) -> VisionResult<Self> {
Ok(Self {
low_threshold,
high_threshold,
})
}
pub fn match_anchors(
&self,
anchors: &[BoundingBox],
targets: &[BoundingBox],
) -> VisionResult<Vec<i32>> {
let mut matches = vec![-1i32; anchors.len()]; // -1: ignore, 0: negative, >0: positive (target idx + 1)
for (anchor_idx, anchor) in anchors.iter().enumerate() {
let mut best_iou = 0.0;
let mut best_target_idx = None;
for (target_idx, target) in targets.iter().enumerate() {
let iou = anchor.iou(target);
if iou > best_iou {
best_iou = iou;
best_target_idx = Some(target_idx);
}
}
if best_iou < self.low_threshold {
matches[anchor_idx] = 0; // Negative
} else if best_iou >= self.high_threshold {
matches[anchor_idx] = (best_target_idx.unwrap() + 1) as i32; // Positive
}
// else: ignore (keep -1)
}
Ok(matches)
}
}