Initial commit
This commit is contained in:
@@ -0,0 +1,293 @@
|
||||
//! ResNet backbone and related building blocks for R-CNN models.
|
||||
|
||||
use crate::{VisionError, VisionResult};
|
||||
use rtx_tensor::{Device, Tensor};
|
||||
use tracing::{debug, info};
|
||||
|
||||
use super::config::BackboneType;
|
||||
|
||||
/// ResNet backbone for R-CNN models
|
||||
pub struct ResNetBackbone {
|
||||
backbone_type: BackboneType,
|
||||
stem: ConvStem,
|
||||
layer1: ResLayer,
|
||||
layer2: ResLayer,
|
||||
layer3: ResLayer,
|
||||
layer4: ResLayer,
|
||||
out_channels: Vec<usize>,
|
||||
}
|
||||
|
||||
impl ResNetBackbone {
|
||||
pub fn new(backbone_type: BackboneType) -> VisionResult<Self> {
|
||||
let (depths, channels) = match backbone_type {
|
||||
BackboneType::ResNet50 => (vec![3, 4, 6, 3], vec![64, 128, 256, 512]),
|
||||
BackboneType::ResNet101 => (vec![3, 4, 23, 3], vec![64, 128, 256, 512]),
|
||||
BackboneType::ResNeXt50 => (vec![3, 4, 6, 3], vec![128, 256, 512, 1024]),
|
||||
BackboneType::ResNeXt101 => (vec![3, 4, 23, 3], vec![128, 256, 512, 1024]),
|
||||
BackboneType::EfficientNet => (vec![2, 2, 2, 3], vec![32, 64, 128, 256]),
|
||||
};
|
||||
|
||||
let stem = ConvStem::new(3, 64)?;
|
||||
let layer1 = ResLayer::new(64, channels[0], depths[0], 1)?;
|
||||
let layer2 = ResLayer::new(channels[0] * 4, channels[1], depths[1], 2)?;
|
||||
let layer3 = ResLayer::new(channels[1] * 4, channels[2], depths[2], 2)?;
|
||||
let layer4 = ResLayer::new(channels[2] * 4, channels[3], depths[3], 2)?;
|
||||
|
||||
let out_channels = channels.iter().map(|&c| c * 4).collect();
|
||||
|
||||
info!("Created ResNet backbone: {:?}", backbone_type);
|
||||
|
||||
Ok(Self {
|
||||
backbone_type,
|
||||
stem,
|
||||
layer1,
|
||||
layer2,
|
||||
layer3,
|
||||
layer4,
|
||||
out_channels,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Vec<Tensor>> {
|
||||
let mut features = Vec::new();
|
||||
|
||||
// Stem
|
||||
let x = self.stem.forward(x)?;
|
||||
|
||||
// ResNet layers
|
||||
let c1 = self.layer1.forward(&x)?;
|
||||
features.push(c1.clone());
|
||||
|
||||
let c2 = self.layer2.forward(&c1)?;
|
||||
features.push(c2.clone());
|
||||
|
||||
let c3 = self.layer3.forward(&c2)?;
|
||||
features.push(c3.clone());
|
||||
|
||||
let c4 = self.layer4.forward(&c3)?;
|
||||
features.push(c4);
|
||||
|
||||
debug!("ResNet backbone produced {} feature maps", features.len());
|
||||
Ok(features)
|
||||
}
|
||||
|
||||
pub fn out_channels(&self) -> &[usize] {
|
||||
&self.out_channels
|
||||
}
|
||||
}
|
||||
|
||||
/// Convolutional stem
|
||||
pub struct ConvStem {
|
||||
conv: Tensor,
|
||||
bn_weight: Tensor,
|
||||
bn_bias: Tensor,
|
||||
maxpool: MaxPoolLayer,
|
||||
}
|
||||
|
||||
impl ConvStem {
|
||||
pub fn new(in_channels: usize, out_channels: usize) -> VisionResult<Self> {
|
||||
let conv_weight = Tensor::randn(&[out_channels, in_channels, 7, 7], &Device::default())?;
|
||||
let bn_weight = Tensor::ones([out_channels], &Device::default())?;
|
||||
let bn_bias = Tensor::zeros([out_channels], &Device::default())?;
|
||||
let maxpool = MaxPoolLayer::new(3, 2, 1)?;
|
||||
|
||||
Ok(Self {
|
||||
conv: conv_weight,
|
||||
bn_weight,
|
||||
bn_bias,
|
||||
maxpool,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
let conv_out = x.conv2d(&self.conv, None, 3, 2, 1, 1)?;
|
||||
let norm_out = conv_out
|
||||
.mul(&self.bn_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
.add(&self.bn_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
|
||||
let activated = norm_out.relu()?;
|
||||
let pooled = self.maxpool.forward(&activated)?;
|
||||
Ok(pooled)
|
||||
}
|
||||
}
|
||||
|
||||
/// ResNet layer containing multiple residual blocks
|
||||
pub struct ResLayer {
|
||||
blocks: Vec<ResidualBlock>,
|
||||
}
|
||||
|
||||
impl ResLayer {
|
||||
pub fn new(
|
||||
in_channels: usize,
|
||||
out_channels: usize,
|
||||
num_blocks: usize,
|
||||
stride: usize,
|
||||
) -> VisionResult<Self> {
|
||||
let mut blocks = Vec::new();
|
||||
|
||||
// First block with potential downsampling
|
||||
blocks.push(ResidualBlock::new(in_channels, out_channels, stride, true)?);
|
||||
|
||||
// Remaining blocks
|
||||
for _ in 1..num_blocks {
|
||||
blocks.push(ResidualBlock::new(
|
||||
out_channels * 4,
|
||||
out_channels,
|
||||
1,
|
||||
false,
|
||||
)?);
|
||||
}
|
||||
|
||||
Ok(Self { blocks })
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
let mut out = x.clone();
|
||||
for block in &self.blocks {
|
||||
out = block.forward(&out)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Residual block
|
||||
pub struct ResidualBlock {
|
||||
conv1: Tensor,
|
||||
bn1_weight: Tensor,
|
||||
bn1_bias: Tensor,
|
||||
conv2: Tensor,
|
||||
bn2_weight: Tensor,
|
||||
bn2_bias: Tensor,
|
||||
conv3: Tensor,
|
||||
bn3_weight: Tensor,
|
||||
bn3_bias: Tensor,
|
||||
downsample: Option<DownsampleLayer>,
|
||||
stride: usize,
|
||||
}
|
||||
|
||||
impl ResidualBlock {
|
||||
pub fn new(
|
||||
in_channels: usize,
|
||||
out_channels: usize,
|
||||
stride: usize,
|
||||
downsample: bool,
|
||||
) -> VisionResult<Self> {
|
||||
let conv1 = Tensor::randn(&[out_channels, in_channels, 1, 1], &Device::default())?;
|
||||
let bn1_weight = Tensor::ones([out_channels], &Device::default())?;
|
||||
let bn1_bias = Tensor::zeros([out_channels], &Device::default())?;
|
||||
|
||||
let conv2 = Tensor::randn(&[out_channels, out_channels, 3, 3], &Device::default())?;
|
||||
let bn2_weight = Tensor::ones([out_channels], &Device::default())?;
|
||||
let bn2_bias = Tensor::zeros([out_channels], &Device::default())?;
|
||||
|
||||
let conv3 = Tensor::randn(&[out_channels * 4, out_channels, 1, 1], &Device::default())?;
|
||||
let bn3_weight = Tensor::ones([out_channels * 4], &Device::default())?;
|
||||
let bn3_bias = Tensor::zeros([out_channels * 4], &Device::default())?;
|
||||
|
||||
let downsample_layer = if downsample {
|
||||
Some(DownsampleLayer::new(in_channels, out_channels * 4, stride)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
conv1,
|
||||
bn1_weight,
|
||||
bn1_bias,
|
||||
conv2,
|
||||
bn2_weight,
|
||||
bn2_bias,
|
||||
conv3,
|
||||
bn3_weight,
|
||||
bn3_bias,
|
||||
downsample: downsample_layer,
|
||||
stride,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
let identity = if let Some(downsample) = &self.downsample {
|
||||
downsample.forward(x)?
|
||||
} else {
|
||||
x.clone()
|
||||
};
|
||||
|
||||
// First conv + bn + relu
|
||||
let out = x.conv2d(&self.conv1, None, 0, 1, 1, 1)?;
|
||||
let out = out
|
||||
.mul(&self.bn1_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
.add(&self.bn1_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
|
||||
let out = out.relu()?;
|
||||
|
||||
// Second conv + bn + relu
|
||||
let out = out.conv2d(&self.conv2, None, 1, self.stride, 1, 1)?;
|
||||
let out = out
|
||||
.mul(&self.bn2_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
.add(&self.bn2_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
|
||||
let out = out.relu()?;
|
||||
|
||||
// Third conv + bn
|
||||
let out = out.conv2d(&self.conv3, None, 0, 1, 1, 1)?;
|
||||
let out = out
|
||||
.mul(&self.bn3_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
.add(&self.bn3_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
|
||||
|
||||
// Add residual connection and apply ReLU
|
||||
let out = out.add(&identity)?;
|
||||
let out = out.relu()?;
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Downsample layer for residual connections
|
||||
pub struct DownsampleLayer {
|
||||
conv: Tensor,
|
||||
bn_weight: Tensor,
|
||||
bn_bias: Tensor,
|
||||
stride: usize,
|
||||
}
|
||||
|
||||
impl DownsampleLayer {
|
||||
pub fn new(in_channels: usize, out_channels: usize, stride: usize) -> VisionResult<Self> {
|
||||
let conv = Tensor::randn(&[out_channels, in_channels, 1, 1], &Device::default())?;
|
||||
let bn_weight = Tensor::ones([out_channels], &Device::default())?;
|
||||
let bn_bias = Tensor::zeros([out_channels], &Device::default())?;
|
||||
|
||||
Ok(Self {
|
||||
conv,
|
||||
bn_weight,
|
||||
bn_bias,
|
||||
stride,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
let conv_out = x.conv2d(&self.conv, None, 0, self.stride, 1, 1)?;
|
||||
let norm_out = conv_out
|
||||
.mul(&self.bn_weight.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
.add(&self.bn_bias.unsqueeze(-1)?.unsqueeze(-1)?)?;
|
||||
Ok(norm_out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Max pooling layer
|
||||
pub struct MaxPoolLayer {
|
||||
kernel_size: usize,
|
||||
stride: usize,
|
||||
padding: usize,
|
||||
}
|
||||
|
||||
impl MaxPoolLayer {
|
||||
pub fn new(kernel_size: usize, stride: usize, padding: usize) -> VisionResult<Self> {
|
||||
Ok(Self {
|
||||
kernel_size,
|
||||
stride,
|
||||
padding,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
x.max_pool2d(self.kernel_size, self.stride, self.padding)
|
||||
.map_err(|e| VisionError::tensor_error_with_source("Max pooling failed", e))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! R-CNN configuration types and variants.
|
||||
|
||||
/// R-CNN model variants
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum RcnnVariant {
|
||||
/// Fast R-CNN with external proposals
|
||||
FastRcnn,
|
||||
/// Faster R-CNN with integrated RPN
|
||||
FasterRcnn,
|
||||
/// Mask R-CNN with instance segmentation
|
||||
MaskRcnn,
|
||||
/// RetinaNet with focal loss
|
||||
RetinaNet,
|
||||
/// FCOS (Fully Convolutional One-Stage)
|
||||
Fcos,
|
||||
}
|
||||
|
||||
impl RcnnVariant {
|
||||
/// Get model configuration
|
||||
pub fn config(&self) -> RcnnConfig {
|
||||
match self {
|
||||
Self::FastRcnn => RcnnConfig {
|
||||
backbone_type: BackboneType::ResNet50,
|
||||
neck_type: Some(NeckType::FPN),
|
||||
rpn_enabled: false,
|
||||
mask_head_enabled: false,
|
||||
roi_head_type: RoiHeadType::StandardRoiHead,
|
||||
anchor_scales: vec![8.0, 16.0, 32.0],
|
||||
anchor_ratios: vec![0.5, 1.0, 2.0],
|
||||
roi_pool_size: 7,
|
||||
mask_pool_size: 14,
|
||||
nms_threshold: 0.7,
|
||||
score_threshold: 0.05,
|
||||
},
|
||||
Self::FasterRcnn => RcnnConfig {
|
||||
backbone_type: BackboneType::ResNet50,
|
||||
neck_type: Some(NeckType::FPN),
|
||||
rpn_enabled: true,
|
||||
mask_head_enabled: false,
|
||||
roi_head_type: RoiHeadType::StandardRoiHead,
|
||||
anchor_scales: vec![8.0, 16.0, 32.0],
|
||||
anchor_ratios: vec![0.5, 1.0, 2.0],
|
||||
roi_pool_size: 7,
|
||||
mask_pool_size: 14,
|
||||
nms_threshold: 0.7,
|
||||
score_threshold: 0.05,
|
||||
},
|
||||
Self::MaskRcnn => RcnnConfig {
|
||||
backbone_type: BackboneType::ResNet50,
|
||||
neck_type: Some(NeckType::FPN),
|
||||
rpn_enabled: true,
|
||||
mask_head_enabled: true,
|
||||
roi_head_type: RoiHeadType::StandardRoiHead,
|
||||
anchor_scales: vec![8.0, 16.0, 32.0],
|
||||
anchor_ratios: vec![0.5, 1.0, 2.0],
|
||||
roi_pool_size: 7,
|
||||
mask_pool_size: 14,
|
||||
nms_threshold: 0.7,
|
||||
score_threshold: 0.05,
|
||||
},
|
||||
Self::RetinaNet => RcnnConfig {
|
||||
backbone_type: BackboneType::ResNet50,
|
||||
neck_type: Some(NeckType::FPN),
|
||||
rpn_enabled: false,
|
||||
mask_head_enabled: false,
|
||||
roi_head_type: RoiHeadType::RetinaHead,
|
||||
anchor_scales: vec![4.0, 5.04, 6.35],
|
||||
anchor_ratios: vec![0.5, 1.0, 2.0],
|
||||
roi_pool_size: 7,
|
||||
mask_pool_size: 14,
|
||||
nms_threshold: 0.5,
|
||||
score_threshold: 0.05,
|
||||
},
|
||||
Self::Fcos => RcnnConfig {
|
||||
backbone_type: BackboneType::ResNet50,
|
||||
neck_type: Some(NeckType::FPN),
|
||||
rpn_enabled: false,
|
||||
mask_head_enabled: false,
|
||||
roi_head_type: RoiHeadType::FcosHead,
|
||||
anchor_scales: vec![],
|
||||
anchor_ratios: vec![],
|
||||
roi_pool_size: 7,
|
||||
mask_pool_size: 14,
|
||||
nms_threshold: 0.6,
|
||||
score_threshold: 0.05,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// R-CNN configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RcnnConfig {
|
||||
pub backbone_type: BackboneType,
|
||||
pub neck_type: Option<NeckType>,
|
||||
pub rpn_enabled: bool,
|
||||
pub mask_head_enabled: bool,
|
||||
pub roi_head_type: RoiHeadType,
|
||||
pub anchor_scales: Vec<f32>,
|
||||
pub anchor_ratios: Vec<f32>,
|
||||
pub roi_pool_size: usize,
|
||||
pub mask_pool_size: usize,
|
||||
pub nms_threshold: f32,
|
||||
pub score_threshold: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BackboneType {
|
||||
ResNet50,
|
||||
ResNet101,
|
||||
ResNeXt50,
|
||||
ResNeXt101,
|
||||
EfficientNet,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum NeckType {
|
||||
FPN,
|
||||
PAFPN,
|
||||
BiFPN,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RoiHeadType {
|
||||
StandardRoiHead,
|
||||
RetinaHead,
|
||||
FcosHead,
|
||||
CascadeRoiHead,
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
//! Complete R-CNN detector implementation.
|
||||
|
||||
use crate::detection::{Detector, DetectorStats};
|
||||
use crate::utils::{nms, preprocessing};
|
||||
use crate::{BoundingBox, DetectionResult, VisionConfig, VisionError, VisionResult};
|
||||
use rtx_tensor::Tensor;
|
||||
use tracing::{debug, info};
|
||||
|
||||
use super::backbone::ResNetBackbone;
|
||||
use super::config::{NeckType, RcnnConfig, RcnnVariant};
|
||||
use super::fpn::FeaturePyramidNetwork;
|
||||
use super::roi_head::RoiHead;
|
||||
use super::rpn::RegionProposalNetwork;
|
||||
|
||||
/// Complete R-CNN detector implementation
|
||||
pub struct RcnnDetector {
|
||||
variant: RcnnVariant,
|
||||
backbone: ResNetBackbone,
|
||||
neck: Option<FeaturePyramidNetwork>,
|
||||
rpn: Option<RegionProposalNetwork>,
|
||||
roi_head: RoiHead,
|
||||
config: RcnnConfig,
|
||||
input_size: (usize, usize),
|
||||
num_classes: usize,
|
||||
}
|
||||
|
||||
impl RcnnDetector {
|
||||
pub fn new(
|
||||
variant: RcnnVariant,
|
||||
weights_path: Option<&str>,
|
||||
vision_config: &VisionConfig,
|
||||
) -> VisionResult<Self> {
|
||||
info!("Initializing R-CNN detector: {:?}", variant);
|
||||
|
||||
let config = variant.config();
|
||||
let input_size = vision_config.input_size;
|
||||
let num_classes = vision_config.num_classes;
|
||||
|
||||
let backbone = ResNetBackbone::new(config.backbone_type.clone())?;
|
||||
|
||||
let neck = if let Some(neck_type) = &config.neck_type {
|
||||
match neck_type {
|
||||
NeckType::FPN => Some(FeaturePyramidNetwork::new(backbone.out_channels(), 256)?),
|
||||
_ => None, // Other neck types not implemented yet
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let rpn = if config.rpn_enabled {
|
||||
let num_anchors = config.anchor_scales.len() * config.anchor_ratios.len();
|
||||
Some(RegionProposalNetwork::new(256, num_anchors)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let roi_head = RoiHead::new(
|
||||
256,
|
||||
num_classes,
|
||||
config.roi_pool_size,
|
||||
config.mask_head_enabled,
|
||||
config.mask_pool_size,
|
||||
)?;
|
||||
|
||||
let mut detector = Self {
|
||||
variant,
|
||||
backbone,
|
||||
neck,
|
||||
rpn,
|
||||
roi_head,
|
||||
config,
|
||||
input_size,
|
||||
num_classes,
|
||||
};
|
||||
|
||||
// Load pre-trained weights if provided
|
||||
if let Some(path) = weights_path {
|
||||
detector.load_weights(path)?;
|
||||
}
|
||||
|
||||
info!("R-CNN detector initialized successfully");
|
||||
Ok(detector)
|
||||
}
|
||||
|
||||
pub fn load_weights(&mut self, path: &str) -> VisionResult<()> {
|
||||
info!("Loading R-CNN weights from: {}", path);
|
||||
|
||||
// Load and apply weights (simplified)
|
||||
let _weights = std::fs::read(path)
|
||||
.map_err(|e| VisionError::model_load_error(format!("Failed to read weights: {e}")))?;
|
||||
|
||||
// Apply weights to model components
|
||||
// Implementation would depend on weight format
|
||||
|
||||
info!("Successfully loaded R-CNN weights");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn generate_proposals_from_rpn(
|
||||
&self,
|
||||
objectness_logits: &[Tensor],
|
||||
bbox_deltas: &[Tensor],
|
||||
anchors: &[Tensor],
|
||||
) -> VisionResult<Vec<BoundingBox>> {
|
||||
let mut proposals = Vec::new();
|
||||
|
||||
// Process each feature level
|
||||
for ((obj_logits, bbox_delta), anchor_tensor) in objectness_logits
|
||||
.iter()
|
||||
.zip(bbox_deltas.iter())
|
||||
.zip(anchors.iter())
|
||||
{
|
||||
// Convert tensors to proposals
|
||||
let obj_scores = obj_logits.sigmoid()?.to_vec()?;
|
||||
let deltas = bbox_delta.to_vec()?;
|
||||
let anchor_coords = anchor_tensor.to_vec()?;
|
||||
|
||||
let num_anchors = obj_scores.len();
|
||||
for i in 0..num_anchors {
|
||||
let score = obj_scores[i];
|
||||
if score > 0.5 {
|
||||
// Objectness threshold
|
||||
let delta_idx = i * 4;
|
||||
let anchor_idx = i * 4;
|
||||
if delta_idx + 3 < deltas.len() && anchor_idx + 3 < anchor_coords.len() {
|
||||
// Apply bbox regression
|
||||
let dx = deltas[delta_idx];
|
||||
let dy = deltas[delta_idx + 1];
|
||||
let dw = deltas[delta_idx + 2];
|
||||
let dh = deltas[delta_idx + 3];
|
||||
|
||||
let anchor_x = anchor_coords[anchor_idx];
|
||||
let anchor_y = anchor_coords[anchor_idx + 1];
|
||||
let anchor_w = anchor_coords[anchor_idx + 2] - anchor_coords[anchor_idx];
|
||||
let anchor_h =
|
||||
anchor_coords[anchor_idx + 3] - anchor_coords[anchor_idx + 1];
|
||||
|
||||
let pred_x = anchor_x + dx * anchor_w;
|
||||
let pred_y = anchor_y + dy * anchor_h;
|
||||
let pred_w = anchor_w * dw.exp();
|
||||
let pred_h = anchor_h * dh.exp();
|
||||
|
||||
proposals.push(BoundingBox::new(
|
||||
pred_x - pred_w / 2.0,
|
||||
pred_y - pred_h / 2.0,
|
||||
pred_w,
|
||||
pred_h,
|
||||
score,
|
||||
0, // RPN doesn't predict class
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter and sort proposals
|
||||
if let Some(rpn) = &self.rpn {
|
||||
let filtered_proposals =
|
||||
rpn.filter_proposals(proposals, &objectness_logits[0], self.input_size, false)?;
|
||||
Ok(filtered_proposals)
|
||||
} else {
|
||||
Ok(proposals)
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_sliding_window_proposals(&self) -> VisionResult<Vec<BoundingBox>> {
|
||||
// Generate sliding window proposals for Fast R-CNN
|
||||
let mut proposals = Vec::new();
|
||||
let (img_h, img_w) = self.input_size;
|
||||
|
||||
let scales = vec![0.25, 0.5, 1.0, 2.0];
|
||||
let ratios: Vec<f32> = vec![0.5, 1.0, 2.0];
|
||||
let stride = 16;
|
||||
|
||||
for y in (0..img_h).step_by(stride) {
|
||||
for x in (0..img_w).step_by(stride) {
|
||||
for &scale in &scales {
|
||||
for &ratio in &ratios {
|
||||
let w = 32.0f32 * scale * ratio.sqrt();
|
||||
let h = 32.0f32 * scale / ratio.sqrt();
|
||||
|
||||
let x1 = (x as f32 - w / 2.0).max(0.0);
|
||||
let y1 = (y as f32 - h / 2.0).max(0.0);
|
||||
let x2 = (x as f32 + w / 2.0).min(img_w as f32);
|
||||
let y2 = (y as f32 + h / 2.0).min(img_h as f32);
|
||||
|
||||
if x2 > x1 && y2 > y1 {
|
||||
proposals.push(BoundingBox::new(
|
||||
x1,
|
||||
y1,
|
||||
x2 - x1,
|
||||
y2 - y1,
|
||||
1.0, // Default confidence for sliding window
|
||||
0, // No class prediction yet
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Limit number of proposals
|
||||
proposals.truncate(2000);
|
||||
Ok(proposals)
|
||||
}
|
||||
|
||||
fn postprocess_detections(
|
||||
&self,
|
||||
class_logits: &Tensor,
|
||||
bbox_deltas: &Tensor,
|
||||
proposals: &[BoundingBox],
|
||||
config: &VisionConfig,
|
||||
) -> VisionResult<Vec<BoundingBox>> {
|
||||
let num_proposals = proposals.len();
|
||||
if num_proposals == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// Get class probabilities
|
||||
let class_probs = class_logits.softmax(-1)?;
|
||||
let prob_data = class_probs.to_vec()?;
|
||||
let delta_data = bbox_deltas.to_vec()?;
|
||||
|
||||
// Assume shape is [num_proposals, num_classes] for class_probs
|
||||
// and [num_proposals, num_classes * 4] for bbox_deltas
|
||||
let num_classes = class_logits.shape()[class_logits.shape().len() - 1];
|
||||
|
||||
let mut detections = Vec::new();
|
||||
|
||||
for (i, proposal) in proposals.iter().enumerate() {
|
||||
let prob_start = i * num_classes;
|
||||
let delta_start_base = i * num_classes * 4;
|
||||
|
||||
if prob_start + num_classes > prob_data.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip background class (index 0)
|
||||
for class_id in 1..num_classes {
|
||||
let confidence = prob_data[prob_start + class_id];
|
||||
|
||||
if confidence > config.confidence_threshold {
|
||||
// Apply bbox regression
|
||||
let delta_start = delta_start_base + class_id * 4;
|
||||
if delta_start + 3 < delta_data.len() {
|
||||
let dx = delta_data[delta_start];
|
||||
let dy = delta_data[delta_start + 1];
|
||||
let dw = delta_data[delta_start + 2];
|
||||
let dh = delta_data[delta_start + 3];
|
||||
|
||||
let pred_x = proposal.x + dx * proposal.width;
|
||||
let pred_y = proposal.y + dy * proposal.height;
|
||||
let pred_w = proposal.width * dw.exp();
|
||||
let pred_h = proposal.height * dh.exp();
|
||||
|
||||
detections.push(BoundingBox::new(
|
||||
pred_x,
|
||||
pred_y,
|
||||
pred_w,
|
||||
pred_h,
|
||||
confidence,
|
||||
class_id - 1, // Adjust for background class
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply NMS
|
||||
let final_detections = nms::apply_nms_per_class(detections, config.nms_threshold)?;
|
||||
|
||||
Ok(final_detections)
|
||||
}
|
||||
}
|
||||
|
||||
impl Detector for RcnnDetector {
|
||||
fn detect(
|
||||
&mut self,
|
||||
image: &Tensor,
|
||||
vision_config: &VisionConfig,
|
||||
) -> VisionResult<DetectionResult> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
// Preprocess image
|
||||
let processed = preprocessing::resize_tensor(image, self.input_size, true)?;
|
||||
let normalized = preprocessing::imagenet_normalize(&processed)?;
|
||||
let batched = normalized.unsqueeze(0)?;
|
||||
|
||||
// Backbone forward pass
|
||||
let backbone_features = self.backbone.forward(&batched)?;
|
||||
|
||||
// Neck forward pass (if present)
|
||||
let features = if let Some(neck) = &self.neck {
|
||||
neck.forward(&backbone_features)?
|
||||
} else {
|
||||
backbone_features
|
||||
};
|
||||
|
||||
// RPN forward pass (if present)
|
||||
let proposals = if let Some(rpn) = &self.rpn {
|
||||
let image_sizes = vec![self.input_size];
|
||||
let (objectness_logits, bbox_deltas, anchors) = rpn.forward(&features, &image_sizes)?;
|
||||
|
||||
// Generate proposals from RPN predictions
|
||||
self.generate_proposals_from_rpn(&objectness_logits, &bbox_deltas, &anchors)?
|
||||
} else {
|
||||
// Use external proposals or sliding window
|
||||
self.generate_sliding_window_proposals()?
|
||||
};
|
||||
|
||||
// RoI head forward pass
|
||||
let (class_logits, bbox_deltas, _masks) = self.roi_head.forward(&features, &proposals)?;
|
||||
|
||||
// Post-process predictions
|
||||
let detections =
|
||||
self.postprocess_detections(&class_logits, &bbox_deltas, &proposals, vision_config)?;
|
||||
|
||||
let processing_time = start_time.elapsed().as_millis() as f32;
|
||||
|
||||
debug!(
|
||||
"R-CNN detection completed: {} boxes in {:.1}ms",
|
||||
detections.len(),
|
||||
processing_time
|
||||
);
|
||||
|
||||
Ok(DetectionResult::new(
|
||||
detections,
|
||||
(image.shape()[2], image.shape()[3]),
|
||||
processing_time,
|
||||
format!("{:?}", self.variant),
|
||||
))
|
||||
}
|
||||
|
||||
fn detect_batch(
|
||||
&mut self,
|
||||
images: &Tensor,
|
||||
config: &VisionConfig,
|
||||
) -> VisionResult<Vec<DetectionResult>> {
|
||||
let batch_size = images.shape()[0];
|
||||
let mut results = Vec::with_capacity(batch_size);
|
||||
|
||||
// Process each image individually for simplicity
|
||||
// In practice, you'd implement proper batch processing
|
||||
for i in 0..batch_size {
|
||||
let image = images.narrow(0, i, 1)?.squeeze(Some(0))?;
|
||||
let result = self.detect(&image, config)?;
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
match self.variant {
|
||||
RcnnVariant::FastRcnn => "Fast R-CNN",
|
||||
RcnnVariant::FasterRcnn => "Faster R-CNN",
|
||||
RcnnVariant::MaskRcnn => "Mask R-CNN",
|
||||
RcnnVariant::RetinaNet => "RetinaNet",
|
||||
RcnnVariant::Fcos => "FCOS",
|
||||
}
|
||||
}
|
||||
|
||||
fn input_size(&self) -> (usize, usize) {
|
||||
self.input_size
|
||||
}
|
||||
|
||||
fn num_classes(&self) -> usize {
|
||||
self.num_classes
|
||||
}
|
||||
|
||||
fn get_stats(&self) -> DetectorStats {
|
||||
let params = match self.variant {
|
||||
RcnnVariant::FastRcnn => 25_600_000,
|
||||
RcnnVariant::FasterRcnn => 41_800_000,
|
||||
RcnnVariant::MaskRcnn => 44_400_000,
|
||||
RcnnVariant::RetinaNet => 36_300_000,
|
||||
RcnnVariant::Fcos => 32_100_000,
|
||||
};
|
||||
|
||||
DetectorStats {
|
||||
total_params: params,
|
||||
flops: params as f64 * 5.0, // Rough estimate
|
||||
model_size_mb: params as f32 * 4.0 / (1024.0 * 1024.0),
|
||||
avg_inference_ms: 0.0,
|
||||
peak_memory_mb: 0.0,
|
||||
features: vec![
|
||||
"Two-stage detection".to_string(),
|
||||
"Feature Pyramid Network".to_string(),
|
||||
"Region Proposal Network".to_string(),
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
//! Feature Pyramid Network (FPN) for multi-scale feature extraction.
|
||||
|
||||
use crate::VisionResult;
|
||||
use rtx_tensor::Tensor;
|
||||
|
||||
use super::layers::ConvBlock;
|
||||
|
||||
/// Feature Pyramid Network (FPN)
|
||||
pub struct FeaturePyramidNetwork {
|
||||
lateral_convs: Vec<ConvBlock>,
|
||||
fpn_convs: Vec<ConvBlock>,
|
||||
out_channels: usize,
|
||||
}
|
||||
|
||||
impl FeaturePyramidNetwork {
|
||||
pub fn new(in_channels: &[usize], out_channels: usize) -> VisionResult<Self> {
|
||||
let mut lateral_convs = Vec::new();
|
||||
let mut fpn_convs = Vec::new();
|
||||
|
||||
for &in_ch in in_channels {
|
||||
lateral_convs.push(ConvBlock::new(in_ch, out_channels, 1, 1, 0)?);
|
||||
fpn_convs.push(ConvBlock::new(out_channels, out_channels, 3, 1, 1)?);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
lateral_convs,
|
||||
fpn_convs,
|
||||
out_channels,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, features: &[Tensor]) -> VisionResult<Vec<Tensor>> {
|
||||
let mut fpn_features = Vec::new();
|
||||
|
||||
// Top-down pathway
|
||||
let mut top_down =
|
||||
self.lateral_convs[features.len() - 1].forward(&features[features.len() - 1])?;
|
||||
fpn_features.push(self.fpn_convs[features.len() - 1].forward(&top_down)?);
|
||||
|
||||
for i in (0..features.len() - 1).rev() {
|
||||
// Upsample
|
||||
let shape = features[i].shape();
|
||||
let upsampled = self.upsample(&top_down, (shape[2], shape[3]))?;
|
||||
|
||||
// Lateral connection
|
||||
let lateral = self.lateral_convs[i].forward(&features[i])?;
|
||||
|
||||
// Add
|
||||
top_down = upsampled.add(&lateral)?;
|
||||
fpn_features.insert(0, self.fpn_convs[i].forward(&top_down)?);
|
||||
}
|
||||
|
||||
Ok(fpn_features)
|
||||
}
|
||||
|
||||
fn upsample(&self, x: &Tensor, target_size: (usize, usize)) -> VisionResult<Tensor> {
|
||||
// Simple nearest neighbor upsampling
|
||||
let shape = x.shape();
|
||||
let scale_h = target_size.0 as f32 / shape[2] as f32;
|
||||
let scale_w = target_size.1 as f32 / shape[3] as f32;
|
||||
|
||||
// For simplicity, we'll create a basic upsampling implementation
|
||||
// In production, you'd use proper interpolation
|
||||
if scale_h.round() as usize == 2 && scale_w.round() as usize == 2 {
|
||||
let upsampled = x
|
||||
.unsqueeze(4)?
|
||||
.unsqueeze(4)?
|
||||
.repeat(&[1, 1, 1, 1, 2, 2])?
|
||||
.view([shape[0], shape[1], target_size.0, target_size.1])?;
|
||||
Ok(upsampled)
|
||||
} else {
|
||||
// Fallback to identity for other scales
|
||||
Ok(x.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//! Common neural network layers used in R-CNN models.
|
||||
|
||||
use crate::{VisionError, VisionResult};
|
||||
use rtx_tensor::{Device, Tensor};
|
||||
|
||||
/// Linear (fully connected) layer
|
||||
pub struct LinearLayer {
|
||||
weight: Tensor,
|
||||
bias: Option<Tensor>,
|
||||
}
|
||||
|
||||
impl LinearLayer {
|
||||
pub fn new(in_features: usize, out_features: usize) -> VisionResult<Self> {
|
||||
let weight = Tensor::randn(&[out_features, in_features], &Device::default())?;
|
||||
let bias = Some(Tensor::zeros([out_features], &Device::default())?);
|
||||
|
||||
Ok(Self { weight, bias })
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
let linear_out = x.matmul(&self.weight.transpose(-2, -1)?)?;
|
||||
|
||||
if let Some(bias) = &self.bias {
|
||||
linear_out.add(bias).map_err(|e| {
|
||||
VisionError::tensor_error_with_source("Linear layer bias addition failed", e)
|
||||
})
|
||||
} else {
|
||||
Ok(linear_out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Conv block used in various parts
|
||||
pub struct ConvBlock {
|
||||
conv: Tensor,
|
||||
bn_weight: Option<Tensor>,
|
||||
bn_bias: Option<Tensor>,
|
||||
kernel_size: usize,
|
||||
stride: usize,
|
||||
padding: usize,
|
||||
}
|
||||
|
||||
impl ConvBlock {
|
||||
pub fn new(
|
||||
in_channels: usize,
|
||||
out_channels: usize,
|
||||
kernel_size: usize,
|
||||
stride: usize,
|
||||
padding: usize,
|
||||
) -> VisionResult<Self> {
|
||||
let conv = Tensor::randn(
|
||||
&[out_channels, in_channels, kernel_size, kernel_size],
|
||||
&Device::default(),
|
||||
)?;
|
||||
let bn_weight = Some(Tensor::ones([out_channels], &Device::default())?);
|
||||
let bn_bias = Some(Tensor::zeros([out_channels], &Device::default())?);
|
||||
|
||||
Ok(Self {
|
||||
conv,
|
||||
bn_weight,
|
||||
bn_bias,
|
||||
kernel_size,
|
||||
stride,
|
||||
padding,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<Tensor> {
|
||||
let conv_out = x.conv2d(&self.conv, None, self.padding, self.stride, 1, 1)?;
|
||||
|
||||
let normalized = if let (Some(weight), Some(bias)) = (&self.bn_weight, &self.bn_bias) {
|
||||
conv_out
|
||||
.mul(&weight.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
.add(&bias.unsqueeze(-1)?.unsqueeze(-1)?)?
|
||||
} else {
|
||||
conv_out
|
||||
};
|
||||
|
||||
Ok(normalized)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
//! R-CNN family detection implementation
|
||||
//!
|
||||
//! Implements:
|
||||
//! - Fast R-CNN: Region-based CNN with RoI pooling
|
||||
//! - Faster R-CNN: End-to-end detection with RPN
|
||||
//! - Mask R-CNN: Instance segmentation with mask heads
|
||||
//! - Feature Pyramid Networks (FPN) for multi-scale detection
|
||||
|
||||
mod backbone;
|
||||
mod config;
|
||||
mod detector;
|
||||
mod fpn;
|
||||
mod layers;
|
||||
mod roi_head;
|
||||
mod rpn;
|
||||
|
||||
pub use backbone::{
|
||||
ConvStem, DownsampleLayer, MaxPoolLayer, ResLayer, ResNetBackbone, ResidualBlock,
|
||||
};
|
||||
pub use config::{BackboneType, NeckType, RcnnConfig, RcnnVariant, RoiHeadType};
|
||||
pub use detector::RcnnDetector;
|
||||
pub use fpn::FeaturePyramidNetwork;
|
||||
pub use layers::{ConvBlock, LinearLayer};
|
||||
pub use roi_head::{BboxHead, MaskHead, RoiExtractor, RoiHead};
|
||||
pub use rpn::{AnchorGenerator, AnchorMatcher, RegionProposalNetwork};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::VisionConfig;
|
||||
|
||||
#[test]
|
||||
fn test_rcnn_config() {
|
||||
let config = RcnnVariant::FasterRcnn.config();
|
||||
assert!(config.rpn_enabled);
|
||||
assert!(!config.mask_head_enabled);
|
||||
assert_eq!(config.roi_pool_size, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_anchor_generator() {
|
||||
let generator = AnchorGenerator::new(vec![8.0, 16.0, 32.0], vec![0.5, 1.0, 2.0]).unwrap();
|
||||
assert_eq!(generator.base_anchors.len(), 9); // 3 scales * 3 ratios
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rcnn_detector_creation() {
|
||||
let config = VisionConfig::default();
|
||||
let detector = RcnnDetector::new(RcnnVariant::FasterRcnn, None, &config);
|
||||
assert!(detector.is_ok());
|
||||
|
||||
let detector = detector.unwrap();
|
||||
assert_eq!(detector.name(), "Faster R-CNN");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
//! RoI head components for R-CNN detection.
|
||||
|
||||
use crate::{BoundingBox, SegmentationMask, VisionError, VisionResult};
|
||||
use rtx_tensor::Tensor;
|
||||
|
||||
use super::layers::{ConvBlock, LinearLayer};
|
||||
|
||||
/// RoI head for final classification and regression
|
||||
pub struct RoiHead {
|
||||
bbox_head: BboxHead,
|
||||
mask_head: Option<MaskHead>,
|
||||
roi_extractor: RoiExtractor,
|
||||
}
|
||||
|
||||
impl RoiHead {
|
||||
pub fn new(
|
||||
in_channels: usize,
|
||||
num_classes: usize,
|
||||
roi_pool_size: usize,
|
||||
enable_mask: bool,
|
||||
mask_pool_size: usize,
|
||||
) -> VisionResult<Self> {
|
||||
let bbox_head = BboxHead::new(in_channels, num_classes, roi_pool_size)?;
|
||||
let mask_head = if enable_mask {
|
||||
Some(MaskHead::new(in_channels, num_classes, mask_pool_size)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let roi_extractor = RoiExtractor::new(roi_pool_size)?;
|
||||
|
||||
Ok(Self {
|
||||
bbox_head,
|
||||
mask_head,
|
||||
roi_extractor,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(
|
||||
&self,
|
||||
features: &[Tensor],
|
||||
proposals: &[BoundingBox],
|
||||
) -> VisionResult<(Tensor, Tensor, Option<Vec<SegmentationMask>>)> {
|
||||
// Extract RoI features
|
||||
let roi_features = self.roi_extractor.forward(features, proposals)?;
|
||||
|
||||
// Bbox head forward
|
||||
let (class_logits, bbox_deltas) = self.bbox_head.forward(&roi_features)?;
|
||||
|
||||
// Mask head forward (if enabled)
|
||||
let masks = if let Some(mask_head) = &self.mask_head {
|
||||
Some(mask_head.forward(&roi_features, proposals)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok((class_logits, bbox_deltas, masks))
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounding box head
|
||||
pub struct BboxHead {
|
||||
fc1: LinearLayer,
|
||||
fc2: LinearLayer,
|
||||
cls_score: LinearLayer,
|
||||
bbox_pred: LinearLayer,
|
||||
pool_size: usize,
|
||||
}
|
||||
|
||||
impl BboxHead {
|
||||
pub fn new(in_channels: usize, num_classes: usize, pool_size: usize) -> VisionResult<Self> {
|
||||
let flattened_size = in_channels * pool_size * pool_size;
|
||||
let hidden_size = 1024;
|
||||
|
||||
let fc1 = LinearLayer::new(flattened_size, hidden_size)?;
|
||||
let fc2 = LinearLayer::new(hidden_size, hidden_size)?;
|
||||
let cls_score = LinearLayer::new(hidden_size, num_classes + 1)?; // +1 for background
|
||||
let bbox_pred = LinearLayer::new(hidden_size, (num_classes + 1) * 4)?;
|
||||
|
||||
Ok(Self {
|
||||
fc1,
|
||||
fc2,
|
||||
cls_score,
|
||||
bbox_pred,
|
||||
pool_size,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(&self, x: &Tensor) -> VisionResult<(Tensor, Tensor)> {
|
||||
// Flatten
|
||||
let batch_size = x.shape()[0];
|
||||
let total_elements = x.numel();
|
||||
let flattened_size = total_elements / batch_size;
|
||||
let flattened = x.view([batch_size, flattened_size])?;
|
||||
|
||||
// FC layers
|
||||
let h1 = self.fc1.forward(&flattened)?.relu()?;
|
||||
let h2 = self.fc2.forward(&h1)?.relu()?;
|
||||
|
||||
// Output heads
|
||||
let cls_score = self.cls_score.forward(&h2)?;
|
||||
let bbox_pred = self.bbox_pred.forward(&h2)?;
|
||||
|
||||
Ok((cls_score, bbox_pred))
|
||||
}
|
||||
}
|
||||
|
||||
/// Mask head for instance segmentation
|
||||
pub struct MaskHead {
|
||||
convs: Vec<ConvBlock>,
|
||||
mask_fcn_logits: ConvBlock,
|
||||
num_classes: usize,
|
||||
pool_size: usize,
|
||||
}
|
||||
|
||||
impl MaskHead {
|
||||
pub fn new(in_channels: usize, num_classes: usize, pool_size: usize) -> VisionResult<Self> {
|
||||
let mut convs = Vec::new();
|
||||
let hidden_channels = 256;
|
||||
|
||||
// 4 conv layers
|
||||
convs.push(ConvBlock::new(in_channels, hidden_channels, 3, 1, 1)?);
|
||||
for _ in 1..4 {
|
||||
convs.push(ConvBlock::new(hidden_channels, hidden_channels, 3, 1, 1)?);
|
||||
}
|
||||
|
||||
let mask_fcn_logits = ConvBlock::new(hidden_channels, num_classes, 1, 1, 0)?;
|
||||
|
||||
Ok(Self {
|
||||
convs,
|
||||
mask_fcn_logits,
|
||||
num_classes,
|
||||
pool_size,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(
|
||||
&self,
|
||||
x: &Tensor,
|
||||
proposals: &[BoundingBox],
|
||||
) -> VisionResult<Vec<SegmentationMask>> {
|
||||
let mut mask_features = x.clone();
|
||||
|
||||
// Apply conv layers
|
||||
for conv in &self.convs {
|
||||
mask_features = conv.forward(&mask_features)?.relu()?;
|
||||
}
|
||||
|
||||
// Final mask prediction
|
||||
let mask_logits = self.mask_fcn_logits.forward(&mask_features)?;
|
||||
|
||||
// Convert to individual masks
|
||||
let mut masks = Vec::new();
|
||||
let batch_size = mask_logits.shape()[0];
|
||||
|
||||
for i in 0..batch_size.min(proposals.len()) {
|
||||
let mask_logit = mask_logits.narrow(0, i, 1)?.squeeze(Some(0))?;
|
||||
|
||||
// Apply sigmoid to get probabilities
|
||||
let mask_prob = mask_logit.sigmoid()?;
|
||||
|
||||
// Create segmentation mask
|
||||
let mask =
|
||||
SegmentationMask::new(mask_prob, proposals[i].class_id, proposals[i].confidence);
|
||||
masks.push(mask);
|
||||
}
|
||||
|
||||
Ok(masks)
|
||||
}
|
||||
}
|
||||
|
||||
/// RoI feature extractor
|
||||
pub struct RoiExtractor {
|
||||
pool_size: usize,
|
||||
}
|
||||
|
||||
impl RoiExtractor {
|
||||
pub fn new(pool_size: usize) -> VisionResult<Self> {
|
||||
Ok(Self { pool_size })
|
||||
}
|
||||
|
||||
pub fn forward(&self, features: &[Tensor], proposals: &[BoundingBox]) -> VisionResult<Tensor> {
|
||||
if proposals.is_empty() {
|
||||
return Err(VisionError::invalid_input(
|
||||
"No proposals provided to RoI extractor",
|
||||
));
|
||||
}
|
||||
|
||||
let feature_channels = features[0].shape()[1];
|
||||
let output_shape = vec![
|
||||
proposals.len(),
|
||||
feature_channels,
|
||||
self.pool_size,
|
||||
self.pool_size,
|
||||
];
|
||||
|
||||
// Simplified RoI pooling implementation
|
||||
// In practice, you'd implement proper RoI align
|
||||
let pooled_features =
|
||||
Tensor::zeros_typed(&output_shape, features[0].dtype(), features[0].device())?;
|
||||
|
||||
for proposal in proposals.iter() {
|
||||
// Find appropriate feature level based on RoI size
|
||||
let roi_area = proposal.width * proposal.height;
|
||||
let feature_level = self.select_feature_level(roi_area, features.len());
|
||||
|
||||
let feature = &features[feature_level.min(features.len() - 1)];
|
||||
|
||||
// Extract and pool RoI
|
||||
let _roi_feature = self.roi_align(feature, proposal)?;
|
||||
|
||||
// Copy to output tensor (simplified)
|
||||
// In practice, you'd use proper tensor assignment
|
||||
}
|
||||
|
||||
Ok(pooled_features)
|
||||
}
|
||||
|
||||
fn select_feature_level(&self, roi_area: f32, num_levels: usize) -> usize {
|
||||
// Heuristic: larger RoIs use lower resolution features
|
||||
let base_area = 224.0f32 * 224.0f32; // Base image size
|
||||
let level = (roi_area.log2() - base_area.log2()) / 2.0f32;
|
||||
(level as usize).min(num_levels - 1)
|
||||
}
|
||||
|
||||
fn roi_align(&self, feature: &Tensor, proposal: &BoundingBox) -> VisionResult<Tensor> {
|
||||
// Simplified RoI align - in practice you'd implement bilinear sampling
|
||||
let feat_shape = feature.shape();
|
||||
let feat_h = feat_shape[2] as f32;
|
||||
let feat_w = feat_shape[3] as f32;
|
||||
|
||||
// Map proposal coordinates to feature map coordinates
|
||||
let x1 = (proposal.x / feat_w * feat_w).max(0.0) as usize;
|
||||
let y1 = (proposal.y / feat_h * feat_h).max(0.0) as usize;
|
||||
let x2 = ((proposal.x + proposal.width) / feat_w * feat_w).min(feat_w) as usize;
|
||||
let y2 = ((proposal.y + proposal.height) / feat_h * feat_h).min(feat_h) as usize;
|
||||
|
||||
// Extract region and resize to pool_size
|
||||
let roi_region = feature
|
||||
.narrow(2, y1, y2.saturating_sub(y1).max(1))?
|
||||
.narrow(3, x1, x2.saturating_sub(x1).max(1))?;
|
||||
|
||||
// Simple average pooling to target size
|
||||
self.adaptive_avg_pool(&roi_region, self.pool_size)
|
||||
}
|
||||
|
||||
fn adaptive_avg_pool(&self, x: &Tensor, target_size: usize) -> VisionResult<Tensor> {
|
||||
let shape = x.shape();
|
||||
if shape[2] == target_size && shape[3] == target_size {
|
||||
return Ok(x.clone());
|
||||
}
|
||||
|
||||
// Simplified adaptive pooling - in practice you'd use proper implementation
|
||||
let stride_h = shape[2] / target_size;
|
||||
let _stride_w = shape[3] / target_size;
|
||||
|
||||
x.avg_pool2d(stride_h.max(1), stride_h.max(1), 0)
|
||||
.map_err(|e| VisionError::tensor_error_with_source("Adaptive pooling failed", e))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//! Region Proposal Network (RPN) and anchor generation.
|
||||
|
||||
use crate::utils::nms;
|
||||
use crate::{BoundingBox, VisionError, VisionResult};
|
||||
use rtx_tensor::{Device, Tensor};
|
||||
use tracing::debug;
|
||||
|
||||
use super::layers::ConvBlock;
|
||||
|
||||
/// Region Proposal Network (RPN)
|
||||
pub struct RegionProposalNetwork {
|
||||
conv: ConvBlock,
|
||||
cls_logits: ConvBlock,
|
||||
bbox_pred: ConvBlock,
|
||||
anchor_generator: AnchorGenerator,
|
||||
anchor_matcher: AnchorMatcher,
|
||||
}
|
||||
|
||||
impl RegionProposalNetwork {
|
||||
pub fn new(in_channels: usize, num_anchors_per_location: usize) -> VisionResult<Self> {
|
||||
let conv = ConvBlock::new(in_channels, in_channels, 3, 1, 1)?;
|
||||
let cls_logits = ConvBlock::new(in_channels, num_anchors_per_location, 1, 1, 0)?;
|
||||
let bbox_pred = ConvBlock::new(in_channels, num_anchors_per_location * 4, 1, 1, 0)?;
|
||||
|
||||
let anchor_generator = AnchorGenerator::new(vec![8.0, 16.0, 32.0], vec![0.5, 1.0, 2.0])?;
|
||||
let anchor_matcher = AnchorMatcher::new(0.3, 0.7)?;
|
||||
|
||||
Ok(Self {
|
||||
conv,
|
||||
cls_logits,
|
||||
bbox_pred,
|
||||
anchor_generator,
|
||||
anchor_matcher,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn forward(
|
||||
&self,
|
||||
features: &[Tensor],
|
||||
image_sizes: &[(usize, usize)],
|
||||
) -> VisionResult<(Vec<Tensor>, Vec<Tensor>, Vec<Tensor>)> {
|
||||
let mut objectness_logits = Vec::new();
|
||||
let mut pred_bbox_deltas = Vec::new();
|
||||
let mut anchors = Vec::new();
|
||||
|
||||
for (feature, &image_size) in features.iter().zip(image_sizes.iter()) {
|
||||
// RPN head
|
||||
let conv_out = self.conv.forward(feature)?;
|
||||
|
||||
// Classification (objectness)
|
||||
let cls_logits = self.cls_logits.forward(&conv_out)?;
|
||||
objectness_logits.push(cls_logits);
|
||||
|
||||
// Regression (bbox deltas)
|
||||
let bbox_deltas = self.bbox_pred.forward(&conv_out)?;
|
||||
pred_bbox_deltas.push(bbox_deltas);
|
||||
|
||||
// Generate anchors
|
||||
let feature_shape = (feature.shape()[2], feature.shape()[3]);
|
||||
let level_anchors = self
|
||||
.anchor_generator
|
||||
.generate_anchors(feature_shape, image_size)?;
|
||||
anchors.push(level_anchors);
|
||||
}
|
||||
|
||||
Ok((objectness_logits, pred_bbox_deltas, anchors))
|
||||
}
|
||||
|
||||
pub fn filter_proposals(
|
||||
&self,
|
||||
proposals: Vec<BoundingBox>,
|
||||
_objectness_logits: &Tensor,
|
||||
_image_size: (usize, usize),
|
||||
training: bool,
|
||||
) -> VisionResult<Vec<BoundingBox>> {
|
||||
let num_proposals_train = 2000;
|
||||
let num_proposals_test = 1000;
|
||||
let nms_threshold = 0.7;
|
||||
|
||||
let max_proposals = if training {
|
||||
num_proposals_train
|
||||
} else {
|
||||
num_proposals_test
|
||||
};
|
||||
|
||||
// Apply NMS to proposals
|
||||
let mut filtered_proposals = proposals;
|
||||
filtered_proposals.retain(|prop| prop.confidence > 0.5);
|
||||
|
||||
if filtered_proposals.len() > max_proposals {
|
||||
filtered_proposals.sort_by(|a, b| b.confidence.total_cmp(&a.confidence));
|
||||
filtered_proposals.truncate(max_proposals);
|
||||
}
|
||||
|
||||
// Apply NMS
|
||||
let final_proposals = nms::apply_nms(filtered_proposals, nms_threshold)?;
|
||||
|
||||
debug!("RPN generated {} proposals", final_proposals.len());
|
||||
Ok(final_proposals)
|
||||
}
|
||||
}
|
||||
|
||||
/// Anchor generator for RPN
|
||||
pub struct AnchorGenerator {
|
||||
scales: Vec<f32>,
|
||||
aspect_ratios: Vec<f32>,
|
||||
pub base_anchors: Vec<BoundingBox>,
|
||||
}
|
||||
|
||||
impl AnchorGenerator {
|
||||
pub fn new(scales: Vec<f32>, aspect_ratios: Vec<f32>) -> VisionResult<Self> {
|
||||
let mut base_anchors = Vec::new();
|
||||
|
||||
for &scale in &scales {
|
||||
for &ratio in &aspect_ratios {
|
||||
let width = scale * ratio.sqrt();
|
||||
let height = scale / ratio.sqrt();
|
||||
|
||||
base_anchors.push(BoundingBox::new(
|
||||
-width / 2.0,
|
||||
-height / 2.0,
|
||||
width,
|
||||
height,
|
||||
1.0, // Default confidence
|
||||
0, // Default class
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
scales,
|
||||
aspect_ratios,
|
||||
base_anchors,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn generate_anchors(
|
||||
&self,
|
||||
feature_shape: (usize, usize),
|
||||
image_size: (usize, usize),
|
||||
) -> VisionResult<Tensor> {
|
||||
let (feat_h, feat_w) = feature_shape;
|
||||
let (img_h, img_w) = image_size;
|
||||
|
||||
let stride_h = img_h as f32 / feat_h as f32;
|
||||
let stride_w = img_w as f32 / feat_w as f32;
|
||||
|
||||
let mut all_anchors = Vec::new();
|
||||
|
||||
for y in 0..feat_h {
|
||||
for x in 0..feat_w {
|
||||
let center_x = (x as f32 + 0.5) * stride_w;
|
||||
let center_y = (y as f32 + 0.5) * stride_h;
|
||||
|
||||
for base_anchor in &self.base_anchors {
|
||||
let anchor_x = center_x + base_anchor.x;
|
||||
let anchor_y = center_y + base_anchor.y;
|
||||
|
||||
all_anchors.push([
|
||||
anchor_x,
|
||||
anchor_y,
|
||||
anchor_x + base_anchor.width,
|
||||
anchor_y + base_anchor.height,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert Vec<[f32; 4]> to Tensor
|
||||
let flattened: Vec<f32> = all_anchors.iter().flat_map(|a| a.iter().copied()).collect();
|
||||
Tensor::from_vec(flattened, &[all_anchors.len(), 4], &Device::default())
|
||||
.map_err(|e| VisionError::tensor_error_with_source("Failed to create anchor tensor", e))
|
||||
}
|
||||
}
|
||||
|
||||
/// Anchor matcher for training
|
||||
pub struct AnchorMatcher {
|
||||
low_threshold: f32,
|
||||
high_threshold: f32,
|
||||
}
|
||||
|
||||
impl AnchorMatcher {
|
||||
pub fn new(low_threshold: f32, high_threshold: f32) -> VisionResult<Self> {
|
||||
Ok(Self {
|
||||
low_threshold,
|
||||
high_threshold,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn match_anchors(
|
||||
&self,
|
||||
anchors: &[BoundingBox],
|
||||
targets: &[BoundingBox],
|
||||
) -> VisionResult<Vec<i32>> {
|
||||
let mut matches = vec![-1i32; anchors.len()]; // -1: ignore, 0: negative, >0: positive (target idx + 1)
|
||||
|
||||
for (anchor_idx, anchor) in anchors.iter().enumerate() {
|
||||
let mut best_iou = 0.0;
|
||||
let mut best_target_idx = None;
|
||||
|
||||
for (target_idx, target) in targets.iter().enumerate() {
|
||||
let iou = anchor.iou(target);
|
||||
|
||||
if iou > best_iou {
|
||||
best_iou = iou;
|
||||
best_target_idx = Some(target_idx);
|
||||
}
|
||||
}
|
||||
|
||||
if best_iou < self.low_threshold {
|
||||
matches[anchor_idx] = 0; // Negative
|
||||
} else if best_iou >= self.high_threshold {
|
||||
matches[anchor_idx] = (best_target_idx.unwrap() + 1) as i32; // Positive
|
||||
}
|
||||
// else: ignore (keep -1)
|
||||
}
|
||||
|
||||
Ok(matches)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user