rustytorch/docs/book/src/deployment/containers.md

Docker & Kubernetes

Container deployment for RustyTorch++ applications.

Docker

Basic Dockerfile

# Build stage
FROM rust:1.92-bookworm AS builder

WORKDIR /app
COPY . .

RUN cargo build --release -p rtx-serving-api

# Runtime stage
FROM nvidia/cuda:12.0-runtime-ubuntu22.04

RUN apt-get update && apt-get install -y \
    libssl-dev \
    ca-certificates \
    && rm -rf /var/lib/apt/lists/*

COPY --from=builder /app/target/release/rtx-serving-api /usr/local/bin/

EXPOSE 8080
CMD ["rtx-serving-api"]

Multi-Stage GPU Build

# CUDA build stage
FROM nvidia/cuda:12.0-devel-ubuntu22.04 AS builder

# Install Rust
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"

WORKDIR /app
COPY . .

# Build with CUDA support
RUN cargo build --release --features cuda

# Runtime stage (smaller image)
FROM nvidia/cuda:12.0-runtime-ubuntu22.04

COPY --from=builder /app/target/release/rtx-serving-api /app/
COPY --from=builder /app/models /app/models

WORKDIR /app
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s \
    CMD curl -f http://localhost:8080/health || exit 1

CMD ["./rtx-serving-api"]

Docker Compose

version: '3.8'

services:
  rtx-api:
    build: .
    ports:
      - "8080:8080"
    volumes:
      - ./models:/app/models
      - model-cache:/app/cache
    environment:
      - RUST_LOG=info
      - MODEL_PATH=/app/models/model.bin
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  prometheus:
    image: prom/prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml

  grafana:
    image: grafana/grafana
    ports:
      - "3000:3000"

volumes:
  model-cache:

Kubernetes

Deployment

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rtx-inference
  labels:
    app: rtx-inference
spec:
  replicas: 3
  selector:
    matchLabels:
      app: rtx-inference
  template:
    metadata:
      labels:
        app: rtx-inference
    spec:
      containers:
      - name: rtx-api
        image: rustytorch/serving-api:latest
        ports:
        - containerPort: 8080
        resources:
          limits:
            nvidia.com/gpu: 1
            memory: "16Gi"
            cpu: "4"
          requests:
            nvidia.com/gpu: 1
            memory: "8Gi"
            cpu: "2"
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
        volumeMounts:
        - name: model-storage
          mountPath: /models
      volumes:
      - name: model-storage
        persistentVolumeClaim:
          claimName: model-pvc

Service

apiVersion: v1
kind: Service
metadata:
  name: rtx-inference
spec:
  selector:
    app: rtx-inference
  ports:
  - protocol: TCP
    port: 80
    targetPort: 8080
  type: LoadBalancer

Horizontal Pod Autoscaler

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: rtx-inference-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: rtx-inference
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Pods
    pods:
      metric:
        name: inference_queue_length
      target:
        type: AverageValue
        averageValue: "100"

GPU Node Pool (GKE)

# GKE GPU node pool configuration
apiVersion: container.google.com/v1
kind: NodePool
metadata:
  name: gpu-pool
spec:
  autoscaling:
    enabled: true
    minNodeCount: 0
    maxNodeCount: 10
  config:
    machineType: n1-standard-8
    accelerators:
    - acceleratorType: nvidia-tesla-t4
      acceleratorCount: 1
    taints:
    - key: nvidia.com/gpu
      value: present
      effect: NoSchedule

Model Storage

Persistent Volume

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: model-pvc
spec:
  accessModes:
    - ReadOnlyMany
  resources:
    requests:
      storage: 100Gi
  storageClassName: standard

Init Container for Model Download

initContainers:
- name: model-downloader
  image: curlimages/curl
  command:
  - sh
  - -c
  - |
    curl -o /models/model.bin https://hub.rustytorch.ai/models/v1/download
  volumeMounts:
  - name: model-storage
    mountPath: /models

Monitoring in Kubernetes

ServiceMonitor (Prometheus Operator)

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: rtx-inference
spec:
  selector:
    matchLabels:
      app: rtx-inference
  endpoints:
  - port: http
    path: /metrics
    interval: 15s

Next Steps

• Serving API - API configuration
• Monitoring - Performance monitoring