511 lines
16 KiB
Rust
511 lines
16 KiB
Rust
//! Tests for TenantManager - Per-tenant isolation and resource quotas
|
|
//! Following strict TDD: These are FAILING tests written FIRST
|
|
|
|
use chrono::Utc;
|
|
use rtx_platform::{
|
|
PlatformConfig, PlatformError,
|
|
error::TenantError,
|
|
tenant::{IsolationLevel, ResourceQuota, TenantConfig, TenantManager},
|
|
};
|
|
use std::collections::HashMap;
|
|
use uuid::Uuid;
|
|
|
|
/// Create test configuration
|
|
fn create_test_config() -> PlatformConfig {
|
|
// Use the same config as region tests for consistency
|
|
let mut regions = HashMap::new();
|
|
regions.insert(
|
|
"us-east-1".to_string(),
|
|
rtx_platform::region::RegionConfig {
|
|
id: "us-east-1".to_string(),
|
|
name: "US East 1".to_string(),
|
|
endpoint: "https://us-east-1.rustytorch.com".to_string(),
|
|
capacity_limits: HashMap::from([
|
|
("gpu".to_string(), 1000),
|
|
("memory_gb".to_string(), 10000),
|
|
]),
|
|
availability_zone_count: 3,
|
|
latency_targets_ms: HashMap::new(),
|
|
},
|
|
);
|
|
|
|
PlatformConfig {
|
|
database_url: "postgresql://test:test@localhost/test".to_string(),
|
|
redis_urls: vec!["redis://localhost:6379".to_string()],
|
|
kafka_brokers: vec!["localhost:9092".to_string()],
|
|
metrics_endpoint: "http://localhost:9090".to_string(),
|
|
regions,
|
|
sla_targets: rtx_platform::slo::SlaTargets {
|
|
availability_percentage: 99.95,
|
|
max_latency_p50_ms: 100,
|
|
max_latency_p99_ms: 500,
|
|
max_latency_p999_ms: 2000,
|
|
max_error_rate_percentage: 0.1,
|
|
error_budget_burn_rate_threshold: 2.0,
|
|
downtime_budget_minutes_per_month: 43.8,
|
|
},
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_tenant_manager_creation_and_basic_operations() {
|
|
let config = create_test_config();
|
|
|
|
// This test WILL FAIL initially - TenantManager doesn't exist yet
|
|
let mut manager = TenantManager::new(&config)
|
|
.await
|
|
.expect("Failed to create TenantManager");
|
|
manager
|
|
.start()
|
|
.await
|
|
.expect("Failed to start TenantManager");
|
|
|
|
// Create a new tenant
|
|
let tenant_id = Uuid::new_v4();
|
|
let tenant_config = TenantConfig {
|
|
id: tenant_id,
|
|
name: "Test Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Strict,
|
|
resource_quotas: HashMap::from([
|
|
(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 100,
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: Some(150),
|
|
},
|
|
),
|
|
(
|
|
"memory_gb".to_string(),
|
|
ResourceQuota {
|
|
limit: 1000,
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: Some(1500),
|
|
},
|
|
),
|
|
]),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
manager
|
|
.create_tenant(tenant_config)
|
|
.await
|
|
.expect("Failed to create tenant");
|
|
|
|
// Verify tenant was created
|
|
let retrieved_tenant = manager
|
|
.get_tenant(tenant_id)
|
|
.await
|
|
.expect("Failed to get tenant");
|
|
assert_eq!(retrieved_tenant.id, tenant_id);
|
|
assert_eq!(retrieved_tenant.name, "Test Tenant");
|
|
assert_eq!(retrieved_tenant.isolation_level, IsolationLevel::Strict);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_resource_quota_enforcement() {
|
|
let config = create_test_config();
|
|
let mut manager = TenantManager::new(&config).await.unwrap();
|
|
manager.start().await.unwrap();
|
|
|
|
let tenant_id = Uuid::new_v4();
|
|
let tenant_config = TenantConfig {
|
|
id: tenant_id,
|
|
name: "Quota Test Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Standard,
|
|
resource_quotas: HashMap::from([(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 50, // Small limit for testing
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: Some(70),
|
|
},
|
|
)]),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
manager.create_tenant(tenant_config).await.unwrap();
|
|
|
|
// This test WILL FAIL - quota enforcement not implemented
|
|
// Request within quota - should succeed
|
|
let allocation_result = manager
|
|
.allocate_resources(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 30)]),
|
|
)
|
|
.await;
|
|
assert!(allocation_result.is_ok());
|
|
|
|
let allocation = allocation_result.unwrap();
|
|
assert_eq!(allocation.allocated_resources.get("gpu"), Some(&30));
|
|
|
|
// Check quota usage updated
|
|
let updated_tenant = manager.get_tenant(tenant_id).await.unwrap();
|
|
assert_eq!(updated_tenant.resource_quotas.get("gpu").unwrap().used, 30);
|
|
|
|
// Request that would exceed quota - should fail
|
|
let over_quota_result = manager
|
|
.allocate_resources(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 25)]), // 30 + 25 = 55 > 50 limit
|
|
)
|
|
.await;
|
|
|
|
assert!(matches!(
|
|
over_quota_result,
|
|
Err(PlatformError::Tenant(TenantError::QuotaExceeded { .. }))
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_burst_quota_mechanism() {
|
|
let config = create_test_config();
|
|
let mut manager = TenantManager::new(&config).await.unwrap();
|
|
manager.start().await.unwrap();
|
|
|
|
let tenant_id = Uuid::new_v4();
|
|
let tenant_config = TenantConfig {
|
|
id: tenant_id,
|
|
name: "Burst Test Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Standard,
|
|
resource_quotas: HashMap::from([(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 40,
|
|
used: 35, // Already near limit
|
|
reserved: 0,
|
|
burst_limit: Some(60), // Allows bursting
|
|
},
|
|
)]),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 2, // Higher priority for burst
|
|
};
|
|
|
|
manager.create_tenant(tenant_config).await.unwrap();
|
|
|
|
// This test WILL FAIL - burst quota mechanism not implemented
|
|
// Request that exceeds normal quota but within burst - should succeed temporarily
|
|
let burst_allocation = manager
|
|
.allocate_resources_with_burst(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 20)]), // 35 + 20 = 55, exceeds 40 but within 60
|
|
std::time::Duration::from_secs(300), // 5 minutes burst duration
|
|
)
|
|
.await;
|
|
|
|
assert!(burst_allocation.is_ok());
|
|
let allocation = burst_allocation.unwrap();
|
|
assert!(allocation.is_burst_allocation);
|
|
assert!(allocation.burst_expires_at.is_some());
|
|
|
|
// Verify burst quota tracking
|
|
let tenant = manager.get_tenant(tenant_id).await.unwrap();
|
|
assert_eq!(tenant.resource_quotas.get("gpu").unwrap().used, 55);
|
|
|
|
// Check burst expiration triggers cleanup
|
|
manager
|
|
.cleanup_expired_burst_allocations()
|
|
.await
|
|
.expect("Failed to cleanup burst");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_tenant_isolation_enforcement() {
|
|
let config = create_test_config();
|
|
let mut manager = TenantManager::new(&config).await.unwrap();
|
|
manager.start().await.unwrap();
|
|
|
|
// Create two tenants with different isolation levels
|
|
let tenant_a = Uuid::new_v4();
|
|
let tenant_b = Uuid::new_v4();
|
|
|
|
let strict_tenant = TenantConfig {
|
|
id: tenant_a,
|
|
name: "Strict Isolation Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Strict, // No resource sharing
|
|
resource_quotas: HashMap::from([(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 50,
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: None,
|
|
},
|
|
)]),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
let standard_tenant = TenantConfig {
|
|
id: tenant_b,
|
|
name: "Standard Isolation Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Standard, // Allows some sharing
|
|
resource_quotas: HashMap::from([(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 50,
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: None,
|
|
},
|
|
)]),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
manager.create_tenant(strict_tenant).await.unwrap();
|
|
manager.create_tenant(standard_tenant).await.unwrap();
|
|
|
|
// This test WILL FAIL - isolation enforcement not implemented
|
|
// Allocate resources for strict tenant
|
|
manager
|
|
.allocate_resources(
|
|
tenant_a,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 25)]),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
// Verify isolation - strict tenant resources should be completely isolated
|
|
let isolation_check = manager.check_isolation_violations(tenant_a).await.unwrap();
|
|
assert_eq!(isolation_check.violations.len(), 0);
|
|
assert!(isolation_check.has_dedicated_resources);
|
|
|
|
// Attempt to access strict tenant's resources from another tenant should fail
|
|
let cross_access_result = manager
|
|
.attempt_cross_tenant_access(tenant_b, tenant_a, "read_tensor_data")
|
|
.await;
|
|
|
|
assert!(matches!(
|
|
cross_access_result,
|
|
Err(PlatformError::Tenant(
|
|
TenantError::IsolationViolation { .. }
|
|
))
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_tenant_authentication_and_authorization() {
|
|
let config = create_test_config();
|
|
let mut manager = TenantManager::new(&config).await.unwrap();
|
|
manager.start().await.unwrap();
|
|
|
|
let tenant_id = Uuid::new_v4();
|
|
let tenant_config = TenantConfig {
|
|
id: tenant_id,
|
|
name: "Auth Test Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Standard,
|
|
resource_quotas: HashMap::new(),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
manager.create_tenant(tenant_config).await.unwrap();
|
|
|
|
// This test WILL FAIL - authentication not implemented
|
|
// Generate API key for tenant
|
|
let api_key = manager
|
|
.generate_tenant_api_key(
|
|
tenant_id,
|
|
vec!["compute:execute".to_string(), "data:read".to_string()],
|
|
Some(chrono::Utc::now() + chrono::Duration::hours(24)), // Expires in 24h
|
|
)
|
|
.await
|
|
.expect("Failed to generate API key");
|
|
|
|
// Verify authentication with valid key
|
|
let auth_result = manager
|
|
.authenticate_request(&api_key.key, "compute:execute")
|
|
.await;
|
|
assert!(auth_result.is_ok());
|
|
|
|
let auth_context = auth_result.unwrap();
|
|
assert_eq!(auth_context.tenant_id, tenant_id);
|
|
assert!(
|
|
auth_context
|
|
.permissions
|
|
.contains(&"compute:execute".to_string())
|
|
);
|
|
|
|
// Test authorization failure for insufficient permissions
|
|
let unauth_result = manager
|
|
.authenticate_request(&api_key.key, "admin:delete")
|
|
.await;
|
|
assert!(matches!(
|
|
unauth_result,
|
|
Err(PlatformError::Tenant(TenantError::AuthFailed { .. }))
|
|
));
|
|
|
|
// Test expired key handling
|
|
let expired_key = manager
|
|
.generate_tenant_api_key(
|
|
tenant_id,
|
|
vec!["compute:execute".to_string()],
|
|
Some(chrono::Utc::now() - chrono::Duration::hours(1)), // Already expired
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
let expired_auth_result = manager
|
|
.authenticate_request(&expired_key.key, "compute:execute")
|
|
.await;
|
|
assert!(expired_auth_result.is_err());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_resource_reservation_system() {
|
|
let config = create_test_config();
|
|
let mut manager = TenantManager::new(&config).await.unwrap();
|
|
manager.start().await.unwrap();
|
|
|
|
let tenant_id = Uuid::new_v4();
|
|
let tenant_config = TenantConfig {
|
|
id: tenant_id,
|
|
name: "Reservation Test Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Standard,
|
|
resource_quotas: HashMap::from([(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 100,
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: None,
|
|
},
|
|
)]),
|
|
allowed_regions: vec!["us-east-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
manager.create_tenant(tenant_config).await.unwrap();
|
|
|
|
// This test WILL FAIL - reservation system not implemented
|
|
// Create a future reservation
|
|
let reservation_time = chrono::Utc::now() + chrono::Duration::hours(2);
|
|
let reservation = manager
|
|
.create_resource_reservation(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 50)]),
|
|
reservation_time,
|
|
chrono::Duration::hours(4), // 4-hour reservation
|
|
)
|
|
.await
|
|
.expect("Failed to create reservation");
|
|
|
|
assert!(reservation.reservation_id.is_some());
|
|
assert_eq!(reservation.scheduled_start, reservation_time);
|
|
|
|
// Check that quota shows reserved resources
|
|
let tenant = manager.get_tenant(tenant_id).await.unwrap();
|
|
assert_eq!(tenant.resource_quotas.get("gpu").unwrap().reserved, 50);
|
|
|
|
// Try to reserve more than quota allows - should fail
|
|
let over_reservation_result = manager
|
|
.create_resource_reservation(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 60)]), // 50 + 60 = 110 > 100 limit
|
|
chrono::Utc::now() + chrono::Duration::hours(3),
|
|
chrono::Duration::hours(2),
|
|
)
|
|
.await;
|
|
|
|
assert!(matches!(
|
|
over_reservation_result,
|
|
Err(PlatformError::Tenant(TenantError::QuotaExceeded { .. }))
|
|
));
|
|
|
|
// Test reservation activation
|
|
manager
|
|
.activate_reservation(reservation.reservation_id.unwrap())
|
|
.await
|
|
.expect("Failed to activate reservation");
|
|
|
|
let updated_tenant = manager.get_tenant(tenant_id).await.unwrap();
|
|
assert_eq!(updated_tenant.resource_quotas.get("gpu").unwrap().used, 50);
|
|
assert_eq!(
|
|
updated_tenant.resource_quotas.get("gpu").unwrap().reserved,
|
|
0
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_multi_region_tenant_coordination() {
|
|
let config = create_test_config();
|
|
let mut manager = TenantManager::new(&config).await.unwrap();
|
|
manager.start().await.unwrap();
|
|
|
|
let tenant_id = Uuid::new_v4();
|
|
let tenant_config = TenantConfig {
|
|
id: tenant_id,
|
|
name: "Multi-Region Tenant".to_string(),
|
|
created_at: Utc::now(),
|
|
isolation_level: IsolationLevel::Standard,
|
|
resource_quotas: HashMap::from([(
|
|
"gpu".to_string(),
|
|
ResourceQuota {
|
|
limit: 200, // Global quota across regions
|
|
used: 0,
|
|
reserved: 0,
|
|
burst_limit: None,
|
|
},
|
|
)]),
|
|
allowed_regions: vec!["us-east-1".to_string(), "us-west-1".to_string()],
|
|
priority: 1,
|
|
};
|
|
|
|
manager.create_tenant(tenant_config).await.unwrap();
|
|
|
|
// This test WILL FAIL - multi-region coordination not implemented
|
|
// Allocate resources in multiple regions simultaneously
|
|
let (east_result, west_result) = tokio::join!(
|
|
manager.allocate_resources(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 80)]),
|
|
),
|
|
manager.allocate_resources(
|
|
tenant_id,
|
|
"us-west-1",
|
|
HashMap::from([("gpu".to_string(), 70)]),
|
|
)
|
|
);
|
|
|
|
assert!(east_result.is_ok());
|
|
assert!(west_result.is_ok());
|
|
|
|
// Verify global quota tracking across regions
|
|
let tenant = manager.get_tenant(tenant_id).await.unwrap();
|
|
assert_eq!(tenant.resource_quotas.get("gpu").unwrap().used, 150); // 80 + 70
|
|
|
|
// Attempt allocation that would exceed global quota
|
|
let over_global_quota = manager
|
|
.allocate_resources(
|
|
tenant_id,
|
|
"us-east-1",
|
|
HashMap::from([("gpu".to_string(), 60)]), // 150 + 60 = 210 > 200
|
|
)
|
|
.await;
|
|
|
|
assert!(matches!(
|
|
over_global_quota,
|
|
Err(PlatformError::Tenant(TenantError::QuotaExceeded { .. }))
|
|
));
|
|
}
|