//! Tests for TenantManager - Per-tenant isolation and resource quotas //! Following strict TDD: These are FAILING tests written FIRST use chrono::Utc; use rtx_platform::{ PlatformConfig, PlatformError, error::TenantError, tenant::{IsolationLevel, ResourceQuota, TenantConfig, TenantManager}, }; use std::collections::HashMap; use uuid::Uuid; /// Create test configuration fn create_test_config() -> PlatformConfig { // Use the same config as region tests for consistency let mut regions = HashMap::new(); regions.insert( "us-east-1".to_string(), rtx_platform::region::RegionConfig { id: "us-east-1".to_string(), name: "US East 1".to_string(), endpoint: "https://us-east-1.rustytorch.com".to_string(), capacity_limits: HashMap::from([ ("gpu".to_string(), 1000), ("memory_gb".to_string(), 10000), ]), availability_zone_count: 3, latency_targets_ms: HashMap::new(), }, ); PlatformConfig { database_url: "postgresql://test:test@localhost/test".to_string(), redis_urls: vec!["redis://localhost:6379".to_string()], kafka_brokers: vec!["localhost:9092".to_string()], metrics_endpoint: "http://localhost:9090".to_string(), regions, sla_targets: rtx_platform::slo::SlaTargets { availability_percentage: 99.95, max_latency_p50_ms: 100, max_latency_p99_ms: 500, max_latency_p999_ms: 2000, max_error_rate_percentage: 0.1, error_budget_burn_rate_threshold: 2.0, downtime_budget_minutes_per_month: 43.8, }, } } #[tokio::test] async fn test_tenant_manager_creation_and_basic_operations() { let config = create_test_config(); // This test WILL FAIL initially - TenantManager doesn't exist yet let mut manager = TenantManager::new(&config) .await .expect("Failed to create TenantManager"); manager .start() .await .expect("Failed to start TenantManager"); // Create a new tenant let tenant_id = Uuid::new_v4(); let tenant_config = TenantConfig { id: tenant_id, name: "Test Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Strict, resource_quotas: HashMap::from([ ( "gpu".to_string(), ResourceQuota { limit: 100, used: 0, reserved: 0, burst_limit: Some(150), }, ), ( "memory_gb".to_string(), ResourceQuota { limit: 1000, used: 0, reserved: 0, burst_limit: Some(1500), }, ), ]), allowed_regions: vec!["us-east-1".to_string()], priority: 1, }; manager .create_tenant(tenant_config) .await .expect("Failed to create tenant"); // Verify tenant was created let retrieved_tenant = manager .get_tenant(tenant_id) .await .expect("Failed to get tenant"); assert_eq!(retrieved_tenant.id, tenant_id); assert_eq!(retrieved_tenant.name, "Test Tenant"); assert_eq!(retrieved_tenant.isolation_level, IsolationLevel::Strict); } #[tokio::test] async fn test_resource_quota_enforcement() { let config = create_test_config(); let mut manager = TenantManager::new(&config).await.unwrap(); manager.start().await.unwrap(); let tenant_id = Uuid::new_v4(); let tenant_config = TenantConfig { id: tenant_id, name: "Quota Test Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Standard, resource_quotas: HashMap::from([( "gpu".to_string(), ResourceQuota { limit: 50, // Small limit for testing used: 0, reserved: 0, burst_limit: Some(70), }, )]), allowed_regions: vec!["us-east-1".to_string()], priority: 1, }; manager.create_tenant(tenant_config).await.unwrap(); // This test WILL FAIL - quota enforcement not implemented // Request within quota - should succeed let allocation_result = manager .allocate_resources( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 30)]), ) .await; assert!(allocation_result.is_ok()); let allocation = allocation_result.unwrap(); assert_eq!(allocation.allocated_resources.get("gpu"), Some(&30)); // Check quota usage updated let updated_tenant = manager.get_tenant(tenant_id).await.unwrap(); assert_eq!(updated_tenant.resource_quotas.get("gpu").unwrap().used, 30); // Request that would exceed quota - should fail let over_quota_result = manager .allocate_resources( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 25)]), // 30 + 25 = 55 > 50 limit ) .await; assert!(matches!( over_quota_result, Err(PlatformError::Tenant(TenantError::QuotaExceeded { .. })) )); } #[tokio::test] async fn test_burst_quota_mechanism() { let config = create_test_config(); let mut manager = TenantManager::new(&config).await.unwrap(); manager.start().await.unwrap(); let tenant_id = Uuid::new_v4(); let tenant_config = TenantConfig { id: tenant_id, name: "Burst Test Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Standard, resource_quotas: HashMap::from([( "gpu".to_string(), ResourceQuota { limit: 40, used: 35, // Already near limit reserved: 0, burst_limit: Some(60), // Allows bursting }, )]), allowed_regions: vec!["us-east-1".to_string()], priority: 2, // Higher priority for burst }; manager.create_tenant(tenant_config).await.unwrap(); // This test WILL FAIL - burst quota mechanism not implemented // Request that exceeds normal quota but within burst - should succeed temporarily let burst_allocation = manager .allocate_resources_with_burst( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 20)]), // 35 + 20 = 55, exceeds 40 but within 60 std::time::Duration::from_secs(300), // 5 minutes burst duration ) .await; assert!(burst_allocation.is_ok()); let allocation = burst_allocation.unwrap(); assert!(allocation.is_burst_allocation); assert!(allocation.burst_expires_at.is_some()); // Verify burst quota tracking let tenant = manager.get_tenant(tenant_id).await.unwrap(); assert_eq!(tenant.resource_quotas.get("gpu").unwrap().used, 55); // Check burst expiration triggers cleanup manager .cleanup_expired_burst_allocations() .await .expect("Failed to cleanup burst"); } #[tokio::test] async fn test_tenant_isolation_enforcement() { let config = create_test_config(); let mut manager = TenantManager::new(&config).await.unwrap(); manager.start().await.unwrap(); // Create two tenants with different isolation levels let tenant_a = Uuid::new_v4(); let tenant_b = Uuid::new_v4(); let strict_tenant = TenantConfig { id: tenant_a, name: "Strict Isolation Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Strict, // No resource sharing resource_quotas: HashMap::from([( "gpu".to_string(), ResourceQuota { limit: 50, used: 0, reserved: 0, burst_limit: None, }, )]), allowed_regions: vec!["us-east-1".to_string()], priority: 1, }; let standard_tenant = TenantConfig { id: tenant_b, name: "Standard Isolation Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Standard, // Allows some sharing resource_quotas: HashMap::from([( "gpu".to_string(), ResourceQuota { limit: 50, used: 0, reserved: 0, burst_limit: None, }, )]), allowed_regions: vec!["us-east-1".to_string()], priority: 1, }; manager.create_tenant(strict_tenant).await.unwrap(); manager.create_tenant(standard_tenant).await.unwrap(); // This test WILL FAIL - isolation enforcement not implemented // Allocate resources for strict tenant manager .allocate_resources( tenant_a, "us-east-1", HashMap::from([("gpu".to_string(), 25)]), ) .await .unwrap(); // Verify isolation - strict tenant resources should be completely isolated let isolation_check = manager.check_isolation_violations(tenant_a).await.unwrap(); assert_eq!(isolation_check.violations.len(), 0); assert!(isolation_check.has_dedicated_resources); // Attempt to access strict tenant's resources from another tenant should fail let cross_access_result = manager .attempt_cross_tenant_access(tenant_b, tenant_a, "read_tensor_data") .await; assert!(matches!( cross_access_result, Err(PlatformError::Tenant( TenantError::IsolationViolation { .. } )) )); } #[tokio::test] async fn test_tenant_authentication_and_authorization() { let config = create_test_config(); let mut manager = TenantManager::new(&config).await.unwrap(); manager.start().await.unwrap(); let tenant_id = Uuid::new_v4(); let tenant_config = TenantConfig { id: tenant_id, name: "Auth Test Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Standard, resource_quotas: HashMap::new(), allowed_regions: vec!["us-east-1".to_string()], priority: 1, }; manager.create_tenant(tenant_config).await.unwrap(); // This test WILL FAIL - authentication not implemented // Generate API key for tenant let api_key = manager .generate_tenant_api_key( tenant_id, vec!["compute:execute".to_string(), "data:read".to_string()], Some(chrono::Utc::now() + chrono::Duration::hours(24)), // Expires in 24h ) .await .expect("Failed to generate API key"); // Verify authentication with valid key let auth_result = manager .authenticate_request(&api_key.key, "compute:execute") .await; assert!(auth_result.is_ok()); let auth_context = auth_result.unwrap(); assert_eq!(auth_context.tenant_id, tenant_id); assert!( auth_context .permissions .contains(&"compute:execute".to_string()) ); // Test authorization failure for insufficient permissions let unauth_result = manager .authenticate_request(&api_key.key, "admin:delete") .await; assert!(matches!( unauth_result, Err(PlatformError::Tenant(TenantError::AuthFailed { .. })) )); // Test expired key handling let expired_key = manager .generate_tenant_api_key( tenant_id, vec!["compute:execute".to_string()], Some(chrono::Utc::now() - chrono::Duration::hours(1)), // Already expired ) .await .unwrap(); let expired_auth_result = manager .authenticate_request(&expired_key.key, "compute:execute") .await; assert!(expired_auth_result.is_err()); } #[tokio::test] async fn test_resource_reservation_system() { let config = create_test_config(); let mut manager = TenantManager::new(&config).await.unwrap(); manager.start().await.unwrap(); let tenant_id = Uuid::new_v4(); let tenant_config = TenantConfig { id: tenant_id, name: "Reservation Test Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Standard, resource_quotas: HashMap::from([( "gpu".to_string(), ResourceQuota { limit: 100, used: 0, reserved: 0, burst_limit: None, }, )]), allowed_regions: vec!["us-east-1".to_string()], priority: 1, }; manager.create_tenant(tenant_config).await.unwrap(); // This test WILL FAIL - reservation system not implemented // Create a future reservation let reservation_time = chrono::Utc::now() + chrono::Duration::hours(2); let reservation = manager .create_resource_reservation( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 50)]), reservation_time, chrono::Duration::hours(4), // 4-hour reservation ) .await .expect("Failed to create reservation"); assert!(reservation.reservation_id.is_some()); assert_eq!(reservation.scheduled_start, reservation_time); // Check that quota shows reserved resources let tenant = manager.get_tenant(tenant_id).await.unwrap(); assert_eq!(tenant.resource_quotas.get("gpu").unwrap().reserved, 50); // Try to reserve more than quota allows - should fail let over_reservation_result = manager .create_resource_reservation( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 60)]), // 50 + 60 = 110 > 100 limit chrono::Utc::now() + chrono::Duration::hours(3), chrono::Duration::hours(2), ) .await; assert!(matches!( over_reservation_result, Err(PlatformError::Tenant(TenantError::QuotaExceeded { .. })) )); // Test reservation activation manager .activate_reservation(reservation.reservation_id.unwrap()) .await .expect("Failed to activate reservation"); let updated_tenant = manager.get_tenant(tenant_id).await.unwrap(); assert_eq!(updated_tenant.resource_quotas.get("gpu").unwrap().used, 50); assert_eq!( updated_tenant.resource_quotas.get("gpu").unwrap().reserved, 0 ); } #[tokio::test] async fn test_multi_region_tenant_coordination() { let config = create_test_config(); let mut manager = TenantManager::new(&config).await.unwrap(); manager.start().await.unwrap(); let tenant_id = Uuid::new_v4(); let tenant_config = TenantConfig { id: tenant_id, name: "Multi-Region Tenant".to_string(), created_at: Utc::now(), isolation_level: IsolationLevel::Standard, resource_quotas: HashMap::from([( "gpu".to_string(), ResourceQuota { limit: 200, // Global quota across regions used: 0, reserved: 0, burst_limit: None, }, )]), allowed_regions: vec!["us-east-1".to_string(), "us-west-1".to_string()], priority: 1, }; manager.create_tenant(tenant_config).await.unwrap(); // This test WILL FAIL - multi-region coordination not implemented // Allocate resources in multiple regions simultaneously let (east_result, west_result) = tokio::join!( manager.allocate_resources( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 80)]), ), manager.allocate_resources( tenant_id, "us-west-1", HashMap::from([("gpu".to_string(), 70)]), ) ); assert!(east_result.is_ok()); assert!(west_result.is_ok()); // Verify global quota tracking across regions let tenant = manager.get_tenant(tenant_id).await.unwrap(); assert_eq!(tenant.resource_quotas.get("gpu").unwrap().used, 150); // 80 + 70 // Attempt allocation that would exceed global quota let over_global_quota = manager .allocate_resources( tenant_id, "us-east-1", HashMap::from([("gpu".to_string(), 60)]), // 150 + 60 = 210 > 200 ) .await; assert!(matches!( over_global_quota, Err(PlatformError::Tenant(TenantError::QuotaExceeded { .. })) )); }