687 lines
19 KiB
Rust
687 lines
19 KiB
Rust
//! # Governance Pipeline
|
|
//!
|
|
//! Provides SBOM generation, provenance tracking, compliance validation,
|
|
//! and audit logging for the unified graph system.
|
|
//!
|
|
//! ## Features
|
|
//! - Software Bill of Materials (SBOM) generation
|
|
//! - Data and model provenance tracking
|
|
//! - Compliance validation against policies
|
|
//! - Comprehensive audit logging
|
|
//! - Cryptographic integrity verification
|
|
|
|
use crate::{NodeId, Result};
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::{BTreeMap, HashMap};
|
|
use uuid::Uuid;
|
|
|
|
/// Software Bill of Materials entry
|
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
pub struct SbomEntry {
|
|
pub name: String,
|
|
pub version: String,
|
|
pub supplier: Option<String>,
|
|
pub license: Option<String>,
|
|
pub checksum: String,
|
|
pub source_location: Option<String>,
|
|
pub dependencies: Vec<String>,
|
|
}
|
|
|
|
/// Software Bill of Materials for a graph or component
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Sbom {
|
|
pub id: Uuid,
|
|
pub name: String,
|
|
pub version: String,
|
|
pub creation_timestamp: DateTime<Utc>,
|
|
pub components: Vec<SbomEntry>,
|
|
pub relationships: Vec<SbomRelationship>,
|
|
pub metadata: BTreeMap<String, String>,
|
|
}
|
|
|
|
/// Relationship between SBOM components
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SbomRelationship {
|
|
pub source: String,
|
|
pub target: String,
|
|
pub relationship_type: RelationshipType,
|
|
}
|
|
|
|
/// Types of relationships between components
|
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
pub enum RelationshipType {
|
|
DependsOn,
|
|
Contains,
|
|
GeneratedFrom,
|
|
DerivedFrom,
|
|
}
|
|
|
|
/// Provenance information for data or models
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ProvenanceRecord {
|
|
pub id: Uuid,
|
|
pub resource_id: String,
|
|
pub resource_type: ResourceType,
|
|
pub creation_timestamp: DateTime<Utc>,
|
|
pub creator: String,
|
|
pub source_data: Vec<ProvenanceLink>,
|
|
pub transformations: Vec<TransformationRecord>,
|
|
pub validation_results: Vec<ValidationResult>,
|
|
pub integrity_hash: String,
|
|
pub metadata: BTreeMap<String, String>,
|
|
}
|
|
|
|
/// Type of resource being tracked
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum ResourceType {
|
|
Dataset,
|
|
Model,
|
|
Artifact,
|
|
Computation,
|
|
}
|
|
|
|
/// Link to source data in provenance chain
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ProvenanceLink {
|
|
pub resource_id: String,
|
|
pub relationship: String,
|
|
pub timestamp: DateTime<Utc>,
|
|
}
|
|
|
|
/// Record of a transformation applied to data
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TransformationRecord {
|
|
pub operation: String,
|
|
pub parameters: BTreeMap<String, String>,
|
|
pub timestamp: DateTime<Utc>,
|
|
pub node_id: Option<NodeId>,
|
|
}
|
|
|
|
/// Result of compliance or validation check
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ValidationResult {
|
|
pub check_type: String,
|
|
pub status: ValidationStatus,
|
|
pub message: String,
|
|
pub timestamp: DateTime<Utc>,
|
|
pub details: BTreeMap<String, String>,
|
|
}
|
|
|
|
/// Status of validation check
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum ValidationStatus {
|
|
Passed,
|
|
Failed,
|
|
Warning,
|
|
Skipped,
|
|
}
|
|
|
|
/// Audit log entry
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AuditLogEntry {
|
|
pub id: Uuid,
|
|
pub timestamp: DateTime<Utc>,
|
|
pub event_type: AuditEventType,
|
|
pub actor: String,
|
|
pub resource: String,
|
|
pub action: String,
|
|
pub outcome: AuditOutcome,
|
|
pub context: BTreeMap<String, String>,
|
|
}
|
|
|
|
/// Type of audit event
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum AuditEventType {
|
|
Access,
|
|
Modification,
|
|
Creation,
|
|
Deletion,
|
|
Execution,
|
|
Validation,
|
|
}
|
|
|
|
/// Outcome of audited action
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum AuditOutcome {
|
|
Success,
|
|
Failure,
|
|
Partial,
|
|
}
|
|
|
|
/// Compliance policy definition
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct CompliancePolicy {
|
|
pub id: String,
|
|
pub name: String,
|
|
pub description: String,
|
|
pub rules: Vec<ComplianceRule>,
|
|
pub severity: PolicySeverity,
|
|
}
|
|
|
|
/// Individual compliance rule
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ComplianceRule {
|
|
pub id: String,
|
|
pub description: String,
|
|
pub rule_type: RuleType,
|
|
pub parameters: BTreeMap<String, String>,
|
|
}
|
|
|
|
/// Type of compliance rule
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum RuleType {
|
|
DataRetention,
|
|
AccessControl,
|
|
DataClassification,
|
|
AuditRequirement,
|
|
IntegrityCheck,
|
|
}
|
|
|
|
/// Severity level for policy violations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum PolicySeverity {
|
|
Critical,
|
|
High,
|
|
Medium,
|
|
Low,
|
|
Info,
|
|
}
|
|
|
|
/// Main governance pipeline
|
|
#[derive(Debug)]
|
|
pub struct GovernancePipeline {
|
|
sbom_generator: SbomGenerator,
|
|
provenance_tracker: ProvenanceTracker,
|
|
compliance_validator: ComplianceValidator,
|
|
audit_logger: AuditLogger,
|
|
}
|
|
|
|
/// SBOM generation component
|
|
#[derive(Debug)]
|
|
pub struct SbomGenerator {
|
|
component_registry: HashMap<String, SbomEntry>,
|
|
}
|
|
|
|
/// Provenance tracking component
|
|
#[derive(Debug)]
|
|
pub struct ProvenanceTracker {
|
|
records: HashMap<String, ProvenanceRecord>,
|
|
}
|
|
|
|
/// Compliance validation component
|
|
#[derive(Debug)]
|
|
pub struct ComplianceValidator {
|
|
policies: Vec<CompliancePolicy>,
|
|
}
|
|
|
|
/// Audit logging component
|
|
#[derive(Debug)]
|
|
pub struct AuditLogger {
|
|
entries: Vec<AuditLogEntry>,
|
|
}
|
|
|
|
impl GovernancePipeline {
|
|
/// Create new governance pipeline
|
|
pub fn new() -> Self {
|
|
Self {
|
|
sbom_generator: SbomGenerator::new(),
|
|
provenance_tracker: ProvenanceTracker::new(),
|
|
compliance_validator: ComplianceValidator::new(),
|
|
audit_logger: AuditLogger::new(),
|
|
}
|
|
}
|
|
|
|
/// Generate SBOM for graph components
|
|
pub fn generate_sbom(
|
|
&mut self,
|
|
name: String,
|
|
version: String,
|
|
components: Vec<SbomEntry>,
|
|
) -> Result<Sbom> {
|
|
self.sbom_generator.generate_sbom(name, version, components)
|
|
}
|
|
|
|
/// Track provenance for a resource
|
|
pub fn track_provenance(&mut self, record: ProvenanceRecord) -> Result<()> {
|
|
self.provenance_tracker.add_record(record)
|
|
}
|
|
|
|
/// Validate compliance against policies
|
|
pub fn validate_compliance(&self, resource_id: &str) -> Result<Vec<ValidationResult>> {
|
|
self.compliance_validator.validate(resource_id)
|
|
}
|
|
|
|
/// Log audit event
|
|
pub fn log_audit_event(&mut self, entry: AuditLogEntry) -> Result<()> {
|
|
self.audit_logger.log_entry(entry)
|
|
}
|
|
|
|
/// Get provenance record by resource ID
|
|
pub fn get_provenance(&self, resource_id: &str) -> Option<&ProvenanceRecord> {
|
|
self.provenance_tracker.get_record(resource_id)
|
|
}
|
|
|
|
/// Get audit log entries
|
|
pub fn get_audit_log(&self) -> &[AuditLogEntry] {
|
|
self.audit_logger.get_entries()
|
|
}
|
|
}
|
|
|
|
impl SbomGenerator {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
component_registry: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
pub fn generate_sbom(
|
|
&mut self,
|
|
name: String,
|
|
version: String,
|
|
components: Vec<SbomEntry>,
|
|
) -> Result<Sbom> {
|
|
let id = Uuid::new_v4();
|
|
let creation_timestamp = Utc::now();
|
|
let relationships = self.extract_relationships(&components)?;
|
|
let metadata = BTreeMap::new();
|
|
|
|
// Register components
|
|
for component in &components {
|
|
self.component_registry
|
|
.insert(component.name.clone(), component.clone());
|
|
}
|
|
|
|
Ok(Sbom {
|
|
id,
|
|
name,
|
|
version,
|
|
creation_timestamp,
|
|
components,
|
|
relationships,
|
|
metadata,
|
|
})
|
|
}
|
|
|
|
fn extract_relationships(&self, components: &[SbomEntry]) -> Result<Vec<SbomRelationship>> {
|
|
let mut relationships = Vec::new();
|
|
|
|
for component in components {
|
|
for dep in &component.dependencies {
|
|
relationships.push(SbomRelationship {
|
|
source: component.name.clone(),
|
|
target: dep.clone(),
|
|
relationship_type: RelationshipType::DependsOn,
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(relationships)
|
|
}
|
|
}
|
|
|
|
impl ProvenanceTracker {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
records: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
pub fn add_record(&mut self, record: ProvenanceRecord) -> Result<()> {
|
|
self.records.insert(record.resource_id.clone(), record);
|
|
Ok(())
|
|
}
|
|
|
|
pub fn get_record(&self, resource_id: &str) -> Option<&ProvenanceRecord> {
|
|
self.records.get(resource_id)
|
|
}
|
|
|
|
pub fn create_record(
|
|
&mut self,
|
|
resource_id: String,
|
|
resource_type: ResourceType,
|
|
creator: String,
|
|
) -> Result<Uuid> {
|
|
let id = Uuid::new_v4();
|
|
let record = ProvenanceRecord {
|
|
id,
|
|
resource_id: resource_id.clone(),
|
|
resource_type,
|
|
creation_timestamp: Utc::now(),
|
|
creator,
|
|
source_data: Vec::new(),
|
|
transformations: Vec::new(),
|
|
validation_results: Vec::new(),
|
|
integrity_hash: self.compute_integrity_hash(&resource_id)?,
|
|
metadata: BTreeMap::new(),
|
|
};
|
|
|
|
self.add_record(record)?;
|
|
Ok(id)
|
|
}
|
|
|
|
fn compute_integrity_hash(&self, resource_id: &str) -> Result<String> {
|
|
use std::collections::hash_map::DefaultHasher;
|
|
use std::hash::{Hash, Hasher};
|
|
|
|
let mut hasher = DefaultHasher::new();
|
|
resource_id.hash(&mut hasher);
|
|
Utc::now().timestamp().hash(&mut hasher);
|
|
Ok(format!("{:x}", hasher.finish()))
|
|
}
|
|
}
|
|
|
|
impl ComplianceValidator {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
policies: Vec::new(),
|
|
}
|
|
}
|
|
|
|
pub fn add_policy(&mut self, policy: CompliancePolicy) {
|
|
self.policies.push(policy);
|
|
}
|
|
|
|
pub fn validate(&self, resource_id: &str) -> Result<Vec<ValidationResult>> {
|
|
let mut results = Vec::new();
|
|
|
|
for policy in &self.policies {
|
|
for rule in &policy.rules {
|
|
let result = self.validate_rule(rule, resource_id)?;
|
|
results.push(result);
|
|
}
|
|
}
|
|
|
|
Ok(results)
|
|
}
|
|
|
|
fn validate_rule(&self, rule: &ComplianceRule, resource_id: &str) -> Result<ValidationResult> {
|
|
// Basic validation logic - in practice this would be more sophisticated
|
|
let status = match rule.rule_type {
|
|
RuleType::DataRetention => ValidationStatus::Passed,
|
|
RuleType::AccessControl => ValidationStatus::Passed,
|
|
RuleType::DataClassification => ValidationStatus::Warning,
|
|
RuleType::AuditRequirement => ValidationStatus::Passed,
|
|
RuleType::IntegrityCheck => ValidationStatus::Passed,
|
|
};
|
|
|
|
Ok(ValidationResult {
|
|
check_type: rule.id.clone(),
|
|
status,
|
|
message: format!("Validation of {} against rule {}", resource_id, rule.id),
|
|
timestamp: Utc::now(),
|
|
details: BTreeMap::new(),
|
|
})
|
|
}
|
|
}
|
|
|
|
impl AuditLogger {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
entries: Vec::new(),
|
|
}
|
|
}
|
|
|
|
pub fn log_entry(&mut self, entry: AuditLogEntry) -> Result<()> {
|
|
self.entries.push(entry);
|
|
Ok(())
|
|
}
|
|
|
|
pub fn get_entries(&self) -> &[AuditLogEntry] {
|
|
&self.entries
|
|
}
|
|
|
|
pub fn create_entry(
|
|
&mut self,
|
|
event_type: AuditEventType,
|
|
actor: String,
|
|
resource: String,
|
|
action: String,
|
|
outcome: AuditOutcome,
|
|
) -> Result<Uuid> {
|
|
let id = Uuid::new_v4();
|
|
let entry = AuditLogEntry {
|
|
id,
|
|
timestamp: Utc::now(),
|
|
event_type,
|
|
actor,
|
|
resource,
|
|
action,
|
|
outcome,
|
|
context: BTreeMap::new(),
|
|
};
|
|
|
|
self.log_entry(entry)?;
|
|
Ok(id)
|
|
}
|
|
}
|
|
|
|
impl Default for GovernancePipeline {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for SbomGenerator {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for ProvenanceTracker {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for ComplianceValidator {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl Default for AuditLogger {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_sbom_generation() {
|
|
let mut pipeline = GovernancePipeline::new();
|
|
|
|
let components = vec![
|
|
SbomEntry {
|
|
name: "rtx-tensor".to_string(),
|
|
version: "1.0.0".to_string(),
|
|
supplier: Some("RustyTorch++".to_string()),
|
|
license: Some("MIT".to_string()),
|
|
checksum: "abc123".to_string(),
|
|
source_location: Some("/crates/rtx-tensor".to_string()),
|
|
dependencies: vec!["serde".to_string()],
|
|
},
|
|
SbomEntry {
|
|
name: "serde".to_string(),
|
|
version: "1.0.136".to_string(),
|
|
supplier: Some("David Tolnay".to_string()),
|
|
license: Some("MIT".to_string()),
|
|
checksum: "def456".to_string(),
|
|
source_location: Some("https://crates.io/crates/serde".to_string()),
|
|
dependencies: vec![],
|
|
},
|
|
];
|
|
|
|
let result = pipeline.generate_sbom(
|
|
"test-graph".to_string(),
|
|
"1.0.0".to_string(),
|
|
components.clone(),
|
|
);
|
|
assert!(result.is_ok());
|
|
|
|
let sbom = result.unwrap();
|
|
assert_eq!(sbom.name, "test-graph");
|
|
assert_eq!(sbom.version, "1.0.0");
|
|
assert_eq!(sbom.components.len(), 2);
|
|
assert_eq!(sbom.relationships.len(), 1);
|
|
assert_eq!(
|
|
sbom.relationships[0].relationship_type,
|
|
RelationshipType::DependsOn
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_provenance_tracking() {
|
|
let mut pipeline = GovernancePipeline::new();
|
|
|
|
let record_id = pipeline
|
|
.provenance_tracker
|
|
.create_record(
|
|
"dataset-123".to_string(),
|
|
ResourceType::Dataset,
|
|
"test-user".to_string(),
|
|
)
|
|
.unwrap();
|
|
|
|
let record = pipeline.get_provenance("dataset-123");
|
|
assert!(record.is_some());
|
|
|
|
let record = record.unwrap();
|
|
assert_eq!(record.id, record_id);
|
|
assert_eq!(record.resource_id, "dataset-123");
|
|
assert_eq!(record.creator, "test-user");
|
|
assert!(!record.integrity_hash.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_compliance_validation() {
|
|
let mut pipeline = GovernancePipeline::new();
|
|
|
|
let policy = CompliancePolicy {
|
|
id: "data-retention".to_string(),
|
|
name: "Data Retention Policy".to_string(),
|
|
description: "Ensures data is retained according to policy".to_string(),
|
|
rules: vec![ComplianceRule {
|
|
id: "retention-30d".to_string(),
|
|
description: "Data must be retained for 30 days".to_string(),
|
|
rule_type: RuleType::DataRetention,
|
|
parameters: BTreeMap::new(),
|
|
}],
|
|
severity: PolicySeverity::High,
|
|
};
|
|
|
|
pipeline.compliance_validator.add_policy(policy);
|
|
|
|
let results = pipeline.validate_compliance("test-resource").unwrap();
|
|
assert_eq!(results.len(), 1);
|
|
assert_eq!(results[0].check_type, "retention-30d");
|
|
assert!(matches!(results[0].status, ValidationStatus::Passed));
|
|
}
|
|
|
|
#[test]
|
|
fn test_audit_logging() {
|
|
let mut pipeline = GovernancePipeline::new();
|
|
|
|
let entry_id = pipeline
|
|
.audit_logger
|
|
.create_entry(
|
|
AuditEventType::Access,
|
|
"user-123".to_string(),
|
|
"dataset-456".to_string(),
|
|
"read".to_string(),
|
|
AuditOutcome::Success,
|
|
)
|
|
.unwrap();
|
|
|
|
let entries = pipeline.get_audit_log();
|
|
assert_eq!(entries.len(), 1);
|
|
assert_eq!(entries[0].id, entry_id);
|
|
assert_eq!(entries[0].actor, "user-123");
|
|
assert_eq!(entries[0].resource, "dataset-456");
|
|
assert_eq!(entries[0].action, "read");
|
|
assert!(matches!(entries[0].outcome, AuditOutcome::Success));
|
|
}
|
|
|
|
#[test]
|
|
fn test_provenance_with_transformations() {
|
|
let mut tracker = ProvenanceTracker::new();
|
|
|
|
let record = ProvenanceRecord {
|
|
id: Uuid::new_v4(),
|
|
resource_id: "model-123".to_string(),
|
|
resource_type: ResourceType::Model,
|
|
creation_timestamp: Utc::now(),
|
|
creator: "ml-engineer".to_string(),
|
|
source_data: vec![ProvenanceLink {
|
|
resource_id: "dataset-456".to_string(),
|
|
relationship: "trained_on".to_string(),
|
|
timestamp: Utc::now(),
|
|
}],
|
|
transformations: vec![TransformationRecord {
|
|
operation: "normalize".to_string(),
|
|
parameters: {
|
|
let mut params = BTreeMap::new();
|
|
params.insert("mean".to_string(), "0.0".to_string());
|
|
params.insert("std".to_string(), "1.0".to_string());
|
|
params
|
|
},
|
|
timestamp: Utc::now(),
|
|
node_id: Some(NodeId::new()),
|
|
}],
|
|
validation_results: vec![],
|
|
integrity_hash: "hash123".to_string(),
|
|
metadata: BTreeMap::new(),
|
|
};
|
|
|
|
tracker.add_record(record.clone()).unwrap();
|
|
|
|
let retrieved = tracker.get_record("model-123").unwrap();
|
|
assert_eq!(retrieved.resource_id, "model-123");
|
|
assert_eq!(retrieved.source_data.len(), 1);
|
|
assert_eq!(retrieved.transformations.len(), 1);
|
|
assert_eq!(retrieved.transformations[0].operation, "normalize");
|
|
}
|
|
|
|
#[test]
|
|
fn test_compliance_policy_severity() {
|
|
let policy = CompliancePolicy {
|
|
id: "critical-policy".to_string(),
|
|
name: "Critical Security Policy".to_string(),
|
|
description: "Critical security requirements".to_string(),
|
|
rules: vec![],
|
|
severity: PolicySeverity::Critical,
|
|
};
|
|
|
|
assert!(matches!(policy.severity, PolicySeverity::Critical));
|
|
}
|
|
|
|
#[test]
|
|
fn test_audit_event_types() {
|
|
let events = vec![
|
|
AuditEventType::Access,
|
|
AuditEventType::Modification,
|
|
AuditEventType::Creation,
|
|
AuditEventType::Deletion,
|
|
AuditEventType::Execution,
|
|
AuditEventType::Validation,
|
|
];
|
|
|
|
assert_eq!(events.len(), 6);
|
|
}
|
|
|
|
#[test]
|
|
fn test_validation_status_variants() {
|
|
let statuses = vec![
|
|
ValidationStatus::Passed,
|
|
ValidationStatus::Failed,
|
|
ValidationStatus::Warning,
|
|
ValidationStatus::Skipped,
|
|
];
|
|
|
|
assert_eq!(statuses.len(), 4);
|
|
}
|
|
}
|