Audit & Compliance Service

import asyncio
import json
from typing import Dict, Any, List, Optional, Union
from dataclasses import dataclass, field, asdict
from datetime import datetime
from enum import Enum
import hashlib
import uuid
 
class AuditEventType(Enum):
    USER_ACTION = "user_action"
    API_ACCESS = "api_access"
    DATA_ACCESS = "data_access"
    DATA_MODIFICATION = "data_modification"
    SYSTEM_EVENT = "system_event"
    SECURITY_EVENT = "security_event"
    ADMIN_ACTION = "admin_action"
    COMPLIANCE_EVENT = "compliance_event"
 
class SeverityLevel(Enum):
    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    CRITICAL = "critical"
 
@dataclass
class AuditEvent:
    event_type: AuditEventType
    timestamp: datetime
    source: str
    actor: Dict[str, Any]
    action: str
    resource: Dict[str, Any]
    outcome: str
    details: Dict[str, Any] = field(default_factory=dict)
    session_id: Optional[str] = None
    request_id: Optional[str] = None
    ip_address: Optional[str] = None
    user_agent: Optional[str] = None
    severity: SeverityLevel = SeverityLevel.LOW
    tags: List[str] = field(default_factory=list)
    event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    
    def __post_init__(self):
        # Generate event hash for integrity
        event_data = {
            'event_type': self.event_type.value,
            'timestamp': self.timestamp.isoformat(),
            'source': self.source,
            'actor': self.actor,
            'action': self.action,
            'resource': self.resource,
            'outcome': self.outcome
        }
        self.details['event_hash'] = hashlib.sha256(
            json.dumps(event_data, sort_keys=True).encode()
        ).hexdigest()
 
class AuditLogger:
    def __init__(self, config: Dict[str, Any]):
        self.config = config
        self.storage = AuditStorage(config.get('storage'))
        self.compliance_engine = ComplianceEngine(config.get('compliance'))
        self.encryption_key = config.get('encryption_key')
        self.buffer = []
        self.buffer_size = config.get('buffer_size', 100)
        
    async def log_event(self, event: AuditEvent) -> bool:
        """Log audit event with compliance processing"""
        try:
            # Enrich event with additional context
            enriched_event = await self._enrich_event(event)
            
            # Encrypt sensitive data
            if self._contains_sensitive_data(enriched_event):
                enriched_event = await self._encrypt_sensitive_fields(enriched_event)
            
            # Add to buffer
            self.buffer.append(enriched_event)
            
            # Process compliance rules
            await self.compliance_engine.process_event(enriched_event)
            
            # Flush buffer if full
            if len(self.buffer) >= self.buffer_size:
                await self._flush_buffer()
            
            return True
            
        except Exception as e:
            print(f"Failed to log audit event: {e}")
            return False
    
    async def log_user_action(self, user_id: str, action: str, 
                             resource: str, outcome: str, 
                             request_context: Dict[str, Any] = None) -> bool:
        """Log user action with automatic context"""
        event = AuditEvent(
            event_type=AuditEventType.USER_ACTION,
            timestamp=datetime.utcnow(),
            source="user-interface",
            actor={
                "type": "user",
                "id": user_id,
                "roles": request_context.get('user_roles', []) if request_context else []
            },
            action=action,
            resource={
                "type": "resource",
                "identifier": resource
            },
            outcome=outcome,
            session_id=request_context.get('session_id') if request_context else None,
            request_id=request_context.get('request_id') if request_context else None,
            ip_address=request_context.get('ip_address') if request_context else None,
            user_agent=request_context.get('user_agent') if request_context else None
        )
        
        return await self.log_event(event)
    
    async def log_api_access(self, endpoint: str, method: str, 
                            user_id: str, status_code: int,
                            request_data: Dict[str, Any] = None,
                            response_data: Dict[str, Any] = None) -> bool:
        """Log API access with request/response details"""
        event = AuditEvent(
            event_type=AuditEventType.API_ACCESS,
            timestamp=datetime.utcnow(),
            source="api-gateway",
            actor={
                "type": "user",
                "id": user_id
            },
            action=f"{method} {endpoint}",
            resource={
                "type": "api_endpoint",
                "endpoint": endpoint,
                "method": method
            },
            outcome="success" if 200 <= status_code < 400 else "failure",
            details={
                "status_code": status_code,
                "request_data": self._sanitize_request_data(request_data) if request_data else {},
                "response_data": self._sanitize_response_data(response_data) if response_data else {}
            },
            severity=SeverityLevel.HIGH if status_code >= 400 else SeverityLevel.LOW
        )
        
        return await self.log_event(event)
    
    async def log_data_access(self, user_id: str, table_name: str, 
                             operation: str, record_ids: List[str],
                             data_classification: str = "internal") -> bool:
        """Log data access with classification"""
        severity = SeverityLevel.HIGH if data_classification in ["confidential", "restricted"] else SeverityLevel.MEDIUM
        
        event = AuditEvent(
            event_type=AuditEventType.DATA_ACCESS,
            timestamp=datetime.utcnow(),
            source="data-service",
            actor={
                "type": "user",
                "id": user_id
            },
            action=f"data_{operation}",
            resource={
                "type": "database_table",
                "table": table_name,
                "records": record_ids,
                "classification": data_classification
            },
            outcome="success",
            severity=severity,
            tags=["data_access", data_classification]
        )
        
        return await self.log_event(event)
    
    async def log_security_event(self, event_type: str, severity: SeverityLevel,
                                description: str, actor: Dict[str, Any],
                                additional_context: Dict[str, Any] = None) -> bool:
        """Log security-related events"""
        event = AuditEvent(
            event_type=AuditEventType.SECURITY_EVENT,
            timestamp=datetime.utcnow(),
            source="security-service",
            actor=actor,
            action=event_type,
            resource={
                "type": "security_context",
                "description": description
            },
            outcome="detected",
            severity=severity,
            details=additional_context or {},
            tags=["security", event_type]
        )
        
        return await self.log_event(event)
    
    async def _enrich_event(self, event: AuditEvent) -> AuditEvent:
        """Enrich event with additional context"""
        # Add geolocation if IP available
        if event.ip_address:
            event.details['geolocation'] = await self._get_geolocation(event.ip_address)
        
        # Add user context
        if event.actor.get('id'):
            user_context = await self._get_user_context(event.actor['id'])
            event.actor.update(user_context)
        
        # Add resource metadata
        if event.resource.get('identifier'):
            resource_metadata = await self._get_resource_metadata(event.resource['identifier'])
            event.resource.update(resource_metadata)
        
        return event
    
    async def _flush_buffer(self):
        """Flush buffered events to storage"""
        if self.buffer:
            await self.storage.store_events(self.buffer)
            self.buffer.clear()
 
# Compliance monitoring engine
class ComplianceEngine:
    def __init__(self, config: Dict[str, Any]):
        self.config = config
        self.rules = {}
        self.frameworks = {}
        self._load_compliance_rules()
    
    def _load_compliance_rules(self):
        """Load compliance rules and frameworks"""
        # SOC2 Type II rules
        self.rules['soc2'] = {
            'data_access_monitoring': {
                'description': 'Monitor all access to sensitive data',
                'conditions': [
                    {
                        'field': 'resource.classification',
                        'operator': 'in',
                        'values': ['confidential', 'restricted']
                    }
                ],
                'actions': ['log_detailed', 'alert_if_unusual']
            },
            'privileged_access_control': {
                'description': 'Monitor privileged user activities',
                'conditions': [
                    {
                        'field': 'actor.roles',
                        'operator': 'contains',
                        'values': ['admin', 'root', 'superuser']
                    }
                ],
                'actions': ['log_detailed', 'require_approval']
            }
        }
        
        # GDPR compliance rules
        self.rules['gdpr'] = {
            'personal_data_access': {
                'description': 'Monitor access to personal data',
                'conditions': [
                    {
                        'field': 'resource.type',
                        'operator': 'equals',
                        'values': ['personal_data']
                    }
                ],
                'actions': ['log_detailed', 'check_consent']
            },
            'data_retention_policy': {
                'description': 'Enforce data retention policies',
                'conditions': [
                    {
                        'field': 'action',
                        'operator': 'equals',
                        'values': ['data_delete']
                    }
                ],
                'actions': ['verify_retention_period', 'log_deletion']
            }
        }
    
    async def process_event(self, event: AuditEvent):
        """Process event against compliance rules"""
        for framework, rules in self.rules.items():
            for rule_name, rule_config in rules.items():
                if await self._evaluate_rule(event, rule_config):
                    await self._execute_rule_actions(event, rule_config, framework, rule_name)
    
    async def _evaluate_rule(self, event: AuditEvent, rule_config: Dict[str, Any]) -> bool:
        """Evaluate if event matches rule conditions"""
        conditions = rule_config.get('conditions', [])
        
        for condition in conditions:
            field_path = condition['field']
            operator = condition['operator']
            values = condition['values']
            
            # Get field value from event
            field_value = self._get_field_value(event, field_path)
            
            # Evaluate condition
            if not self._evaluate_condition(field_value, operator, values):
                return False
        
        return True
    
    async def _execute_rule_actions(self, event: AuditEvent, rule_config: Dict[str, Any],
                                   framework: str, rule_name: str):
        """Execute actions for matched compliance rule"""
        actions = rule_config.get('actions', [])
        
        for action in actions:
            if action == 'log_detailed':
                await self._log_compliance_event(event, framework, rule_name)
            elif action == 'alert_if_unusual':
                await self._check_unusual_activity(event)
            elif action == 'require_approval':
                await self._trigger_approval_workflow(event)
            elif action == 'check_consent':
                await self._verify_data_consent(event)
 
# Usage example
audit_config = {
    'storage': {
        'type': 'postgresql',
        'connection_string': 'postgresql://audit:password@db.sindhan.ai/audit',
        'encryption_enabled': True
    },
    'compliance': {
        'frameworks': ['soc2', 'gdpr', 'hipaa'],
        'alert_webhook': 'https://alerts.sindhan.ai/compliance'
    },
    'buffer_size': 50,
    'encryption_key': 'your-encryption-key'
}
 
# Initialize audit logger
audit_logger = AuditLogger(audit_config)
 
# Log user login
await audit_logger.log_user_action(
    user_id='user-12345',
    action='login',
    resource='authentication_system',
    outcome='success',
    request_context={
        'session_id': 'sess-abc-123',
        'ip_address': '192.168.1.100',
        'user_agent': 'Mozilla/5.0...',
        'user_roles': ['user', 'finance']
    }
)
 
# Log API access
await audit_logger.log_api_access(
    endpoint='/api/users/profile',
    method='GET',
    user_id='user-12345',
    status_code=200,
    request_data={'user_id': 'user-12345'},
    response_data={'profile': {'name': 'John Doe'}}
)
 
# Log sensitive data access
await audit_logger.log_data_access(
    user_id='user-12345',
    table_name='customer_personal_data',
    operation='read',
    record_ids=['customer-456', 'customer-789'],
    data_classification='confidential'
)
 
# Log security event
await audit_logger.log_security_event(
    event_type='failed_login_attempt',
    severity=SeverityLevel.MEDIUM,
    description='Multiple failed login attempts detected',
    actor={
        'type': 'unknown',
        'ip_address': '192.168.1.100'
    },
    additional_context={
        'attempts': 5,
        'time_window': '5 minutes'
    }
)

from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta
import pandas as pd
from dataclasses import dataclass
 
@dataclass
class ComplianceReport:
    report_id: str
    framework: str
    period_start: datetime
    period_end: datetime
    controls_assessed: List[Dict[str, Any]]
    compliance_score: float
    findings: List[Dict[str, Any]]
    recommendations: List[str]
    generated_at: datetime
    generated_by: str
 
class ComplianceReportGenerator:
    def __init__(self, audit_storage: AuditStorage):
        self.audit_storage = audit_storage
        self.report_templates = self._load_report_templates()
    
    async def generate_soc2_report(self, period_start: datetime, 
                                  period_end: datetime) -> ComplianceReport:
        """Generate SOC2 Type II compliance report"""
        
        # Define SOC2 controls to assess
        soc2_controls = {
            'CC6.1': 'Logical and Physical Access Controls',
            'CC6.2': 'Authentication and Authorization',
            'CC6.3': 'System Access Monitoring',
            'CC7.1': 'Data Transmission Controls',
            'CC7.2': 'Data Storage Protection'
        }
        
        controls_assessment = []
        findings = []
        
        for control_id, control_name in soc2_controls.items():
            assessment = await self._assess_soc2_control(
                control_id, control_name, period_start, period_end
            )
            controls_assessment.append(assessment)
            
            if assessment['status'] != 'compliant':
                findings.extend(assessment.get('findings', []))
        
        # Calculate compliance score
        compliant_controls = sum(1 for c in controls_assessment if c['status'] == 'compliant')
        compliance_score = (compliant_controls / len(controls_assessment)) * 100
        
        # Generate recommendations
        recommendations = await self._generate_soc2_recommendations(findings)
        
        return ComplianceReport(
            report_id=f"soc2-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}",
            framework='SOC2 Type II',
            period_start=period_start,
            period_end=period_end,
            controls_assessed=controls_assessment,
            compliance_score=compliance_score,
            findings=findings,
            recommendations=recommendations,
            generated_at=datetime.utcnow(),
            generated_by='compliance-engine'
        )
    
    async def generate_gdpr_report(self, period_start: datetime, 
                                  period_end: datetime) -> ComplianceReport:
        """Generate GDPR compliance report"""
        
        gdpr_requirements = {
            'lawful_basis': 'Lawful Basis for Processing',
            'consent_management': 'Consent Management',
            'data_subject_rights': 'Data Subject Rights',
            'data_protection_impact': 'Data Protection Impact Assessment',
            'breach_notification': 'Breach Notification'
        }
        
        controls_assessment = []
        findings = []
        
        for req_id, req_name in gdpr_requirements.items():
            assessment = await self._assess_gdpr_requirement(
                req_id, req_name, period_start, period_end
            )
            controls_assessment.append(assessment)
            
            if assessment['status'] != 'compliant':
                findings.extend(assessment.get('findings', []))
        
        # Calculate compliance score
        compliant_reqs = sum(1 for c in controls_assessment if c['status'] == 'compliant')
        compliance_score = (compliant_reqs / len(controls_assessment)) * 100
        
        recommendations = await self._generate_gdpr_recommendations(findings)
        
        return ComplianceReport(
            report_id=f"gdpr-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}",
            framework='GDPR',
            period_start=period_start,
            period_end=period_end,
            controls_assessed=controls_assessment,
            compliance_score=compliance_score,
            findings=findings,
            recommendations=recommendations,
            generated_at=datetime.utcnow(),
            generated_by='compliance-engine'
        )
    
    async def _assess_soc2_control(self, control_id: str, control_name: str,
                                  period_start: datetime, period_end: datetime) -> Dict[str, Any]:
        """Assess specific SOC2 control"""
        
        if control_id == 'CC6.3':  # System Access Monitoring
            # Check for comprehensive access logging
            access_events = await self.audit_storage.query_events(
                event_types=[AuditEventType.DATA_ACCESS, AuditEventType.API_ACCESS],
                period_start=period_start,
                period_end=period_end
            )
            
            # Analyze access patterns
            total_access_events = len(access_events)
            logged_events = sum(1 for event in access_events if event.outcome == 'success')
            monitoring_coverage = (logged_events / total_access_events * 100) if total_access_events > 0 else 0
            
            findings = []
            if monitoring_coverage < 95:
                findings.append({
                    'severity': 'medium',
                    'description': f'Access monitoring coverage is {monitoring_coverage:.1f}%, below 95% threshold',
                    'recommendation': 'Implement comprehensive access logging for all system components'
                })
            
            return {
                'control_id': control_id,
                'control_name': control_name,
                'status': 'compliant' if monitoring_coverage >= 95 else 'non_compliant',
                'score': monitoring_coverage,
                'evidence': {
                    'total_access_events': total_access_events,
                    'logged_events': logged_events,
                    'monitoring_coverage': monitoring_coverage
                },
                'findings': findings
            }
        
        # Additional control assessments would be implemented here
        return {
            'control_id': control_id,
            'control_name': control_name,
            'status': 'not_assessed',
            'score': 0,
            'evidence': {},
            'findings': []
        }
    
    async def generate_compliance_dashboard_data(self) -> Dict[str, Any]:
        """Generate real-time compliance dashboard data"""
        now = datetime.utcnow()
        last_30_days = now - timedelta(days=30)
        
        # Get recent audit events
        recent_events = await self.audit_storage.query_events(
            period_start=last_30_days,
            period_end=now
        )
        
        # Calculate metrics
        total_events = len(recent_events)
        security_events = len([e for e in recent_events if e.event_type == AuditEventType.SECURITY_EVENT])
        failed_accesses = len([e for e in recent_events if e.outcome == 'failure'])
        high_risk_events = len([e for e in recent_events if e.severity == SeverityLevel.HIGH])
        
        # Generate compliance score trends
        compliance_trends = await self._calculate_compliance_trends(last_30_days, now)
        
        return {
            'summary': {
                'total_events_30d': total_events,
                'security_events_30d': security_events,
                'failed_accesses_30d': failed_accesses,
                'high_risk_events_30d': high_risk_events,
                'overall_compliance_score': compliance_trends.get('current_score', 0)
            },
            'trends': compliance_trends,
            'recent_violations': await self._get_recent_violations(),
            'control_status': await self._get_control_status_summary(),
            'risk_indicators': await self._calculate_risk_indicators()
        }
 
# Usage example
report_generator = ComplianceReportGenerator(audit_storage)
 
# Generate SOC2 compliance report
period_start = datetime.utcnow() - timedelta(days=90)  # Last quarter
period_end = datetime.utcnow()
 
soc2_report = await report_generator.generate_soc2_report(period_start, period_end)
print(f"SOC2 Compliance Score: {soc2_report.compliance_score:.1f}%")
print(f"Findings: {len(soc2_report.findings)}")
 
# Generate GDPR compliance report
gdpr_report = await report_generator.generate_gdpr_report(period_start, period_end)
print(f"GDPR Compliance Score: {gdpr_report.compliance_score:.1f}%")
 
# Get dashboard data
dashboard_data = await report_generator.generate_compliance_dashboard_data()
print(f"Security events (30d): {dashboard_data['summary']['security_events_30d']}")