Beyond Multi-AZ: Building True Resilience Against AWS Control Plane Failures

{
  "Type": "HTTPS",
  "ResourcePath": "/health",
  "FullyQualifiedDomainName": "primary.example.com",
  "Port": 443,
  "RequestInterval": 30,
  "FailureThreshold": 3,
  "MeasureLatency": true,
  "EnableSNI": true,
  "Regions": [
    "us-east-1",
    "us-west-2",
    "eu-west-1"
  ],
  "AlarmIdentifier": {
    "Region": "us-east-1",
    "Name": "PrimaryRegionHealthAlarm"
  },
  "InsufficientDataHealthStatus": "LastKnownStatus"
}

resource "aws_route53_health_check" "primary" {
  type              = "HTTPS"
  resource_path     = "/health"
  fqdn              = "primary.example.com"
  port              = 443
  request_interval  = 30
  failure_threshold = 3
  measure_latency   = true
  enable_sni        = true

  # Health checks from multiple global locations
  regions = [
    "us-east-1",
    "us-west-2",
    "eu-west-1"
  ]

  # Use last known status during insufficient data periods
  # This prevents premature failover during transient network issues
  insufficient_data_health_status = "LastKnownStatus"

  tags = {
    Name        = "Primary Region Health Check"
    Environment = "production"
    Purpose     = "multi-region-failover"
  }
}

# CloudWatch alarm for additional monitoring
resource "aws_cloudwatch_metric_alarm" "primary_health" {
  alarm_name          = "PrimaryRegionHealthAlarm"
  comparison_operator = "LessThanThreshold"
  evaluation_periods  = "2"
  metric_name         = "HealthCheckStatus"
  namespace           = "AWS/Route53"
  period              = "60"
  statistic           = "Minimum"
  threshold           = "1"
  alarm_description   = "Primary region health check failure"
  treat_missing_data  = "notBreaching"

  dimensions = {
    HealthCheckId = aws_route53_health_check.primary.id
  }
}

# Failover routing policy
resource "aws_route53_record" "primary" {
  zone_id = aws_route53_zone.main.zone_id
  name    = "app.example.com"
  type    = "A"

  alias {
    name                   = aws_lb.primary.dns_name
    zone_id                = aws_lb.primary.zone_id
    evaluate_target_health = true
  }

  set_identifier  = "Primary"
  health_check_id = aws_route53_health_check.primary.id
  failover_routing_policy {
    type = "PRIMARY"
  }
}

resource "aws_route53_record" "secondary" {
  zone_id = aws_route53_zone.main.zone_id
  name    = "app.example.com"
  type    = "A"

  alias {
    name                   = aws_lb.secondary.dns_name
    zone_id                = aws_lb.secondary.zone_id
    evaluate_target_health = true
  }

  set_identifier = "Secondary"
  failover_routing_policy {
    type = "SECONDARY"
  }
}

#!/usr/bin/env python3
"""
STOP Pattern Health Check Endpoint
Implements inverted health check logic based on S3 failover flag
"""

import boto3
import logging
from flask import Flask, jsonify
from botocore.exceptions import ClientError

app = Flask(__name__)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Configuration
FAILOVER_BUCKET = "my-app-failover-control"
FAILOVER_KEY = "failover-active.flag"
REGION = "us-east-1"  # Current region
IS_PRIMARY = True  # Set to False in secondary region

s3_client = boto3.client('s3', region_name=REGION)

def check_failover_flag():
    """
    Check if failover flag exists in S3
    Returns True if flag exists, False otherwise
    """
    try:
        s3_client.head_object(Bucket=FAILOVER_BUCKET, Key=FAILOVER_KEY)
        return True
    except ClientError as e:
        if e.response['Error']['Code'] == '404':
            return False
        else:
            logger.error(f"Error checking failover flag: {e}")
            # Fail open - assume no failover on S3 errors
            return False

def set_failover_flag():
    """
    Create failover flag in S3 to signal region promotion
    """
    try:
        s3_client.put_object(
            Bucket=FAILOVER_BUCKET,
            Key=FAILOVER_KEY,
            Body=b'failover-active',
            Metadata={
                'region': REGION,
                'timestamp': str(time.time())
            }
        )
        logger.info("Failover flag set successfully")
        return True
    except ClientError as e:
        logger.error(f"Error setting failover flag: {e}")
        return False

@app.route('/health', methods=['GET'])
def health_check():
    """
    Health check endpoint with STOP pattern logic
    
    Primary Region Logic:
    - Healthy if no failover flag exists
    - Unhealthy if failover flag exists (secondary has taken over)
    
    Secondary Region Logic:
    - Unhealthy if no failover flag exists (primary is active)
    - Healthy if failover flag exists (we've been promoted)
    """
    failover_active = check_failover_flag()
    
    if IS_PRIMARY:
        # Primary region: healthy when NO failover flag
        if not failover_active:
            return jsonify({
                "status": "healthy",
                "region": REGION,
                "role": "primary-active"
            }), 200
        else:
            logger.warning("Primary region: failover flag detected, reporting unhealthy")
            return jsonify({
                "status": "unhealthy",
                "region": REGION,
                "role": "primary-passive",
                "reason": "failover-flag-active"
            }), 503
    else:
        # Secondary region: healthy when failover flag EXISTS
        if failover_active:
            return jsonify({
                "status": "healthy",
                "region": REGION,
                "role": "secondary-active"
            }), 200
        else:
            return jsonify({
                "status": "unhealthy",
                "region": REGION,
                "role": "secondary-passive",
                "reason": "primary-region-active"
            }), 503

@app.route('/promote', methods=['POST'])
def promote_secondary():
    """
    Endpoint to manually promote secondary region
    Should be protected with authentication in production
    """
    if not IS_PRIMARY:
        if set_failover_flag():
            logger.info("Secondary region promoted to active")
            return jsonify({
                "status": "promoted",
                "region": REGION
            }), 200
        else:
            return jsonify({
                "status": "error",
                "message": "Failed to set failover flag"
            }), 500
    else:
        return jsonify({
            "status": "error",
            "message": "Cannot promote primary region"
        }), 400

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

resource "aws_s3_bucket" "failover_control_primary" {
  bucket = "my-app-failover-control-us-east-1"
  
  tags = {
    Name        = "Failover Control Bucket"
    Environment = "production"
    Purpose     = "stop-pattern-coordination"
  }
}

resource "aws_s3_bucket" "failover_control_secondary" {
  provider = aws.secondary
  bucket   = "my-app-failover-control-us-west-2"
  
  tags = {
    Name        = "Failover Control Bucket"
    Environment = "production"
    Purpose     = "stop-pattern-coordination"
  }
}

# Enable versioning for audit trail
resource "aws_s3_bucket_versioning" "failover_control_primary" {
  bucket = aws_s3_bucket.failover_control_primary.id
  
  versioning_configuration {
    status = "Enabled"
  }
}

# Cross-region replication ensures both regions can read failover state
resource "aws_s3_bucket_replication_configuration" "failover_control" {
  depends_on = [aws_s3_bucket_versioning.failover_control_primary]
  
  role   = aws_iam_role.replication.arn
  bucket = aws_s3_bucket.failover_control_primary.id

  rule {
    id     = "failover-flag-replication"
    status = "Enabled"

    filter {
      prefix = ""
    }

    destination {
      bucket        = aws_s3_bucket.failover_control_secondary.arn
      storage_class = "STANDARD"
      
      # Replicate within 15 minutes for non-critical coordination
      replication_time {
        status = "Enabled"
        time {
          minutes = 15
        }
      }

      metrics {
        status = "Enabled"
        event_threshold {
          minutes = 15
        }
      }
    }

    delete_marker_replication {
      status = "Enabled"
    }
  }
}

# IAM role for replication
resource "aws_iam_role" "replication" {
  name = "s3-failover-bucket-replication"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          Service = "s3.amazonaws.com"
        }
        Action = "sts:AssumeRole"
      }
    ]
  })
}

resource "aws_iam_role_policy" "replication" {
  name = "s3-failover-replication-policy"
  role = aws_iam_role.replication.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "s3:GetReplicationConfiguration",
          "s3:ListBucket"
        ]
        Resource = aws_s3_bucket.failover_control_primary.arn
      },
      {
        Effect = "Allow"
        Action = [
          "s3:GetObjectVersionForReplication",
          "s3:GetObjectVersionAcl",
          "s3:GetObjectVersionTagging"
        ]
        Resource = "${aws_s3_bucket.failover_control_primary.arn}/*"
      },
      {
        Effect = "Allow"
        Action = [
          "s3:ReplicateObject",
          "s3:ReplicateDelete",
          "s3:ReplicateTags"
        ]
        Resource = "${aws_s3_bucket.failover_control_secondary.arn}/*"
      }
    ]
  })
}

"""
Cognito User Sync Lambda
Limitations:
1. Cannot sync password hashes (security by design)
2. Cannot sync MFA seeds (security limitation)
3. Cannot preserve user GUIDs across pools
4. Requires users to re-authenticate after region failover
"""

import boto3
import json
import logging

logger = logging.getLogger()
logger.setLevel(logging.INFO)

cognito_primary = boto3.client('cognito-idp', region_name='us-east-1')
cognito_secondary = boto3.client('cognito-idp', region_name='us-west-2')

PRIMARY_POOL_ID = 'us-east-1_XXXXXXXXX'
SECONDARY_POOL_ID = 'us-west-2_YYYYYYYYY'

def lambda_handler(event, context):
    """
    Triggered by Cognito Post-Confirmation trigger
    Attempts to replicate user to secondary region
    
    CRITICAL LIMITATION: Cannot replicate passwords
    Users MUST reset password after failover to secondary region
    """
    
    user_attributes = event['request']['userAttributes']
    username = event['userName']
    
    try:
        # Attempt to create user in secondary pool
        response = cognito_secondary.admin_create_user(
            UserPoolId=SECONDARY_POOL_ID,
            Username=username,
            UserAttributes=[
                {'Name': k, 'Value': v} 
                for k, v in user_attributes.items()
                if k not in ['sub']  # sub is auto-generated per pool
            ],
            MessageAction='SUPPRESS',  # Don't send welcome email
            DesiredDeliveryMediums=[]
        )
        
        # Mark as confirmed since user already confirmed in primary
        cognito_secondary.admin_confirm_sign_up(
            UserPoolId=SECONDARY_POOL_ID,
            Username=username
        )
        
        # Set permanent password flag (user must reset)
        cognito_secondary.admin_set_user_password(
            UserPoolId=SECONDARY_POOL_ID,
            Username=username,
            Password=generate_random_password(),  # Temporary unusable password
            Permanent=False
        )
        
        logger.info(f"User {username} replicated to secondary pool")
        
    except cognito_secondary.exceptions.UsernameExistsException:
        logger.info(f"User {username} already exists in secondary pool")
    except Exception as e:
        logger.error(f"Error replicating user {username}: {str(e)}")
        # Don't fail primary registration if secondary sync fails
    
    return event

def generate_random_password():
    """Generate cryptographically random password"""
    import secrets
    import string
    alphabet = string.ascii_letters + string.digits + string.punctuation
    return ''.join(secrets.choice(alphabet) for i in range(32))

"""
DynamoDB Global Tables Pattern with Conflict Resolution
Demonstrates handling eventual consistency in multi-region writes
"""

import boto3
from datetime import datetime
from decimal import Decimal

dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
table = dynamodb.Table('GlobalOrdersTable')

def create_order_with_vector_clock(order_id, user_id, items, region):
    """
    Create order with vector clock for conflict detection
    
    Vector clocks help identify concurrent writes across regions
    that might conflict during eventual consistency windows
    """
    timestamp = Decimal(str(datetime.utcnow().timestamp()))
    
    try:
        response = table.put_item(
            Item={
                'order_id': order_id,
                'user_id': user_id,
                'items': items,
                'status': 'pending',
                'region': region,
                'created_at': timestamp,
                'updated_at': timestamp,
                'version': 1,
                # Vector clock: {region: timestamp}
                'vector_clock': {
                    region: timestamp
                }
            },
            # Prevent overwriting existing orders
            ConditionExpression='attribute_not_exists(order_id)',
            ReturnValues='ALL_OLD'
        )
        return {'success': True, 'order': response}
    except dynamodb.meta.client.exceptions.ConditionalCheckFailedException:
        # Order already exists - possible concurrent write
        return handle_concurrent_order_creation(order_id, user_id, items, region)

def handle_concurrent_order_creation(order_id, user_id, items, region):
    """
    Handle scenario where order was created concurrently in another region
    
    This can happen during network partitions or regional failovers
    when users submit duplicate requests
    """
    existing_order = table.get_item(Key={'order_id': order_id})['Item']
    
    # Compare vector clocks to determine which write is more recent
    existing_timestamp = existing_order['vector_clock'].get(
        existing_order['region'], 
        Decimal('0')
    )
    current_timestamp = Decimal(str(datetime.utcnow().timestamp()))
    
    if current_timestamp > existing_timestamp:
        # Our write is more recent - merge the orders
        return merge_orders(existing_order, items, region, current_timestamp)
    else:
        # Existing order is more recent - return it
        return {
            'success': False, 
            'reason': 'duplicate_order',
            'existing_order': existing_order
        }

def merge_orders(existing_order, new_items, region, timestamp):
    """
    Merge concurrent order submissions using application-specific logic
    
    This demonstrates one approach - your business logic may differ
    """
    # Combine items from both orders
    merged_items = existing_order['items'] + new_items
    
    # Update with merged data and vector clock
    existing_order['vector_clock'][region] = timestamp
    
    table.update_item(
        Key={'order_id': existing_order['order_id']},
        UpdateExpression="""
            SET items = :items,
                updated_at = :timestamp,
                version = version + :inc,
                vector_clock = :vector_clock
        """,
        ExpressionAttributeValues={
            ':items': merged_items,
            ':timestamp': timestamp,
            ':inc': 1,
            ':vector_clock': existing_order['vector_clock']
        }
    )
    
    return {'success': True, 'merged': True, 'order': existing_order}

def read_with_consistency_check(order_id, max_staleness_seconds=5):
    """
    Read order with staleness detection
    
    Helps identify when you're reading potentially stale data
    during replication lag
    """
    response = table.get_item(Key={'order_id': order_id})
    
    if 'Item' not in response:
        return None
    
    order = response['Item']
    updated_at = float(order['updated_at'])
    age_seconds = datetime.utcnow().timestamp() - updated_at
    
    return {
        'order': order,
        'age_seconds': age_seconds,
        'potentially_stale': age_seconds > max_staleness_seconds
    }

# Define both regions as providers
provider "aws" {
  alias  = "primary"
  region = "us-east-1"
}

provider "aws" {
  alias  = "secondary"
  region = "us-west-2"
}

# Primary region auto-scaling group
resource "aws_autoscaling_group" "primary" {
  provider = aws.primary
  
  name                = "app-asg-primary"
  vpc_zone_identifier = aws_subnet.primary[*].id
  target_group_arns   = [aws_lb_target_group.primary.arn]
  health_check_type   = "ELB"
  
  min_size         = 3  # Baseline capacity
  max_size         = 20
  desired_capacity = 6  # Production load capacity
  
  launch_template {
    id      = aws_launch_template.primary.id
    version = "$Latest"
  }
  
  tag {
    key                 = "Name"
    value               = "app-instance-primary"
    propagate_at_launch = true
  }
  
  tag {
    key                 = "Region"
    value               = "primary"
    propagate_at_launch = true
  }
}

# Secondary region auto-scaling group
# CRITICAL: Maintain minimum capacity at all times
resource "aws_autoscaling_group" "secondary" {
  provider = aws.secondary
  
  name                = "app-asg-secondary"
  vpc_zone_identifier = aws_subnet.secondary[*].id
  target_group_arns   = [aws_lb_target_group.secondary.arn]
  health_check_type   = "ELB"
  
  # Reduced baseline capacity for cost optimization
  min_size         = 2
  max_size         = 20
  desired_capacity = 3  # 50% of primary capacity
  
  launch_template {
    id      = aws_launch_template.secondary.id
    version = "$Latest"
  }
  
  tag {
    key                 = "Name"
    value               = "app-instance-secondary"
    propagate_at_launch = true
  }
  
  tag {
    key                 = "Region"
    value               = "secondary"
    propagate_at_launch = true
  }
}

# Target tracking scaling policy for secondary region
# This enables automatic scale-up during failover
resource "aws_autoscaling_policy" "secondary_cpu" {
  provider = aws.secondary
  
  name                   = "cpu-target-tracking"
  autoscaling_group_name = aws_autoscaling_group.secondary.name
  policy_type            = "TargetTrackingScaling"
  
  target_tracking_configuration {
    predefined_metric_specification {
      predefined_metric_type = "ASGAverageCPUUtilization"
    }
    target_value = 60.0
  }
}

# Application load balancer in primary region
resource "aws_lb" "primary" {
  provider = aws.primary
  
  name               = "app-alb-primary"
  internal           = false
  load_balancer_type = "application"
  security_groups    = [aws_security_group.alb_primary.id]
  subnets            = aws_subnet.primary[*].id
  
  enable_deletion_protection = true
  enable_http2               = true
  enable_cross_zone_load_balancing = true
  
  tags = {
    Name        = "app-alb-primary"
    Environment = "production"
  }
}

# Application load balancer in secondary region
resource "aws_lb" "secondary" {
  provider = aws.secondary
  
  name               = "app-alb-secondary"
  internal           = false
  load_balancer_type = "application"
  security_groups    = [aws_security_group.alb_secondary.id]
  subnets            = aws_subnet.secondary[*].id
  
  enable_deletion_protection = true
  enable_http2               = true
  enable_cross_zone_load_balancing = true
  
  tags = {
    Name        = "app-alb-secondary"
    Environment = "production"
  }
}

#!/bin/bash
#
# Safe Multi-Region Failover Test
# Tests secondary region functionality without disrupting production traffic
#

set -euo pipefail

PRIMARY_REGION="us-east-1"
SECONDARY_REGION="us-west-2"
TEST_DOMAIN="test.example.com"
PROD_DOMAIN="app.example.com"

echo "=== Multi-Region Failover Test ==="
echo "Testing secondary region without impacting production"

# Step 1: Create temporary test DNS record pointing to secondary
echo "[1/6] Creating test DNS record..."
aws route53 change-resource-record-sets \
  --hosted-zone-id "${HOSTED_ZONE_ID}" \
  --change-batch '{
    "Changes": [{
      "Action": "UPSERT",
      "ResourceRecordSet": {
        "Name": "'"${TEST_DOMAIN}"'",
        "Type": "A",
        "AliasTarget": {
          "HostedZoneId": "'"${SECONDARY_ALB_ZONE_ID}"'",
          "DNSName": "'"${SECONDARY_ALB_DNS}"'",
          "EvaluateTargetHealth": true
        }
      }
    }]
  }'

# Step 2: Wait for DNS propagation
echo "[2/6] Waiting for DNS propagation (60 seconds)..."
sleep 60

# Step 3: Run health checks against secondary region
echo "[3/6] Running health checks against secondary region..."
for i in {1..10}; do
  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "https://${TEST_DOMAIN}/health")
  if [ "${HTTP_CODE}" -eq 200 ]; then
    echo "  ✓ Health check ${i}/10: OK"
  else
    echo "  ✗ Health check ${i}/10: FAILED (HTTP ${HTTP_CODE})"
    exit 1
  fi
  sleep 2
done

# Step 4: Run functional tests against secondary
echo "[4/6] Running functional tests..."
./run-smoke-tests.sh "${TEST_DOMAIN}"

# Step 5: Verify data replication status
echo "[5/6] Checking DynamoDB Global Table replication lag..."
REPLICATION_LAG=$(aws dynamodb describe-table \
  --table-name GlobalTable \
  --region "${SECONDARY_REGION}" \
  --query 'Table.Replicas[?RegionName==`'"${PRIMARY_REGION}"'`].ReplicaStatus' \
  --output text)

if [ "${REPLICATION_LAG}" = "ACTIVE" ]; then
  echo "  ✓ DynamoDB replication: ACTIVE"
else
  echo "  ✗ DynamoDB replication: ${REPLICATION_LAG}"
  exit 1
fi

# Step 6: Verify RDS read replica lag
echo "[6/6] Checking RDS replica lag..."
RDS_LAG=$(aws rds describe-db-instances \
  --db-instance-identifier secondary-replica \
  --region "${SECONDARY_REGION}" \
  --query 'DBInstances[0].StatusInfos[?StatusType==`read replication`].Status' \
  --output text)

echo "  RDS replica lag: ${RDS_LAG:-0} seconds"

# Cleanup test record
echo "Cleaning up test DNS record..."
aws route53 change-resource-record-sets \
  --hosted-zone-id "${HOSTED_ZONE_ID}" \
  --change-batch '{
    "Changes": [{
      "Action": "DELETE",
      "ResourceRecordSet": {
        "Name": "'"${TEST_DOMAIN}"'",
        "Type": "A",
        "AliasTarget": {
          "HostedZoneId": "'"${SECONDARY_ALB_ZONE_ID}"'",
          "DNSName": "'"${SECONDARY_ALB_DNS}"'",
          "EvaluateTargetHealth": true
        }
      }
    }]
  }'

echo ""
echo "✓ Failover test completed successfully"
echo "Secondary region is ready for production failover"

#!/usr/bin/env python3
"""
Downtime Cost Calculator and Multi-Region ROI Analysis
Helps determine if multi-region investment is financially justified
"""

def calculate_downtime_cost(
    annual_revenue: float,
    revenue_during_incident_hours: float = None,
    incident_duration_hours: float = 4.0,
    recovery_impact_hours: float = 8.0,
    recovery_conversion_penalty: float = 0.4,
    support_tickets: int = 1000,
    support_cost_per_ticket: float = 15.0,
    customer_churn_increase: float = 0.05,
    avg_customer_lifetime_value: float = 500.0,
    total_customers: int = 10000
):
    """
    Calculate total cost of a regional outage
    
    Args:
        annual_revenue: Total annual revenue ($)
        revenue_during_incident_hours: Revenue during incident (if known)
        incident_duration_hours: Hours of complete downtime
        recovery_impact_hours: Hours of degraded service post-recovery
        recovery_conversion_penalty: Conversion rate reduction during recovery (0-1)
        support_tickets: Expected support tickets
        support_cost_per_ticket: Cost to handle each ticket
        customer_churn_increase: Additional churn rate increase (0-1)
        avg_customer_lifetime_value: Average LTV per customer
        total_customers: Total customer base
    """
    
    # Direct revenue loss during incident
    if revenue_during_incident_hours:
        direct_revenue_loss = revenue_during_incident_hours
    else:
        hourly_revenue = annual_revenue / 8760  # Hours in year
        direct_revenue_loss = hourly_revenue * incident_duration_hours
    
    # Revenue loss during recovery period (degraded conversion)
    hourly_revenue = annual_revenue / 8760
    recovery_revenue_loss = (
        hourly_revenue * recovery_impact_hours * recovery_conversion_penalty
    )
    
    # Support costs
    support_costs = support_tickets * support_cost_per_ticket
    
    # Customer churn impact
    churned_customers = total_customers * customer_churn_increase
    churn_ltv_loss = churned_customers * avg_customer_lifetime_value
    
    # Total cost
    total_cost = (
        direct_revenue_loss +
        recovery_revenue_loss +
        support_costs +
        churn_ltv_loss
    )
    
    return {
        'direct_revenue_loss': direct_revenue_loss,
        'recovery_revenue_loss': recovery_revenue_loss,
        'support_costs': support_costs,
        'churn_ltv_loss': churn_ltv_loss,
        'total_cost': total_cost
    }

def calculate_multi_region_roi(
    current_monthly_cost: float,
    multi_region_monthly_cost: float,
    annual_outage_probability: float,
    outage_cost: float,
    planning_horizon_years: int = 3
):
    """
    Calculate ROI for multi-region investment over planning horizon
    
    Args:
        current_monthly_cost: Current single-region infrastructure cost
        multi_region_monthly_cost: Projected multi-region cost
        annual_outage_probability: Probability of regional outage per year
        outage_cost: Expected cost per outage (from calculate_downtime_cost)
        planning_horizon_years: Years to analyze
    """
    
    # Current architecture costs
    current_annual_infra = current_monthly_cost * 12
    current_expected_outage_cost = outage_cost * annual_outage_probability
    current_total_annual = current_annual_infra + current_expected_outage_cost
    
    # Multi-region architecture costs
    multi_region_annual_infra = multi_region_monthly_cost * 12
    # Assume multi-region reduces outage probability by 95%
    multi_region_outage_risk = annual_outage_probability * 0.05
    multi_region_expected_outage = outage_cost * multi_region_outage_risk
    multi_region_total_annual = multi_region_annual_infra + multi_region_expected_outage
    
    # ROI calculation
    annual_net_cost_increase = multi_region_total_annual - current_total_annual
    total_investment = annual_net_cost_increase * planning_horizon_years
    
    # Expected savings from avoided outages
    expected_outages_avoided = (
        annual_outage_probability * planning_horizon_years * 0.95
    )
    total_outage_cost_savings = expected_outages_avoided * outage_cost
    
    net_benefit = total_outage_cost_savings - total_investment
    roi_percentage = (net_benefit / total_investment * 100) if total_investment > 0 else 0
    
    return {
        'current_annual_cost': current_total_annual,
        'multi_region_annual_cost': multi_region_total_annual,
        'annual_cost_increase': annual_net_cost_increase,
        'total_investment': total_investment,
        'expected_outages_avoided': expected_outages_avoided,
        'total_savings': total_outage_cost_savings,
        'net_benefit': net_benefit,
        'roi_percentage': roi_percentage,
        'recommendation': 'INVEST' if roi_percentage > 0 else 'DO NOT INVEST'
    }

# Example usage
if __name__ == '__main__':
    # Calculate cost of a typical regional outage
    outage_cost = calculate_downtime_cost(
        annual_revenue=10_000_000,  # $10M annual revenue
        incident_duration_hours=4.0,
        recovery_impact_hours=8.0,
        recovery_conversion_penalty=0.4,
        support_tickets=1200,
        support_cost_per_ticket=15.0,
        customer_churn_increase=0.05,
        avg_customer_lifetime_value=500.0,
        total_customers=10000
    )
    
    print("=== Outage Cost Analysis ===")
    print(f"Direct revenue loss: ${outage_cost['direct_revenue_loss']:,.2f}")
    print(f"Recovery period loss: ${outage_cost['recovery_revenue_loss']:,.2f}")
    print(f"Support costs: ${outage_cost['support_costs']:,.2f}")
    print(f"Customer churn impact: ${outage_cost['churn_ltv_loss']:,.2f}")
    print(f"TOTAL OUTAGE COST: ${outage_cost['total_cost']:,.2f}")
    print()
    
    # Calculate multi-region ROI
    roi = calculate_multi_region_roi(
        current_monthly_cost=12_000,
        multi_region_monthly_cost=19_800,
        annual_outage_probability=1.5,  # 1.5 regional outages per year
        outage_cost=outage_cost['total_cost'],
        planning_horizon_years=3
    )
    
    print("=== Multi-Region ROI Analysis (3-year horizon) ===")
    print(f"Current annual cost: ${roi['current_annual_cost']:,.2f}")
    print(f"Multi-region annual cost: ${roi['multi_region_annual_cost']:,.2f}")
    print(f"Annual cost increase: ${roi['annual_cost_increase']:,.2f}")
    print(f"Total 3-year investment: ${roi['total_investment']:,.2f}")
    print(f"Expected outages avoided: {roi['expected_outages_avoided']:.2f}")
    print(f"Total savings from avoided outages: ${roi['total_savings']:,.2f}")
    print(f"Net benefit: ${roi['net_benefit']:,.2f}")
    print(f"ROI: {roi['roi_percentage']:.1f}%")
    print(f"RECOMMENDATION: {roi['recommendation']}")

# Check Route 53 health check status
aws route53 get-health-check-status --health-check-id 

# Monitor DynamoDB Global Tables replication lag
aws cloudwatch get-metric-statistics \
  --namespace AWS/DynamoDB \
  --metric-name ReplicationLatency \
  --dimensions Name=TableName,Value=YourTable Name=ReceivingRegion,Value=us-west-2 \
  --start-time 2025-11-25T00:00:00Z \
  --end-time 2025-11-25T23:59:59Z \
  --period 300 \
  --statistics Average,Maximum

# Check RDS cross-region replication lag
aws rds describe-db-instances \
  --db-instance-identifier your-replica \
  --region us-west-2 \
  --query 'DBInstances[0].StatusInfos'

# Verify S3 replication status
aws s3api get-bucket-replication --bucket your-bucket

# Test DNS resolution from multiple locations
dig app.example.com @8.8.8.8  # Google DNS
dig app.example.com @1.1.1.1  # Cloudflare DNS

# Check Auto Scaling group health in secondary region
aws autoscaling describe-auto-scaling-groups \
  --auto-scaling-group-names your-asg \
  --region us-west-2

# Monitor control plane API availability
aws ec2 describe-instances --region us-east-1 --max-results 1
# If this times out or fails, control plane is likely impacted