Disaster Recovery as Code: автоматизация RTO (время восстановления) и RPO (точка восстановления) через шаблоны

variable "rto_minutes" {
  description = "Maximum acceptable service downtime in minutes"
  type        = number
  default     = 60
}

variable "rpo_minutes" {
  description = "Maximum acceptable data loss window in minutes"
  type        = number
  default     = 60
}

locals {
  # Align schedule to the nearest practical cadence for your RPO
  backup_schedule = var.rpo_minutes <= 60 ? "cron(0 * * * ? *)" : "cron(0 0 * * ? *)"
}

resource "aws_backup_vault" "dr" {
  name = "application-dr-vault"
}

resource "aws_backup_plan" "application" {
  name = "application-dr-plan"

  rule {
    rule_name         = "rpo-aligned-backup"
    target_vault_name = aws_backup_vault.dr.name
    schedule          = local.backup_schedule

    lifecycle {
      delete_after = 35
    }
  }
}

import time
from datetime import datetime, timezone

import boto3


class BackupVerifier:
    def __init__(self, rto_minutes=60, rpo_minutes=60):
        self.rto_minutes = rto_minutes
        self.rpo_minutes = rpo_minutes
        self.rds = boto3.client("rds")

    def test_restore(self, source_instance_id, drill_instance_id):
        start = time.time()

        snapshots = self.rds.describe_db_snapshots(
            DBInstanceIdentifier=source_instance_id,
            SnapshotType="automated",
        )["DBSnapshots"]

        if not snapshots:
            raise RuntimeError("No automated snapshots found")

        latest = max(snapshots, key=lambda item: item["SnapshotCreateTime"])
        snapshot_time = latest["SnapshotCreateTime"]
        if snapshot_time.tzinfo is None:
            snapshot_time = snapshot_time.replace(tzinfo=timezone.utc)

        snapshot_age = (
            datetime.now(timezone.utc) - snapshot_time
        ).total_seconds() / 60

        if snapshot_age > self.rpo_minutes:
            raise AssertionError(
                f"RPO breach: snapshot age {snapshot_age:.1f}m exceeds {self.rpo_minutes}m"
            )

        self.rds.restore_db_instance_from_db_snapshot(
            DBInstanceIdentifier=drill_instance_id,
            DBSnapshotIdentifier=latest["DBSnapshotIdentifier"],
            DBInstanceClass="db.r6g.large",
            MultiAZ=False,
            PubliclyAccessible=False,
            DeletionProtection=False,
        )

        waiter = self.rds.get_waiter("db_instance_available")
        waiter.wait(DBInstanceIdentifier=drill_instance_id)

        elapsed = (time.time() - start) / 60
        if elapsed > self.rto_minutes:
            raise AssertionError(
                f"RTO breach: restore took {elapsed:.1f}m, target {self.rto_minutes}m"
            )

        return {
            "status": "PASS",
            "rto_actual_minutes": round(elapsed, 1),
            "rpo_actual_minutes": round(snapshot_age, 1),
            "snapshot_id": latest["DBSnapshotIdentifier"],
        }

name: DR Infrastructure Sync
on:
  push:
    paths: ['infrastructure/**']
  schedule:
    - cron: '0 */6 * * *'
  workflow_dispatch:

jobs:
  dr-sync:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_wrapper: false

      - name: Terraform plan (DR region)
        working-directory: infrastructure/dr
        run: |
          terraform init -input=false
          terraform plan -detailed-exitcode -input=false -var="environment=dr" -out=dr.tfplan

      - name: Apply DR infrastructure
        if: github.ref == 'refs/heads/main'
        working-directory: infrastructure/dr
        run: terraform apply -input=false dr.tfplan

resource "aws_route53_health_check" "primary" {
  fqdn              = var.primary_healthcheck_fqdn
  port              = 443
  type              = "HTTPS"
  resource_path     = "/health"
  failure_threshold = 3
  request_interval  = 30

  tags = {
    Name = "primary-api-health-check"
  }
}

resource "aws_route53_record" "failover_primary" {
  zone_id        = var.zone_id
  name           = "api.example.com"
  type           = "A"
  set_identifier = "primary"

  failover_routing_policy {
    type = "PRIMARY"
  }

  alias {
    name                   = var.primary_alb_dns
    zone_id                = var.primary_alb_zone_id
    evaluate_target_health = true
  }

  health_check_id = aws_route53_health_check.primary.id
}

resource "aws_route53_record" "failover_secondary" {
  zone_id        = var.zone_id
  name           = "api.example.com"
  type           = "A"
  set_identifier = "secondary"

  failover_routing_policy {
    type = "SECONDARY"
  }

  alias {
    name                   = var.dr_alb_dns
    zone_id                = var.dr_alb_zone_id
    evaluate_target_health = true
  }
}

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  name: disaster-recovery
spec:
  entrypoint: recover
  templates:
    - name: recover
      dag:
        tasks:
          - name: validate-dr-infra
            template: validate-infra
          - name: restore-databases
            template: restore-db
            dependencies: [validate-dr-infra]
          - name: sync-cache-layer
            template: sync-cache
            dependencies: [restore-databases]
          - name: update-dns
            template: dns-failover
            dependencies: [sync-cache-layer]
          - name: verify-recovery
            template: health-check
            dependencies: [update-dns]

    - name: validate-infra
      container:
        image: registry.example.com/dr-tools:latest
        command: [python, /scripts/validate_dr_infra.py]

    - name: restore-db
      container:
        image: registry.example.com/dr-tools:latest
        command: [python, /scripts/restore_database.py, --from-latest-snapshot, --verify-rpo]

    - name: sync-cache
      container:
        image: registry.example.com/dr-tools:latest
        command: [python, /scripts/warm_cache.py]

    - name: dns-failover
      container:
        image: registry.example.com/dr-tools:latest
        command: [python, /scripts/failover_dns.py]

    - name: health-check
      container:
        image: registry.example.com/dr-tools:latest
        command: [python, /scripts/verify_recovery.py]