Обнаружение аномалий стоимости в Kubernetes: оповещения и автоматическое исправление с OpenCost

apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: opencost
  namespace: opencost
spec:
  interval: 1h
  chart:
    spec:
      chart: opencost
      sourceRef:
        kind: HelmRepository
        name: opencost
        namespace: flux-system
  values:
    opencost:
      exporter:
        defaultClusterId: prod-us-east-1
      prometheus:
        external:
          enabled: true
          url: http://prometheus-kube-prometheus-prometheus.monitoring.svc:9090

scrape_configs:
  - job_name: opencost
    scrape_interval: 1m
    scrape_timeout: 15s
    metrics_path: /metrics
    static_configs:
      - targets:
          - opencost.opencost.svc.cluster.local:9003
    metric_relabel_configs:
      - target_label: cluster
        replacement: prod-us-east-1

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cost-anomaly-detection
  namespace: monitoring
spec:
  groups:
    - name: cost.baseline
      interval: 5m
      rules:
        - record: namespace:cost_hourly_usd
          expr: |
            sum by (namespace) (
              container_cpu_allocation
                * on (node) group_left()
                  node_cpu_hourly_cost
              +
              container_memory_allocation_bytes
                * on (node) group_left()
                  node_ram_hourly_cost
                / 1024 / 1024 / 1024
            )
        - record: namespace:cost_hourly_avg_7d
          expr: avg_over_time(namespace:cost_hourly_usd[7d])
        - record: namespace:cost_hourly_stddev_7d
          expr: stddev_over_time(namespace:cost_hourly_usd[7d])
    - name: cost.anomaly_alerts
      rules:
        - alert: NamespaceCostAnomalyHigh
          expr: |
            namespace:cost_hourly_usd
            > (namespace:cost_hourly_avg_7d + 2 * namespace:cost_hourly_stddev_7d)
          for: 1h
          labels:
            severity: warning
          annotations:
            summary: "Аномалия стоимости в namespace {{ $labels.namespace }}"
            description: >
              Почасовая стоимость ${{ $value | humanize }} превышает
              семидневную базу более чем на две сигмы.
        - alert: NamespaceBudgetExceeded
          expr: namespace:cost_hourly_usd * 24 > 10000
          for: 30m
          labels:
            severity: critical
          annotations:
            summary: "Превышен дневной бюджет в {{ $labels.namespace }}"
            description: >
              Прогнозируемые суточные расходы превышают лимит $10 000 для namespace.

apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  generateName: cost-remediation-
  namespace: argo-workflows
spec:
  entrypoint: remediate
  arguments:
    parameters:
      - name: namespace
      - name: current_cost
  templates:
    - name: remediate
      dag:
        tasks:
          - name: scale-down-non-critical
            template: scale-deployments
          - name: notify-team
            template: send-notification
            dependencies: [scale-down-non-critical]
    - name: scale-deployments
      container:
        image: bitnami/kubectl:1.30
        command: [sh, -c]
        args:
          - |
            set -euo pipefail
            echo "Уменьшаем некритичные deployment в ${NAMESPACE}"
            kubectl scale deployment \
              -n "${NAMESPACE}" \
              -l 'cost-tier=best-effort' \
              --replicas=1
            echo "Исправление завершено для ${NAMESPACE}"
        env:
          - name: NAMESPACE
            value: "{{workflow.parameters.namespace}}"
    - name: send-notification
      container:
        image: curlimages/curl:8.5.0
        command: [sh, -c]
        args:
          - |
            curl -fsS -X POST "${SLACK_WEBHOOK}" \
              -H 'Content-Type: application/json' \
              -d "{\"text\":\"Исправление по стоимости для ${NAMESPACE}: некритичные deployment уменьшены до 1 реплики. Почасовой сигнал: {{workflow.parameters.current_cost}}\"}"
        env:
          - name: NAMESPACE
            value: "{{workflow.parameters.namespace}}"
          - name: SLACK_WEBHOOK
            valueFrom:
              secretKeyRef:
                name: cost-alerts-slack
                key: webhook-url

apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: cost-alerts
  namespace: monitoring
spec:
  route:
    receiver: slack-cost-team
    routes:
      - matchers:
          - name: alertname
            value: NamespaceBudgetExceeded
        receiver: argo-workflows
        continue: true
      - matchers:
          - name: alertname
            value: NamespaceCostAnomalyHigh
        receiver: slack-cost-team
  receivers:
    - name: argo-workflows
      webhookConfigs:
        - url: http://argo-events-eventsource-svc.argo-events.svc:12000/cost-remediation
          sendResolved: true
    - name: slack-cost-team
      slackConfigs:
        - apiURL:
            name: slack-webhook
            key: url
          channel: "#cost-alerts"
          title: '{{ .GroupLabels.alertname }}'
          text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

apiVersion: v1
kind: ResourceQuota
metadata:
  name: cost-guardrails
  namespace: team-backend
  labels:
    team: backend
    cost-center: "cc-2201"
spec:
  hard:
    requests.cpu: "40"
    requests.memory: 80Gi
    limits.cpu: "80"
    limits.memory: 160Gi
    pods: "100"
    services.loadbalancers: "2"