awoooi/ops/reboot-recovery/full-stack-cold-start-baseline.yml

version: 2026-05-06.v1
scope:
  included_hosts:
    "110": "DevOps, registry, observability, Sentry, runners"
    "120": "K3s control plane and VIP"
    "121": "K3s peer node and DR drill cron"
    "188": "Data, AI, web, momo, SignOz, public nginx gateway"
  excluded_hosts:
    "112": "Kali security host; recorded but not part of cold-start release gate"

principles:
  - recover_dependency_chain_before_workloads
  - keep_ai_auto_repair_observe_only_until_green
  - never_generic_restart_stateful_services
  - preserve_corrupt_parts_in_quarantine_not_delete
  - release_runners_and_crawlers_last

phases:
  - id: P0-NETWORK
    order: 0
    gates:
      - ping_110_120_121_188
      - ssh_port_110_120_121_188
      - arp_evidence_or_monitor_mode_fallback

  - id: P0-188-DATA
    order: 10
    required_before:
      - P1-K3S
      - P2-WORKLOAD-ALERTCHAIN
    gates:
      - containerd_docker_postgresql_redis_ollama_nginx_active
      - postgresql_5432_accepting_connections
      - redis_pong
      - momo_db_not_restarting
      - signoz_http_reachable
      - momo_health_200

  - id: P0-110-REGISTRY-OBSERVABILITY
    order: 20
    required_before:
      - P1-K3S
      - P3-RUNNER-CD
    gates:
      - docker_active
      - harbor_v2_200_or_401
      - gitea_200_or_302
      - prometheus_ready
      - alertmanager_healthy
      - sentry_http_reachable
      - docker_containers_all_up
      - runner_watchdog_disabled
      - sentry_clickhouse_not_restarting
      - cadvisor_image_v0_47_0
      - cadvisor_cpu_cap_0_3

  - id: P1-K3S
    order: 30
    gates:
      - 120_can_reach_188_postgres
      - mon_and_mon1_ready
      - no_non_running_non_succeeded_pods
      - awoooi_dev_api_nodeport_200
      - vip_192_168_0_125_present

  - id: P2-WORKLOAD-ALERTCHAIN
    order: 40
    gates:
      - awoooi_api_vip_health_2xx_or_3xx
      - awoooi_web_vip_2xx_or_3xx
      - alertmanager_webhook_e2e_2xx_when_release_gate

  - id: P2-PUBLIC-ROUTES
    order: 50
    public_https_routes:
      - https://awoooi.wooo.work/api/v1/health
      - https://awoooi.wooo.work/
      - https://mo.wooo.work/
      - https://mo.wooo.work/health
      - https://gitea.wooo.work/
      - https://harbor.wooo.work/
      - https://registry.wooo.work/
      - https://sentry.wooo.work/
      - https://signoz.wooo.work/
      - https://stock.wooo.work/
      - https://langfuse.wooo.work/
      - https://bitan.wooo.work/
      - https://aiops.wooo.work/

  - id: P2-SCHEDULES
    order: 60
    gates:
      - cron_active_188_110_120_121
      - docker_restart_textfile_fresh_188
      - docker_stats_textfile_fresh_188_110
      - systemd_units_textfile_fresh_110
      - backup_health_textfile_fresh_188_110
      - backup_from_110_success_under_25h
      - expected_backup_jobs_fresh_188_110
      - host_service_config_backup_success_under_48h
      - sentry_dedicated_backup_success_under_48h
      - backup_integrity_check_success_under_8d
      - backup_restore_drill_success_under_31d
      - velero_schedule_present_and_latest_completed_under_25h
      - velero_restore_test_cron_present
      - momo_scheduler_registered_jobs
      - k8s_cronjobs_unsuspended
      - k8s_failed_jobs_zero
      - dr_drill_cron_present_121

  - id: P3-HIGH-LOAD-WORK
    order: 70
    release_after:
      - P0-NETWORK
      - P0-188-DATA
      - P0-110-REGISTRY-OBSERVABILITY
      - P1-K3S
      - P2-WORKLOAD-ALERTCHAIN
      - P2-PUBLIC-ROUTES
      - P2-SCHEDULES
    release_conditions:
      - host_load_per_core_below_1_0_for_15m
      - no_restart_storm
      - clickhouse_merge_or_kafka_lag_not_increasing_two_checks
    examples:
      - sentry_snuba_consumers
      - momo_scheduler_chrome_crawlers
      - gitea_actions_jobs

  - id: P3-RUNNER-CD
    order: 80
    release_conditions:
      - all_previous_gates_green
      - runner_cpuquota_200_percent
      - runner_memorymax_2g
      - watchdogusec_0
      - active_awoooi_cd_or_gitea_actions_task_containers_cpu_capped_during_cold_start

automation_policy:
  before_green:
    ai_auto_repair: observe_only
    alertmanager_smoke_test: manual_or_release_gate_only
    stateful_service_actions: human_approval_required
    generic_restart: forbidden
  after_green:
    ai_auto_repair: limited_execution_for_stateless_exporters_only
    stateful_service_actions: human_in_the_loop
    runner_cd: controlled_release

resource_guardrails:
  "110":
    cadvisor:
      image: gcr.io/cadvisor/cadvisor:v0.47.0
      cpus: 0.3
      mem_limit: 512m
    sentry_snuba_cold_start_consumers:
      cpus: 0.5
      persist_in: /opt/sentry/docker-compose.override.yml
    sentry_self_hosted_memory_limits:
      taskscheduler_mem_limit: 1g
      relay_mem_limit: 2g
      persist_in: /opt/sentry/docker-compose.override.yml
      note: "taskscheduler/relay 不得回退到 512m/1g 造成長期 >85% memory-limit pressure；110 主機仍以 ClickHouse/Kafka/Snuba CPU caps 防止冷啟動過載。"
    actions_runner_systemd:
      cpu_quota: 200%
      memory_max: 2G
      watchdog: disabled
  "188":
    ollama_systemd:
      cpu_quota: 300%
      memory_high: 20G
      memory_max: 24G
      max_loaded_models: 1
      num_parallel: 1
      note: "188 本機 Ollama 是 cold-start 依賴與 Open-WebUI local endpoint；不得維持 disabled/inactive，也不得保留 700%/45G 無節制 guardrail。"
    litellm:
      cpus: 1.0
      memory: 1G
      mode: stateless
    momo_scheduler:
      cpus: 2.0
      memory: 2G
    signoz_clickhouse:
      memory: 24G
      note: do_not_lower_during_merge_backlog

authoritative_checks:
  read_only_monitor:
    command: bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color
    expected_for_cron: PASS>0 WARN=0 BLOCKED=0
  release_gate:
    command: SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --send-alert-test
    expected: PASS=64 WARN=0 BLOCKED=0
  textfile_metric:
    path: /home/wooo/node_exporter_textfiles/cold_start_recovery.prom
    green_metric: awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1
  backup_baseline:
    path: ops/reboot-recovery/full-stack-backup-baseline.yml
    required_metrics:
      - awoooi_backup_health_monitor_up
      - awoooi_backup_job_fresh
      - awoooi_backup_integrity_fresh
      - awoooi_velero_restore_test_cron_present
      - awoooi_velero_restore_test_last_success_fresh