name: chaos / migration full drill (M6)

# M6 of chaos engineering: 每月跑完整迁移演练 ——
#   开新机 + provision + deploy + **拉最新 OSS backup + restore 到新机 +
#   用真数据 smoke 验证** + destroy。
#
# 相比 M4 migration drill 多了中段的数据迁移 + 真数据验证，相当于
# "epsilon 到期换机时的完整剧本"每月演练一次。
#
# 停机模型：真换机时新机先跑起来，老机停服秒级 snapshot → 新机 restore
# → DNS 切换。维护窗口 1-5 分钟，用户看到 503 几秒到几分钟。zero-downtime
# 对 SQLite 架构做不到，需要 Postgres 主从（v0.5+）。
#
# 设计见 brain/sop/services/migration.md

on:
  schedule:
    - cron: "0 4 15 * *"   # UTC 每月 15 号 04:00（= 北京 12:00）
  workflow_dispatch:
    inputs:
      keep_on_failure:
        description: "失败时保留 ECS 让人 debug（默认销毁）"
        required: false
        type: boolean
        default: false
      instance_type:
        description: "按量 ECS 规格（默认 t6-c1m1，可填 ecs.u1-c1m2.large 验证换机型）"
        required: false
        type: string
        default: "ecs.t6-c1m1.large"

jobs:
  drill:
    name: 开机 + provision + deploy + restore + 真数据 smoke + destroy
    runs-on: ubuntu-latest
    timeout-minutes: 40
    env:
      REGION: cn-hangzhou
      ZONE: cn-hangzhou-h
      IMAGE_ID: ubuntu_22_04_x64_20G_alibase_20260316.vhd
      VSWITCH_ID: vsw-bp1jtmzb0pmhmpezhh5h4
      SG_ID: sg-bp1b0ke3ei1y5n3k2qs1
      INSTANCE_TYPE: ${{ inputs.instance_type || 'ecs.t6-c1m1.large' }}
      BUCKET: agentaily-backup-xiangqin-prod
      OSS_ENDPOINT: oss-cn-hangzhou.aliyuncs.com
    steps:
      - uses: actions/checkout@v4

      - name: 装 aliyun CLI + sqlite3
        run: |
          ARCH=$(uname -m | sed s/x86_64/amd64/ | sed s/aarch64/arm64/)
          cd /tmp
          curl -fsSL "https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-${ARCH}.tgz" -o aliyun-cli.tgz
          tar xzf aliyun-cli.tgz
          sudo mv aliyun /usr/local/bin/
          sudo apt-get install -y -qq sqlite3

      - name: 配 aliyun profile（drill）
        env:
          AK_ID: ${{ secrets.ALIYUN_OSS_AK_ID }}
          AK_SECRET: ${{ secrets.ALIYUN_OSS_AK_SECRET }}
        run: |
          aliyun configure set --profile drill --mode AK \
            --access-key-id "$AK_ID" \
            --access-key-secret "$AK_SECRET" \
            --region "$REGION"

      - name: 生成临时 SSH key + 上传 aliyun keypair
        id: keypair
        run: |
          set -euo pipefail
          TS=$(date +%Y%m%d%H%M%S)
          KEY_NAME="xq-fulldrill-${TS}"
          ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/drill" -C "$KEY_NAME" -q
          chmod 600 "$HOME/.ssh/drill"
          aliyun --profile drill ecs ImportKeyPair \
            --RegionId "$REGION" --KeyPairName "$KEY_NAME" \
            --PublicKeyBody "$(cat $HOME/.ssh/drill.pub)"
          echo "key_name=$KEY_NAME" >> "$GITHUB_OUTPUT"

      - name: RunInstances 按量 ECS
        id: create
        run: |
          set -euo pipefail
          KEY_NAME="${{ steps.keypair.outputs.key_name }}"
          RESP=$(aliyun --profile drill ecs RunInstances \
            --RegionId "$REGION" --ZoneId "$ZONE" \
            --ImageId "$IMAGE_ID" --InstanceType "$INSTANCE_TYPE" \
            --SecurityGroupId "$SG_ID" --VSwitchId "$VSWITCH_ID" \
            --InstanceChargeType PostPaid --InternetChargeType PayByTraffic \
            --InternetMaxBandwidthOut 5 --SystemDisk.Size 20 \
            --Amount 1 --KeyPairName "$KEY_NAME" --InstanceName "$KEY_NAME" \
            --Tag.1.Key purpose --Tag.1.Value chaos-migration-full-drill)
          INSTANCE_ID=$(echo "$RESP" | python3 -c "import json,sys; print(json.load(sys.stdin)['InstanceIdSets']['InstanceIdSet'][0])")
          echo "instance_id=$INSTANCE_ID" >> "$GITHUB_OUTPUT"

      - name: 等 Running + 拿公网 IP
        id: wait
        run: |
          set -euo pipefail
          INSTANCE_ID="${{ steps.create.outputs.instance_id }}"
          # 轮询上限 60 × 10s = 10 min。#008 观察：aliyun RunInstances → Running
          # 正常 15-60s，极端情况（按量池紧张 / 底层资源分配慢）2-5 分钟；10 分钟
          # 兜住极端场景。timeout-minutes: 40 给整个 job 足够余量。
          for i in $(seq 1 60); do
            RESP=$(aliyun --profile drill ecs DescribeInstances --RegionId "$REGION" --InstanceIds "[\"$INSTANCE_ID\"]")
            STATUS=$(echo "$RESP" | python3 -c "import json,sys; print(json.load(sys.stdin)['Instances']['Instance'][0]['Status'])")
            IP=$(echo "$RESP" | python3 -c "import json,sys; d=json.load(sys.stdin)['Instances']['Instance'][0]; print((d.get('PublicIpAddress') or {}).get('IpAddress',[''])[0] if (d.get('PublicIpAddress') or {}).get('IpAddress') else '')")
            echo "[$i/60] status=$STATUS ip=$IP"
            if [ "$STATUS" = "Running" ] && [ -n "$IP" ]; then
              echo "public_ip=$IP" >> "$GITHUB_OUTPUT"
              break
            fi
            sleep 10
          done
          # 注意：不要写成 `[ -z "$IP" ] && { ... exit 1; }`—— 当 IP 已取到时
          # `[ -z ]` 返回 1 → `&&` 短路整行 exit 1 → set -e 杀掉 step（2026-04-23
          # M6 踩坑）。用 if/then/fi 显式一点。
          if [ -z "${IP:-}" ]; then
            echo "::error::10 min 未 Running（status=$STATUS ip=$IP）—— aliyun API 极端抖动，下次 drill 再试"
            exit 1
          fi

      - name: 等 SSH 22 可达
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          for i in $(seq 1 30); do nc -zw3 "$IP" 22 2>/dev/null && break || sleep 5; done

      - name: 配 ~/.ssh/config
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          mkdir -p ~/.ssh
          cat >> ~/.ssh/config <<EOF
          Host $IP
              User root
              IdentityFile ~/.ssh/drill
              StrictHostKeyChecking accept-new
              UserKnownHostsFile ~/.ssh/known_hosts
              ServerAliveInterval 30
              ServerAliveCountMax 20
          EOF
          chmod 600 ~/.ssh/config
          for i in $(seq 1 10); do ssh-keyscan "$IP" >> ~/.ssh/known_hosts 2>/dev/null && break || sleep 3; done

      - name: provision.sh（全新机器从零装环境）
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          bash scripts/provision.sh "$IP"

      - name: deploy.sh（同步代码 + 建 systemd）
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          bash scripts/deploy.sh "$IP"

      # ─── 这里是 M6 相比 M4 的核心增量：真数据迁移 ───

      - name: pull 最新 OSS backup 到 GHA runner
        id: backup
        run: |
          set -euo pipefail
          LATEST=$(aliyun --profile drill oss ls "oss://${BUCKET}/daily/" -e "$OSS_ENDPOINT" \
              | grep -oE 'xiangqin-[0-9]{8}\.db\.gz' | sort -u | tail -1)
          if [ -z "${LATEST:-}" ]; then
            echo "::error::OSS 没 backup"
            exit 1
          fi
          mkdir -p /tmp/restore
          aliyun --profile drill oss cp "oss://${BUCKET}/daily/${LATEST}" "/tmp/restore/${LATEST}" \
              -e "$OSS_ENDPOINT" --force
          gunzip "/tmp/restore/${LATEST}"
          DB_FILE="/tmp/restore/${LATEST%.gz}"
          sqlite3 "$DB_FILE" 'PRAGMA integrity_check' | head -1
          echo "db_file=$DB_FILE" >> "$GITHUB_OUTPUT"

      - name: scp backup 到新机 + 停服 + 覆盖 db + 重启
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          DB_FILE="${{ steps.backup.outputs.db_file }}"
          ssh root@$IP 'systemctl stop xiangqin'
          scp "$DB_FILE" root@$IP:/opt/xiangqin/data/xiangqin.db
          # 用 /health polling 代替 `sleep 3 && systemctl is-active`：
          # 新 db 首次启动可能跑 schema 检查 / 索引加载，3 秒不够（2026-04-23
          # M6 踩坑：is-active 返回 activating + exit 3）。20 秒 polling 兜住。
          ssh root@$IP "set -e; \
            chown xiangqin:xiangqin /opt/xiangqin/data/xiangqin.db; \
            chmod 600 /opt/xiangqin/data/xiangqin.db; \
            systemctl start xiangqin; \
            for i in \$(seq 1 20); do \
              curl -sf http://127.0.0.1/health >/dev/null && { echo \"xiangqin ready after \${i}s\"; exit 0; }; \
              sleep 1; \
            done; \
            echo '::error::xiangqin /health 20s 未 ready —— 下面是诊断 dump'; \
            echo '--- systemctl status ---'; \
            systemctl status --no-pager -l xiangqin; \
            echo '--- journalctl full (last 100 lines) ---'; \
            journalctl -u xiangqin --no-pager -n 100 --output=cat; \
            echo '--- systemd unit 内容 ---'; \
            cat /etc/systemd/system/xiangqin.service; \
            echo '--- systemd override.conf（SALT 应该在这里）---'; \
            cat /etc/systemd/system/xiangqin.service.d/*.conf 2>/dev/null || echo '(无 override.conf)'; \
            echo '--- xiangqin user 存在吗 ---'; \
            getent passwd xiangqin || echo '(xiangqin user 不存在)'; \
            echo '--- data 目录权限 ---'; \
            ls -la /opt/xiangqin/data/; \
            echo '--- 手动复现启动（不进 systemd）看 Python traceback ---'; \
            cd /opt/xiangqin && \
              sudo -u xiangqin /opt/xiangqin/.venv/bin/python -c 'from xiangqin.app import app; print(\"import ok\")' 2>&1 | head -30 || true; \
            exit 1"

      - name: smoke —— 用真数据验证新机能响应
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          # 1. /health 返回 200
          ssh root@$IP 'curl -sf http://127.0.0.1/health'
          # 2. db 里 users 数 > 0（真数据恢复了）
          USERS=$(ssh root@$IP 'sqlite3 /opt/xiangqin/data/xiangqin.db "SELECT COUNT(*) FROM users"')
          echo "恢复的 db 中 users 数: $USERS"
          # 3. 模拟 DNS 切换：从 GHA runner 直连新机 IP 测
          EXT_HEALTH=$(curl -sf -m 5 "http://${IP}/health")
          echo "外网 /health: $EXT_HEALTH"
          cat <<EOF >> "$GITHUB_STEP_SUMMARY"
          ### M6 真数据迁移演练 ✓
          - 新机 IP: \`$IP\`
          - 恢复 users 数: **$USERS**
          - /health: \`$EXT_HEALTH\`
          - 含义：真换机时用这个流程，1-5 分钟维护窗口内完成 backup → restore → DNS 切换，用户体验"几分钟 503 后恢复"
          EOF

      - name: cleanup always
        if: always() && (success() || !inputs.keep_on_failure)
        run: |
          set +e
          INSTANCE_ID="${{ steps.create.outputs.instance_id }}"
          KEY_NAME="${{ steps.keypair.outputs.key_name }}"

          # 主任务失败太早（例如 RunInstances 完但实例还 Initializing）时，
          # DeleteInstance 会返 IncorrectInstanceStatus.Initializing；先轮询等
          # 到 Running 或 Stopped 这种"可删"状态再删，最多 2 分钟。
          if [ -n "$INSTANCE_ID" ]; then
            for i in $(seq 1 24); do
              STATUS=$(aliyun --profile drill ecs DescribeInstances \
                --RegionId "$REGION" --InstanceIds "[\"$INSTANCE_ID\"]" 2>/dev/null \
                | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['Instances']['Instance'][0]['Status'] if d['Instances']['Instance'] else 'Gone')" 2>/dev/null)
              echo "[cleanup $i/24] instance $INSTANCE_ID status=$STATUS"
              case "$STATUS" in
                Running|Stopped) echo "可删，继续"; break ;;
                Gone) echo "实例已不存在，跳 DeleteInstance"; INSTANCE_ID=""; break ;;
                *) sleep 5 ;;
              esac
            done
            if [ -n "$INSTANCE_ID" ]; then
              echo "销毁 ECS $INSTANCE_ID"
              aliyun --profile drill ecs DeleteInstance \
                --RegionId "$REGION" --InstanceId "$INSTANCE_ID" --Force true
            fi
          fi

          # keypair 删除：DeleteInstance 返回后实例可能还在 "Stopping/Deleting"，
          # keypair 还绑着。轮询到实例消失再删 keypair。
          if [ -n "$KEY_NAME" ]; then
            for i in $(seq 1 24); do
              COUNT=$(aliyun --profile drill ecs DescribeInstances \
                --RegionId "$REGION" --KeyPairName "$KEY_NAME" 2>/dev/null \
                | python3 -c "import json,sys; print(len(json.load(sys.stdin)['Instances']['Instance']))" 2>/dev/null || echo 0)
              echo "[cleanup $i/24] keypair $KEY_NAME still attached=$COUNT"
              [ "$COUNT" = "0" ] && break
              sleep 5
            done
            echo "删除 keypair $KEY_NAME"
            aliyun --profile drill ecs DeleteKeyPairs \
              --RegionId "$REGION" --KeyPairNames "[\"$KEY_NAME\"]" || echo "::warning::keypair $KEY_NAME 删除失败，需手动清"
          fi

          exit 0   # cleanup 总是 success；真删失败 warning 给人，不阻断 workflow 汇总状态

      - name: 失败时保留 debug
        if: failure() && inputs.keep_on_failure
        run: |
          IP="${{ steps.wait.outputs.public_ip }}"
          echo "::warning::drill 机器 ${{ steps.create.outputs.instance_id }} ($IP) 保留，debug 完手动删"
