name: redeploy-tenants-on-main

# Ported from .github/workflows/redeploy-tenants-on-main.yml on 2026-05-11 per RFC
# internal#219 §1 sweep. Differences from the GitHub version:
#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
#   - Dropped `merge_group:` (no Gitea merge queue).
#   - Dropped `environment:` blocks (Gitea has no environments).
#   - Workflow-level env.GITHUB_SERVER_URL pinned per
#     feedback_act_runner_github_server_url.
#   - `continue-on-error: true` on each job (RFC §1 contract).
#   - Dropped unsupported `workflow_run` (task #81).
#   - Later changed to manual-only after publish-workspace-server-image.yml
#     gained an integrated ordered production deploy job.
#

# Manual production tenant redeploy/rollback helper.
#
# Why this workflow is manual-only: publish-workspace-server-image now owns
# the ordered build -> push -> production auto-deploy sequence in one workflow.
# A separate push-triggered redeploy workflow races before the new ECR image
# exists and can paint main red with a false deployment failure.
#
# This workflow closes the gap by calling the control-plane admin
# endpoint that performs a canary-first, batched, health-gated rolling
# redeploy across every live tenant. Implemented in molecule-ai/
# molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
# (feat/tenant-auto-redeploy, landing alongside this workflow).
#
# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
# Gitea suspension migration. The staging-verify.yml promote step now
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
#
# Runtime ordering for automatic deploys now lives in
# publish-workspace-server-image.yml:
#   1. build-and-push creates new :staging-<sha> images in ECR.
#   2. deploy-production waits for required push contexts on that SHA.
#   3. deploy-production calls redeploy-fleet canary-first.
#
# Rollback path: set PROD_MANUAL_REDEPLOY_TARGET_TAG as a repo/org
# variable or secret, run workflow_dispatch, then unset it after the
# rollback. That calls redeploy-fleet with target_tag=<value>,
# re-pulling the pinned image on every tenant.

on:
  workflow_dispatch:
permissions:
  contents: read
  # No write scopes needed — the workflow hits an external CP endpoint,
  # not the GitHub API.

# Serialize manual redeploys so two operator-triggered rollbacks do not
# overlap and cause confusing per-tenant SSM state.
#
# NOTE: cancel-in-progress: false removed (Rule 7 fix). Gitea 1.22.6
# cancels queued runs regardless of this setting, so it provides no
# actual protection. Each redeploy-fleet call is idempotent (canary-first
# + batched + health-gated) so a cancelled predecessor is recovered
# automatically by the next run.
concurrency:
  group: redeploy-tenants-on-main

env:
  GITHUB_SERVER_URL: https://git.moleculesai.app

jobs:
  # bp-exempt: production redeploy is a side-effect workflow, not a merge gate.
  redeploy:
    if: ${{ github.event_name == 'workflow_dispatch' }}
    # Dedicated publish/release lane (internal#462 / #394 / #399).
    # Production tenant redeploy — a deploy action, reserved capacity so
    # it never queues behind PR-CI. `publish` -> molecule-runner-publish-*.
    runs-on: publish
    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
    continue-on-error: true
    timeout-minutes: 25
    env:
      # Rule 9 fix: keep the same operational kill switch surface as the
      # integrated auto-deploy workflow.
      PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
    steps:
      - name: Kill-switch guard
        # Rule 9 fix: exit fast if kill switch is set. No redeploy happens.
        if: env.PROD_AUTO_DEPLOY_DISABLED == 'true'
        run: |
          echo "::notice::Production auto-deploy disabled (PROD_AUTO_DEPLOY_DISABLED=true). Skipping redeploy."
          echo "To re-enable: unset the repo variable or set it to false."
      - name: Note on ECR propagation
        # ECR image manifests are consistent immediately after push — no
        # CDN cache to wait for. The old GHCR-based workflow had a 30s
        # sleep to avoid race conditions; ECR makes that unnecessary.
        run: echo "ECR image available immediately after push — proceeding."

      - name: Compute target tag
        id: tag
        # Resolution order:
        #   1. Operator-supplied input (workflow_dispatch with explicit
        #      tag) → used verbatim. Lets ops pin `latest` for emergency
        #      rollback to last canary-verified digest, or pin a specific
        #      `staging-<sha>` to roll back to a known-good build.
        #   2. Default → `staging-<short_head_sha>` for manual reruns from
        #      the current default-branch SHA.
        env:
          PROD_MANUAL_REDEPLOY_TARGET_TAG: ${{ vars.PROD_MANUAL_REDEPLOY_TARGET_TAG || secrets.PROD_MANUAL_REDEPLOY_TARGET_TAG || '' }}
          HEAD_SHA: ${{ github.sha }}
        run: |
          set -euo pipefail
          if [ -n "${PROD_MANUAL_REDEPLOY_TARGET_TAG:-}" ]; then
            echo "target_tag=$PROD_MANUAL_REDEPLOY_TARGET_TAG" >> "$GITHUB_OUTPUT"
            echo "Using operator-pinned tag from PROD_MANUAL_REDEPLOY_TARGET_TAG."
          else
            SHORT="${HEAD_SHA:0:7}"
            echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
            echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
          fi

      - name: Call CP redeploy-fleet
        # CP_ADMIN_API_TOKEN must be set as a repo/org secret on
        # molecule-ai/molecule-core, matching the staging/prod CP's
        # CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
        # repo's secrets for CI.
        env:
          CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
          CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
          CANARY_SLUG: ${{ vars.PROD_REDEPLOY_CANARY_SLUG || secrets.PROD_REDEPLOY_CANARY_SLUG || '' }}
          SOAK_SECONDS: ${{ vars.PROD_REDEPLOY_SOAK_SECONDS || secrets.PROD_REDEPLOY_SOAK_SECONDS || '' }}
          BATCH_SIZE: ${{ vars.PROD_REDEPLOY_BATCH_SIZE || secrets.PROD_REDEPLOY_BATCH_SIZE || '' }}
          DRY_RUN: ${{ vars.PROD_REDEPLOY_DRY_RUN || secrets.PROD_REDEPLOY_DRY_RUN || '' }}
          PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
        run: |
          set -euo pipefail

          case "${PROD_AUTO_DEPLOY_DISABLED,,}" in
            1|true|yes|on)
              echo "::notice::PROD_AUTO_DEPLOY_DISABLED is set; skipping production redeploy."
              exit 0
              ;;
          esac

          CANARY_SLUG="${CANARY_SLUG:-hongming}"
          SOAK_SECONDS="${SOAK_SECONDS:-60}"
          BATCH_SIZE="${BATCH_SIZE:-3}"
          DRY_RUN="${DRY_RUN:-false}"

          if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
            echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
            echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
            exit 1
          fi

          # confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
          # contract (cp#228 / task #308) for fleet-wide intent. Empty body
          # / {confirm:false} / {only_slugs:[]} → 400. This caller redeploys
          # the entire prod fleet (canary + fan-out), no slug scoping, so
          # confirm:true is correct.
          BODY=$(jq -nc \
            --arg tag "$TARGET_TAG" \
            --arg canary "$CANARY_SLUG" \
            --argjson soak "$SOAK_SECONDS" \
            --argjson batch "$BATCH_SIZE" \
            --argjson dry "$DRY_RUN" \
            '{
              target_tag: $tag,
              canary_slug: $canary,
              soak_seconds: $soak,
              batch_size: $batch,
              dry_run: $dry,
              confirm: true
            }')

          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
          echo "  target_tag=$TARGET_TAG canary=$CANARY_SLUG soak_seconds=$SOAK_SECONDS batch_size=$BATCH_SIZE dry_run=$DRY_RUN"

          HTTP_RESPONSE=$(mktemp)
          HTTP_CODE_FILE=$(mktemp)
          # Route -w into its own tempfile so curl's exit code (e.g. 56
          # on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
          # pollute the captured stdout. The previous inline-substitution
          # shape produced "000000" on connection reset (curl wrote
          # "000" via -w, then the inline echo-fallback appended another
          # "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
          # set +e/-e keeps the non-zero curl exit from tripping the
          # outer pipeline. See lint-curl-status-capture.yml for the
          # CI gate that pins this fix shape.
          set +e
          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
            -m 1200 \
            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
            -H "Content-Type: application/json" \
            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
            -d "$BODY" >"$HTTP_CODE_FILE"
          set -e
          # Stderr from curl (e.g. dial errors with -sS) goes to the runner
          # log so operators can see WHY a connection failed. Stdout is
          # captured to $HTTP_CODE_FILE because that's where -w writes.
          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"

          echo "HTTP $HTTP_CODE"
          # Rule 8 fix: redact raw CP response from CI logs. Print only
          # safe fields: ok boolean, result count, error presence (no content).
          jq '{ok, result_count: (.results | length), has_errors: (.results | any(.error != null))}' "$HTTP_RESPONSE" || echo "(jq parse failed)"

          # Pretty-print per-tenant results in the job summary so
          # ops can see which tenants were redeployed without drilling
          # into the raw response.
          {
            echo "## Tenant redeploy fleet"
            echo ""
            echo "**Target tag:** \`$TARGET_TAG\`"
            echo "**Canary:** \`$CANARY_SLUG\` (soak ${SOAK_SECONDS}s)"
            echo "**Batch size:** $BATCH_SIZE"
            echo "**Dry run:** $DRY_RUN"
            echo "**HTTP:** $HTTP_CODE"
            echo ""
            echo "### Per-tenant result"
            echo ""
            echo '| Slug | Phase | SSM Status | Exit | Healthz | Errors |'
            echo '|------|-------|------------|------|---------|-------|'
            # Rule 8 fix: .error field redacted from CI logs/summary. Print only
            # presence boolean so ops know whether to look deeper.
            jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error != null) |"' "$HTTP_RESPONSE" || true
          } >> "$GITHUB_STEP_SUMMARY"

          if [ "$HTTP_CODE" != "200" ]; then
            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
            exit 1
          fi
          OK=$(jq -r '.ok' "$HTTP_RESPONSE")
          if [ "$OK" != "true" ]; then
            echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
            exit 1
          fi
          echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."

          # Stash the response for the verify step. $RUNNER_TEMP outlasts
          # the step boundary; $HTTP_RESPONSE doesn't.
          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"

      - name: Verify each tenant /buildinfo matches published SHA
        # ROOT FIX FOR #2395.
        #
        # `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
        # didn't error" — NOT "the new image is running on the tenant."
        # `:latest` lives in the local Docker daemon's image cache; if
        # the SSM document does `docker compose up -d` without an
        # explicit `docker pull`, the daemon serves the previously-
        # cached digest and the container restarts on stale code.
        # 2026-04-30 incident: hongmingwang's tenant reported
        # ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
        # chat_files for 30+ min — the lazy-heal fix never reached the
        # user despite green deploy + green redeploy.
        #
        # This step closes the gap by curling each tenant's /buildinfo
        # endpoint (added in workspace-server/internal/buildinfo +
        # /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
        # returned git_sha to the SHA the workflow expects. Mismatches
        # fail the workflow, which is what `ok=true` should have
        # guaranteed all along.
        #
        # When the redeploy is triggered manually with a specific tag
        # (target_tag != "latest"), the expected SHA may not equal
        # ${{ github.sha }}.
        env:
          EXPECTED_SHA: ${{ github.sha }}
          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
          # Tenant subdomain template — slugs from the response are
          # appended. Production CP issues `<slug>.moleculesai.app`;
          # staging CP issues `<slug>.staging.moleculesai.app`. This
          # workflow runs on main → prod CP → no `staging.` infix.
          TENANT_DOMAIN: 'moleculesai.app'
        run: |
          set -euo pipefail

          EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
          if [ "$TARGET_TAG" != "latest" ] \
             && [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
             && [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
            # Manual redeploy with a pinned tag that isn't the head
            # SHA — operator is rolling back / pinning. Skip the
            # verification because we don't have the expected SHA in
            # this context (would need to inspect the ECR
            # manifest, which is a follow-up). Failing-open here is
            # safe: the operator chose the tag deliberately.
            #
            # `staging-<short_head_sha>` IS verified — it's the new
            # auto-trigger default (see Compute target tag step) and
            # the digest under that tag SHOULD match EXPECTED_SHA.
            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
            exit 0
          fi

          RESP="$RUNNER_TEMP/redeploy-response.json"
          if [ ! -s "$RESP" ]; then
            echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
            exit 1
          fi

          # Pull only successfully-redeployed tenants. Any tenant that
          # halted the rollout already failed the previous step, so we
          # don't double-count them here.
          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
          if [ ${#SLUGS[@]} -eq 0 ]; then
            echo "::warning::No tenants reported healthz_ok — nothing to verify"
            exit 0
          fi

          echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."

          # Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
          # vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
          # comment for the full rationale; same logic applies on prod even
          # though prod has fewer ephemeral tenants — the asymmetry would be a
          # gratuitous fork.
          STALE_COUNT=0
          UNREACHABLE_COUNT=0
          STALE_LINES=()
          UNREACHABLE_LINES=()
          for slug in "${SLUGS[@]}"; do
            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
            # 30s total: tenant just SSM-restarted, may still be coming
            # up. Retry-on-empty rather than retry-on-status — we want
            # to fail fast on "responded with wrong SHA", not "still
            # warming up".
            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
            if [ -z "$ACTUAL_SHA" ]; then
              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
              continue
            fi
            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
            else
              STALE_COUNT=$((STALE_COUNT + 1))
              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
            fi
          done

          {
            echo ""
            echo "### Per-tenant /buildinfo verification"
            echo ""
            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
            echo ""
            if [ $STALE_COUNT -gt 0 ]; then
              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
              echo ""
              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
              echo "|------|----------------------|----------|--------|"
              for line in "${STALE_LINES[@]}"; do echo "$line"; done
              echo ""
            fi
            if [ $UNREACHABLE_COUNT -gt 0 ]; then
              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
              echo ""
              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
              echo "|------|----------------------|----------|--------|"
              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
              echo ""
            fi
            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
              echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
            fi
          } >> "$GITHUB_STEP_SUMMARY"

          if [ $UNREACHABLE_COUNT -gt 0 ]; then
            echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
          fi

          # Belt-and-suspenders sanity floor: same logic as the staging
          # variant — see that file's comment for the full rationale.
          # Floor only applies when fleet >= 4; below that, staging-verify
          # is the actual gate.
          TOTAL_VERIFIED=${#SLUGS[@]}
          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
            exit 1
          fi

          if [ $STALE_COUNT -gt 0 ]; then
            echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
            exit 1
          fi

          echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
