name: Auto-promote staging → main

# Fires after any of the staging-branch quality gates complete. When ALL
# required gates are green on the same staging SHA, opens (or re-uses)
# a PR `staging → main` and enables auto-merge so the merge queue lands
# it. Closes the gap that historically let features sit on staging for
# weeks waiting for a bulk promotion PR (see molecule-core#1496 for the
# 1172-commit example).
#
# 2026-04-28 rewrite (PR #142): the previous version did a direct
# `git merge --ff-only origin staging && git push origin main`. That
# breaks against main's branch-protection ruleset, which requires
# status checks "set by the expected GitHub apps" — direct pushes
# can't satisfy that condition (only PR merges through the queue can).
# The workflow was failing every tick with:
#   remote: error: GH006: Protected branch update failed for refs/heads/main.
#   remote: - Required status checks ... were not set by the expected GitHub apps.
# Fix: mirror the PR-based pattern from auto-sync-main-to-staging.yml
# (the reverse-direction sync, fixed in #2234 for the same reason).
# Both directions now use the same merge-queue path that humans use,
# no special-case bypass.
#
# Safety model:
# - Runs ONLY on workflow_run events for the staging branch.
# - Requires EVERY named gate workflow to have the same head_sha and
#   all be `conclusion == success`. If any of them is red, skipped,
#   cancelled, or pending, we abort (stay on the current main).
# - The PR base=main head=staging path lets GitHub itself enforce
#   branch protection. If main has diverged from staging or required
#   checks aren't satisfied, the merge queue declines the PR — no
#   need for a manual ff-only ancestry check here.
# - Loop safety: the auto-sync-main-to-staging workflow fires when
#   main lands the auto-promote PR, but its merge into staging is by
#   GITHUB_TOKEN which doesn't trigger downstream workflow_run events
#   (GitHub Actions safety). So this workflow doesn't re-fire from
#   its own promote landing.
#
# Toggle via repo variable AUTO_PROMOTE_ENABLED (true/unset). When
# unset, the workflow logs what it would have done but doesn't open
# the PR — useful for dry-running the gate logic without surfacing
# a noisy PR while staging CI is still flaky.
#
# **One-time repo setting (load-bearing):** this workflow opens the
# staging→main PR via `gh pr create` using the default GITHUB_TOKEN.
# Since GitHub's 2022 default change, that token cannot create or
# approve PRs unless the repo opts in. The toggle is at:
#
#   Settings → Actions → General → Workflow permissions
#   → ✅ Allow GitHub Actions to create and approve pull requests
#
# Without it, every workflow_run fails with:
#
#   pull request create failed: GraphQL: GitHub Actions is not
#   permitted to create or approve pull requests (createPullRequest)
#
# Observed 2026-04-29 01:43 UTC blocking promotion of fcd87b9 (PRs
# #2248 + #2249); manually bridged via PR #2252. Re-check this
# setting if auto-promote starts failing with createPullRequest
# errors after a repo or org admin change.

on:
  workflow_run:
    workflows:
      - CI
      - E2E Staging Canvas (Playwright)
      - E2E API Smoke Test
      - CodeQL
    types: [completed]
  workflow_dispatch:
    inputs:
      force:
        description: "Force promote even when AUTO_PROMOTE_ENABLED is unset (manual override)"
        required: false
        default: "false"

permissions:
  contents: write
  pull-requests: write
  # actions: write is needed by the post-merge dispatch tail step
  # (#2358 / #2357) — `gh workflow run publish-workspace-server-image.yml`
  # POSTs to /actions/workflows/.../dispatches which requires this scope.
  # Without it the call 403s and the publish/canary/redeploy chain still
  # doesn't run on staging→main promotions, undoing #2358.
  actions: write

# Serialize auto-promote runs. Multiple staging gate completions can land
# in quick succession (CI + E2E + CodeQL all finish within seconds of
# each other on a green PR) — without this, two parallel runs both:
#   1. Open / re-use the same promote PR.
#   2. Both call `gh pr merge --auto` (idempotent — fine).
#   3. Both poll for the same mergedAt and both `gh workflow run` publish
#      → 2× redundant publish builds racing for the same `:staging-latest`
#      retag, and 2× canary-verify chains.
# cancel-in-progress: false because we don't want a brand-new run to kill
# a polling-tail that's about to dispatch — the polling tail's 30 min cap
# is the right backstop, not workflow-level cancel.
concurrency:
  group: auto-promote-staging
  cancel-in-progress: false

jobs:
  check-all-gates-green:
    # Only consider staging pushes. PRs into staging don't promote.
    if: >
      (github.event_name == 'workflow_run' &&
       github.event.workflow_run.head_branch == 'staging' &&
       github.event.workflow_run.event == 'push')
      || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    outputs:
      all_green: ${{ steps.gates.outputs.all_green }}
      head_sha: ${{ steps.gates.outputs.head_sha }}
    steps:
      # Skip empty-tree promotes (the perpetual auto-promote↔auto-sync cycle
      # observed 2026-05-03). Sequence: auto-promote merges via the staging
      # merge-queue's MERGE strategy, creating a merge commit on main that
      # staging doesn't have. auto-sync then merges main back into staging
      # via another merge commit (the queue's MERGE strategy applies on
      # the staging side too, even when the workflow's local FF would
      # have sufficed). Now staging has a new merge-commit SHA whose
      # tree == main's tree — but auto-promote sees "staging ahead of
      # main by 1" and opens YET another empty promote PR. Each round
      # costs ~30-40 min wallclock, ~2 manual approvals, and burns a
      # full CodeQL Go run (~15 min). Without this guard the cycle
      # repeats indefinitely.
      #
      # Long-term fix is to switch the merge_queue ruleset's
      # `merge_method` away from MERGE so FF-able PRs land cleanly,
      # but that's a broader change affecting every staging PR's
      # commit shape. This guard is the one-line surgical fix that
      # breaks the cycle without touching merge-queue config.
      #
      # Fail-open: if `git diff` errors for any reason, fall through
      # to the gate check (preserve existing behavior). Only skip
      # when the diff is DEFINITIVELY empty.
      - name: Checkout for tree-diff check
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
          ref: staging
      - name: Skip if staging tree == main tree (perpetual-cycle break)
        id: tree-diff
        env:
          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
        run: |
          set -eu
          git fetch origin main --depth=50 || { echo "::warning::git fetch main failed — proceeding (fail-open)"; exit 0; }
          # Compare staging tip's tree against main's tree. `git diff
          # --quiet` exits 0 if no differences, 1 if there are.
          if git diff --quiet origin/main "$HEAD_SHA" -- 2>/dev/null; then
            {
              echo "## ⏭ Skipped — no code to promote"
              echo
              echo "staging tip (\`${HEAD_SHA:0:8}\`) and \`main\` have identical trees."
              echo "This is the auto-promote↔auto-sync merge-commit cycle: staging has a"
              echo "new SHA (a sync-back merge commit) but the underlying file tree is"
              echo "already on main, so there's no real code to ship."
              echo
              echo "Skipping to avoid opening an empty promote PR. Cycle terminates here."
            } >> "$GITHUB_STEP_SUMMARY"
            echo "::notice::auto-promote: staging tree == main tree — no code to promote, skipping"
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Check all required gates on this SHA
        if: steps.tree-diff.outputs.skip != 'true'
        id: gates
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail

          # Required gate workflow files. Use file paths (relative to
          # .github/workflows/) rather than display names because:
          #
          #   1. `gh run list --workflow=<name>` is ambiguous when two
          #      workflows have the same `name:` — observed 2026-04-28
          #      with "CodeQL" matching both `codeql.yml` (explicit) and
          #      GitHub's UI-configured Code-quality default setup
          #      (internal "codeql"). gh CLI returns "could not resolve
          #      to a unique workflow" → empty result → gate evaluated
          #      as missing/none → auto-promote dead-locked despite all
          #      checks actually passing.
          #
          #   2. File paths are the unique identifier for workflows;
          #      `name:` is just a display string and can collide.
          #
          # When adding/removing a gate, update this list AND the
          # branch-protection required-checks list (which uses check-run
          # display names, not workflow names; the two are decoupled and
          # should be kept in sync manually).
          GATES=(
            "ci.yml"
            "e2e-staging-canvas.yml"
            "e2e-api.yml"
            "codeql.yml"
          )

          echo "head_sha=${HEAD_SHA}" >> "$GITHUB_OUTPUT"
          echo "Checking gates on SHA ${HEAD_SHA}"

          ALL_GREEN=true
          for gate in "${GATES[@]}"; do
            # Query the most recent run of this workflow on this SHA.
            # event=push to avoid picking up PR runs. branch=staging to
            # guard against someone dispatching the gate on a non-staging
            # branch at the same SHA.
            RESULT=$(gh run list \
              --repo "$REPO" \
              --workflow "$gate" \
              --branch staging \
              --event push \
              --commit "$HEAD_SHA" \
              --limit 1 \
              --json status,conclusion \
              --jq '.[0] | "\(.status)/\(.conclusion // "none")"' \
              2>/dev/null || echo "missing/none")

            echo "  $gate → $RESULT"

            # Only completed/success counts. completed/failure or
            # in_progress/anything or no record at all = abort.
            if [ "$RESULT" != "completed/success" ]; then
              ALL_GREEN=false
            fi
          done

          echo "all_green=${ALL_GREEN}" >> "$GITHUB_OUTPUT"
          if [ "$ALL_GREEN" != "true" ]; then
            echo "::notice::auto-promote: not all gates are green on ${HEAD_SHA} — staying on current main"
          fi

  promote:
    needs: check-all-gates-green
    if: needs.check-all-gates-green.outputs.all_green == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Check rollout gate
        env:
          AUTO_PROMOTE_ENABLED: ${{ vars.AUTO_PROMOTE_ENABLED }}
          FORCE_INPUT: ${{ github.event.inputs.force }}
        run: |
          set -eu
          # Repo variable AUTO_PROMOTE_ENABLED=true flips this on. While
          # it's unset, the workflow dry-runs (logs what it would have
          # done) but doesn't open the promote PR. Set the variable in
          # Settings → Secrets and variables → Actions → Variables.
          if [ "${AUTO_PROMOTE_ENABLED:-}" != "true" ] && [ "${FORCE_INPUT:-false}" != "true" ]; then
            {
              echo "## ⏸ Auto-promote disabled"
              echo
              echo "Repo variable \`AUTO_PROMOTE_ENABLED\` is not set to \`true\`."
              echo "All gates are green on staging; would have opened a promote PR to \`main\`."
              echo
              echo "To enable: Settings → Secrets and variables → Actions → Variables → \`AUTO_PROMOTE_ENABLED=true\`."
              echo "To test once manually: workflow_dispatch with \`force=true\`."
            } >> "$GITHUB_STEP_SUMMARY"
            echo "::notice::auto-promote disabled — dry run only"
            exit 0
          fi

      # Mint the App token BEFORE the promote-PR step so the auto-merge
      # call can use it. GITHUB_TOKEN-initiated merges suppress the
      # downstream `push` event on main, breaking the
      # publish-workspace-server-image → canary-verify → redeploy-tenants
      # chain (issue #2357). Using the App token here means the
      # merge-queue-landed merge IS able to fire the cascade naturally;
      # the polling tail below stays as defense-in-depth.
      - name: Mint App token for promote-PR + downstream dispatch
        if: ${{ vars.AUTO_PROMOTE_ENABLED == 'true' || github.event.inputs.force == 'true' }}
        id: app-token
        uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
        with:
          app-id: ${{ secrets.MOLECULE_AI_APP_ID }}
          private-key: ${{ secrets.MOLECULE_AI_APP_PRIVATE_KEY }}

      - name: Open (or reuse) staging → main promote PR + enable auto-merge
        if: ${{ vars.AUTO_PROMOTE_ENABLED == 'true' || github.event.inputs.force == 'true' }}
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
          REPO: ${{ github.repository }}
          TARGET_SHA: ${{ needs.check-all-gates-green.outputs.head_sha }}
        run: |
          set -euo pipefail

          # Look for an existing open promote PR (idempotent on re-run
          # of the workflow). The PR's head IS the staging branch — the
          # whole point is "advance main to staging's tip", so we don't
          # need a per-SHA branch like auto-sync-main-to-staging uses.
          PR_NUM=$(gh pr list --repo "$REPO" \
            --base main --head staging --state open \
            --json number --jq '.[0].number // ""')

          if [ -z "$PR_NUM" ]; then
            TITLE="staging → main: auto-promote ${TARGET_SHA:0:7}"
            BODY_FILE=$(mktemp)
            cat > "$BODY_FILE" <<EOFBODY
          Automated promotion of \`staging\` (\`${TARGET_SHA:0:8}\`) to \`main\`. All required staging gates green at this SHA: CI, E2E Staging Canvas, E2E API Smoke, CodeQL.

          This PR is auto-generated by \`.github/workflows/auto-promote-staging.yml\` whenever every required gate completes green on the same staging SHA. It exists because main's branch protection requires status checks "set by the expected GitHub apps" — direct \`git push\` from a workflow can't satisfy that, only PR merges through the queue can.

          Merge queue lands this; no human action needed unless gates fail. Reverse-direction sync (the merge commit on main → staging) is handled by \`auto-sync-main-to-staging.yml\`.
          EOFBODY
            PR_URL=$(gh pr create --repo "$REPO" \
              --base main --head staging \
              --title "$TITLE" \
              --body-file "$BODY_FILE")
            PR_NUM=$(echo "$PR_URL" | grep -oE '[0-9]+$' | tail -1)
            rm -f "$BODY_FILE"
            echo "::notice::Opened PR #${PR_NUM}"
          else
            echo "::notice::Re-using existing promote PR #${PR_NUM}"
          fi

          # Enable auto-merge — the merge queue picks it up once
          # required gates are green on the merge_group ref.
          if ! gh pr merge "$PR_NUM" --repo "$REPO" --auto --merge 2>&1; then
            echo "::warning::Failed to enable auto-merge on PR #${PR_NUM} — operator may need to merge manually."
          fi

          {
            echo "## ✅ Auto-promote PR opened"
            echo
            echo "- Source: staging at \`${TARGET_SHA:0:8}\`"
            echo "- PR: #${PR_NUM}"
            echo
            echo "Merge queue lands the PR once required gates are green; no human action needed unless gates fail."
          } >> "$GITHUB_STEP_SUMMARY"

          # Hand the PR number to the next step so we can dispatch the
          # tenant-redeploy chain after the merge queue lands the merge.
          echo "promote_pr_num=${PR_NUM}" >> "$GITHUB_OUTPUT"
        id: promote_pr

      # The App token minted above (before the promote-PR step) is
      # also used by the polling tail below. Defense-in-depth: with
      # the merge-queue-landed merge now using the App token, the
      # main-branch push event SHOULD fire the publish/canary/redeploy
      # cascade naturally — but if for any reason it doesn't (e.g. an
      # unrelated event-suppression edge case), the explicit dispatches
      # below still wake the chain.
      - name: Wait for promote merge, then dispatch publish + redeploy (#2357)
        # Defense-in-depth dispatch. With the auto-merge call above
        # now using the App token (this commit), the merge-queue-landed
        # merge SHOULD fire publish-workspace-server-image naturally
        # via on:push:[main] — App-token-initiated pushes DO trigger
        # workflow_run cascades, unlike GITHUB_TOKEN-initiated ones
        # (the documented "no recursion" rule —
        # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow).
        #
        # This explicit dispatch stays as belt-and-suspenders for any
        # edge case where the natural cascade misfires. If it never
        # observably fires after this token swap (i.e. the publish
        # workflow has already started by the time we get here), the
        # second dispatch is a harmless no-op (publish-workspace-server-image
        # has its own concurrency group that dedupes).
        #
        # See PR for #2357: pre-fix the merge action was via
        # GITHUB_TOKEN, suppressing the cascade and forcing this tail
        # to be the SOLE chain trigger. With the auto-merge token swap
        # the tail becomes redundant in the happy path; keep until
        # we've observed >=10 successful natural cascades, then drop.
        if: steps.promote_pr.outputs.promote_pr_num != ''
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
          REPO: ${{ github.repository }}
          PR_NUM: ${{ steps.promote_pr.outputs.promote_pr_num }}
        run: |
          # Poll for merge — max 30 min (60 × 30s). The merge queue
          # typically lands within 5-10 min when gates are green. Break
          # early if the PR is closed without merging (operator action,
          # gates flipped red post-approval, branch-protection rejection)
          # so we don't tie up a runner for the full 30 min on a dead PR.
          MERGED=""
          STATE=""
          for _ in $(seq 1 60); do
            VIEW=$(gh pr view "$PR_NUM" --repo "$REPO" --json mergedAt,state)
            MERGED=$(echo "$VIEW" | jq -r '.mergedAt // ""')
            STATE=$(echo "$VIEW" | jq -r '.state // ""')
            if [ -n "$MERGED" ] && [ "$MERGED" != "null" ]; then
              echo "::notice::Promote PR #${PR_NUM} merged at ${MERGED}"
              break
            fi
            if [ "$STATE" = "CLOSED" ]; then
              echo "::warning::Promote PR #${PR_NUM} was closed without merging — skipping deploy dispatch."
              exit 0
            fi
            sleep 30
          done

          if [ -z "$MERGED" ] || [ "$MERGED" = "null" ]; then
            echo "::warning::Promote PR #${PR_NUM} didn't merge within 30min — skipping deploy dispatch (manually run \`gh workflow run publish-workspace-server-image.yml --ref main\` once it lands)."
            exit 0
          fi

          # Dispatch publish on main using the App token. App-initiated
          # workflow_dispatch DOES propagate the workflow_run cascade,
          # unlike GITHUB_TOKEN-initiated dispatch.
          # publish completes → canary-verify chains via workflow_run →
          # redeploy-tenants-on-main chains via workflow_run + branches:[main].
          if gh workflow run publish-workspace-server-image.yml \
              --repo "$REPO" --ref main 2>&1; then
            echo "::notice::Dispatched publish-workspace-server-image on ref=main as molecule-ai App — canary-verify and redeploy-tenants-on-main will chain via workflow_run."
            {
              echo "## 🚀 Tenant redeploy chain dispatched"
              echo
              echo "- publish-workspace-server-image (workflow_dispatch on \`main\`, actor: \`molecule-ai[bot]\`)"
              echo "- canary-verify will chain on completion"
              echo "- redeploy-tenants-on-main will chain on canary green"
            } >> "$GITHUB_STEP_SUMMARY"
          else
            echo "::error::Failed to dispatch publish-workspace-server-image. Run manually: gh workflow run publish-workspace-server-image.yml --ref main"
          fi

          # ALSO dispatch auto-sync-main-to-staging.yml. Same root cause as
          # publish above (issue #2357): the merge-queue-initiated push to
          # main is by GITHUB_TOKEN → no `on: push` triggers fire downstream.
          # Without this dispatch, every staging→main promote leaves staging
          # one merge commit BEHIND main, which silently dead-locks the NEXT
          # promote PR as `mergeStateStatus: BEHIND` because main's
          # branch-protection has `strict: true`. Verified empirically on
          # 2026-05-02 against PR #2442 (Phase 2 promote): only the explicit
          # publish-workspace-server-image dispatch fired on the previous
          # promote SHA 76c604fb, while auto-sync silently no-op'd, leaving
          # staging behind for ~24h until manually bridged.
          if gh workflow run auto-sync-main-to-staging.yml \
              --repo "$REPO" --ref main 2>&1; then
            echo "::notice::Dispatched auto-sync-main-to-staging on ref=main as molecule-ai App — staging will absorb the new main merge commit via PR + merge queue."
          else
            echo "::error::Failed to dispatch auto-sync-main-to-staging. Run manually: gh workflow run auto-sync-main-to-staging.yml --ref main"
          fi
