name: Canary — staging SaaS smoke (every 30 min)

# Minimum viable health check: provisions one Hermes workspace on a fresh
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
# issue on the next green run.
#
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
# but runs only on provisioning-critical pushes + nightly — this one
# catches drift in the 30-min window between those runs (AMI health, CF
# cert rotation, WorkOS session stability, etc.).
#
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
# peers/activity checks. One parent workspace + one A2A turn is enough
# to signal "SaaS stack end-to-end is alive."

on:
  schedule:
    # Every 30 min. Cron on GitHub-hosted runners has a known drift of
    # a few minutes under load — that's fine for a canary.
    - cron: '*/30 * * * *'
  workflow_dispatch:

# Serialise with the full-SaaS workflow so they don't contend for the
# same org-create quota on staging. Different group key from
# e2e-staging-saas since we don't mind queueing canaries behind one
# full run, but two canaries SHOULD queue against each other.
concurrency:
  group: canary-staging
  cancel-in-progress: false

permissions:
  # Needed to open / close the alerting issue.
  issues: write
  contents: read

jobs:
  canary:
    name: Canary smoke
    runs-on: ubuntu-latest
    # 25 min headroom over the 15-min TLS-readiness deadline in
    # tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
    # the job is killed at the wall-clock 15:00 mark BEFORE the bash
    # `fail` + diagnostic burst can fire, leaving every cancellation
    # silent. Sibling staging E2E jobs run at 20-45 min — keeping
    # canary tighter than them so a true wedge still surfaces here
    # first.
    timeout-minutes: 25

    env:
      MOLECULE_CP_URL: https://staging-api.moleculesai.app
      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
      # MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
      # Switched from hermes+OpenAI after #2578 (the staging OpenAI key
      # account went over quota and stayed dead for 36+ hours, taking
      # the canary red the entire time). claude-code template's
      # `minimax` provider routes ANTHROPIC_BASE_URL to
      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
      # ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
      # billing account, so OpenAI quota collapse no longer wedges the
      # canary. Mirrors the migration continuous-synth-e2e.yml made on
      # 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
      # full_saas.sh branches SECRETS_JSON on which key is present —
      # MiniMax wins when set.
      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
      # Direct-Anthropic alternative for operators who don't want to
      # set up a MiniMax account (priority below MiniMax — first
      # non-empty wins in test_staging_full_saas.sh's secrets-injection
      # block). See #2578 PR comment for the rationale.
      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
      # OpenAI fallback — kept wired so an operator-dispatched run with
      # E2E_RUNTIME=hermes overridden via workflow_dispatch can still
      # exercise the OpenAI path without re-editing the workflow.
      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
      E2E_MODE: canary
      E2E_RUNTIME: claude-code
      # Pin the canary to a specific MiniMax model rather than relying
      # on the per-runtime default (which could resolve to "sonnet" →
      # direct Anthropic and defeat the cost saving). M2.7-highspeed
      # is "Token Plan only" but cheap-per-token and fast.
      E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
      E2E_RUN_ID: "canary-${{ github.run_id }}"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Verify admin token present
        run: |
          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
            exit 2
          fi

      - name: Verify LLM key present
        run: |
          # Per-runtime key check — claude-code uses MiniMax; hermes /
          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
          # rather than soft-skip per the lesson from synth E2E #2578:
          # an empty key silently falls through to the wrong
          # SECRETS_JSON branch and the canary fails 5 min later with
          # a confusing auth error instead of the clean "secret
          # missing" message at the top.
          case "${E2E_RUNTIME}" in
            claude-code)
              # Either MiniMax OR direct-Anthropic works — first
              # non-empty wins in the test script's secrets-injection
              # priority chain. Operators only need to set ONE of these
              # secrets; we don't force a choice between them.
              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
                required_secret_value="${E2E_MINIMAX_API_KEY}"
              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
              else
                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
                required_secret_value=""
              fi
              ;;
            langgraph|hermes)
              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
              required_secret_value="${E2E_OPENAI_API_KEY:-}"
              ;;
            *)
              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
              required_secret_name=""
              required_secret_value="present"
              ;;
          esac
          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
            exit 2
          fi
          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"

      - name: Canary run
        id: canary
        run: bash tests/e2e/test_staging_full_saas.sh

      # Alerting: open an issue only after THREE consecutive failures so
      # transient flakes (Cloudflare DNS hiccup, AWS API blip) don't spam
      # the issue list. If an issue is already open, we still comment on
      # every failure so ops sees the streak. Auto-close on next green.
      #
      # Threshold rationale: canary fires every 30 min, so 3 failures =
      # ~90 min of consecutive red — well past any single-run flake but
      # still tight enough that a real outage gets surfaced before the
      # next deploy window.
      - name: Open issue on failure
        if: failure()
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          # Inject the workflow path explicitly — context.workflow is
          # the *name*, not the file path the actions API needs.
          WORKFLOW_PATH: '.github/workflows/canary-staging.yml'
          CONSECUTIVE_THRESHOLD: '3'
        with:
          script: |
            const title = '🔴 Canary failing: staging SaaS smoke';
            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

            // Find an existing open canary issue (stable title match).
            // If one exists, this isn't a "first failure" — comment and exit.
            const { data: existing } = await github.rest.issues.listForRepo({
              owner: context.repo.owner, repo: context.repo.repo,
              state: 'open', labels: 'canary-staging',
              per_page: 10,
            });
            const match = existing.find(i => i.title === title);
            if (match) {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: match.number,
                body: `Canary still failing. ${runURL}`,
              });
              core.info(`Commented on existing issue #${match.number}`);
              return;
            }

            // No open issue yet — check the last N-1 runs' conclusions.
            // We open the issue only if the last (THRESHOLD-1) runs ALSO
            // failed (so this is the 3rd consecutive red).
            const threshold = parseInt(process.env.CONSECUTIVE_THRESHOLD, 10);
            const { data: runs } = await github.rest.actions.listWorkflowRuns({
              owner: context.repo.owner, repo: context.repo.repo,
              workflow_id: process.env.WORKFLOW_PATH,
              status: 'completed',
              per_page: threshold,
              // Skip the current in-progress run; it isn't 'completed' yet.
            });
            // listWorkflowRuns returns recent first. We need (threshold-1)
            // prior failures (current run is the threshold-th).
            const priorFailures = (runs.workflow_runs || [])
              .slice(0, threshold - 1)
              .filter(r => r.id !== context.runId)
              .filter(r => r.conclusion === 'failure')
              .length;
            if (priorFailures < threshold - 1) {
              core.info(`Below threshold: ${priorFailures + 1}/${threshold} consecutive failures — not filing yet`);
              return;
            }

            const body =
              `Canary run failed at ${new Date().toISOString()}, ` +
              `${threshold} consecutive runs red.\n\n` +
              `Run: ${runURL}\n\n` +
              `This issue auto-closes on the next green canary run. ` +
              `Consecutive failures add a comment here rather than a new issue.`;
            await github.rest.issues.create({
              owner: context.repo.owner, repo: context.repo.repo,
              title, body,
              labels: ['canary-staging', 'bug'],
            });
            core.info(`Opened canary failure issue (${threshold} consecutive reds)`);

      - name: Auto-close canary issue on success
        if: success()
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const title = '🔴 Canary failing: staging SaaS smoke';
            const { data: open } = await github.rest.issues.listForRepo({
              owner: context.repo.owner, repo: context.repo.repo,
              state: 'open', labels: 'canary-staging',
              per_page: 10,
            });
            const match = open.find(i => i.title === title);
            if (match) {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: match.number,
                body: `Canary recovered at ${new Date().toISOString()}. Closing.`,
              });
              await github.rest.issues.update({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: match.number,
                state: 'closed',
              });
              core.info(`Closed recovered canary issue #${match.number}`);
            }

      - name: Teardown safety net
        if: always()
        env:
          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
        run: |
          set +e
          # Slug prefix matches what test_staging_full_saas.sh emits
          # in canary mode:
          #   SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
          # Earlier this was `e2e-{today}-canary-` — that was the
          # full-mode pattern (date FIRST, mode SECOND); canary slugs
          # have mode FIRST, date SECOND. The mismatch silently
          # never matched, leaving every cancelled-canary EC2 alive
          # until the once-an-hour sweep eventually caught it
          # (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
          # cleanup; same gap on three earlier cancellations today).
          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
            | python3 -c "
          import json, sys, os, datetime
          run_id = os.environ.get('GITHUB_RUN_ID', '')
          d = json.load(sys.stdin)
          # Scope to slugs from THIS canary run when GITHUB_RUN_ID is
          # available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
          # so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
          # full-mode safety net's per-run scoping (e2e-staging-saas.yml)
          # added after the 2026-04-21 cross-run cleanup incident.
          # Sweep both today AND yesterday's UTC dates so a run that
          # crosses midnight still cleans up its own slug — see the
          # 2026-04-26→27 canvas-safety-net incident.
          today = datetime.date.today()
          yesterday = today - datetime.timedelta(days=1)
          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
          if run_id:
              prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
          else:
              prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
          candidates = [o['slug'] for o in d.get('orgs', [])
                        if any(o.get('slug','').startswith(p) for p in prefixes)
                        and o.get('status') not in ('purged',)]
          print('\n'.join(candidates))
          " 2>/dev/null)
          # Per-slug DELETE with HTTP-code verification. The previous
          # `... >/dev/null || true` swallowed every failure, so a 5xx
          # or timeout from CP looked identical to "successfully cleaned
          # up" and the tenant kept eating ~2 vCPU until the hourly
          # stale sweep caught it (up to 2h later). Now we capture the
          # response code and surface non-2xx as a workflow warning, so
          # the run page shows which slug leaked. We still don't `exit 1`
          # on cleanup failure — a single-canary cleanup miss shouldn't
          # fail-flag the canary itself when the actual smoke check
          # passed. The sweep-stale-e2e-orgs cron (now every 15 min,
          # 30-min threshold) is the safety net for whatever slips past.
          # See molecule-controlplane#420.
          leaks=()
          for slug in $orgs; do
            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
            # pollution of the captured status (lint-curl-status-capture.yml).
            set +e
            curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
              -H "Authorization: Bearer $ADMIN_TOKEN" \
              -H "Content-Type: application/json" \
              -d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
            set -e
            code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
              echo "[teardown] deleted $slug (HTTP $code)"
            else
              echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
              leaks+=("$slug")
            fi
          done
          if [ ${#leaks[@]} -gt 0 ]; then
            echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
          fi
          exit 0
