molecule-ai-status-cd/.gitea/workflows/deploy.yml

name: molecule-ai-status-cd
# Auto-redeploy for the Enter OS status page (https://status.enteros.ai, tc-2).
#
# Polls molecule-ai/molecule-ai-status main; when its HEAD changes, rebuild + roll
# out on tc-2 and then ASSERT THE LIVE PAGE. Runs on a FLEET runner (robot-1) —
# never on the prod box.
#
# WHAT THIS PIPELINE IS ACTUALLY GUARDING. A status page fails in a way a landing
# page cannot: it keeps returning 200, keeps rendering, and lies. That is not
# hypothetical here — before this move, /data/history/* was proxied to `main`
# instead of the `uptime-probe-results` branch the probe publishes to, so the
# public page sat on the 2026-05-11 heartbeat for 75 days: permanently green,
# structurally unable to go red, while the probes ran perfectly every 5 minutes.
# Any HTTP-200 check would have passed the entire time. deploy/assert-live.sh
# therefore asserts the DATA (heartbeat freshness, per-monitor history freshness,
# config-proxy parity with Gitea main, summary drift) against the REAL public
# hostname, so the Cloudflare + tunnel path is proven too. Any miss repins the
# previous image and fails the job.
#
# THE PROBE IS NOT PART OF THIS PIPELINE and is deliberately untouched: the
# 5-minute `Probe + commit` workflow lives in the app repo, runs on the fleet, and
# publishes to `uptime-probe-results`. Nothing here pushes to any branch.
#
# STATE = the deployed image tag, read back off tc-2 (no state file, nothing to
# drift): the container's molecule-ai-status:<sha12> vs the app repo's main HEAD.
on:
  schedule:
    - cron: "*/5 * * * *"
  workflow_dispatch: {}

concurrency:
  group: molecule-ai-status-cd
  cancel-in-progress: false

jobs:
  deploy:
    runs-on: [docker-host, robot-1]   # fleet runner, NOT the prod box
    timeout-minutes: 30
    steps:
      - name: Checkout CD repo
        uses: actions/checkout@v4

      - name: Poll app HEAD -> build on tc-2 -> roll out -> assert live
        env:
          SRC_REPO: molecule-ai/molecule-ai-status
          SRC_BRANCH: main
          GITEA_HOST: git.moleculesai.app
          TC2: molecule@100.64.0.5
          APP_DIR: /home/molecule/molecule-ai-status
          SITE_URL: https://status.enteros.ai
          TC2_DEPLOY_KEY_B64: ${{ secrets.TC2_DEPLOY_KEY_B64 }}
          GITEA_TOKEN: ${{ secrets.SRC_GITEA_TOKEN }}
        run: |
          set -euo pipefail

          # HEAD sha via the Gitea API. Capture the response FIRST, then grep —
          # piping curl straight into `grep -m1` under pipefail SIGPIPEs curl and
          # fails the step (learned the hard way in the Minori CD).
          #
          # The app repo is PUBLIC, so this read needs no credential and none is
          # configured: storing a broad token here to fetch a public sha would be
          # scope for nothing. The header is still sent when SRC_GITEA_TOKEN is
          # set, so making the repo private later is a one-secret change.
          AUTH=()
          if [ -n "${GITEA_TOKEN:-}" ]; then AUTH=(-H "Authorization: token $GITEA_TOKEN"); fi
          RESP=$(curl -sSL "${AUTH[@]}" \
                 "https://$GITEA_HOST/api/v1/repos/$SRC_REPO/branches/$SRC_BRANCH")
          SHA=$(printf '%s\n' "$RESP" | grep -m1 -oE '"id"[[:space:]]*:[[:space:]]*"[0-9a-f]{40}"' | grep -oE '[0-9a-f]{40}' || true)
          echo "$SHA" | grep -qE '^[0-9a-f]{40}$' \
            || { echo "::error::could not read Gitea HEAD (resp: $(printf '%s' "$RESP" | head -c 160))"; exit 1; }
          TAG=${SHA:0:12}
          echo "$SRC_REPO@$SRC_BRANCH HEAD=$SHA tag=$TAG"

          KEYF="$(pwd)/.tc2_deploy_key"
          trap 'rm -f "$KEYF"' EXIT
          echo "$TC2_DEPLOY_KEY_B64" | base64 -d > "$KEYF"
          chmod 600 "$KEYF"
          SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=15 -i $KEYF $TC2"
          SCP="scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=15 -i $KEYF"

          # Deployed tag, read off the box. The image pin in .env IS the deploy state.
          CUR=$($SSH "sed -n 's/^APP_IMAGE=molecule-ai-status://p' $APP_DIR/.env | tail -1" || true)
          echo "currently deployed tag=${CUR:-<none>}"

          # Even when the code has not moved, the DATA can have gone stale — the
          # probe could have stopped, or the results branch could have stopped
          # being reachable. So a no-op deploy still asserts the live page, and
          # still fails loudly. This is the check that turns a silent dark period
          # into a red job.
          if [ "$CUR" = "$TAG" ]; then
            echo "already at $TAG — re-asserting the live page (data can rot without a code change)"
            bash deploy/assert-live.sh "$SITE_URL"
            echo "::notice::no code change; $SITE_URL still serving fresh data"
            exit 0
          fi

          # Ship the deploy scripts fresh every run: the SSOT is this repo, so a
          # hand-edited copy on the box can never silently become what deploys.
          $SSH "mkdir -p $APP_DIR/.deploy/overlay"
          $SCP deploy/rollout.sh deploy/assert-live.sh deploy/docker-compose.yml \
               "$TC2:$APP_DIR/.deploy/"
          $SCP deploy/overlay/Dockerfile deploy/overlay/nginx.docker.conf \
               "$TC2:$APP_DIR/.deploy/overlay/"
          $SSH "cd $APP_DIR && chmod +x .deploy/*.sh"

          # Roll out. rollout.sh prints PREV_IMAGE= so a failed assertion can repin.
          set +e
          OUT=$($SSH "cd $APP_DIR && ./.deploy/rollout.sh $SHA" 2>&1)
          RC=$?
          set -e
          printf '%s\n' "$OUT"
          PREV_IMAGE=$(printf '%s\n' "$OUT" | sed -n 's/^PREV_IMAGE=//p' | tail -1)
          if [ "$RC" -ne 0 ]; then
            echo "::error::rollout.sh failed on tc-2 (rc=$RC) — previous container left in place"
            exit 1
          fi

          # ---- POST-ROLLOUT ASSERTION (hard gate) -------------------------------
          # Against the PUBLIC hostname, through Cloudflare + the tunnel, so it also
          # proves the edge path — not just that a container answers on localhost.
          # Retried for ~7 minutes: the tunnel can 502/404 for a few seconds while
          # cloudflared re-establishes to the new origin, and a monitor edited on
          # main needs one 5-minute probe cycle before summary.json catches up.
          # Polling the REAL signal, never sleeping a fixed guess and calling it done.
          ok=0
          for a in $(seq 1 28); do
            if bash deploy/assert-live.sh "$SITE_URL" "$SHA"; then ok=1; break; fi
            echo "assertion attempt $a failed; retrying in 15s"
            sleep 15
          done

          if [ "$ok" != 1 ]; then
            echo "::error::POST-ROLLOUT ASSERTION FAILED for $SHA at $SITE_URL — rolling back"
            if [ -n "$PREV_IMAGE" ]; then
              $SSH "cd $APP_DIR && ./.deploy/rollout.sh --pin '$PREV_IMAGE'" || \
                echo "::error::ROLLBACK ALSO FAILED — the status page may be serving bad or stale data; investigate NOW"
              echo "::error::repinned $PREV_IMAGE"
            else
              echo "::error::no previous image recorded — cannot auto-roll-back"
            fi
            exit 1
          fi

          echo "::notice::deployed $TAG to tc-2 and verified live at $SITE_URL"