molecule-ai-status/.gitea/workflows/uptime-probe.yml

name: Uptime probe (Gitea-native replaces upptime)

# Runs the molecule-ai-uptime-probe binary on a 5-minute cadence,
# appends per-site JSONL results to history/, and commits the changes
# back to main. Replaces the five upptime workflows that lived in this
# repo before they were moved to .github/workflows-disabled/ (because
# every upptime call to api.github.com 401s post-2026-05-06 GitHub
# org suspension).
#
# See molecule-ai/molecule-ai-status#2 for the design rationale +
# molecule-ai/molecule-ai-uptime-probe for the probe binary itself.
#
# Self-health (heartbeat): writes history/.monitoring-heartbeat.jsonl with the
# current timestamp on every successful commit. If the last heartbeat is
# older than 15 minutes, the status page renders "Monitoring degraded"
# instead of falsely showing "All systems operational." This prevents a
# silent dark period when the runner pool is down and scheduled workflows
# are queuing (not firing).

on:
  schedule:
    # Every 5 minutes — matches the upptime default cadence.
    - cron: "*/5 * * * *"
  # Manual trigger for ad-hoc checks.
  workflow_dispatch:
  # Re-run when probe-list config changes so a new endpoint gets a
  # baseline immediately, not at the next /5 mark.
  push:
    branches: [main]
    paths: [".upptimerc.yml"]

permissions:
  contents: write   # required to commit history/ updates

jobs:
  probe:
    name: Probe + commit
    runs-on: ubuntu-latest
    # Concurrency: at most one probe run at a time per branch. Two
    # cron firings overlapping would race on history/ commits.
    concurrency:
      group: uptime-probe-${{ github.ref }}
      cancel-in-progress: false
    steps:
      - name: Checkout repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          persist-credentials: true

      - name: Seed latest probe history
        # Probe commits are published to uptime-probe-results instead of main.
        # Seed history/ from that branch so each run appends to the latest
        # persisted probe data, then publish with a lease in the commit step.
        run: |
          set -euo pipefail
          if git fetch --depth 1 origin uptime-probe-results:refs/remotes/origin/uptime-probe-results; then
            RESULTS_BASE=$(git rev-parse refs/remotes/origin/uptime-probe-results)
            echo "RESULTS_BASE=$RESULTS_BASE" >> "$GITHUB_ENV"
            git checkout refs/remotes/origin/uptime-probe-results -- history
            echo "seeded history/ from uptime-probe-results@$RESULTS_BASE"
          else
            echo "RESULTS_BASE=" >> "$GITHUB_ENV"
            echo "::notice::No uptime-probe-results branch yet; starting from main history/"
          fi

      - name: "Self-health: check heartbeat staleness"
        # Read the last heartbeat timestamp from the previous commit.
        # If it is more than 15 minutes old, fail so operators notice
        # before the status page silently goes dark. The workflow keeps
        # running (we still commit fresh data if we got this far), but
        # the failure is visible in the Actions UI and will wake someone.
        run: |
          set -euo pipefail
          HEARTBEAT="history/.monitoring-heartbeat.jsonl"
          NOW_S=$(date +%s)
          # Grace period: allow up to 15 minutes of staleness before alerting.
          # 5-minute cadence + potential runner-queue delay + clock skew = 15 min is safe.
          GRACE_S=900
          if [ ! -f "$HEARTBEAT" ]; then
            echo "::notice::First run — no heartbeat file yet. Creating."
          else
            LAST_TS=$(tail -1 "$HEARTBEAT" | python3 -c "import json,sys; print(json.load(sys.stdin)['timestamp'])" 2>/dev/null || echo "")
            if [ -n "$LAST_TS" ]; then
              LAST_S=$(date -d "$LAST_TS" +%s 2>/dev/null || echo 0)
              DELTA_S=$((NOW_S - LAST_S))
              if [ "$DELTA_S" -gt "$GRACE_S" ]; then
                echo "::error::STALE MONITORING: last heartbeat was ${DELTA_S}s ago (limit: ${GRACE_S}s)"
                echo "::error::Scheduled workflows may be queuing on a down runner pool."
                echo "::error::SSH to operator host (5.78.80.188): docker ps | grep molecule-runner"
                echo "::error::If runners are down: cd /opt/molecule/scripts/runners && sudo ./runners-restart-safe.sh --all"
                # Do NOT exit 1 — we still want to commit fresh data and keep the page alive.
                # The failure is purely for alerting visibility.
                echo "::notice::Continuing anyway to keep the page alive — monitoring has degraded."
              else
                echo "::notice::Heartbeat OK: last run was ${DELTA_S}s ago"
              fi
            fi
          fi

      - name: Write + trim heartbeat
        # Always write a fresh heartbeat so the next run has a baseline.
        # Include the workflow-run URL so operators can click through from an alert.
        # Keep only the last 10 entries to prevent unbounded file growth
        # (~10 entries × ~100 bytes × 288 runs/day × 365 days ≈ 1 MB/year — fine
        # without trim, but trimming keeps diffs cheap and git history clean).
        run: |
          set -euo pipefail
          mkdir -p history
          export HB="history/.monitoring-heartbeat.jsonl"
          python3 - <<'PY'
          import datetime
          import json
          import os

          hb = os.environ["HB"]
          ts = datetime.datetime.now(datetime.timezone.utc).isoformat()
          repo = os.environ.get("GITHUB_REPOSITORY") or "molecule-ai/molecule-ai-status"
          run_id = os.environ.get("GITHUB_RUN_ID") or "0"
          run_url = f"https://git.moleculesai.app/{repo}/actions/runs/{run_id}"
          entry = {"timestamp": ts, "run_url": run_url}
          with open(hb, "a", encoding="utf-8") as f:
              f.write(json.dumps(entry) + "\n")
          # Trim to last 10 entries (the file grows 1 line per run).
          with open(hb, encoding="utf-8") as f:
              lines = [line for line in f if line.strip()]
          with open(hb, "w", encoding="utf-8") as f:
              f.write("".join(lines[-10:]))
          PY

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: '1.23'
          token: ${{ secrets.GITEA_TOKEN }}   # see molecule-ai/internal#75

      - name: Install probe
        # Build directly from the probe's repo at a pinned commit. Pin
        # is updated explicitly in this workflow file when the probe
        # itself ships a new behaviour-changing version. Avoids
        # supply-chain ambiguity.
        run: |
          set -euo pipefail
          GOPROBE_REPO=https://git.moleculesai.app/molecule-ai/molecule-ai-uptime-probe.git
          GOPROBE_REF=main
          tmp=$(mktemp -d)
          git clone --depth 1 --branch "$GOPROBE_REF" "$GOPROBE_REPO" "$tmp/probe"
          (cd "$tmp/probe" && go build -o /usr/local/bin/uptime-probe ./cmd/probe)
          /usr/local/bin/uptime-probe -h 2>&1 | sed -n '1,5p'

      - name: Run probes
        # Exit 1 from the probe when any site fails — but we don't
        # want a single failing site to abort the workflow before the
        # commit step. `|| true` swallows the non-zero exit; the
        # failure shows up as success=false in the JSONL history,
        # where the status page picks it up.
        run: |
          mkdir -p history
          /usr/local/bin/uptime-probe \
            -config .upptimerc.yml \
            -history-dir history \
            -timeout 30s \
            > /tmp/run.json || true
          echo "== run summary =="
          jq -r '.[] | "\(.name): \(.status_code) \(.latency_ms)ms success=\(.success)"' /tmp/run.json || cat /tmp/run.json

      - name: Aggregate probe results to Upptime format
        # Reads history/<slug>.jsonl files, computes rolling uptime/response-time
        # aggregates, and writes history/<slug>.yml + history/summary.json.
        # See molecule-ai/molecule-ai-status#7.
        run: |
          set -euo pipefail
          python3 scripts/aggregate.py --history-dir history

      - name: Commit history changes
        # Fails fast if Gitea is unhealthy rather than silently swallowing
        # the push. The next /5 cron firing picks up where this left off once
        # Gitea recovers. Also guarded against concurrent-run race: the
        # workflow-level concurrency group (line ~42) ensures at most one
        # probe run per branch is in-flight at any time.
        run: |
          set -euo pipefail

          # Health gate: fail fast if Gitea is 502 or otherwise unreachable.
          # The probe ran successfully; we just cannot persist the results yet.
          GATEWAY="https://git.moleculesai.app"
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
            --max-time 10 "$GATEWAY/api/v1/version" || echo 000)
          if [ "$HTTP_CODE" != "200" ]; then
            echo "::error::Gitea unhealthy (HTTP $HTTP_CODE) — cannot push results."
            echo "::error::Probe data is in history/. Next successful push after Gitea"
            echo "::error::recovers will commit all buffered results."
            exit 1
          fi

          git config user.name "uptime-probe[bot]"
          git config user.email "uptime-probe@bots.moleculesai.app"
          git add history/
          if git diff --cached --quiet; then
            echo "no history changes to commit"
            exit 0
          fi
          git commit -m "chore(uptime): probe results $(date -u +%Y-%m-%dT%H:%M:%SZ)"
          if [ -n "${RESULTS_BASE:-}" ]; then
            git push \
              --force-with-lease=refs/heads/uptime-probe-results:"$RESULTS_BASE" \
              origin HEAD:uptime-probe-results
          else
            git push origin HEAD:uptime-probe-results
          fi