diff --git a/.gitea/scripts/tests/test_umbrella_reaper_api.py b/.gitea/scripts/tests/test_umbrella_reaper_api.py new file mode 100644 index 000000000..7c34043c0 --- /dev/null +++ b/.gitea/scripts/tests/test_umbrella_reaper_api.py @@ -0,0 +1,474 @@ +import importlib.util +import json +import pathlib +import urllib.error + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "umbrella-reaper.py" + + +def load_reaper(): + spec = importlib.util.spec_from_file_location("umbrella_reaper", SCRIPT) + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(mod) + mod.API = "https://git.example.test/api/v1" + mod.GITEA_TOKEN = "fixture-token" + mod.GITEA_HOST = "git.example.test" + mod.REPO = "owner/repo" + return mod + + +class FakeResponse: + status = 200 + + def __init__(self, payload): + self.payload = payload + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self): + return json.dumps(self.payload).encode("utf-8") + + +def _pr_fixture(number: int, sha: str) -> dict: + return {"number": number, "head": {"sha": sha}} + + +def _status_entry(context: str, state: str) -> dict: + return {"context": context, "status": state} + + +def test_process_pr_compensates_when_all_sub_jobs_success(monkeypatch): + mod = load_reaper() + posted = [] + + def fake_post_status(sha, context, description): + posted.append((sha, context, description)) + + monkeypatch.setattr(mod, "post_status", fake_post_status) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + + pr = _pr_fixture(1, "abc123") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + ok = mod.process_pr(pr) + assert ok is True + assert len(posted) == 1 + assert posted[0][0] == "abc123" + assert posted[0][1] == "CI / all-required (pull_request)" + assert "Compensating status" in posted[0][2] + + +def test_process_pr_skips_when_umbrella_missing(monkeypatch): + mod = load_reaper() + posted = [] + monkeypatch.setattr(mod, "post_status", lambda *a, **k: posted.append(a)) + monkeypatch.setattr(mod, "REQUIRED_SUB_JOBS", ["CI / Platform (Go) (pull_request)"]) + + pr = _pr_fixture(2, "def456") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + ok = mod.process_pr(pr) + assert ok is True + assert posted == [] + + +def test_process_pr_skips_when_sub_job_pending(monkeypatch): + mod = load_reaper() + posted = [] + monkeypatch.setattr(mod, "post_status", lambda *a, **k: posted.append(a)) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + + pr = _pr_fixture(3, "ghi789") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "pending"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + ok = mod.process_pr(pr) + assert ok is True + assert posted == [] + + +def test_process_pr_skips_when_sub_job_failure(monkeypatch): + mod = load_reaper() + posted = [] + monkeypatch.setattr(mod, "post_status", lambda *a, **k: posted.append(a)) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + + pr = _pr_fixture(4, "jkl012") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "failure"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + ok = mod.process_pr(pr) + assert ok is True + assert posted == [] + + +def test_process_pr_returns_false_on_post_failure(monkeypatch): + mod = load_reaper() + + def fake_post_status(sha, context, description): + raise mod.ApiError("POST /statuses/abc123 -> HTTP 500: simulated failure") + + monkeypatch.setattr(mod, "post_status", fake_post_status) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + + pr = _pr_fixture(5, "abc123") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + ok = mod.process_pr(pr) + assert ok is False + + +def test_main_exits_nonzero_when_any_post_fails(monkeypatch): + mod = load_reaper() + + monkeypatch.setenv("GITEA_TOKEN", "fixture-token") + monkeypatch.setenv("GITEA_HOST", "git.example.test") + monkeypatch.setenv("REPO", "owner/repo") + + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + monkeypatch.setattr( + mod, + "list_open_prs", + lambda limit: [ + _pr_fixture(1, "abc123"), + _pr_fixture(2, "def456"), + ], + ) + + calls = {"n": 0} + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + def fake_post_status(sha, context, description): + calls["n"] += 1 + if calls["n"] == 2: + raise mod.ApiError("simulated failure") + + monkeypatch.setattr(mod, "post_status", fake_post_status) + + exit_code = mod.main() + assert exit_code == 1 + + +def test_main_exits_zero_when_all_posts_succeed(monkeypatch): + mod = load_reaper() + + monkeypatch.setenv("GITEA_TOKEN", "fixture-token") + monkeypatch.setenv("GITEA_HOST", "git.example.test") + monkeypatch.setenv("REPO", "owner/repo") + + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + monkeypatch.setattr( + mod, + "list_open_prs", + lambda limit: [_pr_fixture(1, "abc123")], + ) + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + monkeypatch.setattr(mod, "post_status", lambda *a, **k: None) + + exit_code = mod.main() + assert exit_code == 0 + + +def test_dry_run_does_not_post(monkeypatch): + mod = load_reaper() + api_calls = [] + + def fake_api(method, path, *, body=None, query=None, expect_json=True): + api_calls.append((method, path, body)) + return 200, {"ok": True} + + monkeypatch.setattr(mod, "api", fake_api) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + + pr = _pr_fixture(6, "mno345") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + monkeypatch.setattr(mod, "DRY_RUN", True) + + ok = mod.process_pr(pr) + assert ok is True + # DRY_RUN should prevent the POST /statuses call + assert not any( + method == "POST" and "/statuses/" in path for method, path, _ in api_calls + ) + + +def test_duplicate_contexts_use_latest_state(monkeypatch): + mod = load_reaper() + posted = [] + monkeypatch.setattr(mod, "post_status", lambda *a, **k: posted.append(a)) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + ], + ) + + pr = _pr_fixture(7, "pqr678") + + def fake_combined_status(sha): + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + # duplicate: first pending, then success — the loop overwrites + _status_entry("CI / Detect changes (pull_request)", "pending"), + _status_entry("CI / Detect changes (pull_request)", "success"), + ] + } + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + + ok = mod.process_pr(pr) + assert ok is True + assert len(posted) == 1 + + +def test_load_required_sub_jobs_from_ci_yml_pull_request_event(): + mod = load_reaper() + # UMBRELLA_CONTEXT defaults to pull_request, so derivation should yield + # the pull_request suffix. + jobs = mod._load_required_sub_jobs_from_ci_yml(".gitea/workflows") + assert all(j.endswith(" (pull_request)") for j in jobs) + assert "CI / Detect changes (pull_request)" in jobs + assert "CI / Python Lint & Test (pull_request)" in jobs + + +def test_load_required_sub_jobs_from_ci_yml_push_event(monkeypatch): + mod = load_reaper() + monkeypatch.setattr(mod, "UMBRELLA_CONTEXT", "CI / all-required (push)") + jobs = mod._load_required_sub_jobs_from_ci_yml(".gitea/workflows") + assert all(j.endswith(" (push)") for j in jobs) + assert "CI / Detect changes (push)" in jobs + + +def test_list_open_prs_paginates(monkeypatch): + mod = load_reaper() + calls = [] + + def fake_api(method, path, *, body=None, query=None, expect_json=True): + calls.append(query) + page = int(query.get("page", 1)) + limit = int(query.get("limit", 50)) + if page == 1: + return 200, [{"number": 1}, {"number": 2}] + if page == 2: + return 200, [{"number": 3}] + return 200, [] + + monkeypatch.setattr(mod, "api", fake_api) + prs = mod.list_open_prs(limit=2) + assert len(prs) == 3 + assert prs[0]["number"] == 1 + assert prs[2]["number"] == 3 + assert calls[0]["page"] == "1" + assert calls[1]["page"] == "2" + + +def test_process_pr_returns_false_on_status_fetch_failure(monkeypatch): + mod = load_reaper() + + def fake_get_combined_status(sha): + raise mod.ApiError("GET /statuses/abc123 -> HTTP 500: simulated outage") + + monkeypatch.setattr(mod, "get_combined_status", fake_get_combined_status) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + ["CI / Detect changes (pull_request)"], + ) + + pr = _pr_fixture(8, "abc123") + ok = mod.process_pr(pr) + assert ok is False + + +def test_process_pr_returns_false_on_missing_statuses_array(monkeypatch): + mod = load_reaper() + + def fake_get_combined_status(sha): + return {"state": "success"} # missing 'statuses' array + + monkeypatch.setattr(mod, "get_combined_status", fake_get_combined_status) + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + ["CI / Detect changes (pull_request)"], + ) + + pr = _pr_fixture(9, "def456") + ok = mod.process_pr(pr) + assert ok is False + + +def test_main_exits_nonzero_when_any_status_read_fails(monkeypatch): + mod = load_reaper() + + monkeypatch.setenv("GITEA_TOKEN", "fixture-token") + monkeypatch.setenv("GITEA_HOST", "git.example.test") + monkeypatch.setenv("REPO", "owner/repo") + + monkeypatch.setattr( + mod, + "REQUIRED_SUB_JOBS", + [ + "CI / Detect changes (pull_request)", + "CI / Platform (Go) (pull_request)", + ], + ) + monkeypatch.setattr( + mod, + "list_open_prs", + lambda limit: [ + _pr_fixture(1, "abc123"), + _pr_fixture(2, "def456"), + ], + ) + + def fake_combined_status(sha): + if sha == "abc123": + return { + "statuses": [ + _status_entry("CI / all-required (pull_request)", "failure"), + _status_entry("CI / Detect changes (pull_request)", "success"), + _status_entry("CI / Platform (Go) (pull_request)", "success"), + ] + } + raise mod.ApiError("simulated status fetch failure") + + monkeypatch.setattr(mod, "get_combined_status", fake_combined_status) + monkeypatch.setattr(mod, "post_status", lambda *a, **k: None) + + exit_code = mod.main() + assert exit_code == 1 diff --git a/.gitea/scripts/umbrella-reaper.py b/.gitea/scripts/umbrella-reaper.py new file mode 100644 index 000000000..b3294652c --- /dev/null +++ b/.gitea/scripts/umbrella-reaper.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""umbrella-reaper — auto-recovery for stale CI umbrella statuses on PRs. + +Tracking: molecule-core#1780. + +Sibling to status-reaper.py (default-branch push-suffix compensation), +but scoped to pull_request umbrellas instead of main-branch contexts. + +What this script does, per `.gitea/workflows/umbrella-reaper.yml` invocation: + + 1. List open PRs via GET /repos/{o}/{r}/pulls?state=open&limit={N}. + 2. For EACH PR: + - GET combined commit status for PR head SHA. + - Look for the umbrella context (default: "CI / all-required (pull_request)"). + - If umbrella state is "failure": + - Verify ALL required sub-job contexts are "success". + - If yes → POST compensating success to /statuses/{sha} with the + same umbrella context and an honest description. + - If any required sub-job is NOT success → skip (umbrella correctly + reflects reality; do NOT lie). + - If umbrella state is "success" or "pending" → skip. + 3. Exit 0. Re-running is idempotent — Gitea de-dups by context. + +What it does NOT do: + - Touch non-umbrella contexts. + - Compensate when ANY required sub-job is missing, pending, failure, or + cancelled. Only the "all sub-jobs green, umbrella stale" race. + - Merge PRs. It only posts a status; branch protection still requires + human approval. + - Run on closed PRs. + +Halt conditions: + - Missing required env vars → exit 1 with ::error:: message. + - API 5xx on PR list → fail-loud (can't assess state). + - API 5xx on an individual PR's status → ::warning:: + continue to next PR. +""" +from __future__ import annotations + +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +def _load_required_sub_jobs_from_ci_yml(workflows_dir: str) -> list[str]: + """Parse ci.yml and extract the all-required sentinel's sub-job contexts. + + Supports two shapes of the all-required job run block: + 1. Legacy Python f-string list (pre-2026-06-01): + f"CI / Detect changes ({event})" + 2. Current shell-script shape (post-2026-06-01 scheduler fix): + check "Detect changes" "$CHANGES_RESULT" + + Raises RuntimeError if ci.yml is missing, has no all-required job, or the + run block cannot be parsed. + """ + ci_path = Path(workflows_dir) / "ci.yml" + if not ci_path.exists(): + raise RuntimeError(f"ci.yml not found at {ci_path}") + + # PyYAML is installed by the workflow (same as status-reaper.py). + import yaml + + with ci_path.open() as f: + doc = yaml.safe_load(f) + + jobs = doc.get("jobs", {}) + all_required = jobs.get("all-required") + if not isinstance(all_required, dict): + raise RuntimeError("ci.yml missing 'all-required' job") + + steps = all_required.get("steps", []) + run_block = "" + for step in steps: + if isinstance(step, dict): + run_text = step.get("run", "") + if run_text: + run_block = run_text + break + + if not run_block: + raise RuntimeError("all-required job missing run block") + + # Determine event suffix from the umbrella context we are watching. + if UMBRELLA_CONTEXT.endswith(" (pull_request)"): + suffix = "(pull_request)" + elif UMBRELLA_CONTEXT.endswith(" (push)"): + suffix = "(push)" + else: + m = re.search(r' \(([^)]+)\)$', UMBRELLA_CONTEXT) + suffix = m.group(1) if m else "pull_request" + + # Try legacy f-string format first. + if "({event})" in run_block: + matches = re.findall(r'f["\'](.*?\(\{event\}\))["\']', run_block) + if matches: + return [m.replace("({event})", suffix) for m in matches] + + # Try current shell-script format: check "Name" "$RESULT" + matches = re.findall(r'check\s+"([^"]+)"', run_block) + if matches: + return [f"CI / {name} {suffix}" for name in matches] + + raise RuntimeError("unable to derive required sub-jobs from all-required run block") + + +# -------------------------------------------------------------------------- +# Environment +# -------------------------------------------------------------------------- +def _env(key: str, *, default: str = "") -> str: + return os.environ.get(key, default) + + +GITEA_TOKEN = _env("GITEA_TOKEN") +GITEA_HOST = _env("GITEA_HOST") +REPO = _env("REPO") +DRY_RUN = _env("DRY_RUN", default="").lower() in ("1", "true", "yes") + +# The umbrella context to watch. Must match the branch-protection name +# exactly (Gitea de-dups by context string). +UMBRELLA_CONTEXT = _env("UMBRELLA_CONTEXT", default="CI / all-required (pull_request)") + +# Required sub-job contexts. The umbrella is only compensated when ALL of +# these are "success" on the same SHA. Order does not matter. +# +# Derive from ci.yml at runtime to prevent drift (CR2 blocker #1). +# The env var REQUIRED_SUB_JOBS overrides derivation for emergency +# tuning or local testing. +_REQUIRED_SUB_JOBS_OVERRIDE = _env("REQUIRED_SUB_JOBS") +if _REQUIRED_SUB_JOBS_OVERRIDE: + REQUIRED_SUB_JOBS = [ + ctx.strip() + for ctx in _REQUIRED_SUB_JOBS_OVERRIDE.split(";") + if ctx.strip() + ] +else: + try: + REQUIRED_SUB_JOBS = _load_required_sub_jobs_from_ci_yml(".gitea/workflows") + except Exception as exc: + sys.stderr.write( + f"::error::Failed to derive REQUIRED_SUB_JOBS from ci.yml: {exc}\n" + ) + sys.exit(1) + +OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "") +API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else "" +PR_LIMIT = int(_env("PR_LIMIT", default="50")) + + +def _require_runtime_env() -> None: + for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO"): + if not os.environ.get(key): + sys.stderr.write(f"::error::missing required env var: {key}\n") + sys.exit(1) + + +# -------------------------------------------------------------------------- +# Tiny HTTP helper +# -------------------------------------------------------------------------- +class ApiError(RuntimeError): + pass + + +def api( + method: str, + path: str, + *, + body: dict | None = None, + query: dict[str, str] | None = None, + expect_json: bool = True, +) -> tuple[int, Any]: + url = f"{API}{path}" + if query: + url = f"{url}?{urllib.parse.urlencode(query)}" + data = None + headers = { + "Authorization": f"token {GITEA_TOKEN}", + "Accept": "application/json", + } + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, method=method, data=data, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + status = resp.status + except urllib.error.HTTPError as e: + raw = e.read() + status = e.code + + if not (200 <= status < 300): + snippet = raw[:500].decode("utf-8", errors="replace") if raw else "" + raise ApiError(f"{method} {path} -> HTTP {status}: {snippet}") + + if not raw: + return status, None + try: + return status, json.loads(raw) + except json.JSONDecodeError as e: + if expect_json: + raise ApiError( + f"{method} {path} -> HTTP {status} but body is not JSON: {e}" + ) from e + return status, {"_raw": raw.decode("utf-8", errors="replace")} + + +# -------------------------------------------------------------------------- +# Gitea reads / writes +# -------------------------------------------------------------------------- +def list_open_prs(limit: int = 50) -> list[dict]: + """Paginate through all open PR pages. Fail closed on non-list responses.""" + all_prs: list[dict] = [] + page = 1 + while True: + _, body = api( + "GET", + f"/repos/{OWNER}/{NAME}/pulls", + query={"state": "open", "limit": str(limit), "page": str(page)}, + ) + if not isinstance(body, list): + raise ApiError(f"PR list page {page} response is not a JSON array") + if not body: + break + all_prs.extend(body) + if len(body) < limit: + break + page += 1 + return all_prs + + +def get_combined_status(sha: str) -> dict: + _, body = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status") + if not isinstance(body, dict): + raise ApiError(f"status for {sha} response is not a JSON object") + return body + + +def post_status(sha: str, context: str, description: str) -> None: + payload = { + "context": context, + "state": "success", + "description": description, + } + if DRY_RUN: + print(f"[DRY-RUN] Would POST /statuses/{sha}: {json.dumps(payload)}") + return + api("POST", f"/repos/{OWNER}/{NAME}/statuses/{sha}", body=payload) + + +# -------------------------------------------------------------------------- +# Core logic +# -------------------------------------------------------------------------- +def _entry_state(s: dict) -> str: + return s.get("status") or s.get("state") or "" + + +def process_pr(pr: dict) -> bool: + """Process a single PR. Returns True if the tick succeeded for this PR + (including no-op skips), False if a compensating POST failed. + """ + num = pr.get("number") + sha = pr.get("head", {}).get("sha") + if not sha: + print(f"::warning::PR #{num}: missing head.sha; skipping") + return True + + try: + status = get_combined_status(sha) + except ApiError as e: + print(f"::error::PR #{num}: status fetch failed: {e}") + return False + + statuses = status.get("statuses") + if not isinstance(statuses, list): + print(f"::error::PR #{num}: combined status missing 'statuses' array") + return False + umbrella_entry = None + subjob_states: dict[str, str] = {} + + for s in statuses: + if not isinstance(s, dict): + continue + ctx = s.get("context", "") + state = _entry_state(s) + if ctx == UMBRELLA_CONTEXT: + umbrella_entry = s + if ctx in REQUIRED_SUB_JOBS: + subjob_states[ctx] = state + + if umbrella_entry is None: + print(f"::notice::PR #{num}: no umbrella context '{UMBRELLA_CONTEXT}'; skipping") + return True + + umbrella_state = _entry_state(umbrella_entry) + if umbrella_state != "failure": + print(f"::notice::PR #{num}: umbrella is '{umbrella_state}'; skipping") + return True + + # Verify ALL required sub-jobs are present and success + missing = [ctx for ctx in REQUIRED_SUB_JOBS if ctx not in subjob_states] + if missing: + print( + f"::notice::PR #{num}: umbrella=failure, but missing sub-jobs: {missing}; " + "skipping (sub-jobs may still be running)" + ) + return True + + not_success = [ctx for ctx in REQUIRED_SUB_JOBS if subjob_states[ctx] != "success"] + if not_success: + print( + f"::notice::PR #{num}: umbrella=failure, but sub-jobs not all success: " + f"{[(ctx, subjob_states[ctx]) for ctx in not_success]}; skipping" + ) + return True + + # All checks pass — post compensating status + desc = ( + "Compensating status: all required sub-jobs verified success; " + "umbrella stale due to commit-status propagation race. " + f"Auto-posted by umbrella-reaper for PR #{num}." + ) + try: + post_status(sha, UMBRELLA_CONTEXT, desc) + print(f"::notice::PR #{num}: posted compensating success for {UMBRELLA_CONTEXT}") + return True + except ApiError as e: + print(f"::error::PR #{num}: failed to post compensating status: {e}") + return False + + +def main() -> int: + _require_runtime_env() + + # Drift guard: ci.yml derivation already happened at module load, but + # we sanity-check it is non-empty so the loop below doesn't trivially + # no-op because of a parse bug. + if not REQUIRED_SUB_JOBS: + sys.stderr.write("::error::REQUIRED_SUB_JOBS is empty; bailing out\n") + return 1 + + prs = list_open_prs(limit=PR_LIMIT) + print(f"::notice::Scanning {len(prs)} open PRs for stale umbrella statuses") + compensated = 0 + failed = 0 + for pr in prs: + ok = process_pr(pr) + if not ok: + failed += 1 + print(f"::notice::umbrella-reaper complete (failed POSTs={failed})") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.gitea/workflows/umbrella-reaper.yml b/.gitea/workflows/umbrella-reaper.yml new file mode 100644 index 000000000..421bbfcda --- /dev/null +++ b/.gitea/workflows/umbrella-reaper.yml @@ -0,0 +1,67 @@ +# umbrella-reaper — auto-recovery for stale CI umbrella statuses on open PRs. +# +# Tracking: molecule-core#1780. +# +# Problem: when `CI / all-required (pull_request)` reports failure due to +# a propagation/timing race despite all required sub-jobs being success, +# branch protection blocks the merge. Operators currently recover manually +# per docs/runbooks/ci-umbrella-stale-compensating-status.md. +# +# This workflow automates that recovery: it scans open PRs and posts a +# compensating success status when the umbrella is stale but all sub-jobs +# are verified green. +# +# Trust boundary: the script only reads PR lists + statuses and POSTs to +# /statuses/{sha}. It never checks out PR HEAD code. The Gitea token has +# write:repository scope for statuses only. +# +# Sibling: .gitea/workflows/status-reaper.yml (default-branch push-suffix +# compensation). Same persona provisioning model. + +name: umbrella-reaper + +# IMPORTANT — Schedule moved to operator-config: +# /etc/cron.d/molecule-core-umbrella-reaper -> +# /usr/local/bin/molecule-core-cron-bot.sh umbrella-reaper +# +# This keeps the compensation cadence but stops a maintenance bot from +# consuming Gitea Actions runner slots during PR merge waves. +# Gitea 1.22.6 parser quirk per +# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an +# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as +# "unknown on type" when `workflow_dispatch.inputs.X` is present. +on: + workflow_dispatch: + +permissions: + contents: read + +# NOTE: NO `concurrency:` block is intentional — same reasoning as +# status-reaper.yml. Gitea 1.22.6 doesn't honor cancel-in-progress for +# queued ticks; the POST is idempotent so concurrent ticks are safe. + +jobs: + reap: + runs-on: ubuntu-latest + timeout-minutes: 8 + steps: + - name: Check out repo at default-branch HEAD + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + + - name: Install PyYAML + run: python -m pip install --quiet 'PyYAML==6.0.2' + + - name: Compensate stale PR umbrella statuses + env: + GITEA_TOKEN: ${{ secrets.UMBRELLA_REAPER_TOKEN }} + GITEA_HOST: git.moleculesai.app + REPO: ${{ github.repository }} + PR_LIMIT: "50" + run: python3 .gitea/scripts/umbrella-reaper.py