fix(ci): retry status reaper api timeouts

.gitea/scripts/status-reaper.py

@@ -91,7 +91,9 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import socket
 import sys
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -118,6 +120,9 @@ WORKFLOWS_DIR = _env("WORKFLOWS_DIR", default=".gitea/workflows")
 
 OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
 API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
+API_TIMEOUT_SEC = int(_env("STATUS_REAPER_API_TIMEOUT_SEC", default="30") or "30")
+API_RETRIES = int(_env("STATUS_REAPER_API_RETRIES", default="3") or "3")
+API_RETRY_SLEEP_SEC = float(_env("STATUS_REAPER_API_RETRY_SLEEP_SEC", default="2") or "2")
 
 # Compensating-status description prefix. Used as the marker so a human
 # auditing commit statuses can tell at a glance that the green was
@@ -182,13 +187,27 @@ def api(
         data = json.dumps(body).encode("utf-8")
         headers["Content-Type"] = "application/json"
     req = urllib.request.Request(url, method=method, data=data, headers=headers)
-    try:
-        with urllib.request.urlopen(req, timeout=30) as resp:
-            raw = resp.read()
-            status = resp.status
-    except urllib.error.HTTPError as e:
-        raw = e.read()
-        status = e.code
+    attempts = max(API_RETRIES, 1)
+    for attempt in range(1, attempts + 1):
+        try:
+            with urllib.request.urlopen(req, timeout=API_TIMEOUT_SEC) as resp:
+                raw = resp.read()
+                status = resp.status
+            break
+        except urllib.error.HTTPError as e:
+            raw = e.read()
+            status = e.code
+            break
+        except (TimeoutError, socket.timeout, urllib.error.URLError, OSError) as e:
+            if attempt >= attempts:
+                raise ApiError(
+                    f"{method} {path} failed after {attempts} attempts: {e}"
+                ) from e
+            print(
+                f"::warning::{method} {path} transient API error "
+                f"(attempt {attempt}/{attempts}): {e}; retrying"
+            )
+            time.sleep(API_RETRY_SLEEP_SEC)
 
     if not (200 <= status < 300):
         snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""

.gitea/scripts/tests/test_status_reaper_api.py

@@ -0,0 +1,72 @@
+import importlib.util
+import json
+import pathlib
+import urllib.error
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "status-reaper.py"
+
+
+def load_reaper():
+    spec = importlib.util.spec_from_file_location("status_reaper", SCRIPT)
+    mod = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(mod)
+    mod.API = "https://git.example.test/api/v1"
+    mod.GITEA_TOKEN = "test-token"
+    mod.API_TIMEOUT_SEC = 1
+    mod.API_RETRIES = 3
+    mod.API_RETRY_SLEEP_SEC = 0
+    return mod
+
+
+class FakeResponse:
+    status = 200
+
+    def __init__(self, payload):
+        self.payload = payload
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+    def read(self):
+        return json.dumps(self.payload).encode("utf-8")
+
+
+def test_api_retries_transient_timeout(monkeypatch):
+    mod = load_reaper()
+    calls = {"n": 0}
+
+    def fake_urlopen(req, timeout):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise TimeoutError("simulated slow Gitea API")
+        return FakeResponse({"ok": True})
+
+    monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
+
+    status, body = mod.api("GET", "/repos/o/r/commits")
+
+    assert status == 200
+    assert body == {"ok": True}
+    assert calls["n"] == 2
+
+
+def test_api_raises_after_retry_budget(monkeypatch):
+    mod = load_reaper()
+
+    def fake_urlopen(req, timeout):
+        raise urllib.error.URLError("connection reset")
+
+    monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
+
+    try:
+        mod.api("GET", "/repos/o/r/commits")
+    except mod.ApiError as exc:
+        assert "failed after 3 attempts" in str(exc)
+    else:
+        raise AssertionError("expected ApiError")

.gitea/workflows/status-reaper.yml

@@ -84,7 +84,7 @@ permissions:
 jobs:
   reap:
     runs-on: ubuntu-latest
-    timeout-minutes: 3
+    timeout-minutes: 8
     steps:
       - name: Check out repo at default-branch HEAD
         # BASE checkout per `feedback_pull_request_target_workflow_from_base`.
@@ -118,4 +118,7 @@ jobs:
           REPO: ${{ github.repository }}
           WATCH_BRANCH: ${{ github.event.repository.default_branch }}
           WORKFLOWS_DIR: .gitea/workflows
+          STATUS_REAPER_API_RETRIES: "4"
+          STATUS_REAPER_API_TIMEOUT_SEC: "20"
+          STATUS_REAPER_API_RETRY_SLEEP_SEC: "2"
         run: python3 .gitea/scripts/status-reaper.py