fix(ci): reap shadowed pr statuses on main
Some checks failed
sop-checklist / all-items-acked (pull_request) acked: 7/7
qa-review / approved (pull_request) QA approved
Block internal-flavored paths / Block forbidden paths (pull_request) Successful in 38s
Lint curl status-code capture / Scan workflows for curl status-capture pollution (pull_request) Successful in 25s
CI / Detect changes (pull_request) Successful in 2m0s
E2E API Smoke Test / detect-changes (pull_request) Successful in 2m3s
E2E Staging Canvas (Playwright) / detect-changes (pull_request) Successful in 2m6s
Secret scan / Scan diff for credential-shaped strings (pull_request) Successful in 27s
Handlers Postgres Integration / detect-changes (pull_request) Successful in 1m52s
security-review / approved (pull_request) Successful in 22s
lint-required-no-paths / lint-required-no-paths (pull_request) Successful in 1m28s
sop-checklist-gate / gate (pull_request) Successful in 30s
gate-check-v3 / gate-check (pull_request) Failing after 50s
Runtime PR-Built Compatibility / detect-changes (pull_request) Successful in 1m16s
sop-tier-check / tier-check (pull_request) Successful in 29s
Lint workflow YAML (Gitea-1.22.6-hostile shapes) / Lint workflow YAML for Gitea-1.22.6-hostile shapes (pull_request) Successful in 1m46s
lint-continue-on-error-tracking / lint-continue-on-error-tracking (pull_request) Successful in 2m51s
audit-force-merge / audit (pull_request) Successful in 33s
Lint pre-flip continue-on-error / Verify continue-on-error flips have run-log proof (pull_request) Successful in 2m51s
Ops Scripts Tests / Ops scripts (unittest) (pull_request) Successful in 1m45s
lint-required-context-exists-in-bp / lint-required-context-exists-in-bp (pull_request) Successful in 2m48s
CI / Platform (Go) (pull_request) Successful in 7s
CI / Canvas (Next.js) (pull_request) Successful in 6s
CI / Shellcheck (E2E scripts) (pull_request) Successful in 4s
CI / Python Lint & Test (pull_request) Successful in 9s
E2E API Smoke Test / E2E API Smoke Test (pull_request) Successful in 9s
E2E Staging Canvas (Playwright) / Canvas tabs E2E (pull_request) Successful in 12s
Handlers Postgres Integration / Handlers Postgres Integration (pull_request) Successful in 6s
Runtime PR-Built Compatibility / PR-built wheel + import smoke (pull_request) Successful in 5s
CI / Canvas Deploy Reminder (pull_request) Has been skipped
CI / all-required (pull_request) Successful in 3s

This commit is contained in:
hongming-codex-laptop 2026-05-13 13:48:01 -07:00 committed by devops-engineer
parent accefeb1c6
commit cec0259ba7
2 changed files with 167 additions and 14 deletions

View File

@ -58,9 +58,10 @@ What this script does, per `.gitea/workflows/status-reaper.yml` invocation:
even if another tick happens before the runner finishes.
What it does NOT do:
- Touch any context NOT ending in ` (push)`. The required-checks on
main (verified 2026-05-11) all have ` (pull_request)` suffixes;
they CANNOT be reached by this code path.
- Touch ` (pull_request)` contexts unless the exact same
workflow/job has a successful ` (push)` context on the same
default-branch SHA. That case is post-merge status pollution, not
an unproven PR gate.
- Compensate `error`/`pending` states. Only `failure` the only one
Gitea emits for the hardcoded-suffix bug.
- Write to non-default branches. WATCH_BRANCH is sourced from
@ -128,14 +129,20 @@ API_RETRY_SLEEP_SEC = float(_env("STATUS_REAPER_API_RETRY_SLEEP_SEC", default="2
# auditing commit statuses can tell at a glance that the green was
# synthetic, not a real CI pass. Kept stable; downstream tooling
# (e.g. main-red-watchdog visual diff) MAY key on it.
COMPENSATION_DESCRIPTION = (
PUSH_COMPENSATION_DESCRIPTION = (
"Compensated by status-reaper (workflow has no push: trigger; "
"Gitea 1.22.6 hardcoded-suffix bug — see .gitea/scripts/status-reaper.py)"
)
PR_SHADOW_COMPENSATION_DESCRIPTION = (
"Compensated by status-reaper (default-branch pull_request status "
"shadowed by successful push status on same SHA; see "
".gitea/scripts/status-reaper.py)"
)
# Context suffix the reaper acts on. Gitea hardcodes this for ALL
# default-branch workflow runs.
PUSH_SUFFIX = " (push)"
PULL_REQUEST_SUFFIX = " (pull_request)"
def _require_runtime_env() -> None:
@ -376,24 +383,38 @@ def get_combined_status(sha: str) -> dict:
# --------------------------------------------------------------------------
# Context parsing
# --------------------------------------------------------------------------
def parse_push_context(context: str) -> tuple[str, str] | None:
"""Parse `<workflow_name> / <job_name> (push)` into
def parse_suffixed_context(context: str, suffix: str) -> tuple[str, str] | None:
"""Parse `<workflow_name> / <job_name> (<event>)` into
(workflow_name, job_name).
Returns None if the context doesn't match the shape (caller skips).
Strict: requires the trailing ` (push)` and at least one ` / `
Strict: requires the trailing suffix and at least one ` / `
separator. Anything else is left alone.
"""
if not context.endswith(PUSH_SUFFIX):
if not context.endswith(suffix):
return None
head = context[: -len(PUSH_SUFFIX)] # strip " (push)"
head = context[: -len(suffix)]
if " / " not in head:
# No workflow/job separator — not the bug shape we compensate.
return None
workflow_name, job_name = head.split(" / ", 1)
return workflow_name, job_name
def parse_push_context(context: str) -> tuple[str, str] | None:
"""Parse `<workflow_name> / <job_name> (push)` into
(workflow_name, job_name)."""
return parse_suffixed_context(context, PUSH_SUFFIX)
def push_equivalent_context(context: str) -> str | None:
"""Return the matching `(push)` context for a `(pull_request)` context."""
parsed = parse_suffixed_context(context, PULL_REQUEST_SUFFIX)
if parsed is None:
return None
workflow_name, job_name = parsed
return f"{workflow_name} / {job_name}{PUSH_SUFFIX}"
# --------------------------------------------------------------------------
# Compensating POST
# --------------------------------------------------------------------------
@ -402,6 +423,7 @@ def post_compensating_status(
context: str,
target_url: str | None,
*,
description: str = PUSH_COMPENSATION_DESCRIPTION,
dry_run: bool = False,
) -> None:
"""POST a `state=success` to /repos/{o}/{r}/statuses/{sha} with the
@ -413,7 +435,7 @@ def post_compensating_status(
payload: dict[str, Any] = {
"context": context,
"state": "success",
"description": COMPENSATION_DESCRIPTION,
"description": description,
}
# Echo the original target_url when present so a human auditing
# the (now-green) compensated status can still reach the run logs
@ -450,7 +472,8 @@ def reap(
Returns counters for observability:
{compensated, preserved_real_push, preserved_unknown,
preserved_non_failure, preserved_non_push_suffix,
preserved_unparseable,
preserved_unparseable, compensated_pr_shadowed_by_push_success,
preserved_pr_without_push_success,
compensated_contexts: [<context>, ...]}
`compensated_contexts` is rev2-added so `reap_branch` can build
@ -463,10 +486,17 @@ def reap(
"preserved_non_failure": 0,
"preserved_non_push_suffix": 0,
"preserved_unparseable": 0,
"compensated_pr_shadowed_by_push_success": 0,
"preserved_pr_without_push_success": 0,
"compensated_contexts": [],
}
statuses = combined.get("statuses") or []
successful_contexts = {
(s.get("context") or "")
for s in statuses
if isinstance(s, dict) and (s.get("status") or s.get("state") or "") == "success"
}
for s in statuses:
if not isinstance(s, dict):
continue
@ -490,9 +520,31 @@ def reap(
counters["preserved_non_failure"] += 1
continue
# Default-branch `pull_request` contexts can be stale shadows of
# the exact same workflow/job already proven by the successful
# `push` context on the same SHA. Compensate only that narrow
# shape; a missing or failed push equivalent remains a real gate
# signal and is preserved.
push_equivalent = push_equivalent_context(context)
if push_equivalent is not None:
if push_equivalent in successful_contexts:
post_compensating_status(
sha,
context,
s.get("target_url"),
description=PR_SHADOW_COMPENSATION_DESCRIPTION,
dry_run=dry_run,
)
counters["compensated"] += 1
counters["compensated_pr_shadowed_by_push_success"] += 1
counters["compensated_contexts"].append(context)
else:
counters["preserved_pr_without_push_success"] += 1
continue
# Only `(push)`-suffix contexts hit the hardcoded-suffix bug.
# Branch-protection required checks (e.g. `Secret scan / Scan
# diff (pull_request)`) are NOT reachable from this path.
# Other failed contexts are preserved unless handled by the
# pull-request-shadow rule above.
if not context.endswith(PUSH_SUFFIX):
counters["preserved_non_push_suffix"] += 1
continue
@ -614,6 +666,8 @@ def reap_branch(
"preserved_non_failure": 0,
"preserved_non_push_suffix": 0,
"preserved_unparseable": 0,
"compensated_pr_shadowed_by_push_success": 0,
"preserved_pr_without_push_success": 0,
"compensated_per_sha": {},
}
@ -651,6 +705,8 @@ def reap_branch(
"preserved_non_failure",
"preserved_non_push_suffix",
"preserved_unparseable",
"compensated_pr_shadowed_by_push_success",
"preserved_pr_without_push_success",
):
aggregate[key] += per_sha[key]

View File

@ -70,3 +70,100 @@ def test_api_raises_after_retry_budget(monkeypatch):
assert "failed after 3 attempts" in str(exc)
else:
raise AssertionError("expected ApiError")
def test_reap_compensates_failed_pr_context_when_push_equivalent_passed(monkeypatch):
mod = load_reaper()
posted = []
def fake_post(sha, context, target_url, *, description="", dry_run=False):
posted.append((sha, context, target_url, description, dry_run))
monkeypatch.setattr(mod, "post_compensating_status", fake_post)
counters = mod.reap(
{"CI": True, "Handlers Postgres Integration": True},
{
"statuses": [
{
"context": "CI / Platform (Go) (pull_request)",
"status": "failure",
"target_url": "https://git.example.test/ci-pr",
},
{
"context": "CI / Platform (Go) (push)",
"status": "success",
},
{
"context": (
"Handlers Postgres Integration / "
"Handlers Postgres Integration (pull_request)"
),
"status": "failure",
"target_url": "https://git.example.test/handlers-pr",
},
{
"context": (
"Handlers Postgres Integration / "
"Handlers Postgres Integration (push)"
),
"status": "success",
},
],
},
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
)
assert counters["compensated_pr_shadowed_by_push_success"] == 2
assert posted == [
(
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
"CI / Platform (Go) (pull_request)",
"https://git.example.test/ci-pr",
mod.PR_SHADOW_COMPENSATION_DESCRIPTION,
False,
),
(
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
"Handlers Postgres Integration / Handlers Postgres Integration (pull_request)",
"https://git.example.test/handlers-pr",
mod.PR_SHADOW_COMPENSATION_DESCRIPTION,
False,
),
]
def test_reap_preserves_failed_pr_context_without_push_success(monkeypatch):
mod = load_reaper()
posted = []
monkeypatch.setattr(
mod,
"post_compensating_status",
lambda sha, context, target_url, *, description="", dry_run=False: posted.append(
context
),
)
counters = mod.reap(
{"CI": True},
{
"statuses": [
{
"context": "CI / Platform (Go) (pull_request)",
"status": "failure",
},
{
"context": "CI / Platform (Go) (push)",
"status": "failure",
},
{
"context": "CI / Shellcheck (pull_request)",
"status": "failure",
},
],
},
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
)
assert counters["preserved_pr_without_push_success"] == 2
assert posted == []