diff --git a/.gitea/workflows/gate-check-v3-tests.yml b/.gitea/workflows/gate-check-v3-tests.yml new file mode 100644 index 000000000..bf0304dec --- /dev/null +++ b/.gitea/workflows/gate-check-v3-tests.yml @@ -0,0 +1,40 @@ +name: gate-check-v3-tests + +# Regression unit tests for tools/gate-check-v3/gate_check.py. +# This workflow closes the CI gap identified in molecule-core#2615: +# the gate-check evaluator is load-bearing for merge decisions and must +# have automated coverage for its security-sensitive governance-context +# logic (molecule-core#2589). + +on: + push: + branches: [main, staging] + paths: + - 'tools/gate-check-v3/**' + - '.gitea/workflows/gate-check-v3-tests.yml' + pull_request: + branches: [main, staging] + paths: + - 'tools/gate-check-v3/**' + - '.gitea/workflows/gate-check-v3-tests.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # bp-exempt: test-only regression suite for the gate-check evaluator. + # Merge blocking is enforced by the existing CI / all-required aggregate. + unit-tests: + name: gate-check-v3 unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install pytest + run: pip install pytest -q + + - name: Run gate-check-v3 unit tests + working-directory: tools/gate-check-v3 + run: python3 -m pytest test_gate_check.py -v diff --git a/tools/gate-check-v3/gate_check.py b/tools/gate-check-v3/gate_check.py index 4e6b03d64..417b2c253 100644 --- a/tools/gate-check-v3/gate_check.py +++ b/tools/gate-check-v3/gate_check.py @@ -467,11 +467,19 @@ def signal_6_ci(pr_number: int, repo: str, branch: str | None = None, pr_data: d # Gitea Actions uses "status" (pending/success/failure) not "state" for # individual check entries. "state" is null for pending runs. # Exclude our own prior status to prevent self-referential failure loops. - check_statuses = {} + # Gitea /commits//statuses is non-monotonic by id, so collapse by + # max(id) to guarantee the latest result for each context wins. + latest_by_context: dict[str, dict] = {} for s in combined.get("statuses") or []: ctx = s["context"] - if "gate-check" not in ctx.lower(): - check_statuses[ctx] = s.get("status", "pending") + if "gate-check" in ctx.lower(): + continue + existing = latest_by_context.get(ctx) + if existing is None or s.get("id", 0) > existing.get("id", 0): + latest_by_context[ctx] = s + check_statuses = { + ctx: s.get("status", "pending") for ctx, s in latest_by_context.items() + } # Try to get branch protection for required checks required_checks = [] diff --git a/tools/gate-check-v3/test_gate_check.py b/tools/gate-check-v3/test_gate_check.py index bf9a1396c..ef2516bdd 100644 --- a/tools/gate-check-v3/test_gate_check.py +++ b/tools/gate-check-v3/test_gate_check.py @@ -379,16 +379,19 @@ def test_signal_6_missing_required_context_returns_ci_pending(monkeypatch): monkeypatch.setattr( mod, "api_get", _signal_6_api_get( - required_checks=["qa-review / approved (pull_request)", "security-review / approved (pull_request)"], + required_checks=[ + "qa-review / approved (pull_request_target)", + "security-review / approved (pull_request_target)", + ], statuses=[ - {"context": "qa-review / approved (pull_request)", "status": "success"}, + {"context": "qa-review / approved (pull_request_target)", "status": "success"}, # security-review is completely missing ], ), ) result = mod.signal_6_ci(200, "molecule-ai/molecule-core") assert result["verdict"] == "CI_PENDING" - assert "security-review / approved (pull_request)" in result["pending_required"] + assert "security-review / approved (pull_request_target)" in result["pending_required"] def test_signal_6_pending_required_context_returns_ci_pending(monkeypatch): @@ -398,20 +401,20 @@ def test_signal_6_pending_required_context_returns_ci_pending(monkeypatch): mod, "api_get", _signal_6_api_get( required_checks=[ - "qa-review / approved (pull_request)", - "security-review / approved (pull_request)", - "sop-checklist / all-items-acked (pull_request)", + "qa-review / approved (pull_request_target)", + "security-review / approved (pull_request_target)", + "sop-checklist / all-items-acked (pull_request_target)", ], statuses=[ - {"context": "qa-review / approved (pull_request)", "status": "success"}, - {"context": "security-review / approved (pull_request)", "status": "pending"}, - {"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"}, + {"context": "qa-review / approved (pull_request_target)", "status": "success"}, + {"context": "security-review / approved (pull_request_target)", "status": "pending"}, + {"context": "sop-checklist / all-items-acked (pull_request_target)", "status": "success"}, ], ), ) result = mod.signal_6_ci(200, "molecule-ai/molecule-core") assert result["verdict"] == "CI_PENDING" - assert "security-review / approved (pull_request)" in result["pending_required"] + assert "security-review / approved (pull_request_target)" in result["pending_required"] def test_signal_6_failing_required_context_returns_ci_fail(monkeypatch): @@ -421,22 +424,22 @@ def test_signal_6_failing_required_context_returns_ci_fail(monkeypatch): mod, "api_get", _signal_6_api_get( required_checks=[ - "qa-review / approved (pull_request)", - "security-review / approved (pull_request)", - "sop-checklist / all-items-acked (pull_request)", + "qa-review / approved (pull_request_target)", + "security-review / approved (pull_request_target)", + "sop-checklist / all-items-acked (pull_request_target)", "CI / all-required (pull_request)", ], statuses=[ - {"context": "qa-review / approved (pull_request)", "status": "failure"}, - {"context": "security-review / approved (pull_request)", "status": "success"}, - {"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"}, + {"context": "qa-review / approved (pull_request_target)", "status": "failure"}, + {"context": "security-review / approved (pull_request_target)", "status": "success"}, + {"context": "sop-checklist / all-items-acked (pull_request_target)", "status": "success"}, {"context": "CI / all-required (pull_request)", "status": "success"}, ], ), ) result = mod.signal_6_ci(200, "molecule-ai/molecule-core") assert result["verdict"] == "CI_FAIL" - assert "qa-review / approved (pull_request)" in result["failing_required"] + assert "qa-review / approved (pull_request_target)" in result["failing_required"] def test_signal_6_all_required_green_returns_clear(monkeypatch): @@ -446,15 +449,15 @@ def test_signal_6_all_required_green_returns_clear(monkeypatch): mod, "api_get", _signal_6_api_get( required_checks=[ - "qa-review / approved (pull_request)", - "security-review / approved (pull_request)", - "sop-checklist / all-items-acked (pull_request)", + "qa-review / approved (pull_request_target)", + "security-review / approved (pull_request_target)", + "sop-checklist / all-items-acked (pull_request_target)", "CI / all-required (pull_request)", ], statuses=[ - {"context": "qa-review / approved (pull_request)", "status": "success"}, - {"context": "security-review / approved (pull_request)", "status": "success"}, - {"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"}, + {"context": "qa-review / approved (pull_request_target)", "status": "success"}, + {"context": "security-review / approved (pull_request_target)", "status": "success"}, + {"context": "sop-checklist / all-items-acked (pull_request_target)", "status": "success"}, {"context": "CI / all-required (pull_request)", "status": "success"}, ], ), @@ -481,6 +484,85 @@ def test_signal_6_governance_checks_always_required_even_when_bp_empty(monkeypat ) result = mod.signal_6_ci(200, "molecule-ai/molecule-core") assert result["verdict"] == "CI_PENDING" - assert "qa-review / approved (pull_request)" in result["pending_required"] - assert "security-review / approved (pull_request)" in result["pending_required"] - assert "sop-checklist / all-items-acked (pull_request)" in result["pending_required"] + assert "qa-review / approved (pull_request_target)" in result["pending_required"] + assert "security-review / approved (pull_request_target)" in result["pending_required"] + assert "sop-checklist / all-items-acked (pull_request_target)" in result["pending_required"] + + +# ── Signal 6 regression tests for molecule-core#2589 ───────────────────────── + +TRUSTED_QA = "qa-review / approved (pull_request_target)" +TRUSTED_SECURITY = "security-review / approved (pull_request_target)" +TRUSTED_SOP = "sop-checklist / all-items-acked (pull_request_target)" +UNTRUSTED_QA = "qa-review / approved (pull_request)" +UNTRUSTED_SECURITY = "security-review / approved (pull_request)" +UNTRUSTED_SOP = "sop-checklist / all-items-acked (pull_request)" + + +def test_signal_6_trusted_governance_contexts_clear(monkeypatch): + """#2589 regression: gate is satisfied ONLY by trusted (pull_request_target) + governance contexts.""" + mod = load_gate_check() + monkeypatch.setattr( + mod, "api_get", + _signal_6_api_get( + required_checks=[], + statuses=[ + {"context": TRUSTED_QA, "status": "success"}, + {"context": TRUSTED_SECURITY, "status": "success"}, + {"context": TRUSTED_SOP, "status": "success"}, + ], + ), + ) + result = mod.signal_6_ci(200, "molecule-ai/molecule-core") + assert result["verdict"] == "CLEAR" + assert result["passing_required"] == [TRUSTED_QA, TRUSTED_SECURITY, TRUSTED_SOP] + + +def test_signal_6_untrusted_governance_contexts_do_not_satisfy(monkeypatch): + """#2589 security regression: forged/untrusted (pull_request)-suffixed + governance statuses must NOT satisfy the gate.""" + mod = load_gate_check() + monkeypatch.setattr( + mod, "api_get", + _signal_6_api_get( + required_checks=[], + statuses=[ + # Attacker-controlled PR-head workflow posts the untrusted suffixes. + {"context": UNTRUSTED_QA, "status": "success"}, + {"context": UNTRUSTED_SECURITY, "status": "success"}, + {"context": UNTRUSTED_SOP, "status": "success"}, + ], + ), + ) + result = mod.signal_6_ci(200, "molecule-ai/molecule-core") + assert result["verdict"] in ("CI_PENDING", "CI_FAIL") + # Trusted contexts are still missing/unsatisfied. + for ctx in (TRUSTED_QA, TRUSTED_SECURITY, TRUSTED_SOP): + assert ctx in result["pending_required"] + # Untrusted contexts are NOT counted as passing governance. + for ctx in (UNTRUSTED_QA, UNTRUSTED_SECURITY, UNTRUSTED_SOP): + assert ctx not in result["passing_required"] + + +def test_signal_6_status_collapse_uses_max_id(monkeypatch): + """Gitea /commits//statuses is non-monotonic by id; the gate must + collapse duplicate contexts by max(id), not by list order.""" + mod = load_gate_check() + monkeypatch.setattr( + mod, "api_get", + _signal_6_api_get( + required_checks=[TRUSTED_QA], + statuses=[ + # Older id claims success; newer id claims failure. + # List order is deliberately opposite of id order. + {"id": 3, "context": TRUSTED_QA, "status": "failure"}, + {"id": 1, "context": TRUSTED_QA, "status": "success"}, + {"id": 2, "context": TRUSTED_QA, "status": "success"}, + ], + ), + ) + result = mod.signal_6_ci(200, "molecule-ai/molecule-core") + assert result["verdict"] == "CI_FAIL" + assert TRUSTED_QA in result["failing_required"] + assert result["all_check_statuses"][TRUSTED_QA] == "failure"