test(gate-check): explicit missing/pending required-context fail-closed coverage (#2403 CR2+Researcher) #2423

Merged
agent-dev-a merged 2 commits from feat/2403-remove-sop-tier-system into main 2026-06-08 05:20:30 +00:00
4 changed files with 168 additions and 1 deletions
+3 -1
View File
@@ -1170,7 +1170,9 @@ def enumerate_readiness(*, dry_run: bool = False) -> list[ReadinessEntry]:
post-batch summary can be printed.
"""
bp = get_branch_protection(WATCH_BRANCH)
contexts = bp.required_contexts
# Uniform gate: governance checks are ALWAYS required, even if branch
# protection does not enumerate them. Deduplicate against BP list.
contexts = list(dict.fromkeys(bp.required_contexts + GOVERNANCE_REQUIRED_CONTEXTS))
required_approvals = bp.required_approvals
main_sha = get_branch_head(WATCH_BRANCH)
@@ -333,6 +333,27 @@ def test_governance_red_blocks_merge():
assert "required contexts not green" in decision.reason
def test_non_required_red_does_not_block_merge():
# Uniform gate flip (CTO #2407): qa-review, security-review, sop-checklist
# are REQUIRED for ALL PRs. A PR with these failing/pending must NOT be
# force-mergeable, even if BP-required CI is green and approvals are genuine.
pr_status = {
"state": "failure",
"statuses": [
{"context": "CI / all-required (pull_request)", "status": "success"},
{"context": "qa-review / approved (pull_request)", "status": "failure"},
{"context": "security-review / approved (pull_request)", "status": "pending"},
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "failure"},
{"context": "Staging SaaS / e2e (pull_request)", "status": "failure"},
],
}
decision = mq.evaluate_merge_readiness(**_ready_kwargs(pr_status=pr_status))
assert decision.ready is False
assert decision.action == "wait"
assert "required contexts not green" in decision.reason
assert decision.force is False
def test_non_required_advisory_red_does_not_block_merge():
# Governance checks are green; only advisory non-required reds (Staging SaaS)
# are present → PR is still mergeable with force_merge bypassing the advisory.
+14
View File
@@ -433,6 +433,17 @@ def signal_4_branch_divergence(
# ── Signal 6: CI required-checks awareness ───────────────────────────────────
# Governance checks that are ALWAYS required for every PR, regardless of
# branch-protection configuration. These are the uniform-gate checks that
# must pass before any PR can merge (SOP tier removal makes them mandatory
# for all PRs, not just tier:medium/tier:high).
GOVERNANCE_REQUIRED_CONTEXTS = [
"qa-review / approved (pull_request)",
"security-review / approved (pull_request)",
"sop-checklist / all-items-acked (pull_request)",
]
def signal_6_ci(pr_number: int, repo: str, branch: str | None = None, pr_data: dict | None = None) -> dict:
"""
Query combined CI status for PR head commit.
@@ -470,6 +481,9 @@ def signal_6_ci(pr_number: int, repo: str, branch: str | None = None, pr_data: d
required_checks.append(check["context"])
except GiteaError:
pass # No protection or no read access
# Uniform gate: governance checks are ALWAYS required, even if branch
# protection does not enumerate them. Deduplicate against BP list.
required_checks = list(dict.fromkeys(required_checks + GOVERNANCE_REQUIRED_CONTEXTS))
failing_required = []
passing_required = []
+130
View File
@@ -354,3 +354,133 @@ def test_signal_4_branch_api_error_returns_na(monkeypatch):
assert result["verdict"] == "N/A"
assert "error" in result
# ── Signal 6: CI required checks ────────────────────────────────────────────
def _signal_6_api_get(required_checks, statuses):
"""Return a fake_api_get closure for signal_6 tests."""
def fake_api_get(path):
if path == "/repos/molecule-ai/molecule-core/pulls/200":
return {"base": {"sha": "base000", "ref": "main"}, "head": {"sha": "pr222"}}
if path == "/repos/molecule-ai/molecule-core/commits/pr222/status":
return {"state": "failure", "statuses": statuses}
if path == "/repos/molecule-ai/molecule-core/branches/main/protection":
return {"required_status_checks": {"checks": [{"context": c} for c in required_checks]}}
raise AssertionError(f"unexpected api_get: {path}")
return fake_api_get
def test_signal_6_missing_required_context_returns_ci_pending(monkeypatch):
"""A required check that is ABSENT from the status list is treated as missing,
which is fail-closed CI_PENDING (never ready-by-absence)."""
mod = load_gate_check()
monkeypatch.setattr(
mod, "api_get",
_signal_6_api_get(
required_checks=["qa-review / approved (pull_request)", "security-review / approved (pull_request)"],
statuses=[
{"context": "qa-review / approved (pull_request)", "status": "success"},
# security-review is completely missing
],
),
)
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
assert result["verdict"] == "CI_PENDING"
assert "security-review / approved (pull_request)" in result["pending_required"]
def test_signal_6_pending_required_context_returns_ci_pending(monkeypatch):
"""A required check with status 'pending' blocks the gate with CI_PENDING."""
mod = load_gate_check()
monkeypatch.setattr(
mod, "api_get",
_signal_6_api_get(
required_checks=[
"qa-review / approved (pull_request)",
"security-review / approved (pull_request)",
"sop-checklist / all-items-acked (pull_request)",
],
statuses=[
{"context": "qa-review / approved (pull_request)", "status": "success"},
{"context": "security-review / approved (pull_request)", "status": "pending"},
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"},
],
),
)
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
assert result["verdict"] == "CI_PENDING"
assert "security-review / approved (pull_request)" in result["pending_required"]
def test_signal_6_failing_required_context_returns_ci_fail(monkeypatch):
"""A required check with status 'failure' blocks the gate with CI_FAIL."""
mod = load_gate_check()
monkeypatch.setattr(
mod, "api_get",
_signal_6_api_get(
required_checks=[
"qa-review / approved (pull_request)",
"security-review / approved (pull_request)",
"sop-checklist / all-items-acked (pull_request)",
"CI / all-required (pull_request)",
],
statuses=[
{"context": "qa-review / approved (pull_request)", "status": "failure"},
{"context": "security-review / approved (pull_request)", "status": "success"},
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"},
{"context": "CI / all-required (pull_request)", "status": "success"},
],
),
)
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
assert result["verdict"] == "CI_FAIL"
assert "qa-review / approved (pull_request)" in result["failing_required"]
def test_signal_6_all_required_green_returns_clear(monkeypatch):
"""When every required check is success/neutral, the gate is CLEAR."""
mod = load_gate_check()
monkeypatch.setattr(
mod, "api_get",
_signal_6_api_get(
required_checks=[
"qa-review / approved (pull_request)",
"security-review / approved (pull_request)",
"sop-checklist / all-items-acked (pull_request)",
"CI / all-required (pull_request)",
],
statuses=[
{"context": "qa-review / approved (pull_request)", "status": "success"},
{"context": "security-review / approved (pull_request)", "status": "success"},
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"},
{"context": "CI / all-required (pull_request)", "status": "success"},
],
),
)
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
assert result["verdict"] == "CLEAR"
assert result["pending_required"] == []
assert result["failing_required"] == []
def test_signal_6_governance_checks_always_required_even_when_bp_empty(monkeypatch):
"""Uniform gate: qa/security/sop are REQUIRED even if branch protection
does not enumerate them. A PR with only CI/all-required green but missing
governance contexts must be CI_PENDING (fail-closed)."""
mod = load_gate_check()
monkeypatch.setattr(
mod, "api_get",
_signal_6_api_get(
required_checks=[], # BP lists nothing
statuses=[
{"context": "CI / all-required (pull_request)", "status": "success"},
],
),
)
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
assert result["verdict"] == "CI_PENDING"
assert "qa-review / approved (pull_request)" in result["pending_required"]
assert "security-review / approved (pull_request)" in result["pending_required"]
assert "sop-checklist / all-items-acked (pull_request)" in result["pending_required"]