test(gate-check): explicit missing/pending required-context fail-closed coverage (#2403 CR2+Researcher) #2423
@@ -1170,7 +1170,9 @@ def enumerate_readiness(*, dry_run: bool = False) -> list[ReadinessEntry]:
|
||||
post-batch summary can be printed.
|
||||
"""
|
||||
bp = get_branch_protection(WATCH_BRANCH)
|
||||
contexts = bp.required_contexts
|
||||
# Uniform gate: governance checks are ALWAYS required, even if branch
|
||||
# protection does not enumerate them. Deduplicate against BP list.
|
||||
contexts = list(dict.fromkeys(bp.required_contexts + GOVERNANCE_REQUIRED_CONTEXTS))
|
||||
required_approvals = bp.required_approvals
|
||||
|
||||
main_sha = get_branch_head(WATCH_BRANCH)
|
||||
|
||||
@@ -333,6 +333,27 @@ def test_governance_red_blocks_merge():
|
||||
assert "required contexts not green" in decision.reason
|
||||
|
||||
|
||||
def test_non_required_red_does_not_block_merge():
|
||||
# Uniform gate flip (CTO #2407): qa-review, security-review, sop-checklist
|
||||
# are REQUIRED for ALL PRs. A PR with these failing/pending must NOT be
|
||||
# force-mergeable, even if BP-required CI is green and approvals are genuine.
|
||||
pr_status = {
|
||||
"state": "failure",
|
||||
"statuses": [
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
{"context": "qa-review / approved (pull_request)", "status": "failure"},
|
||||
{"context": "security-review / approved (pull_request)", "status": "pending"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "failure"},
|
||||
{"context": "Staging SaaS / e2e (pull_request)", "status": "failure"},
|
||||
],
|
||||
}
|
||||
decision = mq.evaluate_merge_readiness(**_ready_kwargs(pr_status=pr_status))
|
||||
assert decision.ready is False
|
||||
assert decision.action == "wait"
|
||||
assert "required contexts not green" in decision.reason
|
||||
assert decision.force is False
|
||||
|
||||
|
||||
def test_non_required_advisory_red_does_not_block_merge():
|
||||
# Governance checks are green; only advisory non-required reds (Staging SaaS)
|
||||
# are present → PR is still mergeable with force_merge bypassing the advisory.
|
||||
|
||||
@@ -433,6 +433,17 @@ def signal_4_branch_divergence(
|
||||
|
||||
# ── Signal 6: CI required-checks awareness ───────────────────────────────────
|
||||
|
||||
# Governance checks that are ALWAYS required for every PR, regardless of
|
||||
# branch-protection configuration. These are the uniform-gate checks that
|
||||
# must pass before any PR can merge (SOP tier removal makes them mandatory
|
||||
# for all PRs, not just tier:medium/tier:high).
|
||||
GOVERNANCE_REQUIRED_CONTEXTS = [
|
||||
"qa-review / approved (pull_request)",
|
||||
"security-review / approved (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)",
|
||||
]
|
||||
|
||||
|
||||
def signal_6_ci(pr_number: int, repo: str, branch: str | None = None, pr_data: dict | None = None) -> dict:
|
||||
"""
|
||||
Query combined CI status for PR head commit.
|
||||
@@ -470,6 +481,9 @@ def signal_6_ci(pr_number: int, repo: str, branch: str | None = None, pr_data: d
|
||||
required_checks.append(check["context"])
|
||||
except GiteaError:
|
||||
pass # No protection or no read access
|
||||
# Uniform gate: governance checks are ALWAYS required, even if branch
|
||||
# protection does not enumerate them. Deduplicate against BP list.
|
||||
required_checks = list(dict.fromkeys(required_checks + GOVERNANCE_REQUIRED_CONTEXTS))
|
||||
|
||||
failing_required = []
|
||||
passing_required = []
|
||||
|
||||
@@ -354,3 +354,133 @@ def test_signal_4_branch_api_error_returns_na(monkeypatch):
|
||||
|
||||
assert result["verdict"] == "N/A"
|
||||
assert "error" in result
|
||||
|
||||
|
||||
# ── Signal 6: CI required checks ────────────────────────────────────────────
|
||||
|
||||
|
||||
def _signal_6_api_get(required_checks, statuses):
|
||||
"""Return a fake_api_get closure for signal_6 tests."""
|
||||
def fake_api_get(path):
|
||||
if path == "/repos/molecule-ai/molecule-core/pulls/200":
|
||||
return {"base": {"sha": "base000", "ref": "main"}, "head": {"sha": "pr222"}}
|
||||
if path == "/repos/molecule-ai/molecule-core/commits/pr222/status":
|
||||
return {"state": "failure", "statuses": statuses}
|
||||
if path == "/repos/molecule-ai/molecule-core/branches/main/protection":
|
||||
return {"required_status_checks": {"checks": [{"context": c} for c in required_checks]}}
|
||||
raise AssertionError(f"unexpected api_get: {path}")
|
||||
return fake_api_get
|
||||
|
||||
|
||||
def test_signal_6_missing_required_context_returns_ci_pending(monkeypatch):
|
||||
"""A required check that is ABSENT from the status list is treated as missing,
|
||||
which is fail-closed → CI_PENDING (never ready-by-absence)."""
|
||||
mod = load_gate_check()
|
||||
monkeypatch.setattr(
|
||||
mod, "api_get",
|
||||
_signal_6_api_get(
|
||||
required_checks=["qa-review / approved (pull_request)", "security-review / approved (pull_request)"],
|
||||
statuses=[
|
||||
{"context": "qa-review / approved (pull_request)", "status": "success"},
|
||||
# security-review is completely missing
|
||||
],
|
||||
),
|
||||
)
|
||||
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
|
||||
assert result["verdict"] == "CI_PENDING"
|
||||
assert "security-review / approved (pull_request)" in result["pending_required"]
|
||||
|
||||
|
||||
def test_signal_6_pending_required_context_returns_ci_pending(monkeypatch):
|
||||
"""A required check with status 'pending' blocks the gate with CI_PENDING."""
|
||||
mod = load_gate_check()
|
||||
monkeypatch.setattr(
|
||||
mod, "api_get",
|
||||
_signal_6_api_get(
|
||||
required_checks=[
|
||||
"qa-review / approved (pull_request)",
|
||||
"security-review / approved (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)",
|
||||
],
|
||||
statuses=[
|
||||
{"context": "qa-review / approved (pull_request)", "status": "success"},
|
||||
{"context": "security-review / approved (pull_request)", "status": "pending"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"},
|
||||
],
|
||||
),
|
||||
)
|
||||
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
|
||||
assert result["verdict"] == "CI_PENDING"
|
||||
assert "security-review / approved (pull_request)" in result["pending_required"]
|
||||
|
||||
|
||||
def test_signal_6_failing_required_context_returns_ci_fail(monkeypatch):
|
||||
"""A required check with status 'failure' blocks the gate with CI_FAIL."""
|
||||
mod = load_gate_check()
|
||||
monkeypatch.setattr(
|
||||
mod, "api_get",
|
||||
_signal_6_api_get(
|
||||
required_checks=[
|
||||
"qa-review / approved (pull_request)",
|
||||
"security-review / approved (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)",
|
||||
"CI / all-required (pull_request)",
|
||||
],
|
||||
statuses=[
|
||||
{"context": "qa-review / approved (pull_request)", "status": "failure"},
|
||||
{"context": "security-review / approved (pull_request)", "status": "success"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"},
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
],
|
||||
),
|
||||
)
|
||||
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
|
||||
assert result["verdict"] == "CI_FAIL"
|
||||
assert "qa-review / approved (pull_request)" in result["failing_required"]
|
||||
|
||||
|
||||
def test_signal_6_all_required_green_returns_clear(monkeypatch):
|
||||
"""When every required check is success/neutral, the gate is CLEAR."""
|
||||
mod = load_gate_check()
|
||||
monkeypatch.setattr(
|
||||
mod, "api_get",
|
||||
_signal_6_api_get(
|
||||
required_checks=[
|
||||
"qa-review / approved (pull_request)",
|
||||
"security-review / approved (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)",
|
||||
"CI / all-required (pull_request)",
|
||||
],
|
||||
statuses=[
|
||||
{"context": "qa-review / approved (pull_request)", "status": "success"},
|
||||
{"context": "security-review / approved (pull_request)", "status": "success"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "success"},
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
],
|
||||
),
|
||||
)
|
||||
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
|
||||
assert result["verdict"] == "CLEAR"
|
||||
assert result["pending_required"] == []
|
||||
assert result["failing_required"] == []
|
||||
|
||||
|
||||
def test_signal_6_governance_checks_always_required_even_when_bp_empty(monkeypatch):
|
||||
"""Uniform gate: qa/security/sop are REQUIRED even if branch protection
|
||||
does not enumerate them. A PR with only CI/all-required green but missing
|
||||
governance contexts must be CI_PENDING (fail-closed)."""
|
||||
mod = load_gate_check()
|
||||
monkeypatch.setattr(
|
||||
mod, "api_get",
|
||||
_signal_6_api_get(
|
||||
required_checks=[], # BP lists nothing
|
||||
statuses=[
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
],
|
||||
),
|
||||
)
|
||||
result = mod.signal_6_ci(200, "molecule-ai/molecule-core")
|
||||
assert result["verdict"] == "CI_PENDING"
|
||||
assert "qa-review / approved (pull_request)" in result["pending_required"]
|
||||
assert "security-review / approved (pull_request)" in result["pending_required"]
|
||||
assert "sop-checklist / all-items-acked (pull_request)" in result["pending_required"]
|
||||
|
||||
Reference in New Issue
Block a user