From f1ccd3bb05b9e14e0b0b96cc4f90fda2e72b4ee0 Mon Sep 17 00:00:00 2001 From: devops-engineer Date: Mon, 8 Jun 2026 22:42:43 +0000 Subject: [PATCH] ci: fail-closed when ops-scripts unittest collects 0 tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate-integrity hardening. The "Run scripts/ unittests, if any" step in .gitea/workflows/test-ops-scripts.yml detected "no tests collected" via `rc=$?; if [ "$rc" -eq 5 ]`. But Python 3.12's unittest exits 0 (not 5) when discovery finds 0 tests ("Ran 0 tests ... NO TESTS RAN"), so the guard never fired: the step passed GREEN while running ZERO tests. Any test_*.py added under scripts/ would have been silently never executed. A green job that runs 0 tests is worse than a red one. This fails closed: scripts/ (top-level) step: - genuinely NO test_*.py present -> loud SKIP (legitimate no-op; the runtime-packaging tests moved to molecule-ai-workspace-runtime, so there are none today) - test_*.py present but 0 collected -> FAIL (broken import / empty / discovery error) Count is via TestLoader().discover(...).countTestCases(), not exit code. scripts/ops/ step (real gate, 34 tests today): - assert >0 collected so deleting all test files or breaking an import can't pass GREEN by running 0 tests. ci.yml "Diagnostic — per-package verbose 60s" is continue-on-error and explicitly advisory (the blocking gate is the next step); left functional unchanged, only a clarifying comment added so its `set +e` isn't mistaken for this same bug class. The real `Ops Scripts Tests` pytest gate (.gitea/scripts/tests) is untouched. Proven on the operator: scripts/ unittest exits 0 on 0 tests (the bug); new guard SKIPs on no-files, FAILs on files-present-but-0-collected, PASSes on a real test; ops guard PASSes at 34 and FAILs on empty. Workflow-YAML linter green (0 warnings). Part of a gate-integrity hardening pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitea/workflows/ci.yml | 5 +++ .gitea/workflows/test-ops-scripts.yml | 47 ++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 48ecbe8ee..6555b80b6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -148,6 +148,11 @@ jobs: run: $(go env GOPATH)/bin/golangci-lint run --timeout 3m ./... - if: ${{ needs.changes.outputs.platform == 'true' }} name: Diagnostic — per-package verbose 60s + # DIAGNOSTIC ONLY (continue-on-error below): this step exists to dump + # verbose per-package output for triage, NOT to gate. The blocking gate + # is "Run tests with coverage (blocking gate)" immediately below. The + # `set +e` / swallowed exits here are intentional — do not "fix" them + # like a gate; the real gate is the next step. run: | set +e go test -race -v -timeout 60s ./internal/handlers/... 2>&1 | tee /tmp/test-handlers.log diff --git a/.gitea/workflows/test-ops-scripts.yml b/.gitea/workflows/test-ops-scripts.yml index a788eb72f..92c11e5dd 100644 --- a/.gitea/workflows/test-ops-scripts.yml +++ b/.gitea/workflows/test-ops-scripts.yml @@ -58,22 +58,51 @@ jobs: python-version: '3.11' - name: Install .gitea script test dependencies run: python -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2' - - name: Run scripts/ unittests, if any + - name: Run scripts/ unittests (fail-closed on 0 collected) # Top-level scripts/ tests live alongside their target file. The # runtime packaging tests moved to molecule-ai-workspace-runtime, so - # this pass may legitimately find no tests. + # this pass may legitimately find NO test files today. + # + # Gate-integrity fix: the previous guard keyed off `rc==5` to detect + # "no tests collected", but Python 3.12's unittest exits 0 (not 5) + # when discovery finds 0 tests ("NO TESTS RAN"). The guard therefore + # never fired, so any test_*.py added here would silently run 0 tests + # while this step stayed GREEN. A green step that runs 0 tests is + # worse than a red one. We now fail-closed: + # - genuinely NO test_*.py present -> loud SKIP (legitimate no-op) + # - test_*.py present but 0 collected -> FAIL (broken import/empty) working-directory: scripts run: | - set +e - python -m unittest discover -t . -p 'test_*.py' -v - rc=$? - if [ "$rc" -eq 5 ]; then - echo "No top-level scripts/ unittest files found; skipping." + set -euo pipefail + # Non-recursive count: scripts/ has no __init__.py, so unittest + # discover does not recurse into subdirs (ops/ is run separately + # below) — top-level files are the entire discovery scope here. + nfiles=$(find . -maxdepth 1 -name 'test_*.py' | wc -l | tr -d ' ') + if [ "$nfiles" -eq 0 ]; then + echo "SKIP: no top-level scripts/ test_*.py files present (genuine no-op)." exit 0 fi - exit "$rc" + echo "Found $nfiles top-level scripts/ test_*.py file(s); asserting they collect >0 tests." + ncollected=$(python -c "import unittest; print(unittest.TestLoader().discover('.', pattern='test_*.py', top_level_dir='.').countTestCases())") + echo "Collected $ncollected test case(s)." + if [ "$ncollected" -eq 0 ]; then + echo "FAIL: test_*.py file(s) present but 0 tests collected (broken import / empty file / discovery error)." + exit 1 + fi + python -m unittest discover -t . -p 'test_*.py' -v - name: Run scripts/ops/ unittests (sweep_cf_decide, ...) + # Real gate: scripts/ops/ must always run tests. Assert >0 collected so + # deleting all test files (or breaking an import) can't pass GREEN by + # running 0 tests — same gate-integrity class as the scripts/ step. working-directory: scripts/ops - run: python -m unittest discover -p 'test_*.py' -v + run: | + set -euo pipefail + ncollected=$(python -c "import unittest; print(unittest.TestLoader().discover('.', pattern='test_*.py').countTestCases())") + echo "scripts/ops/ collected $ncollected test case(s)." + if [ "$ncollected" -eq 0 ]; then + echo "FAIL: scripts/ops/ collected 0 tests — this gate must run real tests (deleted/broken import?)." + exit 1 + fi + python -m unittest discover -p 'test_*.py' -v - name: Run .gitea/scripts pytest suite run: python -m pytest .gitea/scripts/tests -q -- 2.52.0