molecule-core/.gitea/workflows/handlers-postgres-integration.yml

name: Handlers Postgres Integration

# Ported from .github/workflows/handlers-postgres-integration.yml on 2026-05-11 per RFC
# internal#219 §1 sweep. Differences from the GitHub version:
#   - Kept flat `workflow_dispatch` (no `inputs`) under repository policy;
#     dispatch callers pass parameters through the established env/var path.
#   - Omitted GitHub's `merge_group:` trigger; this repository uses its
#     serialized Gitea queue plus pull-request/push status gates.
#   - Omitted `environment:` blocks; this repository's approval and secret
#     gates do not depend on Actions environments.
#   - Workflow-level env.GITHUB_SERVER_URL pinned per
#     feedback_act_runner_github_server_url.
#   - `continue-on-error: true` on each job (RFC §1 contract).
#

# Real-Postgres integration tests for workspace-server/internal/handlers/.
# Triggered on every PR/push that touches the handlers package.
#
# Why this workflow exists
# ------------------------
# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
# depend on the row state AFTER the SQL runs. The result_preview-lost
# bug shipped to staging in PR #2854 because every unit test was
# satisfied with "an UPDATE statement fired" — none verified the row's
# preview field actually landed. The local-postgres E2E that retrofit
# self-review caught it took 2 minutes to set up and would have caught
# the bug at PR-time.
#
# Why this workflow does NOT use `services: postgres:` (Class B fix)
# ------------------------------------------------------------------
# The self-hosted Docker runner uses `container.network: host`, which
# act_runner applies to BOTH
# the job container AND every service container. With host-net, two
# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
# second postgres FATALs with `could not create any TCP/IP sockets:
# Address in use`, and Docker auto-removes it (act_runner sets
# AutoRemove:true on service containers). By the time the migrations
# step runs `psql`, the postgres container is gone, hence
# `Connection refused` then `failed to remove container: No such
# container` at cleanup time.
#
# Per-job `container.network` override is silently ignored by
# act_runner — `--network and --net in the options will be ignored.`
# appears in the runner log. Documented constraint.
#
# So we sidestep `services:` entirely. The job container still uses
# host-net (inherited from runner config; required for cache server
# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
# postgres on the existing `molecule-core-net` bridge with a
# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
# read its bridge IP via `docker inspect`. A host-net job container
# can reach a bridge-net container directly via the bridge IP (verified
# on the former runner fleet in May 2026 and preserved by current CI).
#
# Trade-offs vs. the original `services:` shape:
#   + No host-port collision; N parallel runs share the bridge cleanly
#   + `if: always()` cleanup runs even on test-step failure
#   - One more step in the workflow (+~3 lines)
#   - Requires `molecule-core-net` to exist on the selected runner host
#     (it does; declared in docker-compose.yml + docker-compose.infra.yml)
#
# Class B Hongming-owned CICD red sweep, 2026-05-08.
#
# Cost: ~30s job (postgres pull from cache + go build + 4 tests).

on:
  push:
    branches: [main, staging]
  pull_request:
    branches: [main, staging]
concurrency:
  group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
  cancel-in-progress: false

env:
  GITHUB_SERVER_URL: https://git.moleculesai.app

jobs:
  detect-changes:
    name: detect-changes
    # mc#1529 §1: pin to `docker-host` so the integration job runs on the
    # self-hosted Docker runners, which carry the
    # `molecule-core-net` bridge network this workflow depends on. PC2
    # runners (hongming-pc-runner-*) also advertise ubuntu-latest but
    # don't have that network — the previous `runs-on: ubuntu-latest`
    # rolled the dice and hard-failed the bridge-inspect step ~30% of
    # the time. detect-changes itself doesn't need the bridge, but keeping
    # both jobs on the same label avoids workspace-volume cross-host
    # surprises and keeps the routing rule discoverable in one place.
    runs-on: docker-host
    # Path-filter step should be quick; 10 min kills a wedged runner
    # without waiting for the runner-level cap.
    timeout-minutes: 10
    # mc#1982 Phase 3 (RFC §1): surface broken workflows without blocking.
    # mc#1982: mask removed. If regressions appear, root-fix the underlying
    # test — do NOT renew the mask silently.
    continue-on-error: false
    outputs:
      handlers: ${{ steps.filter.outputs.handlers }}
      debug: ${{ steps.filter.outputs.debug }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # A full-history checkout can exceed the runner's quiet/startup
          # window before the path filter emits logs. Fetch the common push
          # case cheaply; the script below fetches the exact BASE SHA if it is
          # not present in the shallow checkout.
          fetch-depth: 2
      - id: filter
        env:
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
          PUSH_BEFORE: ${{ github.event.before }}
        run: |
          BASE_SHA="${PR_BASE_SHA:-$PUSH_BEFORE}"
          # --push-before falls back to PUSH_BEFORE (as ci.yml / e2e-api.yml do):
          # act_runner does not set GITHUB_EVENT_BEFORE, and an EMPTY push base
          # makes detect-changes treat the push as "everything changed", so every
          # push to main started a pgvector container for nothing.
          python3 .gitea/scripts/detect-changes.py \
            --profile handlers-postgres \
            --event-name "${{ github.event_name }}" \
            --pr-base-sha "$PR_BASE_SHA" \
            --base-ref "$PR_BASE_REF" \
            --push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}" || {
            # Script crash / uncaught error: fail open so the integration gate
            # runs rather than silently no-oping a potentially-breaking PR.
            echo "handlers=true" >> "$GITHUB_OUTPUT"
            echo "debug=detect-script-error event=${{ github.event_name }} base=$BASE_SHA" >> "$GITHUB_OUTPUT"
            exit 0
          }
          echo "debug=profile=handlers-postgres event=${{ github.event_name }} base=$BASE_SHA" >> "$GITHUB_OUTPUT"

  # ONE job (no job-level `if:`) that always runs and reports under the
  # required-check name `Handlers Postgres Integration`. Real work is gated
  # per-step on `needs.detect-changes.outputs.handlers`. Reason: Gitea
  # registers a check run for every job that matches `name:`, and a job-level
  # `if: false` produces a SKIPPED check run. Branch protection treats all
  # check runs with a matching context name on the latest commit as a SET —
  # any SKIPPED in the set fails the required-check eval, even with SUCCESS
  # siblings. Collapsing to a single always-running job with conditional
  # steps emits exactly one SUCCESS check run regardless of paths filter —
  # branch-protection-clean. Mirrors e2e-api.yml (PR #2264 incident).
  integration:
    name: Handlers Postgres Integration
    needs: detect-changes
    # mc#1529 §1: must run on the docker-host lane (where `molecule-core-net`
    # exists). See detect-changes for the full routing rationale.
    runs-on: docker-host
    # Real cost is ~30s-to-few-min (postgres cache + build + suites);
    # 15 min kills a wedged run promptly.
    timeout-minutes: 15
    # mc#1982 Phase 3 (RFC §1): surface broken workflows without blocking.
    # mc#1982: mask removed. If regressions appear, root-fix the underlying
    # test — do NOT renew the mask silently.
    continue-on-error: false
    env:
      # Unique name per run so concurrent jobs don't collide on the
      # bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
      # workflow_dispatch reruns of the same run_id.
      PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
      # Bridge network already exists on the selected runner host (declared
      # in docker-compose.yml + docker-compose.infra.yml).
      PG_NETWORK: molecule-core-net
    defaults:
      run:
        working-directory: workspace-server
    steps:
      - name: No-op pass (paths filter excluded this commit)
        if: needs.detect-changes.outputs.handlers != 'true'
        working-directory: .
        run: |
          echo "No handlers/migrations changes — Handlers Postgres Integration gate satisfied without running tests."
          echo "::notice::Handlers Postgres Integration no-op pass (paths filter excluded this commit)."
          echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"

      - if: needs.detect-changes.outputs.handlers == 'true'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - if: needs.detect-changes.outputs.handlers == 'true'
        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
        with:
          go-version: 'stable'
          # cache:false — the self-hosted runner bind-mounts a persistent
          # GOCACHE/GOMODCACHE (/var/cache/ci-go-{build,mod}); actions/cache is
          # redundant and corrupts it by untarring over the bind mount ("File
          # exists" -> "Failed to restore" -> partial cache -> linker/typecheck
          # errors on heavy jobs, e.g. test -race link "too many errors" and
          # go-arch-lint "without types"). Fleet sweep cp#698 missed this
          # workflow (found during the cp ci.yml sweep).
          cache: false

      - if: needs.detect-changes.outputs.handlers == 'true'
        name: Start sibling Postgres on bridge network
        working-directory: .
        run: |
          # Sanity: the bridge network must exist on the selected runner host.
          # Hard-fail loud if it doesn't — easier to spot than a silent
          # auto-create that diverges from the rest of the stack.
          if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
            echo "::error::Bridge network '${PG_NETWORK}' missing on docker-host runner. Check the runner bootstrap and incident handbook."
            exit 1
          fi

          # The shared starter wipes a stale same-named container first
          # (rerun on the same run_id) with `rm -fv`, and puts PGDATA on a
          # tmpfs so no anonymous volume is left behind (see its header).
          bash .gitea/scripts/ci-service-container.sh postgres "${PG_NAME}" pgvector/pgvector:pg15 \
            --network "${PG_NETWORK}" \
            --health-cmd "pg_isready -U postgres" \
            --health-interval 5s \
            --health-timeout 5s \
            --health-retries 10 \
            -e POSTGRES_PASSWORD=test \
            -e POSTGRES_DB=molecule >/dev/null

          # Read back the bridge IP. Always present immediately after
          # `docker run -d` for bridge networks.
          PG_HOST=$(docker inspect "${PG_NAME}" \
            --format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
          if [ -z "${PG_HOST}" ]; then
            echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
            docker logs "${PG_NAME}" || true
            exit 1
          fi
          echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
          echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
          echo "Started ${PG_NAME} at ${PG_HOST}:5432"

      - if: needs.detect-changes.outputs.handlers == 'true'
        name: Apply migrations to Postgres service
        env:
          PGPASSWORD: test
        run: |
          # Wait for postgres to actually accept connections. Docker's
          # health-cmd handles container-side readiness, but the wire
          # to the bridge IP is best-tested with pg_isready directly.
          for i in {1..15}; do
            if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
            echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
          done

          # core#2540: pgvector extension must be present BEFORE any migration
          # or integration test that uses the vector(1536) type. The
          # pgvector/pgvector:pg15 image ships the extension files but the
          # extension is NOT auto-created in each database — we do it here
          # once so both migrations (031_memories_pgvector) and handler
          # integration tests (memories_integration_test.go) see the type.
          psql -h "${PG_HOST}" -U postgres -d molecule -c \
            "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null
          echo "✓ pgvector extension ensured"

          # Apply every .up.sql in lexicographic order with
          # ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
          # blocking the suite. This handles the current schema state
          # where a few historical migrations (e.g. 017_memories_fts_*)
          # depend on tables that were later renamed/dropped and so
          # cannot replay from scratch. The migrations that DO succeed
          # land their tables, which is sufficient for the integration
          # tests in handlers/.
          #
          # Why not maintain a curated allowlist: every new migration
          # touching a handlers/-tested table would have to update this
          # workflow. With apply-all-or-skip, a future migration that
          # adds a column to delegations runs automatically (its base
          # table 049_delegations.up.sql already succeeded above it in
          # the order). Operators only need to revisit this if the
          # migration chain becomes legitimately replayable end-to-end.
          #
          # Per-migration result is logged so a failed migration that
          # SHOULD have been replayable surfaces in the CI log instead
          # of silently failing.
          # Apply both *.sql (legacy, lives next to its module) and
          # *.up.sql (newer up/down convention) in a single
          # lexicographically-sorted pass. Excluding *.down.sql so the
          # newest-naming-convention pairs don't undo themselves mid-run.
          # Pre-#149-followup this loop only globbed *.up.sql, which
          # silently skipped 001_workspaces.sql + 009_activity_logs.sql
          # — fine while no integration test depended on those tables,
          # not fine once a cross-table atomicity test came in.
          set +e
          for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
            if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
                  -f "$migration" >/dev/null 2>&1; then
              echo "✓ $(basename "$migration")"
            else
              echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
            fi
          done
          set -e

          # Sanity: the delegations + workspaces + activity_logs tables
          # MUST exist for the integration tests to be meaningful. Hard-
          # fail if any didn't land — that would be a real regression we
          # want loud.
          # workspace_schedules added for the #2149 scheduler integration tests.
          # workspace_plugin_install_reports added for the plugin-install-report
          # suite: without it a silently-skipped migration 20260730060000 would make
          # those tests fail on a confusing scan error instead of saying the table is
          # missing — and this loop exists precisely so a skipped migration is loud.
          # workspace_auth_tokens + org_api_tokens added for the #2156
          # registry-auth TestIntegration_ suite (#2148). Without this
          # guard, a silently-skipped migration 020 (workspace_auth_tokens)
          # or 035 (org_api_tokens) would let the auth tests run against
          # missing tables and falsely green.
          for tbl in delegations workspaces activity_logs pending_uploads workspace_schedules workspace_auth_tokens org_api_tokens workspace_plugin_install_reports; do
            if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
                -c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
                | grep -q 1; then
              echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
              exit 1
            fi
            echo "✓ $tbl table present"
          done

      - if: needs.detect-changes.outputs.handlers == 'true'
        name: Preflight — INTEGRATION_DB_URL must be present
        run: |
          # Belt-and-suspenders: if the postgres-start step failed to
          # export INTEGRATION_DB_URL, fail loud BEFORE go test can
          # t.Skip its way to a green build. Closes the workflow-level
          # fail-open gap identified in PR #2166 blocker #2.
          if [ -z "${INTEGRATION_DB_URL:-}" ]; then
            echo "::error::INTEGRATION_DB_URL is empty — postgres-start step did not export the connection string"
            exit 1
          fi
          echo "INTEGRATION_DB_URL is set"

      - if: needs.detect-changes.outputs.handlers == 'true'
        name: Run integration tests
        run: |
          set -uo pipefail
          # INTEGRATION_DB_URL is exported by the start-postgres step;
          # points at the per-run bridge IP, not 127.0.0.1, so concurrent
          # workflow runs don't fight over a host-net 5432 port.
          #
          # Anti-vacuous-gate guard. `go test` exits 0 — reporting SUCCESS on
          # this ENFORCED, merge-blocking context — in THREE distinct
          # zero-coverage situations, all verified locally:
          #   1. the -run regex matches nothing (tests renamed/deleted):
          #      "ok ... [no tests to run]", $? == 0
          #   2. the `integration` build tag is dropped/typo'd, so the tagged
          #      files vanish from the build: also "[no tests to run]" (or
          #      "[no test files]" for a fully-tagged package), $? == 0
          #   3. the suite skips wholesale (e.g. no INTEGRATION_DB_URL — every
          #      TestIntegration_ starts with a t.Skip guard): $? == 0
          # In all three the PASS arm is exactly what a BROKEN or ABSENT input
          # produces. Require >0 "--- PASS:" lines (needs -v) and fail closed.
          LOG=$(mktemp)
          rc=0
          go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_" 2>&1 | tee "$LOG" || rc=$?
          npass=$(grep -c -E '^[[:space:]]*--- PASS: ' "$LOG" || true)
          echo "handlers ^TestIntegration_: ${npass} test(s) actually ran and passed"
          if [ "$rc" -ne 0 ]; then
            exit "$rc"
          fi
          if [ "${npass:-0}" -eq 0 ]; then
            echo "::error::Handlers integration gate ran ZERO tests (0 '--- PASS:' lines) but go test still exited 0. A gate that runs no tests must NEVER report success. Check the -run regex ^TestIntegration_, the integration build tag on workspace-server/internal/handlers/*_integration_test.go, and that the suite did not skip wholesale on a missing INTEGRATION_DB_URL."
            exit 1
          fi

      - if: needs.detect-changes.outputs.handlers == 'true'
        name: Run live-container tests (MOLECULE_LIVE_DOCKER)
        env:
          MOLECULE_LIVE_DOCKER: "1"
          # Unique per run for the same reason PG_NAME is: this workflow's
          # concurrency group is keyed on the head SHA, so two handlers-touching
          # PRs share one Docker daemon and a fixed box name would let each
          # `docker rm -f` the other's container mid-test.
          MOLECULE_LIVE_BOX_SUFFIX: ${{ github.run_id }}-${{ github.run_attempt }}
        run: |
          set -uo pipefail
          # The TestLiveBox_* suite proves things NO unit test can reach: that a
          # tar actually EXTRACTS where we think it does, and that a settings
          # file lands in a running container. It exists because
          # buildContainerFilesTar doubled destPath and every save silently
          # wrote to /configs/configs/... while the API answered
          # {"status":"saved"} — a bug whose unit test asserted the tar SHAPE
          # and therefore pinned the bug as the contract.
          #
          # Those tests were gated on MOLECULE_LIVE_DOCKER=1 and NO WORKFLOW SET
          # IT, so they ran once by hand and never again. That is the same
          # "artifact real, wiring absent" class as the path-gated
          # template-delivery lane and the plugin gate meta-ci reported as
          # `planned` for its entire life. One-shot evidence is not coverage.
          #
          # Anti-vacuous guard, mirroring the integration step above but with an
          # EXTRA arm: these tests SKIP rather than fail when the env var is
          # unset or the daemon is unreachable, so `go test` exits 0 on exactly
          # the input this step exists to catch. A SKIP here is a FAILURE.
          LOG=$(mktemp)
          rc=0
          # 2m, not 10m. The observed step time is ~4s and startLiveBoxFor already
          # caps container readiness at 20s, so 10m buys nothing on a healthy run —
          # it only pushes 5m+10m+5m past this job's timeout-minutes: 15, making a
          # JOB-level kill newly reachable. That is strictly worse than a go-level
          # timeout: it skips the `if: always()` stop-Postgres step and leaks a
          # pg-handlers-* container, which the sweeper's ^ws- filter does not collect.
          go test -timeout 2m -v ./internal/handlers/ -run '^TestLiveBox_' 2>&1 | tee "$LOG" || rc=$?
          # Anchored at column 0 ON PURPOSE: `go test -v` indents SUBTEST results
          # ("    --- PASS: Parent/child"). With a leading [[:space:]]* class, one
          # function with six t.Run()s scores npass=7 and hands the floor below a
          # free pass. Only top-level results count.
          npass=$(grep -c -E '^--- PASS: TestLiveBox_' "$LOG" || true)
          nskip=$(grep -c -E '^[[:space:]]*--- SKIP: TestLiveBox_' "$LOG" || true)
          # The live suite spans MORE THAN ONE FILE: the WRITE proof
          # (plugin_settings_writer_live_test.go) and the READ-surface proof
          # (files_read_surface_live_test.go, the 2026-08-02 incident — a listing that
          # reported ABSENCE when it meant COULD-NOT-LOOK). Enumerate them; a
          # single-file `declared` would make `npass > declared` fire the moment
          # the second file's tests ran, which is exactly the error the previous
          # shape emitted and told the reader to fix by widening the grep.
          LIVE_TEST_FILES="./internal/handlers/plugin_settings_writer_live_test.go ./internal/handlers/files_read_surface_live_test.go"
          declared=0
          for f in $LIVE_TEST_FILES; do
            if [ ! -f "$f" ]; then
              echo "::error::live-container gate: $f is missing — a file in the live suite moved or was renamed, so 'declared' no longer describes the suite. Point LIVE_TEST_FILES at the new path; do NOT delete the entry."
              exit 1
            fi
            n=$(grep -c '^func TestLiveBox_' "$f" || true)
            if [ "${n:-0}" -eq 0 ]; then
              echo "::error::found ZERO 'func TestLiveBox_' declarations in $f — its tests were renamed away or the file was emptied, so this arm cannot check anything. Point LIVE_TEST_FILES at the new path; do NOT delete the entry."
              exit 1
            fi
            declared=$((declared + n))
          done
          # Cross-check against the WHOLE package. If a TestLiveBox_* is declared
          # in a file that is NOT in LIVE_TEST_FILES, `declared` silently
          # understates the suite and the `npass > declared` arm below would fire
          # with a confusing message. Catch it here and say what to do instead.
          # grep -rh (no -c) prints the matching LINES across the tree; counting
          # them with `wc -l` needs no `bc`/`paste`, which are not guaranteed in
          # the job container.
          pkgdeclared=$(grep -rh '^func TestLiveBox_' ./internal/handlers/ 2>/dev/null | wc -l | tr -d ' ')
          if [ "${pkgdeclared:-0}" -ne "${declared:-0}" ]; then
            echo "::error::live-container gate: ${pkgdeclared} 'func TestLiveBox_' declarations exist in internal/handlers/ but LIVE_TEST_FILES only accounts for ${declared}. A live test was added in an unenumerated file — add that file to LIVE_TEST_FILES (and bump LIVE_MIN) so the floor and the declared-count still describe the suite this step measures."
            exit 1
          fi
          echo "TestLiveBox_: ${npass} passed, ${nskip} skipped, ${declared} declared"
          if [ "$rc" -ne 0 ]; then
            exit "$rc"
          fi
          if [ "${nskip:-0}" -ne 0 ]; then
            echo "::error::live-container gate SKIPPED ${nskip} test(s). A skip here means MOLECULE_LIVE_DOCKER did not take effect or the Docker daemon was unreachable — which is indistinguishable from a pass in the job summary, and is precisely the failure this gate exists to close."
            exit 1
          fi
          # `npass -gt 0` is NOT enough, and this is not hypothetical: two of the
          # seven TestLiveBox_* (UnsafeInstallNameNeverReachesTheBox,
          # LiveWriteMatchesProvisionRenderByteForByte) never call
          # liveDockerOrSkip — they are pure unit tests that happen to share the
          # prefix. Refactor the five container tests away and this step scores
          # npass=2 nskip=0 rc=0 and reports GREEN having started no container at
          # all. Reproduced by execution, not argued: that is the same
          # "artifact real, wiring absent" shape the step was written to close,
          # reappearing inside the guard itself.
          #
          # So require a FLOOR, and name the tests that must have run.
          # LIVE_MIN is the count of ALL TestLiveBox_* (14), NOT the count of
          # Docker-gated ones. npass counts every matching PASS line, and two
          # of them are pure unit tests, so a floor that excluded them leaves
          # exactly two tests of slack: drop RealWriterLandsSettingsOnARunningWorkspace and
          # DeliverReportsWhenTheReloadDidNotConfirm — the two that drive the ACTUAL
          # entry point including backend dispatch, not just the copy primitive —
          # and the step still scores 5 and reports GREEN. Measured population and
          # threshold population must be the same one.
          # 9 write-side + 5 read-surface = 14.
          LIVE_MIN=14
          # Second, independent arm: EVERY TestLiveBox_* declared in the file must
          # have produced a top-level PASS. The floor alone couples this gate to a
          # constant someone has to remember to bump — a reviewer pointed out that
          # when core#4923 lands its two extra live tests, npass becomes 9 and a
          # floor of 7 silently reacquires two tests of slack. Deriving the expected
          # count from the source removes that coupling: adding tests needs no edit
          # here, and a test that is declared but never runs (a -run typo, a build
          # tag, a rename) is caught even though the floor is satisfied.
          #
          # The hardcoded floor stays BECAUSE it does the opposite job: `declared`
          # falls with a deletion, so on its own it would bless shrinking the suite
          # to nothing. Deletion must remain a conscious edit to LIVE_MIN.
          if [ "${npass:-0}" -lt "$declared" ]; then
            echo "::error::live-container gate: ${declared} TestLiveBox_* are declared across ${LIVE_TEST_FILES} but only ${npass} produced a top-level PASS. A declared test did not run — check the -run regex, a build tag, or a rename."
            exit 1
          fi
          if [ "${npass:-0}" -gt "$declared" ]; then
            echo "::error::live-container gate: ${npass} top-level TestLiveBox_ PASS lines but only ${declared} declared across ${LIVE_TEST_FILES}. More top-level PASS lines than declarations should be impossible now that the package-wide cross-check above passed — re-read that arm before touching this one."
            exit 1
          fi
          if [ "${npass:-0}" -lt "$LIVE_MIN" ]; then
            echo "::error::live-container gate saw ${npass} top-level '--- PASS: TestLiveBox_' line(s), expected ${LIVE_MIN} (every TestLiveBox_* across ${LIVE_TEST_FILES}). A test was removed or renamed without updating LIVE_MIN, or the -run regex no longer matches the suite. Note two of them are pure unit tests, so a nonzero count alone does not prove a container ever started."
            exit 1
          fi
          # The canary: the specific regression guard for the #4922 doubled-destPath
          # bug, where every save silently wrote to /configs/configs/... while the
          # API answered {"status":"saved"}. If THIS test stops running, the gate
          # has lost the thing it was built for, whatever the counts say.
          if ! grep -q -E '^--- PASS: TestLiveBox_FilesAPISaveActuallyLandsOnTheBox' "$LOG"; then
            echo "::error::live-container gate did not run TestLiveBox_FilesAPISaveActuallyLandsOnTheBox — the #4922 regression guard. If it was deliberately renamed, update this check to the new name; do NOT delete the check."
            exit 1
          fi
          # Second canary: the READ surface (2026-08-02). The Files API listed
          # config.yaml with a byte-exact size — and returned `[]` for
          # ?path=plugins while /configs/plugins held 8 entries, with every
          # directory missing from the root listing. An operator read that as
          # "the files API cannot see hermes working directories, so no
          # filesystem check exists" and moved to external artifacts only. The
          # API reported ABSENCE when it meant COULD-NOT-LOOK. The counts above
          # cannot tell you WHICH tests ran, so name the two that carry that
          # proof: one asserts directories appear in a root listing, the other
          # that a subdirectory listing is not empty.
          for canary in \
            TestLiveBox_FilesAPIListingSeesDirectoriesOnTheBox \
            TestLiveBox_FilesAPISubdirectoryListingIsNotEmpty; do
            if ! grep -q -E "^--- PASS: ${canary}" "$LOG"; then
              echo "::error::live-container gate did not run ${canary} — the read-surface false-negative guard (2026-08-02 incident). If it was deliberately renamed, update this check to the new name; do NOT delete the check."
              exit 1
            fi
          done

      # The scheduler integration gate (#2149) was retired with the core cron
      # scheduler itself (scheduler-as-trigger-plugin RFC, P4): internal/scheduler
      # no longer exists — schedules are fired by the runtime kind:trigger daemon.
      # The phantom-busy sweep that step also covered is now its own worker
      # (internal/handlers/phantom_busy_sweeper.go) with unit coverage, exercised
      # by the handlers gate above.

      - if: needs.detect-changes.outputs.handlers == 'true'
        name: Migration replay-from-scratch gate (#2150)
        env:
          PGPASSWORD: test
        run: |
          # Issue #2150 (SOP internal#765): prove the FULL forward migration
          # chain (.up + legacy .sql) replays from a blank schema via the
          # PRODUCTION db.RunMigrations entrypoint — hard-fail on any error.
          #
          # This is the gap the psql apply loop above does NOT cover: that
          # loop deliberately SKIPS failing migrations (`⊘ skipped`), so it
          # stays green even if the chain stops replaying. The Go test below
          # uses the real boot-time runner with hard-fail semantics, catching
          # the #211 .down-wipe class and the 045 non-idempotent crash-loop
          # class (it runs the chain twice).
          #
          # Run against a SEPARATE database so the destructive
          # `DROP SCHEMA public CASCADE` inside the test never touches the
          # `molecule` DB the handlers integration tests above migrated. No
          # ordering coupling with the handlers step.
          createdb -h "${PG_HOST}" -U postgres molecule_replay 2>/dev/null || \
            psql -h "${PG_HOST}" -U postgres -d molecule \
              -c "CREATE DATABASE molecule_replay" >/dev/null 2>&1 || true

          # Anti-vacuous-gate guard — same three zero-coverage exit-0 paths as
          # the handlers step above; see that comment for the full enumeration.
          # This one matters most: the whole POINT of this step is "the chain
          # replays", and a silently-zero-test run would assert precisely
          # nothing while turning the merge-blocking context green.
          set -uo pipefail
          LOG=$(mktemp)
          rc=0
          INTEGRATION_DB_URL="postgres://postgres:test@${PG_HOST}:5432/molecule_replay?sslmode=disable" \
            go test -tags=integration -timeout 5m -v ./internal/db/ \
              -run '^TestIntegration_Migration|^TestIntegration_InitPostgres' 2>&1 | tee "$LOG" || rc=$?
          npass=$(grep -c -E '^[[:space:]]*--- PASS: ' "$LOG" || true)
          echo "db replay-from-scratch: ${npass} test(s) actually ran and passed"
          if [ "$rc" -ne 0 ]; then
            exit "$rc"
          fi
          if [ "${npass:-0}" -eq 0 ]; then
            echo "::error::Migration replay-from-scratch gate ran ZERO tests (0 '--- PASS:' lines) but go test still exited 0. A gate that runs no tests must NEVER report success. Check the -run regex, the integration build tag on workspace-server/internal/db/postgres_replay_integration_test.go, and that the suite did not skip wholesale on a missing INTEGRATION_DB_URL."
            exit 1
          fi

      - if: failure() && needs.detect-changes.outputs.handlers == 'true'
        name: Diagnostic dump on failure
        env:
          PGPASSWORD: test
        run: |
          echo "::group::postgres container status"
          docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
          docker logs "${PG_NAME}" 2>&1 | tail -50 || true
          echo "::endgroup::"
          echo "::group::delegations table state"
          psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
          echo "::endgroup::"

      - if: always() && needs.detect-changes.outputs.handlers == 'true'
        name: Stop sibling Postgres
        working-directory: .
        run: |
          # always() so containers don't leak when migrations or tests
          # fail. The cleanup is best-effort: if the container is
          # already gone (e.g. concurrent rerun race), don't fail the job.
          docker rm -fv "${PG_NAME}" >/dev/null 2>&1 || true
          echo "Cleaned up ${PG_NAME}"