molecule-core/.gitea/workflows/harness-replays.yml

name: Harness Replays

# Ported from .github/workflows/harness-replays.yml on 2026-05-11 per RFC
# internal#219 §1 sweep. Differences from the GitHub version:
#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
#   - Dropped `merge_group:` (no Gitea merge queue).
#   - Dropped `environment:` blocks (Gitea has no environments).
#   - Workflow-level env.GITHUB_SERVER_URL pinned per
#     feedback_act_runner_github_server_url.
#   - `continue-on-error: true` on each job (RFC §1 contract).
#

# Boots tests/harness (production-shape compose topology with TenantGuard,
# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
# every replay under tests/harness/replays/. Fails the PR if any replay
# fails.
#
# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
# a public route in router.go but forgot to add it to TenantGuard's
# allowlist. The handler-level test in buildinfo_test.go constructed a
# minimal gin engine without TenantGuard — green. The harness's
# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
# inject X-Molecule-Org-Id, so the curl path is identical to production's
# redeploy verifier), but no one ran the harness pre-merge. The bug
# shipped; the redeploy verifier silently soft-warned every tenant as
# "unreachable" for ~1 day before being noticed.
#
# This gate makes "did you actually run the harness?" a CI invariant
# instead of a memory-discipline thing.
#
# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
# to staging+main, real work is gated per-step on detect-changes output.
# One job → one check run → branch-protection-clean (the SKIPPED-in-set
# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).

"on":
  push:
    branches: [main, staging]
    paths:
      - 'workspace-server/**'
      - 'canvas/**'
      - 'tests/harness/**'
      - '.gitea/workflows/harness-replays.yml'
  pull_request:
    branches: [main, staging]
    paths:
      - 'workspace-server/**'
      - 'canvas/**'
      - 'tests/harness/**'
      - '.gitea/workflows/harness-replays.yml'
concurrency:
  # Keyed per-PR. This was a single runner-wide group ("harness-replays-runner")
  # because the harness bound FIXED host ports (cf-proxy :8080, cp-stub :9090)
  # and a FIXED compose project (`harness` → harness-tenant-alpha-1, ...), so two
  # overlapping runs CROSS-WIRED: `docker compose up -d` reused the other run's
  # containers and the replays hit a stale tenant binary — RCA 2026-07-12 (main
  # run 477499): foreign /buildinfo git_sha, org-swapped TenantGuard, empty
  # /workspaces. Per-SHA grouping did not prevent that, so the group was widened
  # to runner-wide.
  #
  # The per-job dind (the "Start disposable dind" step below, task #78) removed
  # that constraint — its own comment says so: the whole harness stack now runs
  # inside a throwaway daemon "so ... two concurrent runs can't collide on the
  # fixed compose project/ports". The fixed :8080/:9090 are now INSIDE the dind;
  # dind.sh publishes them on EPHEMERAL host ports (-p 127.0.0.1::8080) and
  # namespaces the dind itself per run (NS defaults to GITHUB_RUN_ID-GITHUB_JOB).
  # Concurrent runs cannot collide on a port or a container name any more, so the
  # runner-wide group is obsolete — and it is actively harmful (#4122).
  #
  # #4122: a global, unkeyed group is where the phantom jobs come from. Observed
  # on this very workflow (run 485591, jobs 706309/706310): cancelled with
  # runner="-" and started_at=1970-01-01 — queue entries that NEVER executed, yet
  # they post `failure — Has been cancelled` and red the PR. The ephemeral gate
  # had the identical shape and the identical symptom; keying it per-PR (#4146)
  # fixed it. This is the second and last PR-triggered, docker-host, unkeyed group
  # in the repo.
  #
  # cancel-in-progress deliberately STAYS false (unlike #4146): cancelling mid-run
  # could still kill a boot and leak the KEEP_UP containers — the 2026-04-28
  # auto-promote deadlock this file guards against. Keying alone removes the
  # CROSS-PR queue that mints the ghosts; same-PR runs still queue, which is safe.
  group: harness-replays-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: false

env:
  GITHUB_SERVER_URL: https://git.moleculesai.app

jobs:
  # bp-exempt: change detector only; downstream Harness Replays is the meaningful gate.
  detect-changes:
    # mc#1529 follow-on: pin to `docker-host` so this lane lands on
    # Linux operator-host runners (the only ones with a working
    # docker.sock + `molecule-core-net`). The bare `ubuntu-latest`
    # label is also matched by hongming-pc-runner-* (Windows act_runner
    # v1.0.3), where the `docker compose ...` exec below fails. Mirror
    # of mc#1543; see internal#512 for class defect.
    runs-on: docker-host
    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
    # mc#3436: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
    continue-on-error: true
    outputs:
      run: ${{ steps.decide.outputs.run }}
      debug: ${{ steps.decide.outputs.debug }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # Shallow clone — we use the Gitea Compare API for changed-file
          # detection, not local git diff. The base SHA is supplied via
          # GitHub event variables, so no local history is needed.
          fetch-depth: 1
      - id: decide
        env:
          # Pass via env block — env values bypass shell quoting so single
          # quotes in merge-commit messages (e.g. "Merge pull request 'fix: ...'
          # from branch into main") cannot break the bash parser. The prior
          # `echo '${{ toJSON(...) }}'` form broke on every main-push because
          # every main commit is a merge commit with single quotes in the
          # message body — the embedded `'` ended the single-quoted shell string
          # mid-JSON, and a subsequent `(` (e.g. in `(#523)`) was parsed as a
          # subshell, causing "syntax error near unexpected token `('".
          COMMITS_JSON: ${{ toJSON(github.event.commits) }}
        run: |
          set -euo pipefail

          # workflow_dispatch: always run (manual trigger)
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "run=true" >> "$GITHUB_OUTPUT"
            echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Determine changed files.
          # workflow_dispatch: always run.
          # pull_request: use Compare API (branch-to-branch works fine).
          # push: use github.event.commits array (Compare API rejects SHA-to-branch).
          # new-branch: run everything.
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE="${{ github.event.pull_request.base.ref }}"
            HEAD="${{ github.event.pull_request.head.ref }}"
          elif [ -n "${{ github.event.before }}" ] && \
               ! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
            # Push event: extract changed files from github.event.commits array.
            # Gitea Compare API rejects SHA-to-branch comparisons (BaseNotExist),
            # so we use the commits array instead. This array contains all commits
            # in the push, each with their added/removed/modified file lists.
            printf '%s' "$COMMITS_JSON" \
              | python3 .gitea/scripts/push-commits-diff-files.py \
              > .push-diff-files.txt 2>/dev/null || true
            DIFF_FILES=$(cat .push-diff-files.txt 2>/dev/null || true)
            DIFF_FILES_FLAT=$(echo "$DIFF_FILES" | tr '\n' ',')
            if [ -n "$DIFF_FILES" ] && echo "$DIFF_FILES" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
              echo "run=true" >> "$GITHUB_OUTPUT"
            else
              echo "run=false" >> "$GITHUB_OUTPUT"
            fi
            echo "debug=push-files=$DIFF_FILES_FLAT" >> "$GITHUB_OUTPUT"
            exit 0
          else
            # New branch or github.event.before unavailable — run everything.
            echo "run=true" >> "$GITHUB_OUTPUT"
            echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Call Gitea Compare API (pull_request path only — branch-to-branch).
          # Push uses github.event.commits array above.
          RESP=$(curl -sS --fail --max-time 30 \
            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
            -H "Accept: application/json" \
            "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/compare/$BASE...$HEAD") || {
            # If Gitea's Compare API is slow/unavailable, choose the conservative
            # behavior: run the harness instead of failing the detector and polluting
            # main with a red non-gate context.
            echo "run=true" >> "$GITHUB_OUTPUT"
            echo "debug=compare-api-unavailable base=$BASE head=$HEAD" >> "$GITHUB_OUTPUT"
            exit 0
          }
          DIFF_FILES=$(echo "$RESP" | python3 .gitea/scripts/compare-api-diff-files.py 2>/dev/null || true)
          DIFF_FILES_FLAT=$(echo "$DIFF_FILES" | tr '\n' ',')

          echo "debug=diff-base=$BASE diff-files=$DIFF_FILES_FLAT" >> "$GITHUB_OUTPUT"

          if echo "$DIFF_FILES" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "run=false" >> "$GITHUB_OUTPUT"
          fi

  # ONE job that always runs. Real work is gated per-step on
  # detect-changes.outputs.run so an unrelated PR (e.g. doc-only
  # change to molecule-controlplane wired here later) emits the
  # required check without spending CI cycles. Single-job pattern
  # matches e2e-api.yml — see that workflow's comment for why a
  # job-level `if: false` would block branch protection via the
  # SKIPPED-in-set bug.
  # bp-exempt: path-filtered replay suite; CI / all-required is the branch-protection aggregate.
  harness-replays:
    needs: detect-changes
    name: Harness Replays
    # mc#1529 follow-on: `docker compose ... ps/logs` against tenant-alpha/
    # beta containers. Must run on operator-host Linux (docker-host).
    runs-on: docker-host
    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
    # mc#3436: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
    continue-on-error: true
    timeout-minutes: 30
    steps:
      - name: No-op pass (paths filter excluded this commit)
        if: needs.detect-changes.outputs.run != 'true'
        run: |
          echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
          echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
          echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"

      - if: needs.detect-changes.outputs.run == 'true'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # Log what files were detected so future failures include the diff.
      - name: Log detected changes
        if: needs.detect-changes.outputs.run == 'true'
        run: |
          echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"

      # github-app-auth sibling-checkout removed 2026-05-07 (#157):
      # the plugin was dropped + Dockerfile.tenant no longer COPYs it.

      # Pre-clone manifest deps before docker compose builds the tenant
      # image (Task #173 followup — same pattern as
      # publish-workspace-server-image.yml's "Pre-clone manifest deps"
      # step).
      #
      # Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
      # and tenant-beta from workspace-server/Dockerfile.tenant with
      # context=../.. (repo root). That Dockerfile expects
      # .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
      # to be present at build context root (post-#173 it COPYs from there
      # instead of running an in-image clone — the in-image clone failed
      # with "could not read Username for https://git.moleculesai.app"
      # because there's no auth path inside the build sandbox).
      #
      # Without this step harness-replays fails before any replay runs,
      # with `failed to calculate checksum of ref ...
      # "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
      # (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
      # symptom, different root cause: staging still has the in-image
      # clone path, hits the auth error directly).
      #
      # 2026-05-08 sub-finding (#192): the clone step ALSO fails when
      # any referenced workspace-template repo is private and the
      # AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
      # access. Root cause: 5 of 9 workspace-template repos
      # (openclaw, codex, crewai, deepagents, gemini-cli) had been
      # marked private with no team grant. Resolution: flipped them
      # to public per `feedback_oss_first_repo_visibility_default`
      # (the OSS surface should be public). Layer-3 (customer-private +
      # marketplace third-party repos) tracked separately in
      # internal#102.
      #
      # Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
      # is the devops-engineer persona PAT, NOT the founder PAT (per
      # `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
      # embeds it as basic-auth for the duration of the clones and strips
      # .git directories — the token never enters the resulting image.
      - name: Fetch AUTO_SYNC_TOKEN from Infisical SSOT
        # Bot-token consolidation (secrets->kms). AUTO_SYNC_TOKEN (the
        # devops-engineer persona PAT, used here as the cross-repo clone token
        # for manifest deps) is now pulled from the Infisical SSOT at
        # /shared/dev-utils::AUTO_SYNC_TOKEN (environment=prod) instead of the
        # duplicated `secrets.AUTO_SYNC_TOKEN` Gitea secret. Exported to
        # $GITHUB_ENV and consumed as env.* by "Pre-clone manifest deps" below.
        # The old Gitea secret stays in place until a green run proves this
        # fetch (validate-before-delete).
        #
        # FORK SAFETY: this workflow fires on pull_request, so the fetch is
        # gated to same-repo PRs only — a fork PR must NEVER harvest the
        # merge-actor token. On a fork PR this step is skipped, env.* stays
        # empty, and the clone step below falls back to anonymous clone (its
        # pre-existing fork behaviour: repos are public per the manifest OSS
        # contract). Gitea also withholds secrets from forks at the runner
        # level (defense in depth).
        id: auto_sync_token
        if: needs.detect-changes.outputs.run == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
        env:
          INFISICAL_CI_CLIENT_ID: ${{ secrets.INFISICAL_CI_CLIENT_ID }}
          INFISICAL_CI_CLIENT_SECRET: ${{ secrets.INFISICAL_CI_CLIENT_SECRET }}
          INFISICAL_PROJECT_ID: ${{ secrets.INFISICAL_CI_PROJECT_ID }}
        run: |
          set -uo pipefail
          BASE="https://key.moleculesai.app"
          TOK=$(curl -fsS -X POST "$BASE/api/v1/auth/universal-auth/login" \
            -H 'Content-Type: application/json' \
            -d "{\"clientId\":\"$INFISICAL_CI_CLIENT_ID\",\"clientSecret\":\"$INFISICAL_CI_CLIENT_SECRET\"}" \
            | python3 -c 'import sys,json; d=json.load(sys.stdin); v=d.get("accessToken"); sys.stdout.write(v if isinstance(v,str) and v else "")')
          read_secret() {
            curl -fsS "$BASE/api/v3/secrets/raw/$1?workspaceId=$INFISICAL_PROJECT_ID&environment=prod&secretPath=$2" \
              -H "Authorization: Bearer $TOK" \
              | python3 -c 'import sys,json; d=json.load(sys.stdin); v=(d.get("secret") or {}).get("secretValue"); sys.stdout.write(v if isinstance(v,str) and v else "")'
          }
          AUTO_SYNC_TOKEN=$(read_secret AUTO_SYNC_TOKEN %2Fshared%2Fdev-utils)
          echo "::add-mask::$AUTO_SYNC_TOKEN"
          if [ -z "$AUTO_SYNC_TOKEN" ]; then
            echo "::error::Infisical returned empty for AUTO_SYNC_TOKEN at /shared/dev-utils"
            exit 1
          fi
          echo "AUTO_SYNC_TOKEN=$AUTO_SYNC_TOKEN" >> "$GITHUB_ENV"
          echo "AUTO_SYNC_TOKEN loaded from Infisical (len=${#AUTO_SYNC_TOKEN})"

      - name: Pre-clone manifest deps
        if: needs.detect-changes.outputs.run == 'true'
        env:
          MOLECULE_GITEA_TOKEN: ${{ env.AUTO_SYNC_TOKEN }}
        run: |
          set -euo pipefail
          if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
            echo "::warning::AUTO_SYNC_TOKEN not set — using anonymous clone (repos are public per manifest.json OSS contract)"
          fi
          mkdir -p .tenant-bundle-deps
          # Strip JSON5 comments before jq parsing — Integration Tester appends
          # `// Triggered by ...` which breaks `jq` in clone-manifest.sh.
          sed '/^[[:space:]]*\/\//d' manifest.json > .manifest-stripped.json
          bash scripts/clone-manifest.sh \
            .manifest-stripped.json \
            .tenant-bundle-deps/workspace-configs-templates \
            .tenant-bundle-deps/org-templates \
            .tenant-bundle-deps/plugins
          # Sanity-check counts so a silent partial clone fails fast
          # instead of producing a half-empty image.
          ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
          org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
          plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
          echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"

      - name: Install Python deps for replays
        # peer-discovery-404 (and future replays) eval Python against the
        # running tenant — importing workspace/a2a_client.py pulls in
        # httpx. tests/harness/requirements.txt holds just the HTTP-client
        # surface to keep CI install fast (~3s) vs the full
        # workspace/requirements.txt (~30s).
        if: needs.detect-changes.outputs.run == 'true'
        # molecule-ai-workspace-runtime lives ONLY on the Gitea package
        # registry (it was removed from public PyPI to close the
        # dependency-confusion shadow), so pip needs the Gitea index as an
        # EXTRA index. A plain `pip install -r` reaches public PyPI only and
        # fails "No matching distribution found for
        # molecule-ai-workspace-runtime>=0.1.78 (from versions: none)" —
        # which reds this gate on every PR + push. httpx still resolves from
        # PyPI. Mirrors e2e-api.yml's "Install standalone runtime parser from
        # Gitea registry" step (proven-green).
        run: |
          pip install \
            --extra-index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ \
            -r tests/harness/requirements.txt

      - name: Start disposable dind (per-job isolation — task #78)
        # Run the ENTIRE harness stack inside a throwaway docker:dind so a
        # cancelled/SIGKILLed run can't orphan harness-* containers/volumes on the
        # shared host, and two concurrent runs can't collide on the fixed compose
        # project/ports — the chronic non-hermetic "Harness Replays" reds. dind.sh
        # exports DOCKER_HOST (every `docker compose` below targets the ISOLATED
        # daemon), BASE + CP_STUB_BASE (the replay curls reach the nested harness
        # via host-loopback port forwards off the dind), and HARNESS_BIND_ADDR.
        # Feasibility proven by dind-smoke.yml; torn down in the always() step.
        if: needs.detect-changes.outputs.run == 'true'
        working-directory: tests/harness
        run: bash dind.sh up

      - name: Rebuild cp-stub from source (RC #11812 + RC #11815 narrow fix)
        # Why this step exists: tests/harness/up.sh's \`build\` block is
        # gated by \`--rebuild\` (no default rebuild), so without an
        # explicit \`docker compose build\` the CI uses the cached cp-stub
        # image — which was built BEFORE the new /cp/admin/orgs handler
        # landed in tests/harness/cp-stub/main.go. Symptom: the
        # canary-smoke-org-create-400-capture replay gets HTTP 501 from
        # the catch-all (handler not implemented), even though the
        # source has the handler. \`--no-cache\` ensures the rebuilt image
        # pulls the current main.go from the checked-out branch.
        #
        # RC #11815 NARROW FIX: only cp-stub is rebuilt here. The
        # tenants (tenant-alpha/tenant-beta) need SECRETS_ENCRYPTION_KEY
        # which tests/harness/up.sh generates (with \`openssl rand -base64
        # 32\`) and exports — up.sh runs AFTER this pre-build step, so
        # building the tenants here would fail with missing env. The
        # tenants get built later by up.sh (the harness's own \`build\`
        # block doesn't need --rebuild, so the cached tenant images are
        # fine for the non-handler-bug surface). cp-stub is the only
        # service with the stale-handler problem AND no env dep.
        if: needs.detect-changes.outputs.run == 'true'
        working-directory: tests/harness
        env:
          # docker compose validates the whole compose.yml even when only
          # building cp-stub; tenant-alpha/beta require SECRETS_ENCRYPTION_KEY.
          # up.sh generates the real key later. A placeholder is fine here
          # because cp-stub does not use it.
          SECRETS_ENCRYPTION_KEY: rebuild-cp-stub-placeholder
        run: |
          docker compose -f compose.yml build --no-cache cp-stub

      - name: Run all replays against the harness
        # run-all-replays.sh: boot via up.sh → seed via seed.sh → run
        # every replays/*.sh → tear down via down.sh on EXIT (trap).
        # Non-zero exit on any replay failure.
        #
        # KEEP_UP=1: without this, the script's trap-on-EXIT tears
        # down containers immediately on failure, leaving the dump
        # step below with nothing to dump (verified on PR #2410's
        # first run — tenant became unhealthy, trap fired, dump
        # step saw empty containers). Keeping them up lets the
        # failure path collect tenant/cp-stub/cf-proxy logs. The
        # always-run "Force teardown" step does the actual cleanup.
        if: needs.detect-changes.outputs.run == 'true'
        working-directory: tests/harness
        env:
          KEEP_UP: "1"
          # Bake the REAL commit sha into the tenant image (GIT_SHA is the
          # Dockerfile build-arg consumed by compose) AND tell the buildinfo
          # canary to expect it (HARNESS_GIT_SHA). Together they turn a stale or
          # cross-wired tenant image into a fast, honest red in
          # buildinfo-stale-image.sh — instead of the confusing org-swap / empty
          # /workspaces downstream failures that a stale image caused on main run
          # 477499 (RCA 2026-07-12). up.sh now always (cache-)builds the tenant,
          # so this sha is compiled into the running binary.
          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          HARNESS_GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: ./run-all-replays.sh

      - name: Dump compose logs on failure
        # SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
        # file even for read-only `logs` calls. up.sh generates a per-run key
        # and exports it to its OWN shell — this step runs in a fresh shell
        # that wouldn't see it, so without a placeholder the validate step
        # errors before logs print (verified against PR #2492's first run:
        # "required variable SECRETS_ENCRYPTION_KEY is missing a value").
        # A placeholder is fine — we're only reading log streams, not booting.
        if: failure() && needs.detect-changes.outputs.run == 'true'
        working-directory: tests/harness
        env:
          SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
        run: |
          echo "=== docker compose ps ==="
          docker compose -f compose.yml ps || true
          echo "=== tenant-alpha logs ==="
          docker compose -f compose.yml logs tenant-alpha || true
          echo "=== tenant-beta logs ==="
          docker compose -f compose.yml logs tenant-beta || true
          echo "=== cp-stub logs ==="
          docker compose -f compose.yml logs cp-stub || true
          echo "=== cf-proxy logs ==="
          docker compose -f compose.yml logs cf-proxy || true
          echo "=== postgres-alpha logs (last 100) ==="
          docker compose -f compose.yml logs --tail 100 postgres-alpha || true
          echo "=== postgres-beta logs (last 100) ==="
          docker compose -f compose.yml logs --tail 100 postgres-beta || true

      - name: Force teardown
        # We pass KEEP_UP=1 to run-all-replays.sh so the dump step
        # above sees real containers — that means we own teardown
        # explicitly here. Always run.
        if: always() && needs.detect-changes.outputs.run == 'true'
        working-directory: tests/harness
        run: ./down.sh || true

      - name: Teardown dind (destroys the whole harness topology atomically)
        # MUST be the LAST step — the dump + `./down.sh` steps above run against
        # DOCKER_HOST=dind (the harness lives inside it). Removing the single dind
        # container then destroys every nested container/volume/network/image in
        # one atomic op, on the HOST daemon (dind.sh down unsets DOCKER_HOST). On a
        # SIGKILLed job this may not run, but the only leftover is ONE uniquely-
        # named dind-harness-* container — trivial to reap, vs the open-ended
        # harness-*/postgres/redis set today. Gated like Force teardown on
        # run=='true': the no-op path never starts a dind (nothing to reap) and its
        # checkout lacks the tests/harness cwd this step chdirs into.
        if: always() && needs.detect-changes.outputs.run == 'true'
        working-directory: tests/harness
        run: bash dind.sh down