molecule-core/.gitea/workflows/template-delivery-e2e.yml

name: template-delivery-e2e

# RFC #2843 regression gate — asserts a fresh tenant DELIVERS + SERVES BACK the
# rendered /configs bundle (config.yaml + prompts). This is the gate that would
# have caught the 2026-06-15 incident family (#2919 model/identity drop, #32
# skill drop) and the #206 docker-less read-back 59-byte stub.
#
# ── ROOT-FIX (deploy-ordering deadlock) ────────────────────────────────────
# The old gate provisioned against DEPLOYED staging (MOLECULE_CP_URL=
# staging-api) and asserted config.yaml. That is a class-D ENV-COUPLING bug: a
# delivery-surface PR could NEVER green its own gate until it was deployed — but
# deploying needed the merge the red gate blocked (circular). It froze #3478
# (the #206 read-back fix) and the staging 59B regression with it.
#
# The gate now validates the PR's OWN artifact:
#   1. Builds the tenant image (workspace-server/Dockerfile.tenant) FROM THIS REF
#      on robot-1 (`runs-on: [docker-host, robot-1]`) — off the local box.
#   2. Brings up the production-shape harness (tests/harness) — the SAME image
#      shipped to prod, running as a real molecules-server tenant (MOLECULE_ORG_ID
#      set → CPProvisioner, docker-less), talking to the harness cp-stub.
#   3. Provisions a FRESH seo-agent workspace THROUGH the tenant, then asserts
#      the tenant SERVES BACK the FULL rendered config.yaml (>1 KiB) + prompts
#      via the Files API — the #206 host-side /configs mirror read-back. A PR
#      that regresses delivery serves the 59-byte stub → the gate fails.
#   4. Tears the harness down (ephemeral; reaper-clean; NO staging org touched,
#      NO staging pin touched).
# So the gate validates the PR's CHANGE with no deployed-staging dependency.
#
# The post-online PLUGIN channel (seo-all reconcile, assertion E) INHERENTLY
# needs a live deployed runtime (the harness cp-stub never brings one online),
# so it CANNOT be a pre-merge PR gate by construction. It is preserved,
# unchanged, in template-delivery-e2e-staging.yml (`workflow_dispatch`, run
# post-deploy against staging) — no assertion was dropped, it was relocated off
# the PR critical path.
#
# Required-workflow shape (unchanged): NO `on: paths:` filter (a required check
# must not be path-filtered — a docs-only PR would wedge forever on an absent
# context). The detect-changes job applies the delivery path-scoping at RUNTIME
# (profile `template-delivery`, in detect-changes.py); the delivery job's real
# steps are gated per-step on its output. A non-delivery PR emits SUCCESS
# cheaply (no build); a delivery PR runs the full PR-build e2e and BLOCKS on
# failure. Mirrors e2e-api / peer-visibility / harness-replays.

on:
  workflow_dispatch: {}
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: template-delivery-e2e-${{ github.event.pull_request.head.sha || github.sha }}
  cancel-in-progress: false

env:
  GITHUB_SERVER_URL: https://git.moleculesai.app

jobs:
  # Runtime path-scoping (replaces the removed `on: paths:`). Outputs
  # `delivery=true` when the diff touches the workspace-server delivery surface;
  # the gate job runs the real PR-build e2e only then. bp-exempt: helper job;
  # the REQUIRED context is the `delivery` job below, not this one.
  detect-changes:
    runs-on: docker-host
    continue-on-error: false
    outputs:
      delivery: ${{ steps.decide.outputs.delivery }}
      debug: ${{ steps.decide.outputs.debug }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
      - id: decide
        env:
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
          PUSH_BEFORE: ${{ github.event.before }}
        run: |
          python3 .gitea/scripts/detect-changes.py \
            --profile template-delivery \
            --event-name "${{ github.event_name }}" \
            --pr-base-sha "$PR_BASE_SHA" \
            --base-ref "$PR_BASE_REF" \
            --push-before "$PUSH_BEFORE" || {
            # Script crash → fail OPEN so the gate runs rather than silently
            # no-oping a potentially-breaking delivery PR.
            echo "delivery=true" >> "$GITHUB_OUTPUT"
            echo "debug=detect-script-error event=${{ github.event_name }}" >> "$GITHUB_OUTPUT"
            exit 0
          }
          echo "debug=profile=template-delivery event=${{ github.event_name }}" >> "$GITHUB_OUTPUT"

  # ONE job (no job-level `if:`) that ALWAYS runs and reports under the
  # required-check name. Real work is gated per-step on
  # `needs.detect-changes.outputs.delivery` — a non-delivery PR runs only the
  # no-op step and emits SUCCESS (branch-protection-clean; a job-level `if:`
  # would emit a SKIPPED check run that fails the required-check eval — see
  # e2e-api.yml's PR#2264 note). A delivery PR runs the full PR-build e2e and,
  # with no continue-on-error, BLOCKS the merge on failure.
  # bp-required: yes — this context is merge-blocking; branch protection lists
  # "<this> (pull_request)" (via the ["*"] all-green gate). Name UNCHANGED from
  # the pre-redesign gate so the required-context identity does not drift (the
  # asset-channel config+prompts assertions it names are exactly what the
  # PR-build path asserts; the "seo-all via plugin reconcile" clause moved to
  # template-delivery-e2e-staging.yml — see this file's header).
  delivery:
    needs: detect-changes
    # No colon in the name — lint-required-context's PyYAML AST parse rejects an
    # unquoted scalar containing a colon.
    name: Template-asset delivery (fresh seo-agent — config+prompts via asset channel, seo-all via plugin reconcile)
    # robot-1's 32c dedicated build box (`[docker-host, robot-1]` = AND, so it
    # lands ONLY on robot-1 while satisfying the docker-host-pinned lint). Keeps
    # the heavy tenant-image build + harness e2e OFF the local molecules-server
    # box (operator-config#373 local-box SSOT).
    runs-on: [docker-host, robot-1]
    # Build (cache-warm minutes; cold ~45) + harness up + provision + read-back.
    timeout-minutes: 75
    steps:
      - name: No-op pass (delivery surface unchanged in this diff)
        if: needs.detect-changes.outputs.delivery != 'true'
        run: |
          echo "No delivery-surface changes — gate satisfied without provisioning."
          echo "::notice::template-delivery-e2e no-op pass (detect-changes: ${{ needs.detect-changes.outputs.debug }})."

      - if: needs.detect-changes.outputs.delivery == 'true'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Log detected changes
        if: needs.detect-changes.outputs.delivery == 'true'
        run: |
          echo "::notice::detect-changes debug ${{ needs.detect-changes.outputs.debug }}"

      # AUTO_SYNC_TOKEN (devops-engineer persona PAT) is (a) the cross-repo clone
      # token for the private workspace-template-* deps in manifest.json, and
      # (b) wired into the tenant env as MOLECULE_TEMPLATE_REPO_TOKEN so the
      # tenant's giteaTemplateAssetFetcher can fetch the PRIVATE seo-agent
      # template at provision time. Sourced from the Infisical SSOT
      # (/shared/dev-utils, environment=prod) via the CI machine identity.
      #
      # FORK SAFETY: gated to same-repo events — a fork PR must never harvest the
      # token. On a fork the step is skipped, the fetch falls back to anonymous
      # clone (public deps), and the private seo-agent fetch would fail loudly in
      # the replay (not silently) — a maintainer re-runs from a trusted ref.
      - name: Fetch AUTO_SYNC_TOKEN from Infisical SSOT
        id: auto_sync_token
        if: >-
          needs.detect-changes.outputs.delivery == 'true' &&
          (github.event_name == 'workflow_dispatch' ||
           (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
           (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository))
        env:
          INFISICAL_CI_CLIENT_ID: ${{ secrets.INFISICAL_CI_CLIENT_ID }}
          INFISICAL_CI_CLIENT_SECRET: ${{ secrets.INFISICAL_CI_CLIENT_SECRET }}
          INFISICAL_PROJECT_ID: ${{ secrets.INFISICAL_CI_PROJECT_ID }}
        run: |
          set -uo pipefail
          BASE="https://key.moleculesai.app"
          TOK=$(curl -fsS -X POST "$BASE/api/v1/auth/universal-auth/login" \
            -H 'Content-Type: application/json' \
            -d "{\"clientId\":\"$INFISICAL_CI_CLIENT_ID\",\"clientSecret\":\"$INFISICAL_CI_CLIENT_SECRET\"}" \
            | python3 -c 'import sys,json; d=json.load(sys.stdin); v=d.get("accessToken"); sys.stdout.write(v if isinstance(v,str) and v else "")')
          read_secret() {
            curl -fsS "$BASE/api/v3/secrets/raw/$1?workspaceId=$INFISICAL_PROJECT_ID&environment=prod&secretPath=$2" \
              -H "Authorization: Bearer $TOK" \
              | python3 -c 'import sys,json; d=json.load(sys.stdin); v=(d.get("secret") or {}).get("secretValue"); sys.stdout.write(v if isinstance(v,str) and v else "")'
          }
          AUTO_SYNC_TOKEN=$(read_secret AUTO_SYNC_TOKEN %2Fshared%2Fdev-utils)
          echo "::add-mask::$AUTO_SYNC_TOKEN"
          if [ -z "$AUTO_SYNC_TOKEN" ]; then
            echo "::error::Infisical returned empty for AUTO_SYNC_TOKEN at /shared/dev-utils"
            exit 1
          fi
          echo "AUTO_SYNC_TOKEN=$AUTO_SYNC_TOKEN" >> "$GITHUB_ENV"
          echo "AUTO_SYNC_TOKEN loaded from Infisical (len=${#AUTO_SYNC_TOKEN})"

      # Pre-clone manifest deps before docker compose builds the tenant image
      # (Dockerfile.tenant COPYs .tenant-bundle-deps/* — private repos can't be
      # cloned inside the build sandbox). Same pattern as harness-replays.yml.
      - name: Pre-clone manifest deps
        if: needs.detect-changes.outputs.delivery == 'true'
        env:
          MOLECULE_GITEA_TOKEN: ${{ env.AUTO_SYNC_TOKEN }}
        run: |
          set -euo pipefail
          if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
            echo "::warning::AUTO_SYNC_TOKEN not set — anonymous clone (public deps only; private seo-agent will fail the read-back assertion loudly on a fork)"
          fi
          mkdir -p .tenant-bundle-deps
          sed '/^[[:space:]]*\/\//d' manifest.json > .manifest-stripped.json
          bash scripts/clone-manifest.sh \
            .manifest-stripped.json \
            .tenant-bundle-deps/workspace-configs-templates \
            .tenant-bundle-deps/org-templates \
            .tenant-bundle-deps/plugins
          ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
          echo "Cloned workspace-configs-templates: $ws_count"

      - name: Install Python deps for replays
        if: needs.detect-changes.outputs.delivery == 'true'
        run: |
          pip install \
            --extra-index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ \
            -r tests/harness/requirements.txt

      # Build the harness images FROM THIS REF explicitly. `docker compose up`
      # reuses an existing image and does NOT rebuild on a source change — on a
      # persistent robot-1 daemon that would test a STALE tenant image. A
      # cache-aware `build` rebuilds only the layers whose inputs changed (the
      # COPY of workspace-server source invalidates on any delivery change), so
      # THIS ref's code is always what runs. GIT_SHA is baked so /buildinfo
      # reports the served SHA (verify-don't-buy).
      - name: Start disposable dind (per-job isolation — task #78)
        # Same pattern as harness-replays.yml: run the harness stack inside a
        # throwaway docker:dind so a cancelled/SIGKILLed run can't orphan harness-*
        # containers/volumes on the shared host and concurrent runs can't collide
        # on the fixed compose project/ports. dind.sh exports DOCKER_HOST (every
        # `docker compose` below targets the ISOLATED daemon) + BASE/CP_STUB_BASE
        # (the buildinfo curl + template-asset-delivery-gate.sh reach the nested
        # harness via host-loopback forwards) + HARNESS_BIND_ADDR. Torn down in the
        # always() step at the end.
        if: needs.detect-changes.outputs.delivery == 'true'
        working-directory: tests/harness
        run: bash dind.sh up

      - name: Build harness images FROM THIS REF (explicit, not up's skip-if-exists)
        if: needs.detect-changes.outputs.delivery == 'true'
        working-directory: tests/harness
        env:
          SECRETS_ENCRYPTION_KEY: build-placeholder
          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: docker compose -f compose.yml build tenant-alpha tenant-beta cp-stub

      - name: Bring up harness + run template-asset delivery gate
        if: needs.detect-changes.outputs.delivery == 'true'
        timeout-minutes: 20
        working-directory: tests/harness
        env:
          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          # Wire the private-template fetch token so the tenant can fetch the
          # PRIVATE seo-agent template at provision time.
          MOLECULE_TEMPLATE_REPO_TOKEN: ${{ env.AUTO_SYNC_TOKEN }}
        run: |
          set -uo pipefail
          ./up.sh
          echo "--- buildinfo (served SHA must match this ref) ---"
          curl -sS -H "Host: harness-tenant-alpha.localhost" "${BASE:-http://localhost:8080}/buildinfo" | head -c 300 || true
          echo ""
          echo "--- running template-asset delivery gate ---"
          bash template-asset-delivery-gate.sh

      - name: Dump tenant logs on failure
        if: failure() && needs.detect-changes.outputs.delivery == 'true'
        working-directory: tests/harness
        env:
          SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
        run: |
          echo "=== docker compose ps ==="; docker compose -f compose.yml ps || true
          echo "=== tenant-alpha logs (last 200) ==="; docker compose -f compose.yml logs --tail 200 tenant-alpha || true
          echo "=== cp-stub logs (last 80) ==="; docker compose -f compose.yml logs --tail 80 cp-stub || true

      - name: Force teardown
        if: always() && needs.detect-changes.outputs.delivery == 'true'
        working-directory: tests/harness
        run: ./down.sh || true

      - name: Teardown dind (destroys the whole harness topology atomically)
        # LAST step — the dump + ./down.sh steps above run against DOCKER_HOST=dind.
        # Removing the single dind destroys every nested container/volume/network in
        # one atomic op on the HOST daemon (dind.sh down unsets DOCKER_HOST). Gated
        # like Force teardown on delivery=='true': the no-op path never starts a
        # dind (nothing to reap), and its checkout lacks the tests/harness cwd.
        if: always() && needs.detect-changes.outputs.delivery == 'true'
        working-directory: tests/harness
        run: bash dind.sh down