Merge branch 'main' into docs/update-remote-agent-tutorial-sdk-api

Co-authored-by: Molecule AI Core-UIUX <core-uiux@agents.moleculesai.app>
Co-committed-by: Molecule AI Core-UIUX <core-uiux@agents.moleculesai.app>

.gitattributes

@@ -13,3 +13,11 @@ workspace/entrypoint.sh text eol=lf
 # but keep LF for consistency across platforms.
 Dockerfile text eol=lf
 *.dockerfile text eol=lf
+
+# Snapshot golden files — workspace/tests/snapshots/*.txt is consumed by
+# byte-exact comparisons in test_platform_tools.py. A Windows contributor
+# with auto-CRLF=true would otherwise convert \n → \r\n on checkout, the
+# snapshot tests would fail mysteriously locally / pass in CI (or vice
+# versa), and the regen instructions in the test-file header would
+# produce LF files that disagree with the working-copy CRLF versions.
+workspace/tests/snapshots/*.txt text eol=lf

.gitea/scripts/audit-force-merge.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# audit-force-merge — detect a §SOP-6 force-merge after PR close, emit
+# `incident.force_merge` to stdout as structured JSON.
+#
+# Vector's docker_logs source picks up runner stdout; the JSON gets
+# shipped to Loki on molecule-canonical-obs, indexable by event_type.
+# Query example:
+#
+#   {host="operator"} |= "event_type" |= "incident.force_merge" | json
+#
+# A force-merge is detected when a PR closed-with-merged=true had at
+# least one of the repo's required-status-check contexts in a state
+# other than "success" at the merge commit's SHA. That's exactly what
+# the Gitea force_merge:true API call lets through, so it's a faithful
+# detector of the override path.
+#
+# Triggers on `pull_request_target: closed` (loaded from base branch
+# per §SOP-6 security model). No-op when merged=false.
+#
+# Required env (set by the workflow):
+#   GITEA_TOKEN, GITEA_HOST, REPO, PR_NUMBER, REQUIRED_CHECKS
+#
+# REQUIRED_CHECKS is a newline-separated list of status-check context
+# names that branch protection requires. Declared in the workflow YAML
+# rather than fetched from /branch_protections (which needs admin
+# scope — sop-tier-bot has read-only). Trade dynamism for simplicity:
+# when the required-check set changes, update both branch protection
+# AND this env. Keeping them in sync is less complexity than granting
+# the audit bot admin perms on every repo.
+
+set -euo pipefail
+
+: "${GITEA_TOKEN:?required}"
+: "${GITEA_HOST:?required}"
+: "${REPO:?required}"
+: "${PR_NUMBER:?required}"
+: "${REQUIRED_CHECKS:?required (newline-separated context names)}"
+
+OWNER="${REPO%%/*}"
+NAME="${REPO##*/}"
+API="https://${GITEA_HOST}/api/v1"
+AUTH="Authorization: token ${GITEA_TOKEN}"
+
+# 1. Fetch the PR. If not merged, no-op.
+PR=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}")
+MERGED=$(echo "$PR" | jq -r '.merged // false')
+if [ "$MERGED" != "true" ]; then
+  echo "::notice::PR #${PR_NUMBER} closed without merge — no audit emission."
+  exit 0
+fi
+
+MERGE_SHA=$(echo "$PR" | jq -r '.merge_commit_sha // empty')
+MERGED_BY=$(echo "$PR" | jq -r '.merged_by.login // "unknown"')
+TITLE=$(echo "$PR" | jq -r '.title // ""')
+BASE_BRANCH=$(echo "$PR" | jq -r '.base.ref // "main"')
+HEAD_SHA=$(echo "$PR" | jq -r '.head.sha // empty')
+
+if [ -z "$MERGE_SHA" ]; then
+  echo "::warning::PR #${PR_NUMBER} merged=true but no merge_commit_sha — cannot evaluate force-merge."
+  exit 0
+fi
+
+# 2. Required status checks declared in the workflow env.
+REQUIRED="$REQUIRED_CHECKS"
+if [ -z "${REQUIRED//[[:space:]]/}" ]; then
+  echo "::notice::REQUIRED_CHECKS empty — force-merge not applicable."
+  exit 0
+fi
+
+# 3. Status-check state at the PR HEAD (where checks ran). The merge
+#    commit doesn't get its own checks; we evaluate the PR's last
+#    commit, which is what branch protection compared against.
+STATUS=$(curl -sS -H "$AUTH" \
+  "${API}/repos/${OWNER}/${NAME}/commits/${HEAD_SHA}/status")
+declare -A CHECK_STATE
+while IFS=$'\t' read -r ctx state; do
+  [ -n "$ctx" ] && CHECK_STATE[$ctx]="$state"
+done < <(echo "$STATUS" | jq -r '.statuses // [] | .[] | "\(.context)\t\(.status)"')
+
+# 4. For each required check, was it green at merge? YAML block scalars
+#    (`|`) leave a trailing newline; skip blank/whitespace-only lines.
+FAILED_CHECKS=()
+while IFS= read -r req; do
+  trimmed="${req#"${req%%[![:space:]]*}"}"   # ltrim
+  trimmed="${trimmed%"${trimmed##*[![:space:]]}"}"  # rtrim
+  [ -z "$trimmed" ] && continue
+  state="${CHECK_STATE[$trimmed]:-missing}"
+  if [ "$state" != "success" ]; then
+    FAILED_CHECKS+=("${trimmed}=${state}")
+  fi
+done <<< "$REQUIRED"
+
+if [ "${#FAILED_CHECKS[@]}" -eq 0 ]; then
+  echo "::notice::PR #${PR_NUMBER} merged with all required checks green — not a force-merge."
+  exit 0
+fi
+
+# 5. Emit structured audit event.
+NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+FAILED_JSON=$(printf '%s\n' "${FAILED_CHECKS[@]}" | jq -R . | jq -s .)
+
+# Print as a single-line JSON so Vector's parse_json transform can pick
+# it up cleanly from docker_logs.
+jq -nc \
+  --arg event_type "incident.force_merge" \
+  --arg ts "$NOW" \
+  --arg repo "$REPO" \
+  --argjson pr "$PR_NUMBER" \
+  --arg title "$TITLE" \
+  --arg base "$BASE_BRANCH" \
+  --arg merged_by "$MERGED_BY" \
+  --arg merge_sha "$MERGE_SHA" \
+  --argjson failed_checks "$FAILED_JSON" \
+  '{event_type: $event_type, ts: $ts, repo: $repo, pr: $pr, title: $title,
+    base_branch: $base, merged_by: $merged_by, merge_sha: $merge_sha,
+    failed_checks: $failed_checks}'
+
+echo "::warning::FORCE-MERGE detected on PR #${PR_NUMBER} by ${MERGED_BY}: ${#FAILED_CHECKS[@]} required check(s) not green at merge time."

.gitea/scripts/ci-required-drift.py

@@ -0,0 +1,591 @@
+#!/usr/bin/env python3
+"""ci-required-drift — RFC internal#219 §4 + §6.
+
+Detects drift between three sources of "what counts as a required check"
+for this repo, files (or updates) a `[ci-drift]` Gitea issue when any
+pair diverges.
+
+Sources:
+  A. `.gitea/workflows/ci.yml` jobs  (CI source — the actual job set)
+  B. `status_check_contexts` in branch_protections (the merge gate)
+  C. `REQUIRED_CHECKS` env in audit-force-merge.yml (the audit env)
+
+Three failure classes:
+  F1  Job in (A) is not under the sentinel's `needs:` — sentinel
+      doesn't gate it, so a red job on that name can sneak through.
+      Ignores jobs whose `if:` references `github.event_name` (those
+      run only on specific events and may be `skipped` legitimately).
+  F2  Context in (B) corresponds to no emitter — i.e. there's no job
+      in ci.yml whose runtime status-name maps to that context.
+      A stale required-check name is silent: protection demands a
+      green it never receives, but Gitea treats absent-as-pending,
+      not absent-as-red. The gate degrades to advisory.
+  F3  (B) and (C) are not set-equal. Audit env wider than protection
+      → audit flags non-force-merges as force; narrower → real
+      force-merges are missed.
+
+Idempotency:
+  Searches OPEN issues by exact title prefix
+  `[ci-drift] {repo}/{branch}: ` and either edits the existing one
+  (if any) or POSTs a new one. Never spawns duplicates.
+
+Behavior-based AST gate per `feedback_behavior_based_ast_gates`:
+  - Job set comes from PyYAML parse of jobs:* keys
+  - Sentinel needs from PyYAML parse of jobs[sentinel].needs (a list)
+  - Audit env from PyYAML parse, NOT grep — so reformatting the YAML
+    (block-scalar `|` vs flow-style list) does not break the gate
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+import yaml  # PyYAML 6.0.2 — installed by the workflow before this runs.
+
+
+# --------------------------------------------------------------------------
+# Environment
+# --------------------------------------------------------------------------
+def env(key: str, *, required: bool = True, default: str | None = None) -> str:
+    val = os.environ.get(key, default)
+    if required and not val:
+        sys.stderr.write(f"::error::missing required env var: {key}\n")
+        sys.exit(2)
+    return val or ""
+
+
+GITEA_TOKEN = env("GITEA_TOKEN", required=False)
+GITEA_HOST = env("GITEA_HOST", required=False)
+REPO = env("REPO", required=False)
+BRANCHES = env("BRANCHES", required=False).split()
+SENTINEL_JOB = env("SENTINEL_JOB", required=False)
+AUDIT_WORKFLOW_PATH = env("AUDIT_WORKFLOW_PATH", required=False)
+CI_WORKFLOW_PATH = env("CI_WORKFLOW_PATH", required=False)
+DRIFT_LABEL = env("DRIFT_LABEL", required=False)
+
+OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
+API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
+
+
+def _require_runtime_env() -> None:
+    """Enforce env contract — called from `main()` only. Tests import
+    individual functions without setting the full env contract."""
+    for key in (
+        "GITEA_TOKEN",
+        "GITEA_HOST",
+        "REPO",
+        "BRANCHES",
+        "SENTINEL_JOB",
+        "AUDIT_WORKFLOW_PATH",
+        "CI_WORKFLOW_PATH",
+        "DRIFT_LABEL",
+    ):
+        if not os.environ.get(key):
+            sys.stderr.write(f"::error::missing required env var: {key}\n")
+            sys.exit(2)
+
+
+# --------------------------------------------------------------------------
+# Tiny HTTP helper (no requests dependency)
+# --------------------------------------------------------------------------
+class ApiError(RuntimeError):
+    """Raised when a Gitea API call cannot be trusted to have succeeded.
+
+    Covers non-2xx HTTP status AND 2xx with an unparseable JSON body on
+    endpoints that are documented to return JSON (search/read). Callers
+    that swallow this and proceed would risk e.g. creating duplicate
+    `[ci-drift]` issues when a transient 500 hides an existing match.
+    The cron retries hourly; one fail-loud cycle is fine — silent
+    duplicate creation is not (per Five-Axis review on PR #112).
+    """
+
+
+def api(
+    method: str,
+    path: str,
+    *,
+    body: dict | None = None,
+    query: dict[str, str] | None = None,
+    expect_json: bool = True,
+) -> tuple[int, Any]:
+    """Tiny HTTP helper around urllib.
+
+    Raises ApiError on any non-2xx response. Callers that want
+    best-effort semantics (e.g. label-apply) must `try/except ApiError`
+    explicitly — making the failure-soft path opt-in rather than the
+    default closes the duplicate-issue regression class.
+
+    For 2xx responses with a JSON body that fails to parse, raises
+    ApiError when `expect_json=True` (the default for read-shaped
+    paths). On endpoints that legitimately return non-JSON success
+    bodies (e.g. some Gitea create echoes — see
+    `feedback_gitea_create_api_unparseable_response`), callers may pass
+    `expect_json=False` to accept a `_raw` fallthrough — but they MUST
+    then verify success via a follow-up GET, not by trusting the body.
+    """
+    url = f"{API}{path}"
+    if query:
+        url = f"{url}?{urllib.parse.urlencode(query)}"
+    data = None
+    headers = {
+        "Authorization": f"token {GITEA_TOKEN}",
+        "Accept": "application/json",
+    }
+    if body is not None:
+        data = json.dumps(body).encode("utf-8")
+        headers["Content-Type"] = "application/json"
+    req = urllib.request.Request(url, method=method, data=data, headers=headers)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            raw = resp.read()
+            status = resp.status
+    except urllib.error.HTTPError as e:
+        raw = e.read()
+        status = e.code
+
+    if not (200 <= status < 300):
+        snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
+        raise ApiError(
+            f"{method} {path} → HTTP {status}: {snippet}"
+        )
+
+    if not raw:
+        return status, None
+    try:
+        return status, json.loads(raw)
+    except json.JSONDecodeError as e:
+        if expect_json:
+            raise ApiError(
+                f"{method} {path} → HTTP {status} but body is not JSON: {e}"
+            ) from e
+        # Opt-in raw fallthrough for endpoints with known echo-quirks.
+        return status, {"_raw": raw.decode("utf-8", errors="replace")}
+
+
+# --------------------------------------------------------------------------
+# YAML loaders — STRICT (reject GitHub-Actions-only syntax)
+# --------------------------------------------------------------------------
+def load_yaml(path: str) -> dict:
+    """Load + parse a workflow YAML. Hard-fail if the file is missing
+    or doesn't parse — drift-detect cannot make decisions without
+    knowing the actual job set."""
+    if not os.path.exists(path):
+        sys.stderr.write(f"::error::file not found: {path}\n")
+        sys.exit(3)
+    with open(path, encoding="utf-8") as f:
+        try:
+            doc = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            sys.stderr.write(f"::error::YAML parse error in {path}: {e}\n")
+            sys.exit(3)
+    if not isinstance(doc, dict):
+        sys.stderr.write(f"::error::{path} is not a YAML mapping\n")
+        sys.exit(3)
+    return doc
+
+
+def ci_jobs_all(ci_doc: dict) -> set[str]:
+    """Every job key in ci.yml minus the sentinel itself. Used for F1b
+    (sentinel.needs typo check) — needs that name a non-existent job
+    is a typo regardless of event-gating."""
+    jobs = ci_doc.get("jobs")
+    if not isinstance(jobs, dict):
+        sys.stderr.write("::error::ci.yml has no jobs: mapping\n")
+        sys.exit(3)
+    return {k for k in jobs if k != SENTINEL_JOB}
+
+
+def ci_job_names(ci_doc: dict) -> set[str]:
+    """Set of job keys in ci.yml MINUS the sentinel itself MINUS jobs
+    whose `if:` gates on `github.event_name` (those are event-scoped
+    and can legitimately be `skipped` for a given trigger; if we
+    required them under the sentinel `needs:`, every PR-only job
+    would be `skipped` on push and the sentinel would interpret
+    `skipped != success` as failure). RFC §4 spec.
+
+    Used for F1 (jobs missing from sentinel needs). NOT used for F1b
+    (typos in needs) — see `ci_jobs_all` for that."""
+    jobs = ci_doc.get("jobs")
+    if not isinstance(jobs, dict):
+        sys.stderr.write("::error::ci.yml has no jobs: mapping\n")
+        sys.exit(3)
+    names: set[str] = set()
+    for k, v in jobs.items():
+        if k == SENTINEL_JOB:
+            continue
+        if isinstance(v, dict):
+            gate = v.get("if")
+            if isinstance(gate, str) and "github.event_name" in gate:
+                continue
+        names.add(k)
+    return names
+
+
+def sentinel_needs(ci_doc: dict) -> set[str]:
+    sentinel = ci_doc.get("jobs", {}).get(SENTINEL_JOB)
+    if not isinstance(sentinel, dict):
+        sys.stderr.write(
+            f"::error::sentinel job '{SENTINEL_JOB}' not found in {CI_WORKFLOW_PATH}\n"
+        )
+        sys.exit(3)
+    needs = sentinel.get("needs", [])
+    if isinstance(needs, str):
+        needs = [needs]
+    if not isinstance(needs, list):
+        sys.stderr.write("::error::sentinel `needs:` is neither list nor string\n")
+        sys.exit(3)
+    return set(needs)
+
+
+def required_checks_env(audit_doc: dict) -> set[str]:
+    """Pull the REQUIRED_CHECKS env value from audit-force-merge.yml.
+    Walks the YAML AST per `feedback_behavior_based_ast_gates`: we do
+    NOT grep for `REQUIRED_CHECKS:` — that breaks under reformatting,
+    multi-job workflows, or a future move of the env to a different
+    step. Instead, look inside every job's every step's `env:` map."""
+    found: list[str] = []
+    jobs = audit_doc.get("jobs", {})
+    if not isinstance(jobs, dict):
+        sys.stderr.write(f"::warning::{AUDIT_WORKFLOW_PATH} has no jobs: mapping\n")
+        return set()
+    for job in jobs.values():
+        if not isinstance(job, dict):
+            continue
+        for step in job.get("steps", []) or []:
+            if not isinstance(step, dict):
+                continue
+            step_env = step.get("env") or {}
+            if isinstance(step_env, dict) and "REQUIRED_CHECKS" in step_env:
+                v = step_env["REQUIRED_CHECKS"]
+                if isinstance(v, str):
+                    found.append(v)
+    if not found:
+        sys.stderr.write(
+            f"::error::REQUIRED_CHECKS env not found in any step of {AUDIT_WORKFLOW_PATH}\n"
+        )
+        sys.exit(3)
+    if len(found) > 1:
+        # Defensive: refuse to guess which one is canonical.
+        sys.stderr.write(
+            f"::error::REQUIRED_CHECKS env present in {len(found)} steps; ambiguous\n"
+        )
+        sys.exit(3)
+    raw = found[0]
+    # YAML block-scalars (`|`) leave a trailing newline + blanks; trim
+    # consistently with audit-force-merge.sh's parser so both sides
+    # produce identical sets.
+    return {line.strip() for line in raw.splitlines() if line.strip()}
+
+
+# --------------------------------------------------------------------------
+# Mapping: ci.yml job-key  →  protection context name
+# --------------------------------------------------------------------------
+def expected_context(job_key: str, workflow_name: str = "ci") -> str:
+    """Gitea Actions reports status-check contexts as
+       "{workflow.name} / {job.name or job.key} ({event})".
+
+    For ci.yml the event is `pull_request` on PRs (that's what
+    `status_check_contexts` records). Job.name defaults to job.key
+    when no `name:` is set. CP's ci.yml does NOT set per-job `name:`
+    so the key equals the human-name."""
+    return f"{workflow_name} / {job_key} (pull_request)"
+
+
+# --------------------------------------------------------------------------
+# Drift detection
+# --------------------------------------------------------------------------
+def detect_drift(branch: str) -> tuple[list[str], dict]:
+    """Returns (findings, debug). Empty findings == no drift."""
+    findings: list[str] = []
+
+    ci_doc = load_yaml(CI_WORKFLOW_PATH)
+    audit_doc = load_yaml(AUDIT_WORKFLOW_PATH)
+
+    jobs = ci_job_names(ci_doc)
+    jobs_all = ci_jobs_all(ci_doc)
+    needs = sentinel_needs(ci_doc)
+    env_set = required_checks_env(audit_doc)
+
+    # Protection
+    # api() raises ApiError on non-2xx; let it propagate so a transient
+    # 500 fails the run loudly rather than producing a "no drift" lie.
+    _, protection = api("GET", f"/repos/{OWNER}/{NAME}/branch_protections/{branch}")
+    if not isinstance(protection, dict):
+        sys.stderr.write(
+            f"::error::protection response for {branch} not a JSON object\n"
+        )
+        sys.exit(4)
+    contexts = set(protection.get("status_check_contexts") or [])
+
+    # ----- F1: job exists in CI but not under sentinel.needs -----
+    missing_from_needs = sorted(jobs - needs)
+    if missing_from_needs:
+        findings.append(
+            "F1 — jobs in ci.yml NOT under sentinel `needs:` (sentinel doesn't gate them):\n"
+            + "\n".join(f"  - {n}" for n in missing_from_needs)
+        )
+
+    # ----- F1b: needs lists a job that doesn't exist (typo) -----
+    # Compare against jobs_all (incl. event-gated jobs); a typo is a
+    # typo regardless of `if:` gating.
+    stale_needs = sorted(needs - jobs_all)
+    if stale_needs:
+        findings.append(
+            "F1b — sentinel `needs:` lists jobs NOT present in ci.yml (typo or removed job):\n"
+            + "\n".join(f"  - {n}" for n in stale_needs)
+        )
+
+    # ----- F2: protection context has no emitting job -----
+    # Compute the contexts the CI YAML actually produces. The sentinel
+    # is in (B) intentionally (`ci / all-required (pull_request)`); we
+    # whitelist it explicitly.
+    emitted_contexts = {expected_context(j) for j in jobs} | {expected_context(SENTINEL_JOB)}
+    # Contexts NOT produced by ci.yml may still come from other
+    # workflows in the repo (Secret scan etc). We can't enumerate
+    # every workflow's emissions cheaply; instead, flag only contexts
+    # whose prefix is `ci / ` (this workflow's emissions) and which
+    # don't appear in `emitted_contexts`. This narrows F2 to the
+    # failure class the RFC actually targets without producing noise
+    # from cross-workflow emitters.
+    stale_protection = sorted(
+        c for c in contexts if c.startswith("ci / ") and c not in emitted_contexts
+    )
+    if stale_protection:
+        findings.append(
+            "F2 — protection `status_check_contexts` entries with `ci / ` prefix that NO "
+            "job in ci.yml emits (stale name → silent advisory gate):\n"
+            + "\n".join(f"  - {c}" for c in stale_protection)
+        )
+
+    # ----- F3: audit env vs protection contexts (set-equal) -----
+    only_in_env = sorted(env_set - contexts)
+    only_in_protection = sorted(contexts - env_set)
+    if only_in_env:
+        findings.append(
+            "F3a — audit-force-merge.yml `REQUIRED_CHECKS` env has contexts NOT in "
+            f"branch_protections/{branch}.status_check_contexts (audit would flag "
+            "non-force-merges as force):\n"
+            + "\n".join(f"  - {c}" for c in only_in_env)
+        )
+    if only_in_protection:
+        findings.append(
+            "F3b — branch_protections/{br}.status_check_contexts has contexts NOT in "
+            "audit-force-merge.yml `REQUIRED_CHECKS` env (real force-merges would be "
+            "missed):\n".format(br=branch)
+            + "\n".join(f"  - {c}" for c in only_in_protection)
+        )
+
+    debug = {
+        "branch": branch,
+        "ci_jobs": sorted(jobs),
+        "sentinel_needs": sorted(needs),
+        "protection_contexts": sorted(contexts),
+        "audit_env_checks": sorted(env_set),
+        "expected_contexts": sorted(emitted_contexts),
+    }
+    return findings, debug
+
+
+# --------------------------------------------------------------------------
+# Issue file/update
+# --------------------------------------------------------------------------
+def title_for(branch: str) -> str:
+    # Idempotency key — keep stable, never include timestamp/SHA.
+    return f"[ci-drift] {REPO}/{branch}: required-checks divergence detected"
+
+
+def find_open_issue(title: str) -> dict | None:
+    """Return the existing open `[ci-drift]` issue for `title`, or None.
+
+    `None` means "search succeeded, no match" — NOT "search failed".
+    Per Five-Axis review on PR #112: returning None on a transient API
+    error caused the caller to POST a duplicate issue. Now api() raises
+    ApiError on any non-2xx; we let it propagate. The cron retries
+    hourly; failing one cycle loudly is strictly better than silently
+    duplicating.
+
+    Gitea issue search returns at most page=50 per page; one page is
+    enough as long as `[ci-drift]` issues are a tiny minority. (See
+    follow-up issue for Link-header pagination.)
+    """
+    _, results = api(
+        "GET",
+        f"/repos/{OWNER}/{NAME}/issues",
+        query={"state": "open", "type": "issues", "limit": "50"},
+    )
+    if not isinstance(results, list):
+        raise ApiError(
+            f"issue search returned non-list body (got {type(results).__name__})"
+        )
+    for issue in results:
+        if issue.get("title") == title:
+            return issue
+    return None
+
+
+def render_body(branch: str, findings: list[str], debug: dict) -> str:
+    body = [
+        f"# Drift detected on `{REPO}/{branch}`",
+        "",
+        "Auto-filed by `.gitea/workflows/ci-required-drift.yml` "
+        "(RFC [internal#219](https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
+        "",
+        "## Findings",
+        "",
+    ]
+    body.extend(findings)
+    body.extend(
+        [
+            "",
+            "## Resolution",
+            "",
+            "- **F1 / F1b**: add the missing job to `all-required.needs:` "
+            "in `.gitea/workflows/ci.yml`, or remove the stale entry.",
+            "- **F2**: rename the protection context to match an emitter, "
+            "or remove it from `status_check_contexts` "
+            "(PATCH `/api/v1/repos/{owner}/{repo}/branch_protections/{branch}`).",
+            "- **F3a / F3b**: bring `REQUIRED_CHECKS` env in "
+            "`.gitea/workflows/audit-force-merge.yml` into set-equality with "
+            "`status_check_contexts` (single PR, both files).",
+            "",
+            "## Debug",
+            "",
+            "```json",
+            json.dumps(debug, indent=2, sort_keys=True),
+            "```",
+            "",
+            "_This issue is idempotent: drift-detect runs hourly at `:17` "
+            "and edits this body in place. Close the issue once the drift "
+            "is fixed; the next hourly run will reopen if drift returns._",
+        ]
+    )
+    return "\n".join(body)
+
+
+def file_or_update(
+    branch: str,
+    findings: list[str],
+    debug: dict,
+    *,
+    dry_run: bool = False,
+) -> None:
+    """File a new `[ci-drift]` issue, or PATCH the existing one in place.
+
+    `dry_run=True` skips every side-effecting Gitea call (issue
+    search, POST, PATCH, label apply) and prints the would-be issue
+    title + body to stdout. Useful for local testing and for
+    debugging drift output without polluting the issue tracker.
+    """
+    title = title_for(branch)
+    body = render_body(branch, findings, debug)
+
+    if dry_run:
+        print(f"::notice::[dry-run] would file/update drift issue for {branch}")
+        print(f"::group::[dry-run] title")
+        print(title)
+        print(f"::endgroup::")
+        print(f"::group::[dry-run] body")
+        print(body)
+        print(f"::endgroup::")
+        return
+
+    existing = find_open_issue(title)
+    if existing:
+        num = existing["number"]
+        api(
+            "PATCH",
+            f"/repos/{OWNER}/{NAME}/issues/{num}",
+            body={"body": body},
+        )
+        print(f"::notice::Updated existing drift issue #{num} for {branch}")
+        return
+
+    _, created = api(
+        "POST",
+        f"/repos/{OWNER}/{NAME}/issues",
+        body={"title": title, "body": body, "labels": []},
+    )
+    if not isinstance(created, dict):
+        sys.stderr.write("::error::POST issue response not a JSON object\n")
+        sys.exit(5)
+    new_num = created.get("number")
+    print(f"::warning::Filed new drift issue #{new_num} for {branch}")
+
+    # Apply label by name (Gitea's add-labels endpoint accepts label IDs;
+    # look up id by name once). Best-effort: failure to label is logged
+    # but does not fail the audit run — the issue itself IS the alarm.
+    try:
+        _, labels = api("GET", f"/repos/{OWNER}/{NAME}/labels")
+    except ApiError as e:
+        sys.stderr.write(f"::warning::could not list labels: {e}\n")
+        return
+    label_id = None
+    if isinstance(labels, list):
+        for lbl in labels:
+            if lbl.get("name") == DRIFT_LABEL:
+                label_id = lbl.get("id")
+                break
+    if label_id is not None and new_num:
+        try:
+            api(
+                "POST",
+                f"/repos/{OWNER}/{NAME}/issues/{new_num}/labels",
+                body={"labels": [label_id]},
+            )
+        except ApiError as e:
+            sys.stderr.write(
+                f"::warning::could not apply label '{DRIFT_LABEL}' to #{new_num}: {e}\n"
+            )
+    else:
+        sys.stderr.write(f"::warning::label '{DRIFT_LABEL}' not found on repo\n")
+
+
+# --------------------------------------------------------------------------
+# Main
+# --------------------------------------------------------------------------
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        prog="ci-required-drift",
+        description="Detect drift between ci.yml, branch_protections, "
+        "and audit-force-merge.yml REQUIRED_CHECKS env.",
+    )
+    p.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Detect + print findings to stdout; do NOT file or PATCH "
+        "the `[ci-drift]` issue. Useful for local testing and for "
+        "previewing output before turning the workflow loose.",
+    )
+    return p.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(argv)
+    _require_runtime_env()
+
+    for branch in BRANCHES:
+        findings, debug = detect_drift(branch)
+        if findings:
+            print(f"::warning::Drift detected on {branch}:")
+            for f in findings:
+                print(f)
+            file_or_update(branch, findings, debug, dry_run=args.dry_run)
+        else:
+            print(f"::notice::No drift on {branch}.")
+            print(json.dumps(debug, indent=2, sort_keys=True))
+    # Exit 0 even on drift — the issue IS the alarm, not a red workflow.
+    # A red workflow here would page on a CI rename until the issue is
+    # opened, doubling the noise. The issue itself is the actionable
+    # surface. (`api()` raising ApiError is the only path that exits
+    # non-zero, by design: a transient Gitea outage should fail loudly.)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.gitea/scripts/main-red-watchdog.py

@@ -0,0 +1,589 @@
+#!/usr/bin/env python3
+"""main-red-watchdog — Option C of the "main NEVER goes red" directive.
+
+Tracking: molecule-core#420.
+
+What it does (one cron tick):
+  1. GET /api/v1/repos/{owner}/{repo}/branches/{watch_branch}
+     → current HEAD SHA on the watched branch.
+  2. GET /api/v1/repos/{owner}/{repo}/commits/{SHA}/status
+     → combined status + per-context statuses.
+  3. If combined state is `failure` (or any individual status is
+     `failure`): open or PATCH an idempotent
+     `[main-red] {repo}: {SHA[:10]}` issue. Body lists each failed
+     status context with `target_url` + `description`.
+  4. If combined state is `success`: close any open `[main-red]
+     {repo}: ...` issue on a previous SHA with a
+     "main returned to green at SHA {current_SHA}" comment.
+  5. Emit one Loki-shaped JSON line via `logger -t main-red-watchdog`
+     so `reference_obs_stack_phase1`'s Vector → Loki path ingests an
+     alert event (queryable in Grafana as
+     `{tenant="operator-host"} |~ "main-red-watchdog"`).
+
+What it does NOT do:
+  - Auto-revert anything. Option B is explicitly rejected per
+    `feedback_no_such_thing_as_flakes` + `feedback_fix_root_not_symptom`.
+  - Page on its own failures. If api() raises ApiError (transient
+    Gitea outage), the workflow run fails LOUDLY by re-raise — exactly
+    the contract `feedback_api_helper_must_raise_not_return_dict`
+    enforces. Silent fallthrough would re-introduce the duplicate-issue
+    regression class.
+  - Exit non-zero on RED. The issue IS the alarm; failing the watchdog
+    on red would double-page (red workflow + open issue) and create
+    silent-loop risk if the watchdog itself flakes.
+
+Idempotency strategy:
+  Title is keyed on `{SHA[:10]}` (commit-scoped), NOT just `main`.
+  Rationale:
+    - A fix-forward changes HEAD → next cron tick sees a new SHA;
+      auto-close logic closes the prior `[main-red] OLD_SHA` issue and
+      (if the new HEAD is also red, e.g. a different test fails) files
+      a fresh `[main-red] NEW_SHA`. Lineage is preserved.
+    - A revert that happens to land back on a previously-red SHA
+      (rare) would refer to a CLOSED issue; the watchdog never reopens.
+      That's a deliberate trade-off — the operator will see the latest
+      open issue's `closed` event in the activity feed.
+
+This module is import-safe: tests import individual functions without
+invoking main(), so module-level reads use env-with-default and the
+runtime contract enforcement lives in `_require_runtime_env()`.
+
+Run locally (dry-run, no API mutation):
+    GITEA_TOKEN=... GITEA_HOST=git.moleculesai.app REPO=owner/repo \\
+      WATCH_BRANCH=main RED_LABEL=tier:high \\
+      python3 .gitea/scripts/main-red-watchdog.py --dry-run
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+
+# --------------------------------------------------------------------------
+# Environment
+# --------------------------------------------------------------------------
+def _env(key: str, *, default: str = "") -> str:
+    """Read an env var with a default. Module-import-safe — tests can
+    import this script without setting the full env contract."""
+    return os.environ.get(key, default)
+
+
+GITEA_TOKEN = _env("GITEA_TOKEN")
+GITEA_HOST = _env("GITEA_HOST")
+REPO = _env("REPO")
+WATCH_BRANCH = _env("WATCH_BRANCH", default="main")
+RED_LABEL = _env("RED_LABEL", default="tier:high")
+
+OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
+API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
+
+# Title prefix — kept short and stable so the idempotency search can
+# match by exact title without parsing.
+TITLE_PREFIX = "[main-red]"
+
+
+def _require_runtime_env() -> None:
+    """Enforce env contract — called from `main()` only.
+
+    Tests import individual functions without setting the full env
+    contract. Mirrors the CP `ci-required-drift.py` pattern so the
+    runtime guard is a single chokepoint.
+    """
+    for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "WATCH_BRANCH", "RED_LABEL"):
+        if not os.environ.get(key):
+            sys.stderr.write(f"::error::missing required env var: {key}\n")
+            sys.exit(2)
+
+
+# --------------------------------------------------------------------------
+# Tiny HTTP helper — raises on non-2xx + on JSON-decode-of-expected-JSON.
+# --------------------------------------------------------------------------
+class ApiError(RuntimeError):
+    """Raised when a Gitea API call cannot be trusted to have succeeded.
+
+    Covers non-2xx HTTP status AND 2xx with an unparseable JSON body on
+    endpoints documented to return JSON. Callers that swallow this and
+    proceed risk e.g. creating duplicate `[main-red]` issues when a
+    transient 500 hides an existing match. Per
+    `feedback_api_helper_must_raise_not_return_dict`: soft-failure is
+    opt-in via `expect_json=False`, never the default.
+    """
+
+
+def api(
+    method: str,
+    path: str,
+    *,
+    body: dict | None = None,
+    query: dict[str, str] | None = None,
+    expect_json: bool = True,
+) -> tuple[int, Any]:
+    """Tiny HTTP helper around urllib.
+
+    Raises ApiError on any non-2xx response, and on JSON-decode failure
+    when `expect_json=True` (the default for read-shaped paths). Mirrors
+    the CP ci-required-drift.py contract exactly so behaviour is
+    cross-checkable.
+    """
+    url = f"{API}{path}"
+    if query:
+        url = f"{url}?{urllib.parse.urlencode(query)}"
+    data = None
+    headers = {
+        "Authorization": f"token {GITEA_TOKEN}",
+        "Accept": "application/json",
+    }
+    if body is not None:
+        data = json.dumps(body).encode("utf-8")
+        headers["Content-Type"] = "application/json"
+    req = urllib.request.Request(url, method=method, data=data, headers=headers)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            raw = resp.read()
+            status = resp.status
+    except urllib.error.HTTPError as e:
+        raw = e.read()
+        status = e.code
+
+    if not (200 <= status < 300):
+        snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
+        raise ApiError(f"{method} {path} → HTTP {status}: {snippet}")
+
+    if not raw:
+        return status, None
+    try:
+        return status, json.loads(raw)
+    except json.JSONDecodeError as e:
+        if expect_json:
+            raise ApiError(
+                f"{method} {path} → HTTP {status} but body is not JSON: {e}"
+            ) from e
+        # Opt-in raw fallthrough for endpoints with known echo-quirks
+        # (`feedback_gitea_create_api_unparseable_response`). Caller
+        # MUST verify success via a follow-up GET, not by trusting body.
+        return status, {"_raw": raw.decode("utf-8", errors="replace")}
+
+
+# --------------------------------------------------------------------------
+# Gitea reads
+# --------------------------------------------------------------------------
+def get_head_sha(branch: str) -> str:
+    """HEAD SHA of `branch`. Raises ApiError on non-2xx."""
+    _, body = api("GET", f"/repos/{OWNER}/{NAME}/branches/{branch}")
+    if not isinstance(body, dict):
+        raise ApiError(f"branch {branch} response not a JSON object")
+    commit = body.get("commit")
+    if not isinstance(commit, dict):
+        raise ApiError(f"branch {branch} response missing `commit` object")
+    sha = commit.get("id") or commit.get("sha")
+    if not isinstance(sha, str) or len(sha) < 7:
+        raise ApiError(f"branch {branch} response has no usable commit SHA")
+    return sha
+
+
+def get_combined_status(sha: str) -> dict:
+    """Combined commit status for `sha`. Gitea returns:
+        {
+          "state": "success" | "failure" | "pending" | "error",
+          "statuses": [
+            {"context": "...", "state": "success|failure|pending|error",
+             "target_url": "...", "description": "..."},
+            ...
+          ],
+          ...
+        }
+    Raises ApiError on non-2xx.
+    """
+    _, body = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
+    if not isinstance(body, dict):
+        raise ApiError(f"status for {sha} response not a JSON object")
+    return body
+
+
+def is_red(status: dict) -> tuple[bool, list[dict]]:
+    """Return (is_red, failed_statuses).
+
+    A commit is "red" if combined state is `failure` OR any individual
+    status entry is in {`failure`, `error`}. `pending` and `success`
+    do not trip the watchdog — pending means CI is still running, and
+    that's the normal state immediately after a merge.
+
+    `failed_statuses` is the list of per-context entries whose own
+    `state` is in the red set; useful for the issue body.
+    """
+    combined = status.get("state")
+    statuses = status.get("statuses") or []
+    red_states = {"failure", "error"}
+    failed = [
+        s for s in statuses
+        if isinstance(s, dict) and s.get("state") in red_states
+    ]
+    return (combined in red_states or bool(failed), failed)
+
+
+# --------------------------------------------------------------------------
+# Issue file / update / close
+# --------------------------------------------------------------------------
+def title_for(sha: str) -> str:
+    """Idempotency key — `[main-red] {repo}: {SHA[:10]}`.
+
+    Commit-scoped. A fix-forward to a new SHA produces a new title; the
+    prior issue auto-closes via `close_open_red_issues_for_other_shas`.
+    """
+    return f"{TITLE_PREFIX} {REPO}: {sha[:10]}"
+
+
+def list_open_red_issues() -> list[dict]:
+    """All open issues whose title starts with `[main-red] {repo}: `.
+
+    Per Five-Axis review on CP#112 (`feedback_api_helper_must_raise_not_return_dict`):
+    api() raises on non-2xx; we let it propagate. Returning [] on a
+    transient 500 would cause auto-close to skip the cleanup AND the
+    file-or-update path to POST a duplicate — exactly the regression
+    class the helper-raises contract closes.
+
+    Gitea issue search returns at most 50/page; we only need open
+    `[main-red]` issues which are by design ≤ 1 at any time per repo,
+    so a single page is enough.
+    """
+    _, results = api(
+        "GET",
+        f"/repos/{OWNER}/{NAME}/issues",
+        query={"state": "open", "type": "issues", "limit": "50"},
+    )
+    if not isinstance(results, list):
+        raise ApiError(
+            f"issue search returned non-list body (got {type(results).__name__})"
+        )
+    prefix = f"{TITLE_PREFIX} {REPO}: "
+    return [i for i in results if isinstance(i, dict)
+            and isinstance(i.get("title"), str)
+            and i["title"].startswith(prefix)]
+
+
+def find_open_issue_for_sha(sha: str) -> dict | None:
+    """Return the existing open `[main-red] {repo}: {SHA[:10]}` issue,
+    or None if no such issue is open.
+
+    `None` means "search succeeded, no match" — NOT "search failed".
+    api() raises ApiError on any non-2xx; the caller can let that
+    propagate so a transient outage fails loudly instead of silently
+    duplicating.
+    """
+    target = title_for(sha)
+    for issue in list_open_red_issues():
+        if issue.get("title") == target:
+            return issue
+    return None
+
+
+def render_body(sha: str, failed: list[dict], debug: dict) -> str:
+    """Issue body. Markdown. Mirrors CP#112's render_body shape."""
+    lines = [
+        f"# Main is RED on `{REPO}` at `{sha[:10]}`",
+        "",
+        f"Commit: <https://{GITEA_HOST}/{REPO}/commit/{sha}>",
+        "",
+        "Auto-filed by `.gitea/workflows/main-red-watchdog.yml` (Option C "
+        "of the [main-never-red directive]"
+        f"(https://{GITEA_HOST}/molecule-ai/molecule-core/issues/420)). "
+        "Per `feedback_no_such_thing_as_flakes` + "
+        "`feedback_fix_root_not_symptom`: investigate the root cause; do "
+        "NOT revert as a reflex. The watchdog itself never reverts.",
+        "",
+        "## Failed status contexts",
+        "",
+    ]
+    if not failed:
+        lines.append(
+            "_(Combined state reported `failure`/`error` but no per-context "
+            "entries were in a red state. This usually means a CI emitter "
+            "set combined-status directly without a per-context status. "
+            "Check the most recent workflow run for `main` and trace from "
+            "there.)_"
+        )
+    else:
+        for s in failed:
+            ctx = s.get("context", "(no context)")
+            state = s.get("state", "(no state)")
+            url = s.get("target_url") or ""
+            desc = (s.get("description") or "").strip()
+            entry = f"- **{ctx}** — `{state}`"
+            if url:
+                entry += f" → [logs]({url})"
+            if desc:
+                entry += f"\n  - {desc}"
+            lines.append(entry)
+    lines.extend([
+        "",
+        "## Resolution path",
+        "",
+        "1. Read the failed logs (links above).",
+        "2. If reproducible locally, fix forward in a PR targeting `main`.",
+        "3. If the failure is a real flake — STOP. Per "
+        "`feedback_no_such_thing_as_flakes`, intermittent failures are "
+        "real bugs. Investigate to root cause; do not mark as flake.",
+        "4. If the failure is blocking unrelated work for >1 hour, file a "
+        "follow-up issue and assign someone. Do NOT revert without a "
+        "human GO per `feedback_prod_apply_needs_hongming_chat_go` "
+        "(branch protection is a prod surface).",
+        "",
+        "## Debug",
+        "",
+        "```json",
+        json.dumps(debug, indent=2, sort_keys=True),
+        "```",
+        "",
+        "_This issue is idempotent: the watchdog runs hourly at `:05` "
+        "and edits this body in place. When `main` returns to green, the "
+        "watchdog will close this issue automatically with a "
+        "\"main returned to green\" comment._",
+    ])
+    return "\n".join(lines)
+
+
+def emit_loki_event(event_type: str, sha: str, failed_contexts: list[str]) -> None:
+    """Emit a JSON line to syslog tag `main-red-watchdog` for
+    `reference_obs_stack_phase1` (Vector → Loki).
+
+    Best-effort: if `logger` isn't on PATH (e.g. local dev macOS without
+    util-linux logger), print to stderr instead. The Gitea Actions
+    Ubuntu runner has util-linux preinstalled.
+
+    Loki labels: the workflow runs on the Ubuntu runner where Vector is
+    NOT configured (Vector lives on the operator host + tenants per
+    `reference_obs_stack_phase1`). The Loki line is still emitted as
+    stdout JSON so the workflow log itself is parseable; treat the
+    syslog call as belt-and-braces for the cases where this script is
+    invoked from a host that DOES have Vector (e.g. operator-host cron
+    fallback in a follow-up PR).
+    """
+    payload = {
+        "event_type": event_type,
+        "repo": REPO,
+        "sha": sha,
+        "failed_contexts": failed_contexts,
+    }
+    line = json.dumps(payload, sort_keys=True)
+    # Always print to stdout so the workflow log captures it (machine-
+    # readable; `gitea run logs` + Loki ingestion via the operator-host
+    # journald → Vector → Loki path will see this from runners that
+    # forward stdout). Loki query:
+    #   {source="gitea-actions"} |~ "main_red_detected"
+    print(f"main-red-watchdog event: {line}")
+    # Best-effort syslog tag so a future "run from operator-host cron"
+    # path picks it up directly via the existing Vector pipeline.
+    if shutil.which("logger"):
+        try:
+            subprocess.run(
+                ["logger", "-t", "main-red-watchdog", line],
+                check=False,
+                timeout=5,
+            )
+        except (OSError, subprocess.SubprocessError) as e:
+            sys.stderr.write(f"::warning::logger call failed: {e}\n")
+
+
+def file_or_update_red(
+    sha: str,
+    failed: list[dict],
+    debug: dict,
+    *,
+    dry_run: bool = False,
+) -> None:
+    """Open a new `[main-red] {repo}: {SHA[:10]}` issue, or PATCH the
+    existing one's body. Idempotent by title."""
+    title = title_for(sha)
+    body = render_body(sha, failed, debug)
+
+    if dry_run:
+        print(f"::notice::[dry-run] would file/update main-red issue for {sha[:10]}")
+        print("::group::[dry-run] title")
+        print(title)
+        print("::endgroup::")
+        print("::group::[dry-run] body")
+        print(body)
+        print("::endgroup::")
+        return
+
+    existing = find_open_issue_for_sha(sha)
+    if existing:
+        num = existing["number"]
+        api("PATCH", f"/repos/{OWNER}/{NAME}/issues/{num}", body={"body": body})
+        print(f"::notice::Updated existing main-red issue #{num} for {sha[:10]}")
+        return
+
+    _, created = api(
+        "POST",
+        f"/repos/{OWNER}/{NAME}/issues",
+        body={"title": title, "body": body, "labels": []},
+    )
+    if not isinstance(created, dict):
+        raise ApiError("POST issue response not a JSON object")
+    new_num = created.get("number")
+    print(f"::warning::Filed new main-red issue #{new_num} for {sha[:10]}")
+
+    # Apply RED_LABEL by id. Gitea's add-labels endpoint takes IDs, not
+    # names (`feedback_gitea_label_delete_by_id` — same rule for add).
+    # Best-effort: label failure is logged but does not fail the run.
+    try:
+        _, labels = api("GET", f"/repos/{OWNER}/{NAME}/labels")
+    except ApiError as e:
+        sys.stderr.write(f"::warning::could not list labels: {e}\n")
+        return
+    label_id = None
+    if isinstance(labels, list):
+        for lbl in labels:
+            if isinstance(lbl, dict) and lbl.get("name") == RED_LABEL:
+                label_id = lbl.get("id")
+                break
+    if label_id is not None and new_num:
+        try:
+            api(
+                "POST",
+                f"/repos/{OWNER}/{NAME}/issues/{new_num}/labels",
+                body={"labels": [label_id]},
+            )
+        except ApiError as e:
+            sys.stderr.write(
+                f"::warning::could not apply label '{RED_LABEL}' to #{new_num}: {e}\n"
+            )
+    else:
+        sys.stderr.write(f"::warning::label '{RED_LABEL}' not found on repo\n")
+
+
+def close_open_red_issues_for_other_shas(
+    current_sha: str,
+    *,
+    dry_run: bool = False,
+) -> int:
+    """When main is green at current_sha, close any open `[main-red]`
+    issues whose title references a different SHA. Returns the number
+    of issues closed.
+
+    Lineage note: we only close issues whose title prefix matches; if
+    a human renamed the issue or added a suffix this won't touch it.
+    That's intentional — manual editorial state takes precedence.
+    """
+    target_title = title_for(current_sha)
+    open_red = list_open_red_issues()
+    closed = 0
+    for issue in open_red:
+        if issue.get("title") == target_title:
+            # Same SHA — caller should not have invoked this if main is
+            # green. Skip defensively.
+            continue
+        num = issue.get("number")
+        if not isinstance(num, int):
+            continue
+        comment = (
+            f"`main` returned to green at SHA `{current_sha}` "
+            f"(<https://{GITEA_HOST}/{REPO}/commit/{current_sha}>). "
+            "Closing automatically. If the underlying root cause is "
+            "not yet understood, reopen this issue and file a "
+            "postmortem — green-by-flake is still a bug per "
+            "`feedback_no_such_thing_as_flakes`."
+        )
+        if dry_run:
+            print(f"::notice::[dry-run] would close issue #{num} ({issue.get('title')})")
+            closed += 1
+            continue
+        # Comment first, then close. Order matters: a closed issue can
+        # still receive comments, but the activity-feed ordering reads
+        # better with the explanation arriving just before the close.
+        api(
+            "POST",
+            f"/repos/{OWNER}/{NAME}/issues/{num}/comments",
+            body={"body": comment},
+        )
+        api(
+            "PATCH",
+            f"/repos/{OWNER}/{NAME}/issues/{num}",
+            body={"state": "closed"},
+        )
+        print(f"::notice::Closed main-red issue #{num} (green at {current_sha[:10]})")
+        closed += 1
+    return closed
+
+
+# --------------------------------------------------------------------------
+# Main
+# --------------------------------------------------------------------------
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        prog="main-red-watchdog",
+        description="Detect post-merge CI red on the watched branch and "
+        "file an idempotent issue. Option C of the main-never-red directive.",
+    )
+    p.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Detect + print the would-be issue title/body to stdout; do "
+        "NOT POST/PATCH/close any issues. Useful for local testing.",
+    )
+    return p.parse_args(argv)
+
+
+def run_once(*, dry_run: bool = False) -> int:
+    """One watchdog tick. Returns 0 on green or red-issue-filed; lets
+    ApiError propagate on transient outage (workflow run fails loudly,
+    which is correct per the helper-raises contract)."""
+    sha = get_head_sha(WATCH_BRANCH)
+    status = get_combined_status(sha)
+    red, failed = is_red(status)
+
+    debug = {
+        "branch": WATCH_BRANCH,
+        "sha": sha,
+        "combined_state": status.get("state"),
+        "failed_contexts": [s.get("context") for s in failed],
+        "all_contexts": [
+            {"context": s.get("context"), "state": s.get("state")}
+            for s in (status.get("statuses") or [])
+            if isinstance(s, dict)
+        ],
+    }
+
+    if red:
+        failed_ctxs = [s.get("context") for s in failed if s.get("context")]
+        emit_loki_event("main_red_detected", sha, failed_ctxs)
+        print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
+              f"{len(failed)} failed context(s)")
+        file_or_update_red(sha, failed, debug, dry_run=dry_run)
+    else:
+        # Green (or pending — pending is treated as not-red so we don't
+        # spam during the post-merge CI window). Close any stale issues
+        # from earlier SHAs only when we're actually green; pending
+        # means CI hasn't finished and the prior issue might still be
+        # accurate.
+        if status.get("state") == "success":
+            closed = close_open_red_issues_for_other_shas(sha, dry_run=dry_run)
+            if closed:
+                emit_loki_event(
+                    "main_returned_to_green", sha,
+                    [],
+                )
+            print(f"::notice::main is GREEN at {sha[:10]} on {WATCH_BRANCH} "
+                  f"(closed {closed} stale issue(s))")
+        else:
+            print(f"::notice::main is PENDING at {sha[:10]} on {WATCH_BRANCH} "
+                  f"(combined state={status.get('state')!r}; no action)")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(argv)
+    _require_runtime_env()
+    return run_once(dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.gitea/scripts/sop-tier-check.sh

@@ -0,0 +1,379 @@
+#!/usr/bin/env bash
+# sop-tier-check — verify a Gitea PR satisfies the §SOP-6 approval gate.
+#
+# Reads the PR's tier label, walks approving reviewers, and checks team
+# membership against the tier's approval expression. Passes only when
+# ALL clauses in the expression are satisfied by the set of approving
+# reviewers (AND-composition; internal#189).
+#
+# Expression syntax:
+#   "team-a"          — OR-set: any ONE of the comma-separated teams
+#   "team-a AND team-b" — AND: BOTH must each have ≥1 approver
+#   "(a,b,c)"         — OR-set wrapped in parens; same as "a,b,c"
+#
+# Example: "qa AND security AND (managers,ceo)" means:
+#   ≥1 approver in team "qa"  AND
+#   ≥1 approver in team "security"  AND
+#   ≥1 approver in team "managers" OR "ceo"
+#
+# Per the spec (internal#189), the hard gate here pairs with the
+# advisory gate of sop-conformance LLM-judge (internal#188): each
+# required-team click must reflect real verification (visible in review
+# body or A2A messages), not rubber-stamp APPROVE. Both gates together
+# close the "teammate clicks APPROVE without verifying" gap.
+#
+# Invoked from `.gitea/workflows/sop-tier-check.yml`. The workflow sets
+# the env vars below; this script does no IO outside of stdout/stderr +
+# the Gitea API.
+#
+# Required env:
+#   GITEA_TOKEN   — bot PAT with read:organization,read:user,
+#                   read:issue,read:repository scopes
+#   GITEA_HOST    — e.g. git.moleculesai.app
+#   REPO          — owner/name (from github.repository)
+#   PR_NUMBER     — int (from github.event.pull_request.number)
+#   PR_AUTHOR     — login (from github.event.pull_request.user.login)
+#
+# Optional:
+#   SOP_DEBUG=1        — print per-API-call diagnostic lines. Default: off.
+#   SOP_LEGACY_CHECK=1 — revert to OR-gate (≥1 approver from any eligible
+#                         team). Grace window for PRs in-flight when the
+#                         new AND-composition was deployed. Expires 2026-05-17
+#                         (7-day burn-in window; internal#189 Phase 1).
+#                         Set by workflow for PRs merged before the deploy.
+
+set -euo pipefail
+
+# Ensure jq is available. Runners may not have it pre-installed, and the
+# workflow-level jq install can fail on runners with network restrictions
+# (GitHub releases not reachable from some runner networks — infra#241
+# follow-up). This fallback is idempotent — no-op when jq is already on PATH.
+# SOP_FAIL_OPEN=1 makes this always exit 0 so CI never blocks on jq absence.
+if ! command -v jq >/dev/null 2>&1; then
+  echo "::notice::jq not found on PATH — attempting install..."
+  _jq_installed="no"
+  # apt-get first (primary) — Ubuntu package mirrors are reliably reachable.
+  if apt-get update -qq && apt-get install -y -qq jq 2>/dev/null; then
+    echo "::notice::jq installed via apt-get: $(jq --version)"
+    _jq_installed="yes"
+  # GitHub binary as secondary fallback — may fail on restricted networks.
+  elif timeout 120 curl -sSL \
+    "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
+    -o /usr/local/bin/jq \
+    && chmod +x /usr/local/bin/jq; then
+    echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
+    _jq_installed="yes"
+  fi
+  if ! command -v jq >/dev/null 2>&1; then
+    echo "::error::jq installation failed — apt-get and GitHub binary both failed."
+    echo "::error::sop-tier-check requires jq for all JSON API parsing."
+    # SOP_FAIL_OPEN=1 is set in the workflow step's env — makes script always
+    # exit 0 so CI never blocks. The SOP-6 tier review gate remains enforced.
+    if [ "${SOP_FAIL_OPEN:-}" = "1" ]; then
+      echo "::warning::SOP_FAIL_OPEN=1 — exiting 0 so CI does not block."
+      exit 0
+    fi
+    exit 1
+  fi
+fi
+
+debug() {
+  if [ "${SOP_DEBUG:-}" = "1" ]; then
+    echo "  [debug] $*" >&2
+  fi
+}
+
+# Validate env
+: "${GITEA_TOKEN:?GITEA_TOKEN required}"
+: "${GITEA_HOST:?GITEA_HOST required}"
+: "${REPO:?REPO required (owner/name)}"
+: "${PR_NUMBER:?PR_NUMBER required}"
+: "${PR_AUTHOR:?PR_AUTHOR required}"
+
+OWNER="${REPO%%/*}"
+NAME="${REPO##*/}"
+API="https://${GITEA_HOST}/api/v1"
+AUTH="Authorization: token ${GITEA_TOKEN}"
+echo "::notice::tier-check start: repo=$OWNER/$NAME pr=$PR_NUMBER author=$PR_AUTHOR"
+
+# Sanity: token resolves to a user
+WHOAMI=$(curl -sS -H "$AUTH" "${API}/user" | jq -r '.login // ""')
+if [ -z "$WHOAMI" ]; then
+  echo "::error::GITEA_TOKEN cannot resolve a user via /api/v1/user — check the token scope and that the secret is wired correctly."
+  exit 1
+fi
+echo "::notice::token resolves to user: $WHOAMI"
+
+# 1. Read tier label
+LABELS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/labels" | jq -r '.[].name')
+TIER=""
+for L in $LABELS; do
+  case "$L" in
+    tier:low|tier:medium|tier:high)
+      if [ -n "$TIER" ]; then
+        echo "::error::Multiple tier labels: $TIER + $L. Apply exactly one."
+        exit 1
+      fi
+      TIER="$L"
+    ;;
+  esac
+done
+if [ -z "$TIER" ]; then
+  echo "::error::PR has no tier:low|tier:medium|tier:high label. Apply one before merge."
+  exit 1
+fi
+debug "tier=$TIER"
+
+# 2. Tier → required team expression (AND-composition; internal#189)
+#
+# Expression syntax:
+#   clause-a AND clause-b AND ...   — ALL clauses must pass
+#   team-a,team-b,team-c            — OR-set: ≥1 approver in ANY of these teams
+#   (team-a,team-b)                 — same as team-a,team-b (parens optional)
+#
+# This map is the single source of truth. Update it when the team structure
+# or policy changes. Teams referenced here but absent in Gitea are treated
+# as unachievable (would always fail) — operators notice the clear error
+# and create the missing team.
+#
+# Current Gitea teams: ceo, engineers, managers
+# Future teams (create before removing "???" fallback): qa, security, security-audit
+declare -A TIER_EXPR=(
+  # tier:low — same as previous OR gate: any engineer, manager, or ceo.
+  ["tier:low"]="engineers,managers,ceo"
+
+  # tier:medium — AND of (managers) AND (engineers) AND (qa???,security???)
+  # The qa+security clause requires both teams to exist; when not yet
+  # created, the PR author is responsible for adding them before requesting
+  # approval on a tier:medium PR. Ops: create qa + security Gitea teams
+  # and update this map to remove the "???" markers (internal#189 follow-up).
+  ["tier:medium"]="managers AND engineers AND qa???,security???"
+
+  # tier:high — ceo only. The AND-composition adds no value for a
+  # single-team gate, but the framework is wired for consistency.
+  ["tier:high"]="ceo"
+)
+
+EXPR="${TIER_EXPR[$TIER]-}"
+if [ -z "$EXPR" ]; then
+  echo "::error::No expression defined for tier $TIER in TIER_EXPR map."
+  exit 1
+fi
+debug "expression=$EXPR"
+
+# 3. Legacy OR-gate override (7-day burn-in grace window; internal#189 Phase 1)
+if [ "${SOP_LEGACY_CHECK:-}" = "1" ]; then
+  LEGACY_ELIGIBLE=""
+  case "$TIER" in
+    tier:low)    LEGACY_ELIGIBLE="engineers managers ceo" ;;
+    tier:medium) LEGACY_ELIGIBLE="managers ceo" ;;
+    tier:high)   LEGACY_ELIGIBLE="ceo" ;;
+  esac
+  echo "::notice::SOP_LEGACY_CHECK=1 — using OR-gate ({$LEGACY_ELIGIBLE}) for this PR."
+  ELIGIBLE="$LEGACY_ELIGIBLE"
+fi
+
+# 4. Resolve all team names → IDs
+# /orgs/{org}/teams/{slug}/... endpoints don't exist on Gitea 1.22;
+# we use /teams/{id}.
+ORG_TEAMS_FILE=$(mktemp)
+trap 'rm -f "$ORG_TEAMS_FILE"' EXIT
+HTTP_CODE=$(curl -sS -o "$ORG_TEAMS_FILE" -w '%{http_code}' -H "$AUTH" \
+  "${API}/orgs/${OWNER}/teams")
+debug "teams-list HTTP=$HTTP_CODE size=$(wc -c <"$ORG_TEAMS_FILE")"
+if [ "${SOP_DEBUG:-}" = "1" ]; then
+  echo "  [debug] teams-list body (first 300 chars):" >&2
+  head -c 300 "$ORG_TEAMS_FILE" >&2; echo >&2
+fi
+if [ "$HTTP_CODE" != "200" ]; then
+  echo "::error::GET /orgs/${OWNER}/teams returned HTTP $HTTP_CODE — token likely lacks read:org scope."
+  exit 1
+fi
+
+# Collect every team name that appears in the expression.
+# Bash word-splitting on $EXPR splits on spaces, so "AND" appears as a
+# token. We skip it explicitly.
+declare -A TEAM_ID
+_all_teams=""
+for _raw_clause in $EXPR; do
+  # Strip parens and split on comma.
+  _clause=${_raw_clause//[()]/}
+  for _t in $(echo "$_clause" | tr ',' '\n'); do
+    _t=$(echo "$_t" | tr -d '[:space:]')
+    [ -z "$_t" ] && continue
+    # Skip AND / OR operator tokens (bash word-split produced them from
+    # spaces in the expression string).
+    [ "$_t" = "AND" ] || [ "$_t" = "OR" ] && continue
+    # Skip if already in set.
+    case " $_all_teams " in
+      *" $_t "*) ;;  # already present
+      *) _all_teams="${_all_teams} $_t " ;;
+    esac
+  done
+done
+
+for _t in $_all_teams; do
+  _t=$(echo "$_t" | tr -d ' ')
+  [ -z "$_t" ] && continue
+  _id=$(jq -r --arg t "$_t" '.[] | select(.name==$t) | .id' <"$ORG_TEAMS_FILE" | head -1)
+  if [ -z "$_id" ] || [ "$_id" = "null" ]; then
+    # "??" suffix marks teams that don't exist yet (tier:medium qa/security).
+    # Treat as permanently failing clause; clear error message guides ops.
+    if [[ "$_t" == *"???" ]]; then
+      debug "team \"$_t\" not found (expected — pending team creation per internal#189)"
+      continue
+    fi
+    _visible=$(jq -r '.[]?.name? // empty' <"$ORG_TEAMS_FILE" 2>/dev/null | tr '\n' ' ')
+    echo "::error::Team \"$_t\" referenced in tier $TIER expression but not found in org $OWNER. Teams visible: $_visible"
+    exit 1
+  fi
+  TEAM_ID[$_t]="$_id"
+  debug "team-id: $_t → $_id"
+done
+
+# 5. Read approving reviewers
+REVIEWS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews")
+APPROVERS=$(echo "$REVIEWS" | jq -r '[.[] | select(.state=="APPROVED") | .user.login] | unique | .[]')
+if [ -z "$APPROVERS" ]; then
+  echo "::error::No approving reviews on this PR. Set SOP_DEBUG=1 and re-run for diagnostics."
+  exit 1
+fi
+debug "approvers: $(echo "$APPROVERS" | tr '\n' ' ')"
+
+# 6. For each approver: skip self-review; probe team membership by id.
+# Build $APPROVER_TEAMS[<user>]=space-surrounded team names (e.g. " managers ").
+# Pre/post spaces ensure case patterns *${_t}* match even when the name
+# is the first or last entry (bash case *word* needs delimiters on both sides).
+#
+# FALLBACK: if ALL team probes return 403 (token lacks read:org scope),
+# fall back to /orgs/{org}/members/{user}. This returns 204 for any org
+# member — a superset of team membership. Accepting it as a fallback means
+# the gate passes when the token is scoped to repo+user only (core-bot PAT).
+# This is safe because: (a) org membership is a prerequisite for every
+# eligible team; (b) the AND-composition of internal#189 still requires
+# multiple independent approvers; (c) any token with read:repository can
+# see the approving reviews, so bypass requires a colluding approver.
+declare -A APPROVER_TEAMS
+for U in $APPROVERS; do
+  [ "$U" = "$PR_AUTHOR" ] && debug "skip self-review by $U" && continue
+  _any_team_success="no"
+  for T in "${!TEAM_ID[@]}"; do
+    ID="${TEAM_ID[$T]}"
+    CODE=$(curl -sS -o /dev/null -w '%{http_code}' -H "$AUTH" \
+      "${API}/teams/${ID}/members/${U}")
+    debug "probe: $U in team $T (id=$ID) → HTTP $CODE"
+    if [ "$CODE" = "200" ] || [ "$CODE" = "204" ]; then
+      APPROVER_TEAMS[$U]="${APPROVER_TEAMS[$U]:- } ${APPROVER_TEAMS[$U]:+ }$T "
+      debug "$U qualifies for team $T"
+      _any_team_success="yes"
+    fi
+  done
+  # Fallback: if every team probe returned 403, try org membership.
+  # "??" teams were never resolved to IDs so they never entered the loop.
+  # If the user is an org member, credit them as being in each queried team
+  # (engineers, managers, ceo are all org-level). This is safe because org
+  # membership is a prerequisite for all three, and bypass requires a colluding
+  # approver (same risk as before the AND-composition).
+  if [ "$_any_team_success" = "no" ]; then
+    ORG_CODE=$(curl -sS -o /dev/null -w '%{http_code}' -H "$AUTH" \
+      "${API}/orgs/${OWNER}/members/${U}")
+    debug "probe: $U in org $OWNER (fallback) → HTTP $ORG_CODE"
+    if [ "$ORG_CODE" = "204" ]; then
+      for T in "${!TEAM_ID[@]}"; do
+        APPROVER_TEAMS[$U]="${APPROVER_TEAMS[$U]:- } ${APPROVER_TEAMS[$U]:+ }$T "
+      done
+      debug "$U credited as org member for all queried teams (fallback — token may lack read:org)"
+    fi
+  fi
+done
+
+# 7. Evaluate the tier expression.
+#
+# legacy OR-gate: use the simplified loop from before internal#189.
+if [ -n "${LEGACY_ELIGIBLE:-}" ]; then
+  OK=""
+  for _u in "${!APPROVER_TEAMS[@]}"; do
+    for _t2 in $LEGACY_ELIGIBLE; do
+      case "${APPROVER_TEAMS[$_u]}" in
+        *${_t2}*)
+          echo "::notice::approver $_u is in team $_t2 (eligible for $TIER)"
+          OK="yes"
+          break
+        ;;
+      esac
+    done
+    [ -n "$OK" ] && break
+  done
+  if [ -z "$OK" ]; then
+    echo "::error::Tier $TIER requires approval from a non-author member of {$LEGACY_ELIGIBLE}. Set SOP_DEBUG=1 to see per-probe HTTP codes."
+    exit 1
+  fi
+  echo "::notice::sop-tier-check passed: $TIER (legacy OR-gate)"
+  exit 0
+fi
+
+# AND-gate: evaluate the expression clause by clause.
+# _passed_clauses and _failed_clauses accumulate for the status description.
+_passed_clauses=""
+_failed_clauses=""
+
+for _raw_clause in $EXPR; do
+  # Normalise: strip parens, replace commas with spaces so bash word-split
+  # can iterate the OR-set members. The previous form
+  #   _clause=$(echo ... | tr ',' '\n' | tr -d '[:space:]' | grep -v '^$')
+  # collapsed every member into one concatenated token because
+  # `tr -d '[:space:]'` strips the very newlines that just separated them
+  # ("engineers,managers,ceo" -> "engineersmanagersceo"), so the OR-clause
+  # only ever evaluated as a single nonsense team name and never matched
+  # APPROVER_TEAMS. Fixed in #229: leave the comma-separated members as
+  # space-separated tokens for `for _t in $_clause`.
+  _no_parens=${_raw_clause//[()]/}
+  _clause=${_no_parens//,/ }
+  _clause_passed="no"
+  _clause_names=""
+  for _t in $_clause; do
+    # Append (don't overwrite) team name to the human-readable accumulator.
+    # The previous form `_clause_names="${_clause_names:+, }${_t}"`
+    # rewrote the variable on every iteration, so the FAIL message only
+    # ever showed the LAST team. Fixed: prepend prior value before the
+    # comma-separator, then append the new team name.
+    _clause_names="${_clause_names}${_clause_names:+, }${_t}"
+    # Skip teams not yet in Gitea (qa??? / security??? placeholders).
+    [[ "$_t" == *"???" ]] && debug "clause \"$_t\": skipped (team pending creation)" && continue
+    [ -z "${TEAM_ID[$_t]:-}" ] && debug "clause \"$_t\": no ID resolved, skipping" && continue
+    for _u in "${!APPROVER_TEAMS[@]}"; do
+      # Note: APPROVER_TEAMS values are space-surrounded (e.g. " managers ").
+      # Pattern *${_t}* matches team name anywhere in the space-padded string.
+      case "${APPROVER_TEAMS[$_u]}" in
+        *${_t}*)
+          _clause_passed="yes"
+          debug "clause \"$_t\": satisfied by $_u"
+          break
+        ;;
+      esac
+    done
+  done
+
+  # Label for display: strip "???" from pending teams.
+  _label=$(echo "$_raw_clause" | tr -d '()' | tr ',' '/' | tr -d '[:space:]' | sed 's/???//g')
+
+  if [ "$_clause_passed" = "yes" ]; then
+    # Append (don't overwrite) — same accumulator bug as _clause_names above.
+    _passed_clauses="${_passed_clauses}${_passed_clauses:+, }$_label"
+    echo "::notice::clause [$_label]: PASS — satisfied by approving reviewer(s)"
+  else
+    _failed_clauses="${_failed_clauses}${_failed_clauses:+, }$_label"
+    echo "::error::clause [$_label]: FAIL — no approving reviewer belongs to any of these teams (${_clause_names}). Set SOP_DEBUG=1 to see per-team probe results."
+  fi
+done
+
+if [ -n "$_failed_clauses" ]; then
+  echo ""
+  echo "::error::sop-tier-check FAILED for $TIER."
+  echo "  Passed :${_passed_clauses}"
+  echo "  Missing:${_failed_clauses}"
+  echo "  All clauses must be satisfied. Each missing team needs an APPROVED review from one of its members."
+  exit 1
+fi
+
+echo "::notice::sop-tier-check PASSED: $TIER — all required clauses satisfied [${_passed_clauses}]"

.gitea/scripts/tests/test_sop_tier_check_clause_split.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# Regression test for #229 — sop-tier-check tier:low OR-clause splitter.
+#
+# Bug (PR #225 → still broken after PR #231):
+#   Line ~289 of sop-tier-check.sh used:
+#     _clause=$(echo "$_raw_clause" | tr -d '()' | tr ',' '\n' | tr -d '[:space:]' | grep -v '^$')
+#   `tr -d '[:space:]'` strips the newlines that `tr ',' '\n'` just
+#   inserted, collapsing "engineers,managers,ceo" into a single token
+#   "engineersmanagersceo". The for-loop then iterates ONCE on a name
+#   that matches no team, so every tier:low PR fails:
+#     ::error::clause [engineers/managers/ceo]: FAIL — no approving
+#     reviewer belongs to any of these teamsengineersmanagersceo
+#   (note also: missing separators in the error string is bug #2 —
+#    `_clause_names` used "${var:+, }$x" which OVERWRITES per iteration).
+#
+# Fix shape (this PR):
+#   _no_parens=${_raw_clause//[()]/}
+#   _clause=${_no_parens//,/ }    # comma -> space, bash word-split iterates
+#   _clause_names="${_clause_names}${_clause_names:+, }${_t}"  # APPEND, not overwrite
+#
+# This test extracts the splitter logic and asserts it produces the right
+# token list for each of the three tier expressions live in the script.
+
+set -euo pipefail
+
+PASS=0
+FAIL=0
+
+assert_eq() {
+  local label="$1"
+  local expected="$2"
+  local got="$3"
+  if [ "$expected" = "$got" ]; then
+    echo "  PASS  $label"
+    PASS=$((PASS + 1))
+  else
+    echo "  FAIL  $label"
+    echo "        expected: <$expected>"
+    echo "        got:      <$got>"
+    FAIL=$((FAIL + 1))
+  fi
+}
+
+# ----- Splitter under test (mirrors the fixed sop-tier-check.sh block) -----
+split_clause() {
+  local raw="$1"
+  local no_parens=${raw//[()]/}
+  local clause=${no_parens//,/ }
+  local out=""
+  for _t in $clause; do
+    out="${out}${out:+|}$_t"
+  done
+  echo "$out"
+}
+
+echo "test: tier:low OR-clause splits to 3 tokens"
+assert_eq "tier:low" "engineers|managers|ceo" "$(split_clause "engineers,managers,ceo")"
+
+echo "test: tier:medium AND-expression — bash word-split on \$EXPR yields 5 tokens"
+EXPR="managers AND engineers AND qa???,security???"
+out=""
+for _raw in $EXPR; do
+  out="${out}${out:+ ; }$(split_clause "$_raw")"
+done
+assert_eq "tier:medium" "managers ; AND ; engineers ; AND ; qa???|security???" "$out"
+
+echo "test: tier:high single-team OR-clause"
+assert_eq "tier:high" "ceo" "$(split_clause "ceo")"
+
+echo "test: paren-wrapped OR-set unwraps + splits"
+assert_eq "paren OR" "managers|ceo" "$(split_clause "(managers,ceo)")"
+
+# ----- _clause_names accumulator (was overwriting per iteration) -----
+acc=""
+for t in engineers managers ceo; do
+  acc="${acc}${acc:+, }${t}"
+done
+assert_eq "_clause_names append" "engineers, managers, ceo" "$acc"
+
+# ----- _failed_clauses / _passed_clauses accumulator across raw clauses -----
+acc=""
+for c in clauseA clauseB clauseC; do
+  acc="${acc}${acc:+, }${c}"
+done
+assert_eq "_failed_clauses append" "clauseA, clauseB, clauseC" "$acc"
+
+# ----- End-to-end OR-gate: simulate APPROVER_TEAMS[core-lead]=' managers ' -----
+# The script's case pattern is *${_t}* with a space-padded value.
+APPROVER_TEAMS_VAL=" managers "
+matched=""
+for _t in $(split_clause "engineers,managers,ceo" | tr '|' ' '); do
+  case "$APPROVER_TEAMS_VAL" in
+    *${_t}*) matched="$_t"; break ;;
+  esac
+done
+assert_eq "OR-gate matches managers" "managers" "$matched"
+
+echo
+echo "------"
+echo "PASS=$PASS FAIL=$FAIL"
+[ "$FAIL" -eq 0 ]

.gitea/workflows/audit-force-merge.yml

@@ -0,0 +1,88 @@
+# audit-force-merge — emit `incident.force_merge` to the runner log when
+# a PR is merged with required-status checks NOT all green. Vector picks
+# the JSON line off docker_logs and ships to Loki on
+# molecule-canonical-obs (per `reference_obs_stack_phase1`); query as:
+#
+#   {host="operator"} |= "event_type" |= "incident.force_merge" | json
+#
+# Companion to `audit-force-merge.sh` (script-extract pattern, same as
+# sop-tier-check). The audit observes BOTH UI-merged and REST-merged PRs
+# uniformly per `feedback_gh_cli_merge_lies_use_rest`.
+#
+# Closes the §SOP-6 audit gap for the molecule-core repo. RFC:
+# internal#219 §6. Mirrors the same-named workflow in
+# molecule-controlplane; design rationale lives in the RFC, not here,
+# to keep the workflow file scannable.
+
+name: audit-force-merge
+
+# pull_request_target loads from the base branch — same security model
+# as sop-tier-check. Without this, a PR author could rewrite the
+# workflow on their own PR and skip the audit emission for their own
+# force-merge. The base-branch checkout below ALSO uses
+# `base.sha`, not `base.ref`, so a fast-moving base can't slip a
+# different audit script in under us.
+on:
+  pull_request_target:
+    types: [closed]
+
+# `pull-requests: read` + `contents: read` covers everything the script
+# needs (fetch PR + commit statuses). `issues:` deliberately omitted —
+# audit fires-and-forgets to stdout, never opens issues.
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    # Skip when PR is closed without merge — saves a runner.
+    if: github.event.pull_request.merged == true
+    steps:
+      - name: Check out base branch (for the script)
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          # base.sha pinning, NOT base.ref — see header rationale.
+          ref: ${{ github.event.pull_request.base.sha }}
+      - name: Detect force-merge + emit audit event
+        env:
+          # Same org-level secret the sop-tier-check workflow uses;
+          # falls back to the auto-injected GITHUB_TOKEN if the
+          # org-level SOP_TIER_CHECK_TOKEN isn't set on a transitional
+          # repo.
+          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          # Required-status-check contexts to evaluate at merge time.
+          # Newline-separated. MUST mirror branch protection's
+          # status_check_contexts for protected branches
+          # (currently `main`; `staging` protection forthcoming per
+          # RFC internal#219 Phase 4).
+          #
+          # Initialized 2026-05-11 from the current molecule-core `main`
+          # branch protection:
+          #
+          #   GET /api/v1/repos/molecule-ai/molecule-core/
+          #       branch_protections/main
+          #   → status_check_contexts = [
+          #       "Secret scan / Scan diff for credential-shaped strings (pull_request)",
+          #       "sop-tier-check / tier-check (pull_request)"
+          #     ]
+          #
+          # Declared here rather than fetched from /branch_protections
+          # because that endpoint requires admin write — sop-tier-bot
+          # is read-only by design (least-privilege per
+          # `feedback_least_privilege_via_workflow_env` / internal#257).
+          # Drift between this env and the real protection list is
+          # auto-detected by `ci-required-drift.yml` (RFC §4 + §6),
+          # which opens a `[ci-drift]` issue within one hour.
+          #
+          # When the protection set changes (e.g. Phase 4 adds the
+          # `ci / all-required (pull_request)` sentinel), update BOTH
+          # branch protection AND this env in the SAME PR; drift-detect
+          # will otherwise file an issue for you.
+          REQUIRED_CHECKS: |
+            Secret scan / Scan diff for credential-shaped strings (pull_request)
+            sop-tier-check / tier-check (pull_request)
+        run: bash .gitea/scripts/audit-force-merge.sh

.gitea/workflows/block-internal-paths.yml

@@ -0,0 +1,148 @@
+name: Block internal-flavored paths
+
+# Ported from .github/workflows/block-internal-paths.yml on 2026-05-11 per
+# RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `merge_group: { types: [checks_requested] }` (Gitea has no
+#     merge queue; no `gh-readonly-queue/...` refs).
+#   - Workflow-level env.GITHUB_SERVER_URL set per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on the job (RFC §1 contract — surface
+#     defects without blocking; follow-up PR flips after triage).
+#
+# Hard CI gate. Internal content (positioning, competitive briefs, sales
+# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
+# this public monorepo must never re-acquire those paths. CEO directive
+# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
+#
+# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
+# briefs into the easiest path their cwd resolves to (root /research,
+# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
+# or a stale gitignore line. This workflow is the mechanical backstop.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches: [main, staging]
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  check:
+    name: Block forbidden paths
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 2  # need previous commit to diff against on push events
+
+      # For pull_request events the diff base is github.event.pull_request.base.sha,
+      # which may be many commits behind HEAD and therefore absent from the
+      # shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
+      - name: Fetch PR base SHA (pull_request events only)
+        if: github.event_name == 'pull_request'
+        run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
+
+      - name: Refuse if forbidden paths appear
+        env:
+          # Plumb event-specific SHAs through env so the script doesn't
+          # need conditional `${{ ... }}` interpolation per event type.
+          # github.event.before/after only exist on push events;
+          # pull_request has pull_request.base.sha / pull_request.head.sha.
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PUSH_BEFORE: ${{ github.event.before }}
+          PUSH_AFTER: ${{ github.event.after }}
+        run: |
+          # Paths that must NEVER live in the public monorepo. Add to this
+          # list narrowly — broader patterns belong in .gitignore so day-to-day
+          # docs work isn't accidentally blocked.
+          FORBIDDEN_PATTERNS=(
+            "^research/"
+            "^marketing/"
+            "^docs/marketing/"
+            "^comment-[0-9]+\.json$"
+            "^test-pmm.*\.(txt|md)$"
+            "^tick-reflections.*\.(txt|md)$"
+            ".*-temp\.(md|txt)$"
+          )
+
+          # Determine the diff base. Each event type stores its SHAs in
+          # a different place — see the env block above.
+          case "${{ github.event_name }}" in
+            pull_request)
+              BASE="$PR_BASE_SHA"
+              HEAD="$PR_HEAD_SHA"
+              ;;
+            *)
+              BASE="$PUSH_BEFORE"
+              HEAD="$PUSH_AFTER"
+              ;;
+          esac
+
+          # On push events with shallow clones, BASE may be present in
+          # the event payload but absent from the local object DB
+          # (fetch-depth=2 doesn't always reach the previous commit
+          # across true merges). Try fetching it on demand. If the
+          # fetch fails — e.g. the SHA was force-overwritten — we fall
+          # through to the empty-BASE branch below, which scans the
+          # entire tree as if every file were new. Correct, just slow.
+          if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
+            if ! git cat-file -e "$BASE" 2>/dev/null; then
+              git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+            fi
+          fi
+
+          # Files added or modified in this change.
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
+            # New branch / no previous SHA / BASE unreachable — check
+            # the entire tree as if every file were new. Slower but
+            # correct on first push or post-fetch-failure recovery.
+            CHANGED=$(git ls-tree -r --name-only HEAD)
+          else
+            CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
+          fi
+
+          if [ -z "$CHANGED" ]; then
+            echo "No changed files to inspect."
+            exit 0
+          fi
+
+          OFFENDING=""
+          for path in $CHANGED; do
+            for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
+              if echo "$path" | grep -qE "$pattern"; then
+                OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
+                break
+              fi
+            done
+          done
+
+          if [ -n "$OFFENDING" ]; then
+            echo "::error::Forbidden internal-flavored paths detected:"
+            printf "$OFFENDING"
+            echo ""
+            echo "These paths belong in molecule-ai/internal, not this public repo."
+            echo "See docs/internal-content-policy.md for canonical locations."
+            echo ""
+            echo "If your file is genuinely public-facing (e.g. a blog post"
+            echo "ready to ship), use one of these alternatives instead:"
+            echo "  - Public-bound blog posts:  docs/blog/<slug>.md"
+            echo "  - Public-bound tutorials:   docs/tutorials/<slug>.md"
+            echo "  - Public devrel content:    docs/devrel/<slug>.md"
+            echo ""
+            echo "If you legitimately need to add a new top-level path that"
+            echo "happens to match a forbidden pattern, edit"
+            echo ".gitea/workflows/block-internal-paths.yml and update the"
+            echo "FORBIDDEN_PATTERNS list with reviewer signoff."
+            exit 1
+          fi
+
+          echo "OK No forbidden paths in this change."

.gitea/workflows/canary-staging.yml

@@ -0,0 +1,310 @@
+name: Canary — staging SaaS smoke (every 30 min)
+
+# Ported from .github/workflows/canary-staging.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Minimum viable health check: provisions one Hermes workspace on a fresh
+# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
+# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
+# issue on the next green run.
+#
+# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
+# but runs only on provisioning-critical pushes + nightly — this one
+# catches drift in the 30-min window between those runs (AMI health, CF
+# cert rotation, WorkOS session stability, etc.).
+#
+# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
+# peers/activity checks. One parent workspace + one A2A turn is enough
+# to signal "SaaS stack end-to-end is alive."
+
+on:
+  schedule:
+    # Every 30 min. Cron on GitHub-hosted runners has a known drift of
+    # a few minutes under load — that's fine for a canary.
+    - cron: '*/30 * * * *'
+# Serialise with the full-SaaS workflow so they don't contend for the
+# same org-create quota on staging. Different group key from
+# e2e-staging-saas since we don't mind queueing canaries behind one
+# full run, but two canaries SHOULD queue against each other.
+concurrency:
+  group: canary-staging
+  cancel-in-progress: false
+
+permissions:
+  # Needed to open / close the alerting issue.
+  issues: write
+  contents: read
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  canary:
+    name: Canary smoke
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    # 25 min headroom over the 15-min TLS-readiness deadline in
+    # tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
+    # the job is killed at the wall-clock 15:00 mark BEFORE the bash
+    # `fail` + diagnostic burst can fire, leaving every cancellation
+    # silent. Sibling staging E2E jobs run at 20-45 min — keeping
+    # canary tighter than them so a true wedge still surfaces here
+    # first.
+    timeout-minutes: 25
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      # MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
+      # Switched from hermes+OpenAI after #2578 (the staging OpenAI key
+      # account went over quota and stayed dead for 36+ hours, taking
+      # the canary red the entire time). claude-code template's
+      # `minimax` provider routes ANTHROPIC_BASE_URL to
+      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
+      # ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
+      # billing account, so OpenAI quota collapse no longer wedges the
+      # canary. Mirrors the migration continuous-synth-e2e.yml made on
+      # 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
+      # full_saas.sh branches SECRETS_JSON on which key is present —
+      # MiniMax wins when set.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so an operator-dispatched run with
+      # E2E_RUNTIME=hermes overridden via workflow_dispatch can still
+      # exercise the OpenAI path without re-editing the workflow.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
+      E2E_MODE: canary
+      E2E_RUNTIME: claude-code
+      # Pin the canary to a specific MiniMax model rather than relying
+      # on the per-runtime default (which could resolve to "sonnet" →
+      # direct Anthropic and defeat the cost saving). M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
+      E2E_RUN_ID: "canary-${{ github.run_id }}"
+      # Debug-only: when an operator dispatches with keep_on_failure=true,
+      # the canary script's E2E_KEEP_ORG=1 path skips teardown so the
+      # tenant org + EC2 stay alive for SSM-based log capture. Cron runs
+      # never set this (the input only exists on workflow_dispatch) so
+      # unattended cron always tears down. See molecule-core#129
+      # failure mode #1 — capturing the actual exception requires
+      # docker logs from the live container.
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
+            exit 2
+          fi
+
+      - name: Verify LLM key present
+        run: |
+          # Per-runtime key check — claude-code uses MiniMax; hermes /
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
+          # rather than soft-skip per the lesson from synth E2E #2578:
+          # an empty key silently falls through to the wrong
+          # SECRETS_JSON branch and the canary fails 5 min later with
+          # a confusing auth error instead of the clean "secret
+          # missing" message at the top.
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              # Either MiniMax OR direct-Anthropic works — first
+              # non-empty wins in the test script's secrets-injection
+              # priority chain. Operators only need to set ONE of these
+              # secrets; we don't force a choice between them.
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
+            exit 2
+          fi
+          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
+
+      - name: Canary run
+        id: canary
+        run: bash tests/e2e/test_staging_full_saas.sh
+
+      # Alerting: open a sticky issue on the FIRST failure; comment on
+      # subsequent failures; auto-close on next green. Comment-on-existing
+      # de-duplicates so a single open issue accumulates the streak —
+      # ops sees one issue with N comments rather than N issues.
+      #
+      # Why no consecutive-failures threshold (e.g., wait 3 runs before
+      # filing): the prior threshold check used
+      # `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
+      # not expose (returns 404). On Gitea Actions the threshold call
+      # ALWAYS failed, breaking the entire alerting step and going days
+      # silent on real regressions (38h+ chronic red on 2026-05-07/08
+      # before this fix; tracked in molecule-core#129). Filing on first
+      # failure is also better UX — we want to know about the first red,
+      # not wait 90 min for it to "count." Real flakes get one issue +
+      # a quick close-on-green; persistent reds accumulate comments.
+      - name: Open issue on failure (Gitea API)
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          API="${SERVER_URL%/}/api/v1"
+          TITLE="Canary failing: staging SaaS smoke"
+          RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
+
+          EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
+            "${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
+            | jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
+
+          if [ -n "$EXISTING" ]; then
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${EXISTING}/comments" \
+              -d "$(jq -nc --arg run "$RUN_URL" '{body: ("Canary still failing. " + $run)}')" >/dev/null
+            echo "Commented on existing issue #${EXISTING}"
+          else
+            NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+            BODY=$(jq -nc --arg t "$TITLE" --arg now "$NOW" --arg run "$RUN_URL" \
+              '{title: $t, body: ("Canary run failed at " + $now + ".\n\nRun: " + $run + "\n\nThis issue auto-closes on the next green canary run. Consecutive failures add a comment here rather than a new issue.")}')
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues" -d "$BODY" >/dev/null
+            echo "Opened canary failure issue (first red)"
+          fi
+
+      - name: Auto-close canary issue on success (Gitea API)
+        if: success()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          API="${SERVER_URL%/}/api/v1"
+          TITLE="Canary failing: staging SaaS smoke"
+
+          NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
+            "${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
+            | jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number')
+
+          NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          for N in $NUMS; do
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${N}/comments" \
+              -d "$(jq -nc --arg now "$NOW" '{body: ("Canary recovered at " + $now + ". Closing.")}')" >/dev/null
+            curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
+            echo "Closed recovered canary issue #${N}"
+          done
+
+      - name: Teardown safety net
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          # Slug prefix matches what test_staging_full_saas.sh emits
+          # in canary mode:
+          #   SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
+          # Earlier this was `e2e-{today}-canary-` — that was the
+          # full-mode pattern (date FIRST, mode SECOND); canary slugs
+          # have mode FIRST, date SECOND. The mismatch silently
+          # never matched, leaving every cancelled-canary EC2 alive
+          # until the once-an-hour sweep eventually caught it
+          # (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
+          # cleanup; same gap on three earlier cancellations today).
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # Scope to slugs from THIS canary run when GITHUB_RUN_ID is
+          # available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
+          # so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
+          # full-mode safety net's per-run scoping (e2e-staging-saas.yml)
+          # added after the 2026-04-21 cross-run cleanup incident.
+          # Sweep both today AND yesterday's UTC dates so a run that
+          # crosses midnight still cleans up its own slug — see the
+          # 2026-04-26→27 canvas-safety-net incident.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if run_id:
+              prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
+          else:
+              prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
+          candidates = [o['slug'] for o in d.get('orgs', [])
+                        if any(o.get('slug','').startswith(p) for p in prefixes)
+                        and o.get('status') not in ('purged',)]
+          print('\n'.join(candidates))
+          " 2>/dev/null)
+          # Per-slug DELETE with HTTP-code verification. The previous
+          # `... >/dev/null || true` swallowed every failure, so a 5xx
+          # or timeout from CP looked identical to "successfully cleaned
+          # up" and the tenant kept eating ~2 vCPU until the hourly
+          # stale sweep caught it (up to 2h later). Now we capture the
+          # response code and surface non-2xx as a workflow warning, so
+          # the run page shows which slug leaked. We still don't `exit 1`
+          # on cleanup failure — a single-canary cleanup miss shouldn't
+          # fail-flag the canary itself when the actual smoke check
+          # passed. The sweep-stale-e2e-orgs cron (now every 15 min,
+          # 30-min threshold) is the safety net for whatever slips past.
+          # See molecule-controlplane#420.
+          leaks=()
+          for slug in $orgs; do
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
+            set -e
+            code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
+          done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
+          exit 0

.gitea/workflows/canary-verify.yml

@@ -0,0 +1,276 @@
+name: canary-verify
+
+# Ported from .github/workflows/canary-verify.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#   - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
+#     for the `workflow_run` event is partial. If this never fires on a
+#     real publish-workspace-server-image completion, the follow-up
+#     triage PR should replace the trigger with a push-with-paths-filter
+#     on the same publish workflow's path (i.e. `.gitea/workflows/publish-workspace-server-image.yml`).
+#
+
+# Runs the canary smoke suite against the staging canary tenant fleet
+# after a new :staging-<sha> image lands in ECR. On green, calls the
+# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
+# the prod tenant fleet's 5-minute auto-updater picks up the verified
+# digest. On red, :latest stays on the prior known-good digest and
+# prod is untouched.
+#
+# Registry note (2026-05-10): This workflow previously used GHCR
+# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
+# during the 2026-05-06 Gitea suspension migration when publish-
+# workspace-server-image.yml switched to the operator's ECR org
+# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
+# platform-tenant). The GHCR → ECR migration was never applied to
+# this file, so canary-verify was silently smoke-testing the stale
+# GHCR image while the actual staging/prod tenants ran the ECR image.
+# Result: smoke tests could not catch a broken ECR build. Fix:
+#   - Wait step: reads SHA from running canary /health (tenant-
+#     agnostic, works regardless of registry).
+#   - Promote step: calls CP redeploy-fleet endpoint with target_tag=
+#     staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
+#     No longer attempts GHCR crane ops.
+#
+# Dependencies:
+#   - publish-workspace-server-image.yml publishes :staging-<sha>
+#     to ECR on staging and main merges.
+#   - Canary tenants are configured to pull :staging-<sha> from ECR
+#     (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
+#   - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
+#     CANARY_CP_SHARED_SECRET are populated.
+
+on:
+  workflow_run:
+    workflows: ["publish-workspace-server-image"]
+    types: [completed]
+permissions:
+  contents: read
+  packages: write
+  actions: read
+
+env:
+  # ECR registry (post-2026-05-06 SSOT for tenant images).
+  # publish-workspace-server-image.yml pushes here.
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
+  TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
+  # CP endpoint for redeploy-fleet (used in promote step below).
+  CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  canary-smoke:
+    # Skip when the upstream workflow failed — no image to test against.
+    # workflow_dispatch trigger dropped in this Gitea port; only the
+    # workflow_run path remains.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    outputs:
+      sha: ${{ steps.compute.outputs.sha }}
+      smoke_ran: ${{ steps.smoke.outputs.ran }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Compute sha
+        id: compute
+        run: echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
+      - name: Wait for canary tenants to pick up :staging-<sha>
+        # Poll canary health endpoints every 30s for up to 7 min instead
+        # of a fixed 6-min sleep. Exits as soon as ALL canaries report
+        # the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
+        # proceeding after 7 min even if not all canaries responded —
+        # the smoke suite will catch any that didn't update.
+        #
+        # NOTE: The SHA is read from the running tenant's /health response,
+        # NOT from a registry lookup. This is registry-agnostic and works
+        # regardless of whether the tenant pulls from ECR, GHCR, or any
+        # other registry — the canary is telling us what it's actually
+        # running, which is the ground truth for smoke testing.
+        env:
+          CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
+          EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
+        run: |
+          if [ -z "$CANARY_TENANT_URLS" ]; then
+            echo "No canary URLs configured — falling back to 60s wait"
+            sleep 60
+            exit 0
+          fi
+          IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
+          MAX_WAIT=420  # 7 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $MAX_WAIT ]; do
+            ALL_READY=true
+            for url in "${URLS[@]}"; do
+              HEALTH=$(curl -s --max-time 5 "${url}/health" 2>/dev/null || echo "{}")
+              SHA=$(echo "$HEALTH" | grep -o "\"sha\":\"[^\"]*\"" | head -1 | cut -d'"' -f4)
+              if [ "$SHA" != "$EXPECTED_SHA" ]; then
+                ALL_READY=false
+                break
+              fi
+            done
+            if $ALL_READY; then
+              echo "All canaries running staging-${EXPECTED_SHA} after ${ELAPSED}s"
+              exit 0
+            fi
+            echo "Waiting for canaries... (${ELAPSED}s / ${MAX_WAIT}s)"
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
+
+      - name: Run canary smoke suite
+        id: smoke
+        # Graceful-skip when no canary fleet is configured (Phase 2 not yet
+        # stood up — see molecule-controlplane/docs/canary-tenants.md).
+        # Sets `ran=false` on skip so promote-to-latest stays off (we don't
+        # want every main merge auto-promoting without gating). Manual
+        # promote-latest.yml is the release gate while canary is absent.
+        # Once the fleet is real: delete the early-exit branch.
+        env:
+          CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
+          CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
+          CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
+          CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
+        run: |
+          set -euo pipefail
+          if [ -z "${CANARY_TENANT_URLS:-}" ] \
+            || [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
+            || [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
+            {
+              echo "## ⚠️ canary-verify skipped"
+              echo
+              echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
+              echo "Phase 2 canary fleet has not been stood up yet —"
+              echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
+              echo
+              echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "ran=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::canary-verify: skipped — no canary fleet configured"
+            exit 0
+          fi
+          bash scripts/canary-smoke.sh
+          echo "ran=true" >> "$GITHUB_OUTPUT"
+
+      - name: Summary on failure
+        if: ${{ failure() }}
+        run: |
+          {
+            echo "## Canary smoke FAILED"
+            echo
+            echo "Canary tenants rejected image \`staging-${{ steps.compute.outputs.sha }}\`."
+            echo ":latest stays pinned to the prior good digest — prod is untouched."
+            echo
+            echo "Fix forward and merge again, or investigate the specific failed"
+            echo "assertions in the canary-smoke step log above."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  promote-to-latest:
+    # On green, calls the CP redeploy-fleet endpoint with target_tag=
+    # staging-<sha> to promote the verified ECR image. This is the same
+    # mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
+    #
+    # Pre-fix history: the old GHCR promote step used `crane tag` against
+    # ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
+    # image.yml had already migrated to ECR on 2026-05-07 (commit
+    # 10e510f5). The GHCR tags were never updated, so this step was
+    # silently promoting a stale GHCR image while actual prod tenants
+    # pulled from ECR. Canary smoke tests were GHCR-targeted and could
+    # not catch a broken ECR build.
+    needs: canary-smoke
+    if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    env:
+      SHA: ${{ needs.canary-smoke.outputs.sha }}
+      CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
+      # CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
+      # Stored at the repo level so all workflows pick it up automatically.
+      CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+      # canary_slug pin: deploy the verified :staging-<sha> to the canary
+      # first (soak 120s), then fan out to the rest of the fleet.
+      CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
+      SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
+      BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
+    steps:
+      - name: Check CP credentials
+        run: |
+          if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
+            echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
+            echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
+            exit 1
+          fi
+
+      - name: Promote verified ECR image to :latest
+        run: |
+          set -euo pipefail
+
+          TARGET_TAG="staging-${SHA}"
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --argjson soak "${SOAK_SECONDS:-120}" \
+            --argjson batch "${BATCH_SIZE:-3}" \
+            --argjson dry false \
+            '{
+              target_tag: $tag,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          if [ -n "${CANARY_SLUG:-}" ]; then
+            BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
+          fi
+
+          echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  target_tag: $TARGET_TAG"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          CURL_EXIT=$?
+          set -e
+
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
+            exit 1
+          fi
+
+      - name: Summary
+        run: |
+          {
+            echo "## Canary verified — :latest promoted via CP redeploy-fleet"
+            echo ""
+            echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
+            echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
+            echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
+            echo "- **Batch size:** ${BATCH_SIZE:-3}"
+            echo ""
+            echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
+            echo "The fleet's 5-minute health-check loop will pick up the update automatically."
+          } >> "$GITHUB_STEP_SUMMARY"

.gitea/workflows/cascade-list-drift-gate.yml

@@ -0,0 +1,58 @@
+name: cascade-list-drift-gate
+
+# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - on.paths reference .gitea/workflows/publish-runtime.yml (the active
+#     Gitea workflow file) instead of .github/workflows/publish-runtime.yml
+#     (which Category A of this sweep deletes).
+#   - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
+#     .gitea/ workflow (the script's default is still .github/... which
+#     will not exist post-Cat-A).
+#   - Workflow-level env.GITHUB_SERVER_URL set per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on the job (RFC §1 contract — surface
+#     defects without blocking; follow-up PR flips after triage).
+#
+# Structural gate: TEMPLATES list in publish-runtime.yml must match
+# manifest.json's workspace_templates exactly. Closes the recurrence
+# path of PR #2556 (the data fix) and is the first concrete deliverable
+# of RFC #388 PR-3.
+#
+# Triggers narrowly to keep CI quiet: only on PRs that actually change
+# one of the two files. The path-filtered split + always-emit-result
+# pattern (memory: "Required check names need a job that always runs")
+# is unnecessary here because the workflow IS the check name and PR
+# branch protection should require it directly. Future-proof: if this
+# becomes a required check, add a no-op aggregator with always() so the
+# name still emits when paths don't match.
+
+on:
+  pull_request:
+    branches: [staging, main]
+    paths:
+      - manifest.json
+      - .gitea/workflows/publish-runtime.yml
+      - scripts/check-cascade-list-vs-manifest.sh
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Check cascade list matches manifest
+        # Pass the .gitea/ workflow path explicitly — the script's
+        # default still points at .github/... which Category A of this
+        # sweep removes.
+        run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml

.gitea/workflows/check-migration-collisions.yml

@@ -0,0 +1,74 @@
+name: Check migration collisions
+
+# Ported from .github/workflows/check-migration-collisions.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - on.paths includes .gitea/workflows/check-migration-collisions.yml
+#     (this file) instead of the .github/ one.
+#   - Workflow-level env.GITHUB_SERVER_URL pinned to https://git.moleculesai.app
+#     so scripts/ops/check_migration_collisions.py can derive the Gitea API
+#     base (the script already supports this; see _gitea_api_url()).
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# Hard gate (#2341): fails a PR that adds a migration prefix already
+# claimed by the base branch or another open PR. Caught manually 2026-04-30
+# during PR #2276 rebase: 044_runtime_image_pins collided with
+# 044_platform_inbound_secret from RFC #2312. This workflow makes that
+# check automatic.
+#
+# Trigger model: pull_request only — there's no value running this on
+# pushes to staging or main (those are post-merge; the gate must fire
+# pre-merge to be useful). Path filter scopes to PRs that actually touch
+# migrations.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'workspace-server/migrations/**'
+      - 'scripts/ops/check_migration_collisions.py'
+      - '.gitea/workflows/check-migration-collisions.yml'
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+permissions:
+  contents: read
+  # API needs read access to other PRs to detect cross-PR collisions
+  pull-requests: read
+
+jobs:
+  check:
+    name: Migration version collision check
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    continue-on-error: true
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Need history to diff against base ref
+          fetch-depth: 0
+
+      - name: Detect collisions
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: origin/${{ github.event.pull_request.base.ref }}
+          HEAD_REF: ${{ github.event.pull_request.head.sha }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          # Auto-injected; Gitea aliases this for in-repo API access.
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Ensure the named base ref exists locally. checkout@v4 with
+          # fetch-depth=0 pulls full history, but the explicit fetch is
+          # cheap insurance against form-of-ref differences across runs.
+          #
+          # IMPORTANT: do NOT pass --depth=1 here. The script below uses
+          # `git diff origin/<base>...<head>` (three-dot, merge-base form),
+          # which fails with "fatal: no merge base" if the base ref is
+          # shallow.
+          git fetch origin "${{ github.event.pull_request.base.ref }}" || true
+          python3 scripts/ops/check_migration_collisions.py

.gitea/workflows/ci-required-drift.yml

@@ -0,0 +1,107 @@
+# ci-required-drift — hourly sentinel for drift between the canonical
+# "what counts as required" sources of truth in this repo:
+#
+#   1. `.gitea/workflows/ci.yml` jobs                       (CI source)
+#   2. `branch_protections/{main,staging}.status_check_contexts`
+#                                                           (protection)
+#   3. `.gitea/workflows/audit-force-merge.yml` REQUIRED_CHECKS env
+#                                                           (audit env)
+#
+# RFC: internal#219 §4 (jobs ↔ protection) + §6 (audit env ↔ protection).
+# Ported verbatim-then-adapted from molecule-controlplane PR#112
+# (SHA 0adf2098) per RFC internal#219 Phase 2b+c — replicate repo-by-repo.
+#
+# When any pair diverges, a `[ci-drift]` issue is opened or updated
+# (idempotent by title) and labelled `tier:high`. This is the
+# auto-detection that closes the regression class identified in
+# RFC §1 finding 3 (protection only listed 2 of 6 real jobs for
+# ~weeks, undetected) and §6 (audit env drifts silently from
+# protection).
+#
+# Diff logic lives in `.gitea/scripts/ci-required-drift.py`. The
+# Python file does YAML AST parsing + `needs:` graph walking per
+# `feedback_behavior_based_ast_gates` — NOT grep-by-name. That way
+# job renames or matrix-expansion-induced churn produce honest signal.
+#
+# IMPORTANT — TRANSITIONAL STATE: molecule-core's ci.yml does NOT yet
+# contain the `all-required` sentinel job (RFC §4 Phase 4 adds it).
+# Until Phase 4 lands the detector will hard-fail with exit 3 on the
+# missing sentinel. That's intentional: a red workflow on a 5-min cron
+# is louder than a silent issue and forces Phase 4 to land soon.
+
+name: ci-required-drift
+
+# IMPORTANT — Gitea 1.22.6 parser quirk per
+# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
+# `inputs:` block here, even though stock GitHub Actions allows it.
+# Gitea 1.22.6 flattens `workflow_dispatch.inputs.X` into a sibling of
+# the `on:` event keys and rejects the entire workflow as
+# "unknown on type". The whole file then registers for ZERO events
+# (no schedule, no dispatch). When Gitea ≥ 1.23 lands fleet-wide,
+# this constraint can be revisited.
+on:
+  schedule:
+    # Hourly at :17 — offset from :00 to spread load away from the
+    # peak when N cron workflows fire on the hour-boundary, per
+    # RFC §4 cadence ("off-zero").
+    - cron: '17 * * * *'
+  workflow_dispatch:
+
+# Read protection + read CI YAML + write issue. No write on contents.
+permissions:
+  contents: read
+  issues: write
+
+# Serialise — two simultaneous drift runs would duel on the issue
+# create/update path. The audit is idempotent, but parallel POSTs
+# can produce duplicate comments before the title-search dedup wins.
+concurrency:
+  group: ci-required-drift
+  cancel-in-progress: false
+
+jobs:
+  drift:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Check out repo (we read the YAML files locally)
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - name: Set up Python (PyYAML for AST parsing)
+        # Avoid a system-pip install on the runner; setup-python pins
+        # a hermetic interpreter + cache. PyYAML is small enough that
+        # the install is sub-2s — no need to cache wheels.
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: '3.12'
+      - name: Install PyYAML
+        run: python -m pip install --quiet 'PyYAML==6.0.2'
+      - name: Run drift detector
+        env:
+          # GITEA_TOKEN reads protection + writes issues. molecule-core
+          # uses `SOP_TIER_CHECK_TOKEN` as the org-level secret name for
+          # read-only Gitea API access from CI (set by audit-force-merge
+          # and sop-tier-check too). Falls back to the auto-injected
+          # GITHUB_TOKEN if the org-level secret isn't set
+          # (transitional repos).
+          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          # Branches whose protection we compare against. molecule-core
+          # currently has main protected; staging protection is
+          # forthcoming. Keep this list in sync if a new long-lived
+          # branch gets protected (e.g. release/* if introduced later).
+          BRANCHES: 'main staging'
+          # The sentinel job's name inside ci.yml. If the aggregator
+          # is ever renamed, update this too (the drift detector
+          # currently treats `all-required` as the source of "what
+          # the sentinel claims to require").
+          SENTINEL_JOB: 'all-required'
+          # Path to the audit workflow whose REQUIRED_CHECKS env we
+          # cross-check against protection (RFC §6).
+          AUDIT_WORKFLOW_PATH: '.gitea/workflows/audit-force-merge.yml'
+          # Path to the CI workflow with the sentinel + the jobs.
+          CI_WORKFLOW_PATH: '.gitea/workflows/ci.yml'
+          # Issue label applied on file/update. `tier:high` exists in
+          # the molecule-core label set (verified 2026-05-11, label id 9).
+          DRIFT_LABEL: 'tier:high'
+        run: python3 .gitea/scripts/ci-required-drift.py

.gitea/workflows/ci.yml

@@ -0,0 +1,453 @@
+# Ported from .github/workflows/ci.yml on 2026-05-11 per RFC internal#219 §1.
+# continue-on-error: true on every job; follow-up PR will flip required after
+# surfaced bugs are fixed (per RFC §1 — "surface broken workflows without
+# blocking"). The four-surface migration audit
+# (feedback_gitea_actions_migration_audit_pattern) was performed against this
+# port:
+#
+#   1. YAML — dropped `merge_group` trigger (no Gitea merge queue); no
+#      `workflow_dispatch.inputs` to drop (Gitea 1.22.6 rejects those —
+#      feedback_gitea_workflow_dispatch_inputs_unsupported); no `environment:`
+#      blocks; kept `runs-on: ubuntu-latest` (Gitea runner pool advertises
+#      this label per agent_labels in action_runner table). Workflow-level
+#      env.GITHUB_SERVER_URL set as belt-and-suspenders against runner
+#      defaults (feedback_act_runner_github_server_url).
+#
+#   2. Cache — `actions/upload-artifact@v3.2.2` was already pinned to v3 for
+#      Gitea act_runner v0.6 compatibility (a comment in the original called
+#      this out). v4+ is incompatible with Gitea 1.22.x. No `actions/cache`
+#      usage to audit. `actions/setup-python@v6` `cache: pip` is left in
+#      place — works against Gitea's built-in cache server when runner.cache
+#      is configured (currently is, /opt/molecule/runners/config.yaml).
+#
+#   3. Token — workflow uses no custom dispatch tokens. The auto-injected
+#      `GITHUB_TOKEN` (which Gitea aliases to a runner-scoped token) is
+#      sufficient for `actions/checkout` against this same repo.
+#
+#   4. Docs — no docs/scripts reference github.com URLs that need swapping.
+#      The canvas-deploy-reminder step writes a `ghcr.io/...` image
+#      reference into the step summary text — that's documentation prose
+#      pointing at the ECR-mirrored canvas image and stays unchanged for
+#      this port (a separate cleanup if ghcr→ECR sweep is in scope).
+#
+# Cross-links:
+#   - RFC: internal#219 (CI/CD hard-gate hardening)
+#   - Reference port style: molecule-controlplane/.gitea/workflows/ci.yml
+#   - Bugs that may surface immediately and are tracked separately:
+#     internal#214 (Go-side vanity-import / go.sum drift, if any)
+#   - Phase 4 (this PR's follow-up): flip `continue-on-error: false` once
+#     surfaced defects are fixed, then add `all-required` aggregator
+#     sentinel (RFC §2) and PATCH branch protection (Phase 4 scope).
+
+name: CI
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  # `merge_group` (GitHub merge-queue trigger) dropped — Gitea has no merge
+  # queue. The .github/ original retains it; this Gitea-side copy drops it.
+
+# Cancel in-progress CI runs when a new commit arrives on the same ref.
+# Stale runs queue up otherwise. PR refs and main/staging refs each get
+# their own group because github.ref differs.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Belt-and-suspenders against the runner-default trap
+  # (feedback_act_runner_github_server_url). Runners are configured with
+  # this env via /opt/molecule/runners/config.yaml runner.envs, but pinning
+  # at the workflow level protects against a runner regenerated without
+  # the config file (feedback_act_runner_needs_config_file_env).
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  # Detect which paths changed so downstream jobs can skip when only
+  # docs/markdown files were modified.
+  changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after the surfaced defects
+    # (if any) are triaged.
+    continue-on-error: true
+    outputs:
+      platform: ${{ steps.check.outputs.platform }}
+      canvas: ${{ steps.check.outputs.canvas }}
+      python: ${{ steps.check.outputs.python }}
+      scripts: ${{ steps.check.outputs.scripts }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: check
+        run: |
+          # For PR events: diff against the base branch (not HEAD~1 of the branch,
+          # which may be unrelated after force-pushes). When a push updates a PR,
+          # both pull_request and push events fire — prefer the PR base so that
+          # the diff is always computed against the actual merge base, not the
+          # previous SHA on the branch which may be on a different history line.
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          # GITHUB_BASE_REF is set for PR events (the base branch name).
+          # For pull_request events we use the stored base.sha; for push events
+          # (or when base.sha is unavailable) fall back to github.event.before.
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          # Fallback: if BASE is empty or all zeros (new branch), run everything
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "platform=true" >> "$GITHUB_OUTPUT"
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            echo "python=true" >> "$GITHUB_OUTPUT"
+            echo "scripts=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # Both .github/workflows/ci.yml AND .gitea/workflows/ci.yml count
+          # as "this workflow changed" — either edit should force-run every
+          # downstream job. The Gitea port follows the same shape as the
+          # GitHub original so behavior matches when triggered on either
+          # platform.
+          DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".gitea/workflows/ci.yml")
+          echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+
+  # Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
+  # + per-step gating shape preserves the GitHub-side required-check name
+  # contract (so when this Gitea port becomes a required check in Phase 4,
+  # the name match works on PRs that don't touch workspace-server/).
+  platform-build:
+    name: Platform (Go)
+    needs: changes
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    defaults:
+      run:
+        working-directory: workspace-server
+    steps:
+      - if: needs.changes.outputs.platform != 'true'
+        working-directory: .
+        run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.platform == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.platform == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+      - if: needs.changes.outputs.platform == 'true'
+        run: go mod download
+      - if: needs.changes.outputs.platform == 'true'
+        run: go build ./cmd/server
+      # CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
+      - if: needs.changes.outputs.platform == 'true'
+        run: go vet ./... || true
+      - if: needs.changes.outputs.platform == 'true'
+        name: Run golangci-lint
+        run: golangci-lint run --timeout 3m ./... || true
+      - if: needs.changes.outputs.platform == 'true'
+        name: Run tests with race detection and coverage
+        run: go test -race -coverprofile=coverage.out ./...
+
+      - if: needs.changes.outputs.platform == 'true'
+        name: Per-file coverage report
+        # Advisory — lists every source file with its coverage so reviewers
+        # can see at-a-glance where gaps are. Sorted ascending so the worst
+        # offenders float to the top. Does NOT fail the build; the hard
+        # gate is the threshold check below. (#1823)
+        run: |
+          echo "=== Per-file coverage (worst first) ==="
+          go tool cover -func=coverage.out \
+            | grep -v '^total:' \
+            | awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
+                   END {for (f in s) printf "%6.1f%%  %s\n", s[f]/c[f], f}' \
+            | sort -n
+
+      - if: needs.changes.outputs.platform == 'true'
+        name: Check coverage thresholds
+        # Enforces two gates from #1823 Layer 1:
+        #   1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
+        #   2. Per-file floor — non-test .go files in security-critical
+        #      paths with coverage <10% fail the build, UNLESS the file
+        #      path is listed in .coverage-allowlist.txt (acknowledged
+        #      historical debt with a tracking issue + expiry).
+        run: |
+          set -e
+          TOTAL_FLOOR=25
+          # Security-critical paths where a 0%-coverage file is a real risk.
+          CRITICAL_PATHS=(
+            "internal/handlers/tokens"
+            "internal/handlers/workspace_provision"
+            "internal/handlers/a2a_proxy"
+            "internal/handlers/registry"
+            "internal/handlers/secrets"
+            "internal/middleware/wsauth"
+            "internal/crypto"
+          )
+
+          TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
+          echo "Total coverage: ${TOTAL}%"
+          if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
+            echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
+            exit 1
+          fi
+
+          # Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
+          go tool cover -func=coverage.out \
+            | grep -v '^total:' \
+            | awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
+                   END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
+            > /tmp/perfile.txt
+
+          # Build allowlist — paths relative to workspace-server, one per line.
+          # Lines starting with # are comments.
+          ALLOWLIST=""
+          if [ -f ../.coverage-allowlist.txt ]; then
+            ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
+          fi
+
+          FAILED=0
+          WARNED=0
+          for path in "${CRITICAL_PATHS[@]}"; do
+            while read -r file pct; do
+              [[ "$file" == *_test.go ]] && continue
+              [[ "$file" == *"$path"* ]] || continue
+              awk "BEGIN{exit !($pct < 10)}" || continue
+
+              # Strip the package-import prefix so we can match .coverage-allowlist.txt
+              # entries written as paths relative to workspace-server/.
+              # Handle both module paths: platform/workspace-server/... and platform/...
+              rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
+
+              if echo "$ALLOWLIST" | grep -qxF "$rel"; then
+                echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
+                WARNED=$((WARNED+1))
+              else
+                echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
+                FAILED=$((FAILED+1))
+              fi
+            done < /tmp/perfile.txt
+          done
+
+          echo ""
+          echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "$FAILED security-critical file(s) have <10% test coverage and are"
+            echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
+            echo "workspace provisioning — a 0% file here is the exact gap that let"
+            echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
+            echo "  (a) add tests to raise coverage above 10%, or"
+            echo "  (b) add the path to .coverage-allowlist.txt with an expiry date"
+            echo "      and a tracking issue reference."
+            exit 1
+          fi
+
+  # Canvas (Next.js) — required check, always runs. Same always-run +
+  # per-step gating shape as platform-build. The two-job-sharing-name
+  # pattern attempted in PR #2321 doesn't satisfy branch protection
+  # (SKIPPED siblings count as not-passed regardless of SUCCESS
+  # siblings — verified empirically on PR #2314).
+  canvas-build:
+    name: Canvas (Next.js)
+    needs: changes
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    defaults:
+      run:
+        working-directory: canvas
+    steps:
+      - if: needs.changes.outputs.canvas != 'true'
+        working-directory: .
+        run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.canvas == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.canvas == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '22'
+      - if: needs.changes.outputs.canvas == 'true'
+        run: rm -f package-lock.json && npm install
+      - if: needs.changes.outputs.canvas == 'true'
+        run: npm run build
+      - if: needs.changes.outputs.canvas == 'true'
+        name: Run tests with coverage
+        # Coverage instrumentation is configured in canvas/vitest.config.ts
+        # (provider: v8, reporters: text + html + json-summary). Step 2 of
+        # #1815 — wires coverage into CI so we get a baseline visible on
+        # every PR. No threshold gate yet; thresholds dial in (Step 3, also
+        # tracked in #1815) after the team sees what current coverage is.
+        run: npx vitest run --coverage
+      - name: Upload coverage summary as artifact
+        if: needs.changes.outputs.canvas == 'true' && always()
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
+        # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
+        # implement, surfacing as `GHESNotSupportedError: @actions/artifact
+        # v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
+        # currently supported on GHES`. Drop this pin when Gitea ships
+        # the v4 protocol (tracked: post-Gitea-1.23 followup).
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: canvas-coverage-${{ github.run_id }}
+          path: canvas/coverage/
+          retention-days: 7
+          if-no-files-found: warn
+
+  # Shellcheck (E2E scripts) — required check, always runs.
+  shellcheck:
+    name: Shellcheck (E2E scripts)
+    needs: changes
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - if: needs.changes.outputs.scripts != 'true'
+        run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.scripts == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
+        # shellcheck is pre-installed on ubuntu-latest runners (via apt).
+        # infra/scripts/ is included because setup.sh + nuke.sh gate the
+        # README quickstart — a shellcheck regression there silently breaks
+        # new-user onboarding. scripts/ is intentionally excluded until its
+        # pre-existing SC3040/SC3043 warnings are cleaned up.
+        run: |
+          find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
+            | xargs -0 shellcheck --severity=warning
+
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Lint cleanup-trap hygiene (RFC #2873)
+        run: bash tests/e2e/lint_cleanup_traps.sh
+
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Run E2E bash unit tests (no live infra)
+        run: |
+          bash tests/e2e/test_model_slug.sh
+
+  canvas-deploy-reminder:
+    name: Canvas Deploy Reminder
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    needs: [changes, canvas-build]
+    # Only fires on direct pushes to main (i.e. after staging→main promotion).
+    if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - name: Write deploy reminder to step summary
+        env:
+          COMMIT_SHA: ${{ github.sha }}
+          # github.server_url resolves via the workflow-level env override
+          # to the Gitea instance, so the RUN_URL points at the Gitea run
+          # page (not github.com). See feedback_act_runner_github_server_url.
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          # Write body to a temp file — avoids backtick escaping in shell.
+          cat > /tmp/deploy-reminder.md << 'BODY'
+          ## Canvas build passed — deploy required
+
+          The `publish-canvas-image` workflow is now building a fresh Docker image
+          (`ghcr.io/molecule-ai/canvas:latest`) in the background.
+
+          Once it completes (~3–5 min), apply on the host machine with:
+          ```bash
+          cd <runner-workspace>
+          git pull origin main
+          docker compose pull canvas && docker compose up -d canvas
+          ```
+
+          If you need to rebuild from local source instead (e.g. testing unreleased
+          changes or a new `NEXT_PUBLIC_*` URL), use:
+          ```bash
+          docker compose build canvas && docker compose up -d canvas
+          ```
+          BODY
+          printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
+            "$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
+
+          # Gitea has no commit-comments API; write to GITHUB_STEP_SUMMARY,
+          # which both GitHub Actions and Gitea Actions render as the
+          # workflow run's summary page. (#75 / PR-D)
+          cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
+
+  # Python Lint & Test — required check, always runs.
+  python-lint:
+    name: Python Lint & Test
+    needs: changes
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    env:
+      WORKSPACE_ID: test
+    defaults:
+      run:
+        working-directory: workspace
+    steps:
+      - if: needs.changes.outputs.python != 'true'
+        working-directory: .
+        run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.python == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.python == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - if: needs.changes.outputs.python == 'true'
+        run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
+      # Coverage flags + fail-under floor moved into workspace/pytest.ini
+      # (issue #1817) so local `pytest` and CI use identical config.
+      - if: needs.changes.outputs.python == 'true'
+        run: python -m pytest --tb=short
+
+      - if: needs.changes.outputs.python == 'true'
+        name: Per-file critical-path coverage (MCP / inbox / auth)
+        # MCP-critical Python files have a per-file floor on top of the
+        # 86% total floor in pytest.ini. See issue #2790 for full rationale.
+        run: |
+          set -e
+          PER_FILE_FLOOR=75
+          CRITICAL_FILES=(
+            "a2a_mcp_server.py"
+            "mcp_cli.py"
+            "a2a_tools.py"
+            "a2a_tools_inbox.py"
+            "inbox.py"
+            "platform_auth.py"
+          )
+
+          # pytest already wrote .coverage; emit a JSON view scoped to
+          # the critical files so jq/python can read the per-file pct
+          # without parsing tabular text.
+          INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
+          INCLUDES="${INCLUDES%,}"
+          python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
+
+          FAILED=0
+          for f in "${CRITICAL_FILES[@]}"; do
+            pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
+            if [ "$pct" = "MISSING" ]; then
+              echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
+              FAILED=$((FAILED+1))
+              continue
+            fi
+            echo "$f: ${pct}%"
+            if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
+              echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
+              FAILED=$((FAILED+1))
+            fi
+          done
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
+            echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
+            echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
+            echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
+            echo "  (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
+            echo "  (b) if this is unavoidable historical debt, file an issue and propose"
+            echo "      adjusting the floor with rationale in COVERAGE_FLOOR.md."
+            exit 1
+          fi

.gitea/workflows/continuous-synth-e2e.yml

@@ -0,0 +1,255 @@
+name: Continuous synthetic E2E (staging)
+
+# Ported from .github/workflows/continuous-synth-e2e.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
+# regressions visible only at runtime — schema drift, deployment-pipeline
+# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
+#
+# Why this gate exists:
+#   PR-time CI catches code-level regressions but not deployment-time or
+#   integration-time ones. Today's empirical data:
+#     • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
+#       JSON-RPC parse layer between sender and receiver. Visible only
+#       to a sender exercising the full path.
+#     • RFC #2312 chat upload — landed on staging-branch but never
+#       reached staging tenants because publish-workspace-server-image
+#       was main-only. Caught by manual dogfooding hours after deploy.
+#   Both would have surfaced within 15-20 min of regression if a
+#   continuous synth-E2E was running.
+#
+# Cadence: every 20 min (3x/hour). The script is conservatively
+# bounded at 10 min wall-clock; even on degraded staging it should
+# finish before the next firing. cron-overlap is guarded by the
+# concurrency group below.
+#
+# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
+# Plus a fresh tenant provisioned + torn down each run (Railway +
+# AWS pennies). Negligible.
+#
+# Failure handling: when the run fails, the workflow exits non-zero
+# and GitHub's standard email/notification path fires. Operators
+# can subscribe to this workflow's failure channel for paging-grade
+# alerting.
+
+on:
+  schedule:
+    # Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
+    #   1. Stay off the top-of-hour. GitHub Actions scheduler drops
+    #      :00 firings under high load (own docs:
+    #      https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
+    #      Prior history: cron was '0,20,40' (2026-05-02) — only :00
+    #      ever survived. Bumped to '10,30,50' (2026-05-03) on the
+    #      theory that further-from-:00 wins. Empirically 2026-05-04
+    #      that ALSO dropped to ~60 min effective cadence (only ~1
+    #      schedule fire per hour — see molecule-core#2726). Detection
+    #      latency was claimed 20 min, actual 60 min.
+    #   2. Avoid colliding with the existing :15 sweep-cf-orphans
+    #      and :45 sweep-cf-tunnels — both hit the CF API and we
+    #      don't want to fight for rate-limit tokens.
+    #   3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
+    #      secrets, sweep-stale-e2e-orgs every :15) — multiple
+    #      overlapping cron registrations on the same minute is part
+    #      of what GH drops under load.
+    # Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
+    # lanes (1-3 min away from any other cron). Even with empirically-
+    # observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
+    # fires = ~30 min cadence; closer to the 20-min target than the
+    # current shape and provides a real degradation alarm if drops
+    # get worse.
+    - cron: '2,12,22,32,42,52 * * * *'
+permissions:
+  contents: read
+  # No issue-write here — failures surface as red runs in the workflow
+  # history. If you want auto-issue-on-fail, add a follow-up step that
+  # uses gh issue create gated on `if: failure()`. Keeping the surface
+  # minimal until that's actually wanted.
+
+# Serialize so two firings can never overlap. Cron firing every 20 min
+# but scripts conservatively bounded at 10 min — overlap shouldn't
+# happen in steady state, but if a run hangs we don't want N more
+# stacking up.
+concurrency:
+  group: continuous-synth-e2e
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  synth:
+    name: Synthetic E2E against staging
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    # Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
+    # (apt-get update + install docker.io/jq/awscli/caddy + snap install
+    # ssm-agent) runs from raw Ubuntu on every boot — none of it is
+    # pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
+    # across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
+    # latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
+    # A 12min budget leaves only ~2min for the workspace (which needs
+    # ~3.5min for claude-code cold boot) on slow-apt days, blowing the
+    # budget. 20min absorbs the worst tenant tail so the workspace probe
+    # gets the full ~7min it needs even on a slow apt day. Real fix:
+    # pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
+    timeout-minutes: 20
+    env:
+      # claude-code default: cold-start ~5 min (comparable to langgraph),
+      # but uses MiniMax-M2.7-highspeed via the template's third-party-
+      # Anthropic-compat path (workspace-configs-templates/claude-code-
+      # default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
+      # gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
+      # exhaustion class that took the canary down 2026-05-03 (#265).
+      # Operators can pick langgraph / hermes via workflow_dispatch
+      # when they specifically need to exercise the OpenAI or SDK-
+      # native paths.
+      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
+      # Pin the canary to a specific MiniMax model rather than relying
+      # on the per-runtime default ("sonnet" → routes to direct
+      # Anthropic, defeats the cost saving). Operators can override
+      # via workflow_dispatch by setting a different E2E_MODEL_SLUG
+      # input if they need to exercise a specific model. M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
+      # Bound to 10 min so a stuck provision fails the run instead of
+      # holding up the next cron firing. 15-min default in the script
+      # is for the on-PR full lifecycle where we have more headroom.
+      E2E_PROVISION_TIMEOUT_SECS: '600'
+      # Slug suffix — namespaced "synth-" so these runs are
+      # distinguishable from PR-driven runs in CP admin.
+      E2E_RUN_ID: synth-${{ github.run_id }}
+      # Forced false for cron; respected for manual dispatch
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
+      MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      # MiniMax key is the canary's PRIMARY auth path. claude-code
+      # template's `minimax` provider routes ANTHROPIC_BASE_URL to
+      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
+      # tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
+      # which key is present — MiniMax wins when set.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so operators can dispatch with
+      # E2E_RUNTIME=langgraph or =hermes and still have a working
+      # canary path. The script picks the right blob shape based on
+      # which key is non-empty.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        run: |
+          # Hard-fail on missing secret REGARDLESS of trigger. Previously
+          # this step soft-skipped on workflow_dispatch via `exit 0`, but
+          # `exit 0` only ends the STEP — subsequent steps still ran with
+          # the empty secret, the synth script fell through to the wrong
+          # SECRETS_JSON branch, and the canary failed 5 min later with a
+          # confusing "Agent error (Exception)" instead of the clean
+          # "secret missing" message at the top. Caught 2026-05-04 by
+          # dispatched run 25296530706: claude-code + missing MINIMAX
+          # silently used OpenAI keys but kept model=MiniMax-M2.7, then
+          # the workspace 401'd against MiniMax once it tried to call.
+          # Fix: exit 1 in both cron and dispatch paths. Operators who
+          # want to verify a YAML change without setting up the secret
+          # can read the verify-secrets step's stderr — the failure is
+          # itself the verification signal.
+          if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
+            echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
+            echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+            exit 1
+          fi
+
+          # LLM-key requirement is per-runtime: claude-code accepts
+          # EITHER MiniMax OR direct-Anthropic (whichever is set first),
+          # langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_API_KEY).
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
+            echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
+            exit 1
+          fi
+
+      - name: Install required tools
+        run: |
+          # The script depends on jq + curl (already on ubuntu-latest)
+          # and python3 (likewise). Verify they're all present so we
+          # fail fast on a runner image regression rather than mid-script.
+          for cmd in jq curl python3; do
+            command -v "$cmd" >/dev/null 2>&1 || {
+              echo "::error::required tool '$cmd' not on PATH — runner image regression?"
+              exit 1
+            }
+          done
+
+      - name: Run synthetic E2E
+        # The script handles its own teardown via EXIT trap; even on
+        # failure (timeout, assertion), the org is deprovisioned and
+        # leaks are reported. Exit code propagates from the script.
+        run: |
+          bash tests/e2e/test_staging_full_saas.sh
+
+      - name: Failure summary
+        # Runs only on failure. Adds a job summary so the workflow run
+        # page shows a quick "what happened" instead of forcing readers
+        # to scroll through script output.
+        if: failure()
+        run: |
+          {
+            echo "## Continuous synth E2E failed"
+            echo ""
+            echo "**Run ID:** ${{ github.run_id }}"
+            echo "**Trigger:** ${{ github.event_name }}"
+            echo "**Runtime:** ${E2E_RUNTIME}"
+            echo "**Slug:** synth-${{ github.run_id }}"
+            echo ""
+            echo "### What this means"
+            echo ""
+            echo "Staging just regressed on a path that previously worked. Likely classes:"
+            echo "- Schema mismatch between sender and receiver (#2345 class)"
+            echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
+            echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
+            echo "- Staging-CP env var rotation"
+            echo ""
+            echo "### Next steps"
+            echo ""
+            echo "1. Check the script output above for the assertion that failed"
+            echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
+            echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
+            echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
+          } >> "$GITHUB_STEP_SUMMARY"

.gitea/workflows/e2e-api.yml

@@ -0,0 +1,333 @@
+name: E2E API Smoke Test
+
+# Ported from .github/workflows/e2e-api.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+# Extracted from ci.yml so workflow-level concurrency can protect this job
+# from run-level cancellation (issue #458).
+#
+# Trigger model (revised 2026-04-29):
+#
+# Always FIRES on push/pull_request to staging+main. Real work is gated
+# per-step on `needs.detect-changes.outputs.api` — when paths under
+# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
+# changed, the no-op step alone runs and emits SUCCESS for the
+# `E2E API Smoke Test` check, satisfying branch protection without
+# spending CI cycles. See the in-job comment on the `e2e-api` job for
+# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
+# PR #2264 incident that drove the consolidation.
+#
+# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
+# -------------------------------------------------------------------
+# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
+# Gitea act_runner runs with `container.network: host` (operator host
+# `/opt/molecule/runners/config.yaml`), which means:
+#
+#   * Two concurrent runs both try to bind their `-p 15432:5432` /
+#     `-p 16379:6379` host ports — the second postgres/redis FATALs
+#     with `Address in use` and `docker run` returns exit 125 with
+#     `Conflict. The container name "/molecule-ci-postgres" is already
+#     in use by container ...`. Verified in run a7/2727 on 2026-05-07.
+#   * The fixed container names `molecule-ci-postgres` / `-redis` (the
+#     pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
+#     `docker rm -f` at the start of the second job KILLS the first
+#     job's still-running postgres/redis.
+#
+# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
+# platform-server is a Go binary on the host, not a containerised
+# step):
+#
+#   1. Unique container names per run:
+#         pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
+#         redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
+#      `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
+#      same run_id.
+#   2. Ephemeral host port per run (`-p 0:5432`), then read the actual
+#      bound port via `docker port` and export DATABASE_URL/REDIS_URL
+#      pointing at it. No fixed host-port → no port collision.
+#   3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
+#      the original flake fixed in #92 and the script's still IPv6-
+#      enabled.
+#   4. `if: always()` cleanup so containers don't leak when test steps
+#      fail.
+#
+# Issue #94 items #2 + #3 (also fixed here):
+#   * Pre-pull `alpine:latest` so the platform-server's provisioner
+#     (`internal/handlers/container_files.go`) can stand up its
+#     ephemeral token-write helper without a daemon.io round-trip.
+#   * Create `molecule-core-net` bridge network if missing so the
+#     provisioner's container.HostConfig {NetworkMode: ...} attach
+#     succeeds.
+# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
+# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
+# they DO come up. Timeouts are not the bottleneck; not bumped.
+#
+# Item explicitly NOT fixed here: failing test `Status back online`
+# fails because the platform's langgraph workspace template image
+# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
+# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
+# template-registry resolution issue (ADR-002 / local-build mode) and
+# belongs in a separate change that touches workspace-server, not
+# this workflow file.
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+concurrency:
+  # Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
+  # same auto-promote-staging brittleness as e2e-staging-canvas — back-
+  # to-back staging pushes share refs/heads/staging, so the older push's
+  # queued run gets cancelled when a newer push lands. Auto-promote-
+  # staging then sees `completed/cancelled` for the older SHA and stays
+  # put; the newer SHA's gates may eventually save the day, but if the
+  # newer push gets cancelled too, we deadlock.
+  #
+  # See e2e-staging-canvas.yml's identical concurrency block for the full
+  # rationale and the 2026-04-28 incident reference.
+  group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    outputs:
+      api: ${{ steps.decide.outputs.api }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: decide
+        # Inline replacement for dorny/paths-filter — same pattern PR#372's
+        # ci.yml port used. Diffs against the PR base or push BEFORE SHA,
+        # then matches against the api-relevant path set.
+        run: |
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "api=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `E2E API Smoke Test`. Real work is gated per-step
+  # on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
+  # check run for every job that matches `name:`, and a job-level
+  # `if: false` produces a SKIPPED check run. Branch protection treats
+  # all check runs with a matching context name on the latest commit as a
+  # SET — any SKIPPED in the set fails the required-check eval, even with
+  # SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
+  # 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
+  # promotion despite all real work succeeding. Collapsing to a single
+  # always-running job with conditional steps emits exactly one SUCCESS
+  # check run regardless of paths filter — branch-protection-clean.
+  e2e-api:
+    needs: detect-changes
+    name: E2E API Smoke Test
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 15
+    env:
+      # Unique per-run container names so concurrent runs on the host-
+      # network act_runner don't collide on name OR port.
+      # `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
+      # same run_id. PORT is set later (after docker port lookup) since
+      # we let Docker assign an ephemeral host port.
+      PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
+      REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
+      PORT: "8080"
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.api != 'true'
+        run: |
+          echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
+          echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.api == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.api == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+          cache: true
+          cache-dependency-path: workspace-server/go.sum
+      - name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          # Provisioner uses alpine:latest for ephemeral token-write
+          # containers (workspace-server/internal/handlers/container_files.go).
+          # Pre-pull so the first provision in test_api.sh doesn't race
+          # the daemon's pull cache. Idempotent — `docker pull` is a no-op
+          # when the image is already present.
+          docker pull alpine:latest >/dev/null
+          # Provisioner attaches workspace containers to
+          # molecule-core-net (workspace-server/internal/provisioner/
+          # provisioner.go::DefaultNetwork). The bridge already exists on
+          # the operator host's docker daemon — `network create` is
+          # idempotent via `|| true`.
+          docker network create molecule-core-net >/dev/null 2>&1 || true
+          echo "alpine:latest pre-pulled; molecule-core-net ensured."
+      - name: Start Postgres (docker)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          # Defensive cleanup — only matches THIS run's container name,
+          # so it cannot kill a sibling run's postgres. (Pre-fix the
+          # name was static and this rm hit other runs' containers.)
+          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          # `-p 0:5432` requests an ephemeral host port; we read it back
+          # below and export DATABASE_URL.
+          docker run -d --name "$PG_CONTAINER" \
+            -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
+            -p 0:5432 postgres:16 >/dev/null
+          # Resolve the host-side port assignment. `docker port` prints
+          # `0.0.0.0:NNNN` (and on host-net runners may also print an
+          # IPv6 line — take the first IPv4 line).
+          PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
+          if [ -z "$PG_PORT" ]; then
+            # Fallback: any first line. Some Docker versions print only
+            # one line.
+            PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
+          fi
+          if [ -z "$PG_PORT" ]; then
+            echo "::error::Could not resolve host port for $PG_CONTAINER"
+            docker port "$PG_CONTAINER" 5432/tcp || true
+            docker logs "$PG_CONTAINER" || true
+            exit 1
+          fi
+          # 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
+          echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
+          echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
+          echo "Postgres host port: ${PG_PORT}"
+          for i in $(seq 1 30); do
+            if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
+              echo "Postgres ready after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::Postgres did not become ready in 30s"
+          docker logs "$PG_CONTAINER" || true
+          exit 1
+      - name: Start Redis (docker)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
+          docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
+          REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
+          if [ -z "$REDIS_PORT" ]; then
+            REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
+          fi
+          if [ -z "$REDIS_PORT" ]; then
+            echo "::error::Could not resolve host port for $REDIS_CONTAINER"
+            docker port "$REDIS_CONTAINER" 6379/tcp || true
+            docker logs "$REDIS_CONTAINER" || true
+            exit 1
+          fi
+          echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
+          echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
+          echo "Redis host port: ${REDIS_PORT}"
+          for i in $(seq 1 15); do
+            if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
+              echo "Redis ready after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::Redis did not become ready in 15s"
+          docker logs "$REDIS_CONTAINER" || true
+          exit 1
+      - name: Build platform
+        if: needs.detect-changes.outputs.api == 'true'
+        working-directory: workspace-server
+        run: go build -o platform-server ./cmd/server
+      - name: Start platform (background)
+        if: needs.detect-changes.outputs.api == 'true'
+        working-directory: workspace-server
+        run: |
+          # DATABASE_URL + REDIS_URL exported by the start-postgres /
+          # start-redis steps point at this run's per-run host ports.
+          ./platform-server > platform.log 2>&1 &
+          echo $! > platform.pid
+      - name: Wait for /health
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          for i in $(seq 1 30); do
+            if curl -sf http://127.0.0.1:8080/health > /dev/null; then
+              echo "Platform up after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::Platform did not become healthy in 30s"
+          cat workspace-server/platform.log || true
+          exit 1
+      - name: Assert migrations applied
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
+          if [ "$tables" != "1" ]; then
+            echo "::error::Migrations did not apply"
+            cat workspace-server/platform.log || true
+            exit 1
+          fi
+          echo "Migrations OK"
+      - name: Run E2E API tests
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_api.sh
+      - name: Run notify-with-attachments E2E
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_notify_attachments_e2e.sh
+      - name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_priority_runtimes_e2e.sh
+      - name: Run poll-mode + since_id cursor E2E (#2339)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_poll_mode_e2e.sh
+      - name: Run poll-mode chat upload E2E (RFC #2891)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
+      - name: Dump platform log on failure
+        if: failure() && needs.detect-changes.outputs.api == 'true'
+        run: cat workspace-server/platform.log || true
+      - name: Stop platform
+        if: always() && needs.detect-changes.outputs.api == 'true'
+        run: |
+          if [ -f workspace-server/platform.pid ]; then
+            kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
+          fi
+      - name: Stop service containers
+        # always() so containers don't leak when test steps fail. The
+        # cleanup is best-effort: if the container is already gone
+        # (e.g. concurrent rerun race), don't fail the job.
+        if: always() && needs.detect-changes.outputs.api == 'true'
+        run: |
+          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true

.gitea/workflows/e2e-staging-canvas.yml

@@ -0,0 +1,247 @@
+name: E2E Staging Canvas (Playwright)
+
+# Ported from .github/workflows/e2e-staging-canvas.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Playwright test suite that provisions a fresh staging org per run and
+# verifies every workspace-panel tab renders without crashing. Complements
+# e2e-staging-saas.yml (which tests the API shape) by exercising the
+# actual browser + canvas bundle against live staging.
+#
+# Triggers: push to main/staging or PR touching canvas sources + this workflow,
+# manual dispatch, and weekly cron to catch browser/runtime drift even
+# when canvas is quiet.
+# Added staging to push/pull_request branches so the auto-promote gate
+# check (--event push --branch staging) can see a completed run for this
+# workflow — mirrors what PR #1891 does for e2e-api.yml.
+
+on:
+  # Trigger model (revised 2026-04-29):
+  #
+  # Always fires on push/pull_request; real work is gated per-step on
+  # `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
+  # changed, the no-op step alone runs and emits SUCCESS for the
+  # `Canvas tabs E2E` check, satisfying branch protection without
+  # spending CI cycles. See e2e-api.yml for the rationale on why this
+  # is a single job rather than two-jobs-sharing-name.
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
+    # release-note-shaped regressions that don't ride in with a PR.
+    - cron: '0 8 * * 0'
+
+concurrency:
+  # Per-SHA grouping (changed 2026-04-28 from a single global group). The
+  # global group made auto-promote-staging brittle: when a staging push
+  # queued behind an in-flight run and a third entrant (a PR run, a
+  # follow-on push) entered the group, the staging push got cancelled —
+  # leaving auto-promote-staging looking at `completed/cancelled` for a
+  # required gate and refusing to advance main. Observed 2026-04-28
+  # 23:51-23:53 on staging tip 3f99fede.
+  #
+  # The original intent of the global group was to throttle parallel
+  # E2E provisions (each spins a fresh EC2). At our scale that throttle
+  # isn't worth the correctness cost — fresh-org-per-run isolates the
+  # state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
+  # is rounding error vs. the cost of a stuck pipeline.
+  #
+  # Per-SHA still dedupes accidental double-triggers for the SAME SHA.
+  # It does NOT cancel obsolete-PR-version runs on force-push; that
+  # wasted CI is acceptable given the alternative is losing staging-tip
+  # data that auto-promote-staging needs.
+  group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    outputs:
+      canvas: ${{ steps.decide.outputs.canvas }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: decide
+        # Inline replacement for dorny/paths-filter — see e2e-api.yml.
+        # Cron triggers always run real work (no diff context).
+        run: |
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "canvas=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `Canvas tabs E2E`. Real work is gated per-step on
+  # `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
+  # rationale — same path-filter check-name parity issue blocked PR #2264
+  # (staging→main) on 2026-04-29 because branch protection treats matching-
+  # name check runs as a SET, and any SKIPPED member fails the eval.
+  playwright:
+    needs: detect-changes
+    name: Canvas tabs E2E
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 40
+
+    env:
+      CANVAS_E2E_STAGING: '1'
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+
+    defaults:
+      run:
+        working-directory: canvas
+
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.canvas != 'true'
+        working-directory: .
+        run: |
+          echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
+          echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
+
+      - if: needs.detect-changes.outputs.canvas == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
+            exit 2
+          fi
+
+      - name: Set up Node
+        if: needs.detect-changes.outputs.canvas == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: canvas/package-lock.json
+
+      - name: Install canvas deps
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: npm ci
+
+      - name: Install Playwright browsers
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: npx playwright install --with-deps chromium
+
+      - name: Run staging canvas E2E
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: npx playwright test --config=playwright.staging.config.ts
+
+      - name: Upload Playwright report on failure
+        if: failure() && needs.detect-changes.outputs.canvas == 'true'
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
+        # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
+        # implement (see ci.yml upload step for the canonical error
+        # cite). Drop this pin when Gitea ships the v4 protocol.
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: playwright-report-staging
+          path: canvas/playwright-report-staging/
+          retention-days: 14
+
+      - name: Upload screenshots on failure
+        if: failure() && needs.detect-changes.outputs.canvas == 'true'
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: playwright-screenshots
+          path: canvas/test-results/
+          retention-days: 14
+
+      # Safety-net teardown — fires only when Playwright's globalTeardown
+      # didn't (worker crash, runner cancel). Reads the slug from
+      # canvas/.playwright-staging-state.json (written by staging-setup
+      # as its first action, before any CP call) and deletes only that
+      # slug.
+      #
+      # Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
+      # orgs to compensate for setup-crash-before-state-file-write. That
+      # over-aggressive cleanup raced concurrent canvas-E2E runs and
+      # poisoned each other's tenants — observed 2026-04-30 when three
+      # real-test runs killed each other mid-test, surfacing as
+      # `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
+      # DNS record. Pattern-sweep removed; setup now writes the state
+      # file before any CP work, so the slug is always recoverable.
+      - name: Teardown safety net
+        if: always() && needs.detect-changes.outputs.canvas == 'true'
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          STATE_FILE=".playwright-staging-state.json"
+          if [ ! -f "$STATE_FILE" ]; then
+            echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
+            exit 0
+          fi
+          slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
+          if [ -z "$slug" ]; then
+            echo "::warning::State file present but slug missing; nothing to clean up."
+            exit 0
+          fi
+          echo "Deleting orphan tenant: $slug"
+          # Verify HTTP 2xx instead of `>/dev/null || true` swallowing
+          # failures. A 5xx or timeout previously looked identical to
+          # success, leaving the tenant alive for up to ~45 min until
+          # sweep-stale-e2e-orgs caught it. Surface failures as
+          # workflow warnings naming the slug. Don't `exit 1` — a single
+          # cleanup miss shouldn't fail-flag the canvas test when the
+          # actual smoke check passed; the sweeper is the safety net.
+          # See molecule-controlplane#420.
+          # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+          # pollution of the captured status (lint-curl-status-capture.yml).
+          set +e
+          curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
+            -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
+          set -e
+          code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
+          if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+            echo "[teardown] deleted $slug (HTTP $code)"
+          else
+            echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
+          fi
+          exit 0

.gitea/workflows/e2e-staging-external.yml

@@ -0,0 +1,189 @@
+name: E2E Staging External Runtime
+
+# Ported from .github/workflows/e2e-staging-external.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Regression for the four/five workspaces.status=awaiting_agent transitions
+# that silently failed in production for five days before migration 046
+# extended the workspace_status enum (see
+# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
+#
+# Why this is its own workflow (not folded into e2e-staging-saas.yml):
+#   - The full-saas harness defaults to runtime=hermes, never exercises
+#     external-runtime. Adding an `external` parameter to that script
+#     would force every push to staging through both lifecycles in
+#     series, doubling the EC2 cold-start budget.
+#   - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
+#     window, 90s default + sweep interval), which we wait through
+#     deliberately. Folding it into hermes would make the long path
+#     even longer.
+#   - It can run in parallel with the hermes E2E since both create
+#     fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
+#     `e2e-...`).
+#
+# Triggers:
+#   - Push to staging when any source affecting external runtime,
+#     hibernation, or the migration set changes.
+#   - PR review for the same set.
+#   - Manual workflow_dispatch.
+#   - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
+#     30 min after e2e-staging-saas.yml's 07:00 UTC cron).
+#
+# Concurrency: serialized so two staging pushes don't fight for the
+# same EC2 quota window. cancel-in-progress=false so a half-rolled
+# tenant always finishes its teardown.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_restart.go'
+      - 'workspace-server/internal/registry/healthsweep.go'
+      - 'workspace-server/internal/registry/liveness.go'
+      - 'workspace-server/migrations/**'
+      - 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
+      - 'tests/e2e/test_staging_external_runtime.sh'
+      - '.gitea/workflows/e2e-staging-external.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_restart.go'
+      - 'workspace-server/internal/registry/healthsweep.go'
+      - 'workspace-server/internal/registry/liveness.go'
+      - 'workspace-server/migrations/**'
+      - 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
+      - 'tests/e2e/test_staging_external_runtime.sh'
+      - '.gitea/workflows/e2e-staging-external.yml'
+  schedule:
+    - cron: '30 7 * * *'
+
+concurrency:
+  group: e2e-staging-external
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  e2e-staging-external:
+    name: E2E Staging External Runtime
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 25
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
+      E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            # Schedule + push triggers must hard-fail when the token is
+            # missing — silent skip would mask infra rot. Manual dispatch
+            # gets the same hard-fail; an operator running this on a fork
+            # without secrets configured needs to know up-front.
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: CP staging health preflight
+        run: |
+          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
+          if [ "$code" != "200" ]; then
+            echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
+            exit 1
+          fi
+          echo "Staging CP healthy ✓"
+
+      - name: Run external-runtime E2E
+        id: e2e
+        run: bash tests/e2e/test_staging_external_runtime.sh
+
+      # Mirror the e2e-staging-saas.yml safety net: if the runner is
+      # cancelled (e.g. concurrent staging push), the test script's
+      # EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
+      # *this* run id.
+      - name: Teardown safety net (runs on cancel/failure)
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
+          # so concurrent runs and unrelated dev probes are not touched.
+          # Sweep today AND yesterday so a midnight-crossing run still
+          # cleans up its own slug.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if not run_id:
+              # Without a run id we cannot scope safely; bail rather
+              # than risk deleting unrelated tenants.
+              sys.exit(0)
+          prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
+          for o in d.get('orgs', []):
+              s = o.get('slug', '')
+              if s.startswith(prefixes) and o.get('status') != 'purged':
+                  print(s)
+          " 2>/dev/null)
+          if [ -n "$orgs" ]; then
+            echo "Safety-net sweep: deleting leftover orgs:"
+            echo "$orgs"
+            # Per-slug verified DELETE — see molecule-controlplane#420.
+            # `>/dev/null 2>&1` previously hid every failure; surface
+            # non-2xx as workflow warnings so the run page names what
+            # leaked. Sweeper catches the rest within ~45 min.
+            leaks=()
+            for slug in $orgs; do
+              # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+              # pollution of the captured status (lint-curl-status-capture.yml).
+              set +e
+              curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
+                -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+                -H "Authorization: Bearer $ADMIN_TOKEN" \
+                -H "Content-Type: application/json" \
+                -d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
+              set -e
+              code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
+              if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+                echo "[teardown] deleted $slug (HTTP $code)"
+              else
+                echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
+                leaks+=("$slug")
+              fi
+            done
+            if [ ${#leaks[@]} -gt 0 ]; then
+              echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+            fi
+          else
+            echo "Safety-net sweep: no leftover orgs to clean."
+          fi

.gitea/workflows/e2e-staging-saas.yml

@@ -0,0 +1,251 @@
+name: E2E Staging SaaS (full lifecycle)
+
+# Ported from .github/workflows/e2e-staging-saas.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Dedicated workflow that provisions a fresh staging org per run, exercises
+# the full workspace lifecycle (register → heartbeat → A2A → delegation →
+# HMA memory → activity → peers), then tears down and asserts leak-free.
+#
+# Why a separate workflow (not folded into ci.yml):
+#   - The run takes ~25-35 min (EC2 boot + cloudflared DNS + provision sweeps +
+#     agent bootstrap), way too slow for every PR.
+#   - Needs its own concurrency group so two pushes don't fight over the
+#     same staging org slug prefix.
+#   - Has its own required secrets (session cookie, admin token) that most
+#     PRs don't need to read.
+#
+# Triggers:
+#   - Push to main (regression guard)
+#   - workflow_dispatch (manual re-run from UI)
+#   - Nightly cron (catches drift even when no pushes land)
+#   - Changes to any provisioning-critical file under PR review (opt-in
+#     via the same paths watcher that e2e-api.yml uses)
+
+on:
+  # Trunk-based (Phase 3 of internal#81): main is the only branch.
+  # Previously this fired on staging push too because staging was a
+  # superset of main and ran the gate ahead of auto-promote; with no
+  # staging branch, main is where E2E gates the deploy.
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_provision.go'
+      - 'workspace-server/internal/handlers/a2a_proxy.go'
+      - 'workspace-server/internal/middleware/**'
+      - 'workspace-server/internal/provisioner/**'
+      - 'tests/e2e/test_staging_full_saas.sh'
+      - '.gitea/workflows/e2e-staging-saas.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_provision.go'
+      - 'workspace-server/internal/handlers/a2a_proxy.go'
+      - 'workspace-server/internal/middleware/**'
+      - 'workspace-server/internal/provisioner/**'
+      - 'tests/e2e/test_staging_full_saas.sh'
+      - '.gitea/workflows/e2e-staging-saas.yml'
+  schedule:
+    # 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
+    # Cloudflare API regressions, etc. even on quiet days.
+    - cron: '0 7 * * *'
+
+# Serialize: staging has a finite per-hour org creation quota. Two pushes
+# landing in quick succession should queue, not race. `cancel-in-progress:
+# false` mirrors e2e-api.yml — GitHub would otherwise cancel the running
+# teardown step and leave orphan EC2s.
+concurrency:
+  group: e2e-staging-saas
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  e2e-staging-saas:
+    name: E2E Staging SaaS
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 45
+    permissions:
+      contents: read
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      # Single admin-bearer secret drives provision + tenant-token
+      # retrieval + teardown. Configure in
+      # Settings → Secrets and variables → Actions → Repository secrets.
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      # MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
+      # from hermes+OpenAI default after #2578 (the staging OpenAI key
+      # account went over quota and stayed dead for 36+ hours, taking
+      # the full-lifecycle E2E red on every provisioning-critical push).
+      # claude-code template's `minimax` provider routes
+      # ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
+      # MINIMAX_API_KEY at boot — separate billing account so an
+      # OpenAI quota collapse no longer wedges the gate. Mirrors the
+      # canary-staging.yml + continuous-synth-e2e.yml migrations.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so an operator-dispatched run with
+      # E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
+      # exercise the OpenAI path.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
+      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
+      # Pin the model when running on the default claude-code path —
+      # the per-runtime default ("sonnet") routes to direct Anthropic
+      # and defeats the cost saving. Operators can override via the
+      # workflow_dispatch flow (no input wired here yet — runtime
+      # override is enough for ad-hoc).
+      E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
+      E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: Verify LLM key present
+        run: |
+          # Per-runtime key check — claude-code uses MiniMax; hermes /
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
+          # rather than soft-skip per #2578's lesson — empty key
+          # silently falls through to the wrong SECRETS_JSON branch and
+          # produces a confusing auth error 5 min later instead of the
+          # clean "secret missing" message at the top.
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              # Either MiniMax OR direct-Anthropic works — first
+              # non-empty wins in the test script's secrets-injection
+              # priority chain.
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
+            exit 2
+          fi
+          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
+
+      - name: CP staging health preflight
+        run: |
+          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
+          if [ "$code" != "200" ]; then
+            echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
+            exit 1
+          fi
+          echo "Staging CP healthy ✓"
+
+      - name: Run full-lifecycle E2E
+        id: e2e
+        run: bash tests/e2e/test_staging_full_saas.sh
+
+      # Belt-and-braces teardown: the test script itself installs a trap
+      # for EXIT/INT/TERM, but if the GH runner itself is cancelled (e.g.
+      # someone pushes a new commit and workflow concurrency is set to
+      # cancel), the trap may not fire. This `always()` step runs even on
+      # cancellation and attempts the delete a second time. The admin
+      # DELETE endpoint is idempotent so double-invoking is safe.
+      - name: Teardown safety net (runs on cancel/failure)
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          # Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
+          # nuke them. Catches the case where the script died before
+          # exporting its slug.
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # ONLY sweep slugs from *this* CI run. Previously the filter was
+          # f'e2e-{today}-' which stomped on parallel CI runs AND any manual
+          # E2E probes a dev was running against staging (incident 2026-04-21
+          # 15:02Z: this workflow's safety net deleted an unrelated manual
+          # run's tenant 1s after it hit 'running').
+          # Sweep both today AND yesterday's UTC dates so a run that crosses
+          # midnight still matches its own slug — see the 2026-04-26→27
+          # canvas-safety-net incident for the same bug class.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if run_id:
+              prefixes = tuple(f'e2e-{d}-{run_id}-' for d in dates)
+          else:
+              prefixes = tuple(f'e2e-{d}-' for d in dates)
+          candidates = [o['slug'] for o in d.get('orgs', [])
+                        if any(o.get('slug','').startswith(p) for p in prefixes)
+                        and o.get('instance_status') not in ('purged',)]
+          print('\n'.join(candidates))
+          " 2>/dev/null)
+          # Per-slug verified DELETE (was `>/dev/null || true` — see
+          # molecule-controlplane#420). Surface non-2xx as a workflow
+          # warning naming the leaked slug; don't exit 1 (sweeper is
+          # the safety net within ~45 min).
+          leaks=()
+          for slug in $orgs; do
+            echo "Safety-net teardown: $slug"
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
+            set -e
+            code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
+          done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
+          exit 0

.gitea/workflows/e2e-staging-sanity.yml

@@ -0,0 +1,157 @@
+name: E2E Staging Sanity (leak-detection self-check)
+
+# Ported from .github/workflows/e2e-staging-sanity.yml on 2026-05-11 per
+# RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `workflow_dispatch:` (Gitea 1.22.6 finicky on bare dispatch).
+#   - `actions/github-script@v9` issue-open block replaced with curl
+#     calls to the Gitea REST API (/api/v1/repos/.../issues|comments).
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# Periodic assertion that the teardown safety nets in e2e-staging-saas
+# and canary-staging actually work. Runs the E2E harness with
+# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
+# the org is provisioned. The workspace-provision step then fails, the
+# script exits non-zero, and the EXIT trap + workflow always()-step
+# must still tear down cleanly.
+
+on:
+  schedule:
+    - cron: '0 6 * * 1'
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  group: e2e-staging-sanity
+  cancel-in-progress: false
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  sanity:
+    name: Intentional-failure teardown sanity
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 20
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      E2E_MODE: canary
+      E2E_RUNTIME: hermes
+      E2E_RUN_ID: "sanity-${{ github.run_id }}"
+      E2E_INTENTIONAL_FAILURE: "1"
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
+            exit 2
+          fi
+
+      # Inverted assertion: the run MUST fail. If it passes, the
+      # E2E_INTENTIONAL_FAILURE path is broken.
+      - name: Run harness — expecting exit !=0
+        id: harness
+        run: |
+          set +e
+          bash tests/e2e/test_staging_full_saas.sh
+          rc=$?
+          echo "harness_rc=$rc" >> "$GITHUB_OUTPUT"
+          if [ "$rc" = "1" ]; then
+            echo "OK Harness failed as expected (rc=1); teardown trap ran, leak-check passed"
+            exit 0
+          elif [ "$rc" = "0" ]; then
+            echo "::error::Harness succeeded under E2E_INTENTIONAL_FAILURE=1 — the poisoning path is broken"
+            exit 1
+          elif [ "$rc" = "4" ]; then
+            echo "::error::LEAK DETECTED (rc=4) — teardown failed to clean up the org. Safety net broken."
+            exit 4
+          else
+            echo "::error::Unexpected rc=$rc — neither clean-failure nor leak. Investigate harness."
+            exit 1
+          fi
+
+      - name: Open issue if safety net is broken (Gitea API)
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          API="${SERVER_URL%/}/api/v1"
+          TITLE="E2E teardown safety net broken"
+          RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
+
+          BODY_JSON=$(jq -nc --arg t "$TITLE" --arg run "$RUN_URL" '
+            {title: $t,
+             body: ("The weekly sanity run (E2E_INTENTIONAL_FAILURE=1) did not exit as expected. This means one of:\n  - poisoning did not actually cause failure (test harness regression), OR\n  - teardown left an orphan org (leak detection caught a real bug)\n\nRun: " + $run + "\n\nThis is higher priority than a canary failure — the whole E2E safety net cannot be trusted until this is resolved.")}')
+
+          EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
+            "${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
+            | jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
+
+          if [ -n "$EXISTING" ]; then
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${EXISTING}/comments" \
+              -d "$(jq -nc --arg run "$RUN_URL" '{body: ("Still broken. " + $run)}')" >/dev/null
+            echo "Commented on existing issue #${EXISTING}"
+          else
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues" -d "$BODY_JSON" >/dev/null
+            echo "Filed new issue"
+          fi
+
+      # Belt-and-braces: if teardown left anything behind, nuke it here
+      # so we don't bleed staging quota.
+      - name: Teardown safety net
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys
+          d = json.load(sys.stdin)
+          today = __import__('datetime').date.today().strftime('%Y%m%d')
+          candidates = [o['slug'] for o in d.get('orgs', [])
+                        if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
+                        and o.get('status') not in ('purged',)]
+          print('\n'.join(candidates))
+          " 2>/dev/null)
+          leaks=()
+          for slug in $orgs; do
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
+            set -e
+            code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
+          done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
+          exit 0

.gitea/workflows/handlers-postgres-integration.yml

@@ -0,0 +1,282 @@
+name: Handlers Postgres Integration
+
+# Ported from .github/workflows/handlers-postgres-integration.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Real-Postgres integration tests for workspace-server/internal/handlers/.
+# Triggered on every PR/push that touches the handlers package.
+#
+# Why this workflow exists
+# ------------------------
+# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
+# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
+# depend on the row state AFTER the SQL runs. The result_preview-lost
+# bug shipped to staging in PR #2854 because every unit test was
+# satisfied with "an UPDATE statement fired" — none verified the row's
+# preview field actually landed. The local-postgres E2E that retrofit
+# self-review caught it took 2 minutes to set up and would have caught
+# the bug at PR-time.
+#
+# Why this workflow does NOT use `services: postgres:` (Class B fix)
+# ------------------------------------------------------------------
+# Our act_runner config has `container.network: host` (operator host
+# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
+# the job container AND every service container. With host-net, two
+# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
+# second postgres FATALs with `could not create any TCP/IP sockets:
+# Address in use`, and Docker auto-removes it (act_runner sets
+# AutoRemove:true on service containers). By the time the migrations
+# step runs `psql`, the postgres container is gone, hence
+# `Connection refused` then `failed to remove container: No such
+# container` at cleanup time.
+#
+# Per-job `container.network` override is silently ignored by
+# act_runner — `--network and --net in the options will be ignored.`
+# appears in the runner log. Documented constraint.
+#
+# So we sidestep `services:` entirely. The job container still uses
+# host-net (inherited from runner config; required for cache server
+# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
+# postgres on the existing `molecule-core-net` bridge with a
+# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
+# read its bridge IP via `docker inspect`. A host-net job container
+# can reach a bridge-net container directly via the bridge IP (verified
+# manually on operator host 2026-05-08).
+#
+# Trade-offs vs. the original `services:` shape:
+#   + No host-port collision; N parallel runs share the bridge cleanly
+#   + `if: always()` cleanup runs even on test-step failure
+#   - One more step in the workflow (+~3 lines)
+#   - Requires `molecule-core-net` to exist on the operator host
+#     (it does; declared in docker-compose.yml + docker-compose.infra.yml)
+#
+# Class B Hongming-owned CICD red sweep, 2026-05-08.
+#
+# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+concurrency:
+  group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  detect-changes:
+    name: detect-changes
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    outputs:
+      handlers: ${{ steps.filter.outputs.handlers }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: filter
+        # Inline replacement for dorny/paths-filter — see e2e-api.yml.
+        run: |
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "handlers=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            echo "handlers=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
+            echo "handlers=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "handlers=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # Single-job-with-per-step-if pattern: always runs to satisfy the
+  # required-check name on branch protection; real work gates on the
+  # paths filter. See ci.yml's Platform (Go) for the same shape.
+  integration:
+    name: Handlers Postgres Integration
+    needs: detect-changes
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    env:
+      # Unique name per run so concurrent jobs don't collide on the
+      # bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
+      # workflow_dispatch reruns of the same run_id.
+      PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
+      # Bridge network already exists on the operator host (declared
+      # in docker-compose.yml + docker-compose.infra.yml).
+      PG_NETWORK: molecule-core-net
+    defaults:
+      run:
+        working-directory: workspace-server
+    steps:
+      - if: needs.detect-changes.outputs.handlers != 'true'
+        working-directory: .
+        run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Start sibling Postgres on bridge network
+        working-directory: .
+        run: |
+          # Sanity: the bridge network must exist on the operator host.
+          # Hard-fail loud if it doesn't — easier to spot than a silent
+          # auto-create that diverges from the rest of the stack.
+          if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
+            echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
+            exit 1
+          fi
+
+          # If a stale container with the same name exists (rerun on
+          # the same run_id), wipe it first.
+          docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
+
+          docker run -d \
+            --name "${PG_NAME}" \
+            --network "${PG_NETWORK}" \
+            --health-cmd "pg_isready -U postgres" \
+            --health-interval 5s \
+            --health-timeout 5s \
+            --health-retries 10 \
+            -e POSTGRES_PASSWORD=test \
+            -e POSTGRES_DB=molecule \
+            postgres:15-alpine >/dev/null
+
+          # Read back the bridge IP. Always present immediately after
+          # `docker run -d` for bridge networks.
+          PG_HOST=$(docker inspect "${PG_NAME}" \
+            --format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
+          if [ -z "${PG_HOST}" ]; then
+            echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
+            docker logs "${PG_NAME}" || true
+            exit 1
+          fi
+          echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
+          echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
+          echo "Started ${PG_NAME} at ${PG_HOST}:5432"
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Apply migrations to Postgres service
+        env:
+          PGPASSWORD: test
+        run: |
+          # Wait for postgres to actually accept connections. Docker's
+          # health-cmd handles container-side readiness, but the wire
+          # to the bridge IP is best-tested with pg_isready directly.
+          for i in {1..15}; do
+            if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
+            echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
+          done
+
+          # Apply every .up.sql in lexicographic order with
+          # ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
+          # blocking the suite. This handles the current schema state
+          # where a few historical migrations (e.g. 017_memories_fts_*)
+          # depend on tables that were later renamed/dropped and so
+          # cannot replay from scratch. The migrations that DO succeed
+          # land their tables, which is sufficient for the integration
+          # tests in handlers/.
+          #
+          # Why not maintain a curated allowlist: every new migration
+          # touching a handlers/-tested table would have to update this
+          # workflow. With apply-all-or-skip, a future migration that
+          # adds a column to delegations runs automatically (its base
+          # table 049_delegations.up.sql already succeeded above it in
+          # the order). Operators only need to revisit this if the
+          # migration chain becomes legitimately replayable end-to-end.
+          #
+          # Per-migration result is logged so a failed migration that
+          # SHOULD have been replayable surfaces in the CI log instead
+          # of silently failing.
+          # Apply both *.sql (legacy, lives next to its module) and
+          # *.up.sql (newer up/down convention) in a single
+          # lexicographically-sorted pass. Excluding *.down.sql so the
+          # newest-naming-convention pairs don't undo themselves mid-run.
+          # Pre-#149-followup this loop only globbed *.up.sql, which
+          # silently skipped 001_workspaces.sql + 009_activity_logs.sql
+          # — fine while no integration test depended on those tables,
+          # not fine once a cross-table atomicity test came in.
+          set +e
+          for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
+            if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
+                  -f "$migration" >/dev/null 2>&1; then
+              echo "✓ $(basename "$migration")"
+            else
+              echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
+            fi
+          done
+          set -e
+
+          # Sanity: the delegations + workspaces + activity_logs tables
+          # MUST exist for the integration tests to be meaningful. Hard-
+          # fail if any didn't land — that would be a real regression we
+          # want loud.
+          for tbl in delegations workspaces activity_logs pending_uploads; do
+            if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
+                -c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
+                | grep -q 1; then
+              echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
+              exit 1
+            fi
+            echo "✓ $tbl table present"
+          done
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Run integration tests
+        run: |
+          # INTEGRATION_DB_URL is exported by the start-postgres step;
+          # points at the per-run bridge IP, not 127.0.0.1, so concurrent
+          # workflow runs don't fight over a host-net 5432 port.
+          go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
+
+      - if: failure() && needs.detect-changes.outputs.handlers == 'true'
+        name: Diagnostic dump on failure
+        env:
+          PGPASSWORD: test
+        run: |
+          echo "::group::postgres container status"
+          docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
+          docker logs "${PG_NAME}" 2>&1 | tail -50 || true
+          echo "::endgroup::"
+          echo "::group::delegations table state"
+          psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
+          echo "::endgroup::"
+
+      - if: always() && needs.detect-changes.outputs.handlers == 'true'
+        name: Stop sibling Postgres
+        working-directory: .
+        run: |
+          # always() so containers don't leak when migrations or tests
+          # fail. The cleanup is best-effort: if the container is
+          # already gone (e.g. concurrent rerun race), don't fail the job.
+          docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
+          echo "Cleaned up ${PG_NAME}"

.gitea/workflows/harness-replays.yml

@@ -0,0 +1,262 @@
+name: Harness Replays
+
+# Ported from .github/workflows/harness-replays.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Boots tests/harness (production-shape compose topology with TenantGuard,
+# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
+# every replay under tests/harness/replays/. Fails the PR if any replay
+# fails.
+#
+# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
+# a public route in router.go but forgot to add it to TenantGuard's
+# allowlist. The handler-level test in buildinfo_test.go constructed a
+# minimal gin engine without TenantGuard — green. The harness's
+# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
+# inject X-Molecule-Org-Id, so the curl path is identical to production's
+# redeploy verifier), but no one ran the harness pre-merge. The bug
+# shipped; the redeploy verifier silently soft-warned every tenant as
+# "unreachable" for ~1 day before being noticed.
+#
+# This gate makes "did you actually run the harness?" a CI invariant
+# instead of a memory-discipline thing.
+#
+# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
+# to staging+main, real work is gated per-step on detect-changes output.
+# One job → one check run → branch-protection-clean (the SKIPPED-in-set
+# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'tests/harness/**'
+      - '.gitea/workflows/harness-replays.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'tests/harness/**'
+      - '.gitea/workflows/harness-replays.yml'
+concurrency:
+  # Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
+  # cancellation deadlock — see e2e-api.yml's concurrency block for
+  # the 2026-04-28 incident that codified this pattern.
+  group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    outputs:
+      run: ${{ steps.decide.outputs.run }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - id: decide
+        run: |
+          # workflow_dispatch: always run (manual trigger)
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Determine the base commit to diff against.
+          # For pull_request: use base.sha (the merge-base with main/staging).
+          # For push: use github.event.before (the previous tip of the branch).
+          # Fallback for new branches (all-zeros SHA): run everything.
+          if [ "${{ github.event_name }}" = "pull_request" ] && \
+             [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          elif [ -n "${{ github.event.before }}" ] && \
+               ! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
+            BASE="${{ github.event.before }}"
+          else
+            # New branch or github.event.before unavailable — run everything.
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # GitHub Actions and Gitea Actions both expose github.sha for HEAD.
+          DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
+          echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
+
+          if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job that always runs. Real work is gated per-step on
+  # detect-changes.outputs.run so an unrelated PR (e.g. doc-only
+  # change to molecule-controlplane wired here later) emits the
+  # required check without spending CI cycles. Single-job pattern
+  # matches e2e-api.yml — see that workflow's comment for why a
+  # job-level `if: false` would block branch protection via the
+  # SKIPPED-in-set bug.
+  harness-replays:
+    needs: detect-changes
+    name: Harness Replays
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 30
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.run != 'true'
+        run: |
+          echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
+          echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
+          echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
+
+      - if: needs.detect-changes.outputs.run == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # Log what files were detected so future failures include the diff.
+      - name: Log detected changes
+        if: needs.detect-changes.outputs.run == 'true'
+        run: |
+          echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
+
+      # github-app-auth sibling-checkout removed 2026-05-07 (#157):
+      # the plugin was dropped + Dockerfile.tenant no longer COPYs it.
+
+      # Pre-clone manifest deps before docker compose builds the tenant
+      # image (Task #173 followup — same pattern as
+      # publish-workspace-server-image.yml's "Pre-clone manifest deps"
+      # step).
+      #
+      # Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
+      # and tenant-beta from workspace-server/Dockerfile.tenant with
+      # context=../.. (repo root). That Dockerfile expects
+      # .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
+      # to be present at build context root (post-#173 it COPYs from there
+      # instead of running an in-image clone — the in-image clone failed
+      # with "could not read Username for https://git.moleculesai.app"
+      # because there's no auth path inside the build sandbox).
+      #
+      # Without this step harness-replays fails before any replay runs,
+      # with `failed to calculate checksum of ref ...
+      # "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
+      # (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
+      # symptom, different root cause: staging still has the in-image
+      # clone path, hits the auth error directly).
+      #
+      # 2026-05-08 sub-finding (#192): the clone step ALSO fails when
+      # any referenced workspace-template repo is private and the
+      # AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
+      # access. Root cause: 5 of 9 workspace-template repos
+      # (openclaw, codex, crewai, deepagents, gemini-cli) had been
+      # marked private with no team grant. Resolution: flipped them
+      # to public per `feedback_oss_first_repo_visibility_default`
+      # (the OSS surface should be public). Layer-3 (customer-private +
+      # marketplace third-party repos) tracked separately in
+      # internal#102.
+      #
+      # Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
+      # is the devops-engineer persona PAT, NOT the founder PAT (per
+      # `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
+      # embeds it as basic-auth for the duration of the clones and strips
+      # .git directories — the token never enters the resulting image.
+      - name: Pre-clone manifest deps
+        if: needs.detect-changes.outputs.run == 'true'
+        env:
+          MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
+            echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
+            exit 1
+          fi
+          mkdir -p .tenant-bundle-deps
+          bash scripts/clone-manifest.sh \
+            manifest.json \
+            .tenant-bundle-deps/workspace-configs-templates \
+            .tenant-bundle-deps/org-templates \
+            .tenant-bundle-deps/plugins
+          # Sanity-check counts so a silent partial clone fails fast
+          # instead of producing a half-empty image.
+          ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
+          echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
+
+      - name: Install Python deps for replays
+        # peer-discovery-404 (and future replays) eval Python against the
+        # running tenant — importing workspace/a2a_client.py pulls in
+        # httpx. tests/harness/requirements.txt holds just the HTTP-client
+        # surface to keep CI install fast (~3s) vs the full
+        # workspace/requirements.txt (~30s).
+        if: needs.detect-changes.outputs.run == 'true'
+        run: pip install -r tests/harness/requirements.txt
+
+      - name: Run all replays against the harness
+        # run-all-replays.sh: boot via up.sh → seed via seed.sh → run
+        # every replays/*.sh → tear down via down.sh on EXIT (trap).
+        # Non-zero exit on any replay failure.
+        #
+        # KEEP_UP=1: without this, the script's trap-on-EXIT tears
+        # down containers immediately on failure, leaving the dump
+        # step below with nothing to dump (verified on PR #2410's
+        # first run — tenant became unhealthy, trap fired, dump
+        # step saw empty containers). Keeping them up lets the
+        # failure path collect tenant/cp-stub/cf-proxy logs. The
+        # always-run "Force teardown" step does the actual cleanup.
+        if: needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        env:
+          KEEP_UP: "1"
+        run: ./run-all-replays.sh
+
+      - name: Dump compose logs on failure
+        # SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
+        # file even for read-only `logs` calls. up.sh generates a per-run key
+        # and exports it to its OWN shell — this step runs in a fresh shell
+        # that wouldn't see it, so without a placeholder the validate step
+        # errors before logs print (verified against PR #2492's first run:
+        # "required variable SECRETS_ENCRYPTION_KEY is missing a value").
+        # A placeholder is fine — we're only reading log streams, not booting.
+        if: failure() && needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        env:
+          SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
+        run: |
+          echo "=== docker compose ps ==="
+          docker compose -f compose.yml ps || true
+          echo "=== tenant-alpha logs ==="
+          docker compose -f compose.yml logs tenant-alpha || true
+          echo "=== tenant-beta logs ==="
+          docker compose -f compose.yml logs tenant-beta || true
+          echo "=== cp-stub logs ==="
+          docker compose -f compose.yml logs cp-stub || true
+          echo "=== cf-proxy logs ==="
+          docker compose -f compose.yml logs cf-proxy || true
+          echo "=== postgres-alpha logs (last 100) ==="
+          docker compose -f compose.yml logs --tail 100 postgres-alpha || true
+          echo "=== postgres-beta logs (last 100) ==="
+          docker compose -f compose.yml logs --tail 100 postgres-beta || true
+
+      - name: Force teardown
+        # We pass KEEP_UP=1 to run-all-replays.sh so the dump step
+        # above sees real containers — that means we own teardown
+        # explicitly here. Always run.
+        if: always() && needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        run: ./down.sh || true

.gitea/workflows/lint-curl-status-capture.yml

@@ -0,0 +1,104 @@
+name: Lint curl status-code capture
+
+# Ported from .github/workflows/lint-curl-status-capture.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - on.paths and the lint scanner target .gitea/workflows/**.yml (the
+#     active Gitea workflow directory) instead of .github/workflows/**.yml
+#     (which the rest of this sweep is emptying out).
+#   - Self-skip path updated to the .gitea/ version of this file.
+#   - Dropped `merge_group:` trigger.
+#   - Workflow-level env.GITHUB_SERVER_URL set per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
+# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
+#
+#   HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
+#
+# When curl exits non-zero (connection reset -> 56, --fail-with-body 4xx/5xx
+# -> 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
+# "000" for connection failures or the actual code for HTTP errors. The
+# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
+# stdout, producing values like "000000" or "409000" that fail string
+# comparisons against "200" while looking superficially right.
+#
+# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
+# #2797). Memory: feedback_curl_status_capture_pollution.md.
+
+on:
+  pull_request:
+    paths: ['.gitea/workflows/**']
+  push:
+    branches: [main, staging]
+    paths: ['.gitea/workflows/**']
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  scan:
+    name: Scan workflows for curl status-capture pollution
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
+        run: |
+          set -uo pipefail
+          # Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
+          # subshell where the entire command-substitution wraps a curl that
+          # ends with `|| echo "000"`. Must distinguish from the SAFE shape
+          # `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
+          # tempfile produces empty stdout, no pollution.
+          python3 <<'PY'
+          import os, re, sys, glob
+
+          BAD_FILES = []
+
+          # Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
+          # The `\\n` is the bash line-continuation that lets curl flags span lines.
+          # We collapse continuation lines first, then look for the single-line bad pattern.
+          PATTERN = re.compile(
+              r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
+              re.DOTALL,
+          )
+
+          # Self-skip: this lint workflow contains the literal anti-pattern in
+          # its own docstring — that's intentional, not a bug.
+          SELF = ".gitea/workflows/lint-curl-status-capture.yml"
+
+          for f in sorted(glob.glob(".gitea/workflows/*.yml")):
+              if f == SELF:
+                  continue
+              with open(f) as fh:
+                  content = fh.read()
+              # Collapse bash line-continuations (\\\n + leading whitespace)
+              # into a single logical line so the regex can see the full
+              # curl invocation as one chunk.
+              flat = re.sub(r'\\\s*\n\s*', ' ', content)
+              for m in PATTERN.finditer(flat):
+                  BAD_FILES.append((f, m.group(0)[:120]))
+
+          if not BAD_FILES:
+              print("OK No curl-status-capture pollution patterns detected")
+              sys.exit(0)
+
+          print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
+          for f, snippet in BAD_FILES:
+              print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
+              print(f"   matched: {snippet}...")
+          print()
+          print("Fix template:")
+          print('  set +e')
+          print('  curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
+          print('  set -e')
+          print('  HTTP_CODE=$(cat code.txt 2>/dev/null)')
+          print('  [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
+          sys.exit(1)
+          PY

.gitea/workflows/main-red-watchdog.yml

@@ -0,0 +1,94 @@
+# main-red-watchdog — hourly sentinel for post-merge CI red on `main`.
+#
+# RFC: hongming "main NEVER goes red" directive, Option C of the four-
+# option ladder (B = auto-revert is explicitly rejected per
+# `feedback_no_such_thing_as_flakes` + `feedback_fix_root_not_symptom`).
+# Tracking issue: molecule-core#420.
+#
+# What it does:
+#   1. GET branches/main → HEAD SHA
+#   2. GET commits/{SHA}/status → combined status
+#   3. If combined is `failure` (or any individual status is `failure`):
+#      open or PATCH an idempotent `[main-red] {repo}: {SHA[:10]}` issue
+#      with each failed context + target_url + description.
+#   4. If combined is `success` and a prior `[main-red] ...` issue exists,
+#      close it with a "main returned to green at SHA ..." comment.
+#   5. Emit a Loki-shaped JSON line via `logger -t main-red-watchdog` for
+#      `reference_obs_stack_phase1` ingestion via Vector.
+#
+# What it does NOT do:
+#   - Auto-revert anything. Option B is rejected by directive.
+#   - Mutate branch protection. (See AGENTS.md boundaries.)
+#   - Fail the workflow on red. The issue IS the alarm — failing the
+#     watchdog would create a silent-loop where a flake in the watchdog
+#     itself hides actual main-red signal. Exit 0 unless api() raises
+#     ApiError (transient Gitea outage → fail loudly per
+#     `feedback_api_helper_must_raise_not_return_dict`).
+#
+# Pattern source: molecule-controlplane `0adf2098`'s ci-required-drift.yml
+# (just merged 2026-05-11). Same shape (cron + dispatch + sidecar Python +
+# idempotent-by-title issue), simpler scope (1 source, not 3).
+
+name: main-red-watchdog
+
+# IMPORTANT — Gitea 1.22.6 parser quirk per
+# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
+# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
+# "unknown on type" when `workflow_dispatch.inputs.X` is present. Revisit
+# when Gitea ≥ 1.23 is fleet-wide.
+on:
+  schedule:
+    # Hourly at :05 — task spec calls for "off-zero" (`5 * * * *`),
+    # offset from :17 (ci-required-drift) and :00 (peak cron load).
+    - cron: '5 * * * *'
+  workflow_dispatch:
+
+# Read commit status + branch ref + issues; write issues (open/PATCH/close).
+permissions:
+  contents: read
+  issues: write
+
+# Workflow-scoped serialisation — two simultaneous runs would race on the
+# `[main-red] {SHA}` open/PATCH path. Idempotent by title, but parallel
+# POSTs can produce duplicates before the title search dedup wins.
+concurrency:
+  group: main-red-watchdog
+  cancel-in-progress: false
+
+jobs:
+  watchdog:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Check out repo (script lives at .gitea/scripts/)
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+
+      - name: Set up Python (stdlib only — no PyYAML needed here)
+        # The script uses stdlib urllib + json. No PyYAML required (CP's
+        # drift detector needs it for AST parsing; we don't). Pin to the
+        # same 3.12 hermetic interpreter CP uses so the test/runtime
+        # versions stay aligned across watchdog suites.
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: '3.12'
+
+      - name: Run main-red watchdog
+        env:
+          # GITEA_TOKEN reads commit status + writes issues. Falls back
+          # to the auto-injected GITHUB_TOKEN if the org-level secret
+          # isn't set (transitional repos), matching the same pattern
+          # used by deploy-pipeline.yml + ci-required-drift.yml.
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          # Branch under watch. `main` per directive; staging not
+          # included here — staging green is a separate gate
+          # (`feedback_staging_e2e_merge_gate`).
+          WATCH_BRANCH: 'main'
+          # Issue label applied on file/open. `tier:high` exists in the
+          # molecule-core label set (verified 2026-05-11, label id 9).
+          # Rationale for high: main red blocks the promotion train and
+          # poisons every PR's auto-rebase base; treat as a fire even
+          # if intermittent.
+          RED_LABEL: 'tier:high'
+        run: python3 .gitea/scripts/main-red-watchdog.py

.gitea/workflows/publish-canvas-image.yml

@@ -0,0 +1,138 @@
+name: publish-canvas-image
+
+# Ported from .github/workflows/publish-canvas-image.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#   - **Open question for review**: this workflow pushes the canvas
+#     image to `ghcr.io`. GHCR was retired during the 2026-05-06
+#     Gitea migration in favor of ECR (per canary-verify.yml header
+#     notes). The image may not be consumable post-migration. Two
+#     options for follow-up: (a) retarget to
+#     `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas`,
+#     or (b) retire this workflow entirely and route canvas deploys
+#     via the operator-host build path. tier:low + continue-on-error
+#     means failed pushes do not block PRs.
+#
+
+# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
+# on main that touches canvas code. Previously canvas changes were visible in
+# CI (npm run build passed) but the live container was never updated —
+# operators had to manually run `docker compose build canvas` each time.
+#
+# Mirror of publish-platform-image.yml, adapted for the Next.js canvas layer.
+# See that workflow for inline notes on macOS Keychain isolation and QEMU.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      # Only rebuild when canvas source changes — saves GHA minutes on
+      # platform-only / docs-only / MCP-only merges.
+      - 'canvas/**'
+      - '.gitea/workflows/publish-canvas-image.yml'
+  # NOTE (Gitea port): the original GitHub workflow had a
+  # `workflow_dispatch:` manual trigger for the
+  # non-canvas-merge-but-need-fresh-image scenario. Dropped in the
+  # Gitea port (1.22.6 parser-finicky). Manual rebuilds require
+  # pushing an empty commit to canvas/ or running the operator-host
+  # build directly.
+
+permissions:
+  contents: read
+  packages: write  # required to push to ghcr.io/${{ github.repository_owner }}/*
+
+env:
+  IMAGE_NAME: ghcr.io/molecule-ai/canvas
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  build-and-push:
+    name: Build & push canvas image
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Log in to GHCR
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+
+      # Health check: verify Docker daemon is accessible before attempting any
+      # build steps. This fails loudly at step 1 when the runner's docker.sock
+      # is inaccessible rather than silently continuing to the build step
+      # where docker build fails deep in ECR auth with a cryptic error.
+      - name: Verify Docker daemon access
+        run: |
+          set -euo pipefail
+          echo "::group::Docker daemon health check"
+          docker info 2>&1 | head -5 || {
+            echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
+            echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
+            exit 1
+          }
+          echo "Docker daemon OK"
+          echo "::endgroup::"
+
+      - name: Compute tags
+        id: tags
+        shell: bash
+        run: |
+          echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve build args
+        id: build_args
+        # Priority: workflow_dispatch input > repo secret > hardcoded default.
+        # NEXT_PUBLIC_* env vars are baked into the JS bundle at build time by
+        # Next.js — they cannot be changed at runtime without a full rebuild.
+        # For local docker-compose deployments the defaults (localhost:8080)
+        # work as-is; production deployments should set CANVAS_PLATFORM_URL
+        # and CANVAS_WS_URL as repository secrets.
+        #
+        # Inputs are passed via env vars (not direct ${{ }} interpolation) to
+        # prevent shell injection from workflow_dispatch string inputs.
+        shell: bash
+        env:
+          INPUT_PLATFORM_URL: ${{ github.event.inputs.platform_url }}
+          SECRET_PLATFORM_URL: ${{ secrets.CANVAS_PLATFORM_URL }}
+          INPUT_WS_URL: ${{ github.event.inputs.ws_url }}
+          SECRET_WS_URL: ${{ secrets.CANVAS_WS_URL }}
+        run: |
+          PLATFORM_URL="${INPUT_PLATFORM_URL:-${SECRET_PLATFORM_URL:-http://localhost:8080}}"
+          WS_URL="${INPUT_WS_URL:-${SECRET_WS_URL:-ws://localhost:8080/ws}}"
+
+          echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
+          echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
+
+      - name: Build & push canvas image to GHCR
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
+        with:
+          context: ./canvas
+          file: ./canvas/Dockerfile
+          platforms: linux/amd64
+          push: true
+          build-args: |
+            NEXT_PUBLIC_PLATFORM_URL=${{ steps.build_args.outputs.platform_url }}
+            NEXT_PUBLIC_WS_URL=${{ steps.build_args.outputs.ws_url }}
+          tags: |
+            ${{ env.IMAGE_NAME }}:latest
+            ${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)

.gitea/workflows/publish-runtime-autobump.yml

@@ -0,0 +1,100 @@
+name: publish-runtime-autobump
+
+# Auto-bump-on-workspace-edit half of the publish pipeline.
+#
+# Why this file exists (issue #351):
+#   Gitea Actions does not correctly disambiguate `paths:` from `tags:`
+#   when both are bundled under a single `on.push` key. The result is
+#   that tag pushes get filtered out and `publish-runtime.yml` never
+#   fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
+#   because PYPI_TOKEN was absent (publishes would have failed anyway).
+#
+#   Split design:
+#     - publish-runtime.yml         : on.push.tags only        (the publisher)
+#     - publish-runtime-autobump.yml: on.push.branches+paths   (this file — the version-bumper)
+#
+#   This file computes the next version from PyPI's latest, pushes a
+#   `runtime-v$VERSION` tag, and exits. The tag push then triggers
+#   publish-runtime.yml via its tags-only trigger.
+#
+# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
+# so concurrent workspace pushes serialize at the bump step. Without
+# this, two pushes minutes apart could both read PyPI latest=0.1.129
+# and try to tag 0.1.130 simultaneously, only one of which would land.
+
+on:
+  push:
+    branches:
+      - main
+      - staging
+    paths:
+      - "workspace/**"
+
+permissions:
+  contents: write  # required to push tags back
+
+concurrency:
+  group: publish-runtime
+  cancel-in-progress: false
+
+jobs:
+  autobump-and-tag:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Fetch full tag list so the bump logic can sanity-check against
+          # what's already in this repo (catches collision with prior
+          # manual tag pushes).
+          fetch-depth: 0
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Compute next version from PyPI latest
+        id: bump
+        run: |
+          set -eu
+          LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
+            | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
+          MAJOR=$(echo "$LATEST" | cut -d. -f1)
+          MINOR=$(echo "$LATEST" | cut -d. -f2)
+          PATCH=$(echo "$LATEST" | cut -d. -f3)
+          VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
+          echo "PyPI latest=$LATEST -> next=$VERSION"
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
+            exit 1
+          fi
+          if git tag --list | grep -qx "runtime-v$VERSION"; then
+            echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
+            exit 1
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Push runtime-v$VERSION tag
+        env:
+          DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+          VERSION: ${{ steps.bump.outputs.version }}
+          GITEA_URL: https://git.moleculesai.app
+        run: |
+          set -eu
+          if [ -z "$DISPATCH_TOKEN" ]; then
+            echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
+            exit 1
+          fi
+          git config user.name  "publish-runtime autobump"
+          git config user.email "publish-runtime@moleculesai.app"
+          git tag -a "runtime-v$VERSION" \
+            -m "Auto-bump on workspace/** edit on $GITHUB_REF" \
+            -m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
+            -m "publish-runtime.yml will pick up this tag and upload to PyPI"
+          # Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
+          # ensures the resulting tag-push event is dispatched to
+          # publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
+          # trigger downstream workflows.
+          git remote set-url origin "${GITEA_URL#https://}"
+          git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
+          git push origin "runtime-v$VERSION"
+          echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"

.gitea/workflows/publish-runtime.yml

@@ -0,0 +1,339 @@
+name: publish-runtime
+
+# Gitea Actions port of .github/workflows/publish-runtime.yml.
+#
+# Ported 2026-05-10 (issue #206). Key differences from the GitHub version:
+#   - Gitea Actions reads .gitea/workflows/, not .github/workflows/
+#   - Dropped `environment: pypi-publish` — Gitea Actions does not support
+#     named environments or OIDC trusted publishers
+#   - Replaced `pypa/gh-action-pypi-publish@release/v1` (OIDC) with
+#     `twine upload` using PYPI_TOKEN secret — same mechanism as a local
+#     `python -m twine upload` with a PyPI token
+#   - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
+#     — Gitea Actions exposes github.ref (the full ref) but not ref_name
+#   - Dropped `merge_group` trigger (Gitea has no merge queue)
+#
+# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
+# `workspace/**` path-filter trigger in PR #349.
+#
+# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
+# file. Bundling `paths` with `tags` under a single `on.push` key caused
+# Gitea Actions to never dispatch the workflow for tag-push events (0
+# runs in `action_run` for workflow_id='publish-runtime.yml' since the
+# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
+# 0.1.129 despite a v1.0.0 Gitea tag existing).
+#
+# The auto-bump-on-workspace-edit trigger now lives in
+# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
+# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
+# which THIS file then picks up via the tags-only trigger below.
+#
+# This decoupling means Gitea's path-vs-tag evaluator never has to
+# disambiguate — each file has a single unambiguous trigger shape.
+#
+# PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
+# Set via: repo Settings → Actions → Variables and Secrets → New Secret.
+# The token should be a PyPI API token scoped to molecule-ai-workspace-runtime.
+#
+# The DISPATCH_TOKEN cascade (git push to template repos) is unchanged —
+# it uses the Gitea API directly and was already Gitea-compatible.
+
+on:
+  push:
+    tags:
+      - "runtime-v*"
+  workflow_dispatch:
+  # 2026-05-11 (root cause of #351 / 0 runs ever):
+  # Gitea 1.22.6's workflow parser rejects `workflow_dispatch.inputs.version`
+  # with "unknown on type" — it mis-treats the inputs sub-keys as top-level
+  # `on:` event types. Log line:
+  #   actions/workflows.go:DetectWorkflows() [W] ignore invalid workflow
+  #   "publish-runtime.yml": unknown on type: map["version": {...}]
+  # That `[W] ignore invalid workflow` is silent UX — the workflow never
+  # registers, so it never fires for ANY event (push.tags included).
+  # Removing the inputs block restores parsing. Manual dispatch from the
+  # Gitea UI now triggers the PyPI auto-bump fallback in `Derive version`
+  # below (no `inputs.version` to read).
+
+permissions:
+  contents: read
+
+# Serialize publishes so two concurrent tag pushes don't both compute
+# "latest+1" and race on PyPI upload. The second one waits.
+concurrency:
+  group: publish-runtime
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Derive version (tag or PyPI auto-bump)
+        id: version
+        run: |
+          if echo "$GITHUB_REF" | grep -q "^refs/tags/runtime-v"; then
+            # Tag is `runtime-vX.Y.Z` — strip the prefix.
+            VERSION="${GITHUB_REF#refs/tags/runtime-v}"
+          else
+            # workflow_dispatch path (no inputs supported on Gitea 1.22.6) or
+            # any other non-tag trigger: derive from PyPI latest + patch bump.
+            LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
+              | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
+            MAJOR=$(echo "$LATEST" | cut -d. -f1)
+            MINOR=$(echo "$LATEST" | cut -d. -f2)
+            PATCH=$(echo "$LATEST" | cut -d. -f3)
+            VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
+            echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
+          fi
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
+            echo "::error::version $VERSION does not match PEP 440"
+            exit 1
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Publishing molecule-ai-workspace-runtime $VERSION"
+
+      - name: Install build tooling
+        run: pip install build twine
+
+      - name: Build package from workspace/
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "${{ steps.version.outputs.version }}" \
+            --out "${{ runner.temp }}/runtime-build"
+
+      - name: Build wheel + sdist
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: python -m build
+
+      - name: Capture wheel SHA256 for cascade content-verification
+        id: wheel_hash
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: |
+          set -eu
+          WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
+          if [ -z "$WHEEL" ]; then
+            echo "::error::No .whl in dist/ — \`python -m build\` must have failed silently"
+            exit 1
+          fi
+          HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
+          echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
+          echo "Local wheel SHA256 (pre-upload): ${HASH}"
+          echo "Wheel filename: $(basename "$WHEEL")"
+
+      - name: Verify package contents (sanity)
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: |
+          python -m twine check dist/*
+          python -m venv /tmp/smoke
+          /tmp/smoke/bin/pip install --quiet dist/*.whl
+          /tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
+
+      - name: Publish to PyPI
+        # working-directory matches the preceding Build/Verify steps. Without
+        # this, twine runs from the default workspace checkout dir where
+        # `dist/` doesn't exist and fails with:
+        #   ERROR InvalidDistribution: Cannot find file (or expand pattern): 'dist/*'
+        # Caught on the first-ever successful dispatch of this workflow
+        # (run 5097, 2026-05-11 02:08Z) — every other step in the publish
+        # job already had this working-directory; Publish was missing it.
+        working-directory: ${{ runner.temp }}/runtime-build
+        env:
+          # PYPI_TOKEN: repository secret scoped to molecule-ai-workspace-runtime.
+          # Set via: Settings → Actions → Variables and Secrets → New Secret.
+          # Format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          if [ -z "$PYPI_TOKEN" ]; then
+            echo "::error::PYPI_TOKEN secret is not set — set it at Settings → Actions → Variables and Secrets → New Secret."
+            echo "::error::Required format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+            exit 1
+          fi
+          python -m twine upload \
+            --repository pypi \
+            --username __token__ \
+            --password "$PYPI_TOKEN" \
+            dist/*
+
+  cascade:
+    needs: publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for PyPI to propagate the new version
+        env:
+          RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
+          EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
+        run: |
+          set -eu
+          if [ -z "$EXPECTED_SHA256" ]; then
+            echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
+            exit 1
+          fi
+          python -m venv /tmp/propagation-probe
+          PROBE=/tmp/propagation-probe/bin
+          $PROBE/pip install --upgrade --quiet pip
+          for i in $(seq 1 30); do
+            if $PROBE/pip install \
+                  --quiet \
+                  --no-cache-dir \
+                  --force-reinstall \
+                  --no-deps \
+                  "molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
+                  >/dev/null 2>&1; then
+              INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
+                          | awk -F': ' '/^Version:/{print $2}')
+              if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
+                echo "✓ PyPI resolved $RUNTIME_VERSION (install check)"
+                break
+              fi
+            fi
+            if [ $i -eq 30 ]; then
+              echo "::error::pip install --no-cache-dir molecule-ai-workspace-runtime==${RUNTIME_VERSION} never resolved within ~5 min."
+              echo "::error::Refusing to fan out cascade against a potentially stale PyPI index."
+              exit 1
+            fi
+            echo "  [$i/30] waiting for PyPI to propagate ${RUNTIME_VERSION}..."
+            sleep 4
+          done
+
+          # Stage (b): download wheel + SHA256 compare against what we built.
+          # Catches Fastly stale-content serving old bytes under a new version URL.
+          #
+          # Caught run 5196 (first-ever successful publish, 2026-05-11): the
+          # previous one-liner `HASH=$(pip download ... && sha256sum ...)`
+          # captured pip's stdout (`Collecting molecule-ai-workspace-runtime
+          # ==X.Y.Z`) into HASH, then the SHA comparison failed against the
+          # leaked `Collecting...` string. `2>/dev/null` silences stderr but
+          # NOT stdout; pip writes its progress to stdout by default.
+          # Fix: split into two steps, silence pip's stdout explicitly, capture
+          # only sha256sum's output into HASH.
+          python -m pip download \
+            --no-deps \
+            --no-cache-dir \
+            --dest /tmp/wheel-probe \
+            --quiet \
+            "molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
+            >/dev/null 2>&1
+          HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
+          if [ "$HASH" != "$EXPECTED_SHA256" ]; then
+            echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
+            echo "::error::Expected: $EXPECTED_SHA256"
+            echo "::error::Got:      $HASH"
+            echo "::error::Fastly may be serving stale content. Refusing to fan out cascade."
+            exit 1
+          fi
+          echo "✓ PyPI CDN verified (SHA256 match)"
+
+      - name: Fan out via push to .runtime-version
+        env:
+          # Gitea PAT with write:repository scope on the 8 cascade-active
+          # template repos. Used for git push to each template repo's main
+          # branch, which trips their `on: push: branches: [main]` trigger
+          # on publish-image.yml.
+          DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+          RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
+        run: |
+          set +e   # don't abort on a single repo failure — collect them all
+
+          if [ -z "$DISPATCH_TOKEN" ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
+              echo "::warning::set it at Settings → Actions → Variables and Secrets → New Secret."
+              exit 0
+            fi
+            echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
+            echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version."
+            exit 1
+          fi
+          VERSION="$RUNTIME_VERSION"
+          if [ -z "$VERSION" ]; then
+            echo "::error::publish job did not expose a version output"
+            exit 1
+          fi
+
+          GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
+          TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
+          FAILED=""
+          SKIPPED=""
+
+          git config --global user.name  "publish-runtime cascade"
+          git config --global user.email "publish-runtime@moleculesai.app"
+
+          WORKDIR="$(mktemp -d)"
+          for tpl in $TEMPLATES; do
+            REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
+            CLONE="$WORKDIR/$tpl"
+
+            HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
+              -H "Authorization: token $DISPATCH_TOKEN" \
+              "$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
+            if [ "$HTTP" = "404" ]; then
+              echo "↷ $tpl has no publish-image.yml — soft-skip"
+              SKIPPED="$SKIPPED $tpl"
+              continue
+            fi
+
+            attempt=0
+            success=false
+            while [ $attempt -lt 3 ]; do
+              attempt=$((attempt + 1))
+              rm -rf "$CLONE"
+              if ! git clone --depth=1 \
+                  "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
+                  "$CLONE" >/tmp/clone.log 2>&1; then
+                echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
+                sleep 2
+                continue
+              fi
+
+              cd "$CLONE"
+              echo "$VERSION" > .runtime-version
+
+              if git diff --quiet -- .runtime-version; then
+                echo "✓ $tpl already at $VERSION — no commit needed"
+                success=true
+                cd - >/dev/null
+                break
+              fi
+
+              git add .runtime-version
+              git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
+                -m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
+                >/dev/null
+
+              if git push origin HEAD:main >/tmp/push.log 2>&1; then
+                echo "✓ $tpl pushed $VERSION on attempt $attempt"
+                success=true
+                cd - >/dev/null
+                break
+              fi
+
+              echo "::warning::push $tpl attempt $attempt failed, pull-rebasing"
+              git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
+              cd - >/dev/null
+            done
+
+            if [ "$success" != "true" ]; then
+              FAILED="$FAILED $tpl"
+            fi
+          done
+          rm -rf "$WORKDIR"
+
+          if [ -n "$FAILED" ]; then
+            echo "::error::Cascade incomplete after 3 retries each. Failed:$FAILED"
+            exit 1
+          fi
+          if [ -n "$SKIPPED" ]; then
+            echo "Cascade complete: pinned $VERSION. Soft-skipped (no publish-image.yml):$SKIPPED"
+          else
+            echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
+          fi

.gitea/workflows/publish-workspace-server-image.yml

@@ -0,0 +1,174 @@
+name: publish-workspace-server-image
+
+# Gitea Actions port of .github/workflows/publish-workspace-server-image.yml.
+#
+# Ported 2026-05-10 (issue #228). Key differences from the GitHub version:
+#   - Gitea Actions reads .gitea/workflows/, not .github/workflows/
+#   - Dropped `environment:` declarations — Gitea Actions does not support
+#     named environments (used by GitHub OIDC token gates)
+#   - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/heads/}`
+#     — Gitea Actions exposes GITHUB_REF in the same format as GitHub Actions
+#   - docker/setup-buildx-action and aws-actions/configure-aws-credentials are
+#     GitHub Marketplace actions; they are installed by Gitea Actions runners and
+#     work identically here
+#   - All other variables (GITHUB_SHA, GITHUB_REPOSITORY, GITHUB_OUTPUT,
+#     secrets.*) use the same syntax as GitHub Actions
+#
+# Image tags produced:
+#   :staging-<sha> — per-commit digest, stable for canary verify
+#   :staging-latest — tracks most recent build on this branch
+#
+# ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
+# Required secrets: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AUTO_SYNC_TOKEN
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'manifest.json'
+      - 'scripts/**'
+      - '.gitea/workflows/publish-workspace-server-image.yml'
+  workflow_dispatch:
+
+# Serialize per-branch so two rapid staging pushes don't race the same
+# :staging-latest tag retag. Allow staging and main to run in parallel
+# (different GITHUB_REF → different concurrency group) since they
+# produce different :staging-<sha> tags and last-write-wins on
+# :staging-latest is acceptable across branches.
+#
+# cancel-in-progress: false → in-flight builds finish; the next push's
+# build queues. This avoids a partially-pushed image.
+concurrency:
+  group: publish-workspace-server-image-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
+  TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # Health check: verify Docker daemon is accessible before attempting any
+      # build steps. This fails loudly at step 1 when the runner's docker.sock
+      # is inaccessible (e.g. permission change, daemon restart, or group-membership
+      # drift) rather than silently continuing to step 2 where `docker build`
+      # fails deep in the process with a cryptic ECR auth error that doesn't
+      # surface the root cause.  Also reports the daemon version so operator
+      # can correlate with runner host logs.
+      - name: Verify Docker daemon access
+        run: |
+          set -euo pipefail
+          echo "::group::Docker daemon health check"
+          docker info 2>&1 | head -5 || {
+            echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
+            echo "::error::Check: (1) daemon is running, (2) runner user is in docker group, (3) sock permissions are 660+"
+            exit 1
+          }
+          echo "Docker daemon OK"
+          echo "::endgroup::"
+
+      # Pre-clone manifest deps before docker build.
+      #
+      # Why: workspace-template-* repos on Gitea are private. The pre-fix
+      # Dockerfile.tenant ran `git clone` inside an in-image stage with no
+      # auth path — every CI build failed. We clone in the trusted CI
+      # context where AUTO_SYNC_TOKEN is available and Dockerfile.tenant
+      # just COPYs from .tenant-bundle-deps/.
+      #
+      # Token: AUTO_SYNC_TOKEN is the devops-engineer persona PAT.
+      # clone-manifest.sh embeds it as basic-auth for the clones, then
+      # strips .git dirs — the token never enters the image.
+      - name: Pre-clone manifest deps
+        env:
+          MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
+            echo "::error::AUTO_SYNC_TOKEN secret is empty"
+            exit 1
+          fi
+          mkdir -p .tenant-bundle-deps
+          bash scripts/clone-manifest.sh \
+            manifest.json \
+            .tenant-bundle-deps/workspace-configs-templates \
+            .tenant-bundle-deps/org-templates \
+            .tenant-bundle-deps/plugins
+          ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
+          echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
+
+      - name: Compute tags
+        id: tags
+        run: |
+          echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
+      # Build + push platform image (inline ECR auth — mirrors the operator-host
+      # approach; credentials come from GITHUB_SECRET_AWS_ACCESS_KEY_ID /
+      # GITHUB_SECRET_AWS_SECRET_ACCESS_KEY in Gitea Actions).
+      - name: Build & push platform image to ECR (staging-<sha> + staging-latest)
+        env:
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
+          TAG_SHA: staging-${{ steps.tags.outputs.sha }}
+          TAG_LATEST: staging-latest
+          GIT_SHA: ${{ github.sha }}
+          REPO: ${{ github.repository }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-2
+        run: |
+          set -euo pipefail
+          ECR_REGISTRY="${IMAGE_NAME%%/*}"
+          aws ecr get-login-password --region us-east-2 | \
+            docker login --username AWS --password-stdin "${ECR_REGISTRY}"
+          docker build \
+            --file ./workspace-server/Dockerfile \
+            --build-arg GIT_SHA="${GIT_SHA}" \
+            --label "org.opencontainers.image.source=https://github.com/${REPO}" \
+            --label "org.opencontainers.image.revision=${GIT_SHA}" \
+            --label "org.opencontainers.image.description=Molecule AI platform — pending canary verify" \
+            --tag "${IMAGE_NAME}:${TAG_SHA}" \
+            --tag "${IMAGE_NAME}:${TAG_LATEST}" \
+            .
+          docker push "${IMAGE_NAME}:${TAG_SHA}"
+          docker push "${IMAGE_NAME}:${TAG_LATEST}"
+
+      # Build + push tenant image (Go platform + Next.js canvas in one image).
+      - name: Build & push tenant image to ECR (staging-<sha> + staging-latest)
+        env:
+          TENANT_IMAGE_NAME: ${{ env.TENANT_IMAGE_NAME }}
+          TAG_SHA: staging-${{ steps.tags.outputs.sha }}
+          TAG_LATEST: staging-latest
+          GIT_SHA: ${{ github.sha }}
+          REPO: ${{ github.repository }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-2
+        run: |
+          set -euo pipefail
+          ECR_REGISTRY="${TENANT_IMAGE_NAME%%/*}"
+          aws ecr get-login-password --region us-east-2 | \
+            docker login --username AWS --password-stdin "${ECR_REGISTRY}"
+          docker build \
+            --file ./workspace-server/Dockerfile.tenant \
+            --build-arg NEXT_PUBLIC_PLATFORM_URL= \
+            --build-arg GIT_SHA="${GIT_SHA}" \
+            --label "org.opencontainers.image.source=https://github.com/${REPO}" \
+            --label "org.opencontainers.image.revision=${GIT_SHA}" \
+            --label "org.opencontainers.image.description=Molecule AI tenant platform + canvas — pending canary verify" \
+            --tag "${TENANT_IMAGE_NAME}:${TAG_SHA}" \
+            --tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}" \
+            .
+          docker push "${TENANT_IMAGE_NAME}:${TAG_SHA}"
+          docker push "${TENANT_IMAGE_NAME}:${TAG_LATEST}"

.gitea/workflows/railway-pin-audit.yml

@@ -0,0 +1,181 @@
+name: Railway pin audit (drift detection)
+
+# Ported from .github/workflows/railway-pin-audit.yml on 2026-05-11 per
+# RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `workflow_dispatch:` (Gitea 1.22.6 trigger handling).
+#     Manual runs go via cron-trigger bump or push the workflow file
+#     itself.
+#   - `actions/github-script@v9` blocks (which call github.rest.* — a
+#     GitHub-specific JS API) replaced with curl calls against the
+#     Gitea REST API (/api/v1/repos/.../issues, .../labels,
+#     .../comments). Same behaviour: open issue on drift, comment on
+#     repeat-drift, close on clean run.
+#   - Workflow-level env.GITHUB_SERVER_URL set so the curl calls can
+#     derive `git.moleculesai.app` from the runner env (with
+#     hard-coded fallback inside the steps).
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# Daily audit of Railway env vars for drift-prone image-tag pins —
+# automation-cadence layer over the detection script + regression test
+# shipped in PR #2168 (#2001 closure).
+#
+# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
+# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
+# "every fix didn't propagate" — really the tenant image was so old it
+# didn't read the env vars those fixes produced.
+#
+# Cadence: once a day, 13:00 UTC (06:00 PT).
+#
+# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
+# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN.
+
+on:
+  schedule:
+    - cron: '0 13 * * *'
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  group: railway-pin-audit
+  cancel-in-progress: false
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  audit:
+    name: Audit Railway env vars for drift-prone pins
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify RAILWAY_AUDIT_TOKEN present
+        env:
+          RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        id: secret_check
+        run: |
+          set -euo pipefail
+          if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
+            echo "have_secret=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "have_secret=false" >> "$GITHUB_OUTPUT"
+          echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
+          exit 1
+
+      - name: Install Railway CLI
+        if: steps.secret_check.outputs.have_secret == 'true'
+        run: |
+          set -euo pipefail
+          curl -fsSL https://railway.com/install.sh | sh
+          echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
+
+      - name: Verify Railway CLI authenticated
+        if: steps.secret_check.outputs.have_secret == 'true'
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set -euo pipefail
+          if ! railway whoami >/dev/null 2>&1; then
+            echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
+            exit 2
+          fi
+
+      - name: Link molecule-platform project
+        if: steps.secret_check.outputs.have_secret == 'true'
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set -euo pipefail
+          railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
+
+      - name: Run drift audit
+        if: steps.secret_check.outputs.have_secret == 'true'
+        id: audit
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set +e
+          bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
+          rc=${PIPESTATUS[0]}
+          echo "rc=$rc" >> "$GITHUB_OUTPUT"
+          # Capture the audit log for the issue body.
+          {
+            echo 'log<<AUDIT_EOF'
+            cat /tmp/audit.log
+            echo 'AUDIT_EOF'
+          } >> "$GITHUB_OUTPUT"
+          case "$rc" in
+            0) exit 0 ;;
+            1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
+            2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
+            *) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
+          esac
+
+      - name: Open / update drift issue (Gitea API)
+        if: failure() && steps.audit.outputs.rc == '1'
+        env:
+          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          AUDIT_LOG: ${{ steps.audit.outputs.log }}
+          SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          API="${SERVER_URL%/}/api/v1"
+          TITLE="Railway env-var drift detected"
+          RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
+          BODY=$(jq -nc --arg t "$TITLE" --arg log "${AUDIT_LOG:-(log unavailable)}" --arg run "$RUN_URL" '
+            {body: ("Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n**What this means:** an env var (likely on `controlplane`) is pinned to a SHA-shaped or semver tag instead of a floating tag. Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service does not pick them up.\n\n**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (:staging-latest, :main) unless the pin is intentional and documented in the ops runbook.\n\n**Audit output:**\n\n```\n" + $log + "\n```\n\nRun: " + $run + "\n\nCloses automatically when a subsequent daily run reports clean.")}')
+
+          # Look for existing open drift issue with the title.
+          EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
+            "${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
+            | jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
+
+          if [ -n "$EXISTING" ]; then
+            COMMENT_BODY=$(jq -nc --arg log "${AUDIT_LOG:-(log unavailable)}" --arg run "$RUN_URL" \
+              '{body: ("Still drifting. " + $run + "\n\n```\n" + $log + "\n```")}')
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${EXISTING}/comments" -d "$COMMENT_BODY" >/dev/null
+            echo "Commented on existing issue #${EXISTING}"
+          else
+            CREATE_BODY=$(echo "$BODY" | jq --arg t "$TITLE" '. + {title: $t, labels: []}')
+            NUM=$(curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues" -d "$CREATE_BODY" | jq -r .number)
+            echo "Filed issue #${NUM}"
+          fi
+
+      - name: Close stale drift issue on clean run (Gitea API)
+        if: success() && steps.audit.outputs.rc == '0'
+        env:
+          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          API="${SERVER_URL%/}/api/v1"
+          TITLE="Railway env-var drift detected"
+          RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
+
+          NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
+            "${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
+            | jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number')
+
+          for N in $NUMS; do
+            curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${N}/comments" \
+              -d "$(jq -nc --arg run "$RUN_URL" '{body: ("Daily audit clean — drift resolved. " + $run)}')" >/dev/null
+            curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
+              "${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
+            echo "Closed #${N}"
+          done

.gitea/workflows/redeploy-tenants-on-main.yml

@@ -0,0 +1,375 @@
+name: redeploy-tenants-on-main
+
+# Ported from .github/workflows/redeploy-tenants-on-main.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#   - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
+#     for the `workflow_run` event is partial. If this never fires on a
+#     real publish-workspace-server-image completion, the follow-up
+#     triage PR should replace the trigger with a push-with-paths-filter
+#     on .gitea/workflows/publish-workspace-server-image.yml. Until
+#     then continue-on-error+dead-workflow doesn't break anything.
+#
+
+# Auto-refresh prod tenant EC2s after every main merge.
+#
+# Why this workflow exists: publish-workspace-server-image builds and
+# pushes a new platform-tenant :<sha> to ECR on every merge to main,
+# but running tenants pulled their image once at boot and never re-pull.
+# Users see stale code indefinitely.
+#
+# This workflow closes the gap by calling the control-plane admin
+# endpoint that performs a canary-first, batched, health-gated rolling
+# redeploy across every live tenant. Implemented in molecule-ai/
+# molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
+# (feat/tenant-auto-redeploy, landing alongside this workflow).
+#
+# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
+# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
+# Gitea suspension migration. The canary-verify.yml promote step now
+# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
+#
+# Runtime ordering:
+#   1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
+#   2. This workflow fires via workflow_run, calls redeploy-fleet with
+#      target_tag=staging-<sha>. No CDN propagation wait needed —
+#      ECR image manifest is consistent immediately after push.
+#   3. Calls redeploy-fleet with canary_slug (if set) and a soak
+#      period. Canary proves the image boots; batches follow.
+#   4. Any failure aborts the rollout and leaves older tenants on the
+#      prior image — safer default than half-and-half state.
+#
+# Rollback path: re-run this workflow with a specific SHA pinned via
+# the workflow_dispatch input. That calls redeploy-fleet with
+# target_tag=<sha>, re-pulling the older image on every tenant.
+
+on:
+  workflow_run:
+    workflows: ['publish-workspace-server-image']
+    types: [completed]
+    branches: [main]
+permissions:
+  contents: read
+  # No write scopes needed — the workflow hits an external CP endpoint,
+  # not the GitHub API.
+
+# Serialize redeploys so two rapid main pushes' redeploys don't overlap
+# and cause confusing per-tenant SSM state. Without this, GitHub's
+# implicit workflow_run queueing would *probably* serialize them, but
+# the explicit block makes the invariant defensible. Mirrors the
+# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
+#
+# cancel-in-progress: false → aborting a half-rolled-out fleet would
+# leave tenants stuck on whatever image they happened to be on when
+# cancelled. Better to finish the in-flight rollout before starting
+# the next one.
+concurrency:
+  group: redeploy-tenants-on-main
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  redeploy:
+    # Skip the auto-trigger if publish-workspace-server-image didn't
+    # actually succeed. workflow_run fires on any completion state; we
+    # don't want to redeploy against a half-built image.
+    # NOTE (Gitea port): workflow_dispatch trigger dropped; only the
+    # workflow_run path remains.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 25
+    steps:
+      - name: Note on ECR propagation
+        # ECR image manifests are consistent immediately after push — no
+        # CDN cache to wait for. The old GHCR-based workflow had a 30s
+        # sleep to avoid race conditions; ECR makes that unnecessary.
+        run: echo "ECR image available immediately after push — proceeding."
+
+      - name: Compute target tag
+        id: tag
+        # Resolution order:
+        #   1. Operator-supplied input (workflow_dispatch with explicit
+        #      tag) → used verbatim. Lets ops pin `latest` for emergency
+        #      rollback to last canary-verified digest, or pin a specific
+        #      `staging-<sha>` to roll back to a known-good build.
+        #   2. Default → `staging-<short_head_sha>`. The just-published
+        #      digest. Bypasses the `:latest` retag path that's currently
+        #      dead (canary-verify soft-skips without canary fleet, so
+        #      the only thing retagging `:latest` today is the manual
+        #      promote-latest.yml — last run 2026-04-28). Auto-trigger
+        #      from workflow_run uses workflow_run.head_sha; manual
+        #      dispatch with no input falls through to github.sha.
+        env:
+          INPUT_TAG: ${{ inputs.target_tag }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+        run: |
+          set -euo pipefail
+          if [ -n "${INPUT_TAG:-}" ]; then
+            echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
+            echo "Using operator-pinned tag: $INPUT_TAG"
+          else
+            SHORT="${HEAD_SHA:0:7}"
+            echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
+            echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
+          fi
+
+      - name: Call CP redeploy-fleet
+        # CP_ADMIN_API_TOKEN must be set as a repo/org secret on
+        # molecule-ai/molecule-core, matching the staging/prod CP's
+        # CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
+        # repo's secrets for CI.
+        env:
+          CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
+          CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
+          CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
+          SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
+          BATCH_SIZE: ${{ inputs.batch_size || '3' }}
+          DRY_RUN: ${{ inputs.dry_run || false }}
+        run: |
+          set -euo pipefail
+
+          if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
+            echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
+            echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
+            exit 1
+          fi
+
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --arg canary "$CANARY_SLUG" \
+            --argjson soak "$SOAK_SECONDS" \
+            --argjson batch "$BATCH_SIZE" \
+            --argjson dry "$DRY_RUN" \
+            '{
+              target_tag: $tag,
+              canary_slug: $canary,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          # Route -w into its own tempfile so curl's exit code (e.g. 56
+          # on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
+          # pollute the captured stdout. The previous inline-substitution
+          # shape produced "000000" on connection reset (curl wrote
+          # "000" via -w, then the inline echo-fallback appended another
+          # "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
+          # set +e/-e keeps the non-zero curl exit from tripping the
+          # outer pipeline. See lint-curl-status-capture.yml for the
+          # CI gate that pins this fix shape.
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          set -e
+          # Stderr from curl (e.g. dial errors with -sS) goes to the runner
+          # log so operators can see WHY a connection failed. Stdout is
+          # captured to $HTTP_CODE_FILE because that's where -w writes.
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          # Pretty-print per-tenant results in the job summary so
+          # ops can see which tenants were redeployed without drilling
+          # into the raw response.
+          {
+            echo "## Tenant redeploy fleet"
+            echo ""
+            echo "**Target tag:** \`$TARGET_TAG\`"
+            echo "**Canary:** \`$CANARY_SLUG\` (soak ${SOAK_SECONDS}s)"
+            echo "**Batch size:** $BATCH_SIZE"
+            echo "**Dry run:** $DRY_RUN"
+            echo "**HTTP:** $HTTP_CODE"
+            echo ""
+            echo "### Per-tenant result"
+            echo ""
+            echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
+            echo '|------|-------|------------|------|---------|-------|'
+            jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
+            exit 1
+          fi
+          OK=$(jq -r '.ok' "$HTTP_RESPONSE")
+          if [ "$OK" != "true" ]; then
+            echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
+            exit 1
+          fi
+          echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
+
+          # Stash the response for the verify step. $RUNNER_TEMP outlasts
+          # the step boundary; $HTTP_RESPONSE doesn't.
+          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
+
+      - name: Verify each tenant /buildinfo matches published SHA
+        # ROOT FIX FOR #2395.
+        #
+        # `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
+        # didn't error" — NOT "the new image is running on the tenant."
+        # `:latest` lives in the local Docker daemon's image cache; if
+        # the SSM document does `docker compose up -d` without an
+        # explicit `docker pull`, the daemon serves the previously-
+        # cached digest and the container restarts on stale code.
+        # 2026-04-30 incident: hongmingwang's tenant reported
+        # ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
+        # chat_files for 30+ min — the lazy-heal fix never reached the
+        # user despite green deploy + green redeploy.
+        #
+        # This step closes the gap by curling each tenant's /buildinfo
+        # endpoint (added in workspace-server/internal/buildinfo +
+        # /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
+        # returned git_sha to the SHA the workflow expects. Mismatches
+        # fail the workflow, which is what `ok=true` should have
+        # guaranteed all along.
+        #
+        # When the redeploy was triggered by workflow_dispatch with a
+        # specific tag (target_tag != "latest"), the expected SHA may
+        # not equal ${{ github.sha }} — in that case we resolve via
+        # GHCR's manifest. For workflow_run (default :latest) the
+        # workflow_run.head_sha is the SHA that just published.
+        env:
+          EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
+          # Tenant subdomain template — slugs from the response are
+          # appended. Production CP issues `<slug>.moleculesai.app`;
+          # staging CP issues `<slug>.staging.moleculesai.app`. This
+          # workflow runs on main → prod CP → no `staging.` infix.
+          TENANT_DOMAIN: 'moleculesai.app'
+        run: |
+          set -euo pipefail
+
+          EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
+          if [ "$TARGET_TAG" != "latest" ] \
+             && [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
+             && [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
+            # workflow_dispatch with a pinned tag that isn't the head
+            # SHA — operator is rolling back / pinning. Skip the
+            # verification because we don't have the expected SHA in
+            # this context (would need to crane-inspect the GHCR
+            # manifest, which is a follow-up). Failing-open here is
+            # safe: the operator chose the tag deliberately.
+            #
+            # `staging-<short_head_sha>` IS verified — it's the new
+            # auto-trigger default (see Compute target tag step) and
+            # the digest under that tag SHOULD match EXPECTED_SHA.
+            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
+            exit 0
+          fi
+
+          RESP="$RUNNER_TEMP/redeploy-response.json"
+          if [ ! -s "$RESP" ]; then
+            echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
+            exit 1
+          fi
+
+          # Pull only successfully-redeployed tenants. Any tenant that
+          # halted the rollout already failed the previous step, so we
+          # don't double-count them here.
+          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
+          if [ ${#SLUGS[@]} -eq 0 ]; then
+            echo "::warning::No tenants reported healthz_ok — nothing to verify"
+            exit 0
+          fi
+
+          echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
+
+          # Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
+          # vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
+          # comment for the full rationale; same logic applies on prod even
+          # though prod has fewer ephemeral tenants — the asymmetry would be a
+          # gratuitous fork.
+          STALE_COUNT=0
+          UNREACHABLE_COUNT=0
+          STALE_LINES=()
+          UNREACHABLE_LINES=()
+          for slug in "${SLUGS[@]}"; do
+            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
+            # 30s total: tenant just SSM-restarted, may still be coming
+            # up. Retry-on-empty rather than retry-on-status — we want
+            # to fail fast on "responded with wrong SHA", not "still
+            # warming up".
+            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
+            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
+            if [ -z "$ACTUAL_SHA" ]; then
+              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
+              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
+              continue
+            fi
+            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
+              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
+            else
+              STALE_COUNT=$((STALE_COUNT + 1))
+              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
+            fi
+          done
+
+          {
+            echo ""
+            echo "### Per-tenant /buildinfo verification"
+            echo ""
+            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
+            echo ""
+            if [ $STALE_COUNT -gt 0 ]; then
+              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${STALE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $UNREACHABLE_COUNT -gt 0 ]; then
+              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
+              echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ $UNREACHABLE_COUNT -gt 0 ]; then
+            echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
+          fi
+
+          # Belt-and-suspenders sanity floor: same logic as the staging
+          # variant — see that file's comment for the full rationale.
+          # Floor only applies when fleet >= 4; below that, canary-verify
+          # is the actual gate.
+          TOTAL_VERIFIED=${#SLUGS[@]}
+          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
+            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
+            exit 1
+          fi
+
+          if [ $STALE_COUNT -gt 0 ]; then
+            echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
+            exit 1
+          fi
+
+          echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."

.gitea/workflows/redeploy-tenants-on-staging.yml

@@ -0,0 +1,356 @@
+name: redeploy-tenants-on-staging
+
+# Ported from .github/workflows/redeploy-tenants-on-staging.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#   - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
+#     for the `workflow_run` event is partial. If this never fires on a
+#     real publish-workspace-server-image completion, the follow-up
+#     triage PR should replace the trigger with a push-with-paths-filter
+#     on .gitea/workflows/publish-workspace-server-image.yml. Until
+#     then continue-on-error+dead-workflow doesn't break anything.
+#
+
+# Auto-refresh staging tenant EC2s after every staging-branch merge.
+#
+# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
+# the :staging-latest tag. Sister workflow exists for prod (rolls
+# :latest after canary-verify). Both share the same shape — just
+# different CP_URL + target_tag + admin token secret.
+#
+# Why this workflow exists: publish-workspace-server-image now builds
+# on every staging-branch push (PR #2335), pushing
+# platform-tenant:staging-latest to GHCR. Existing tenants pulled
+# their image once at boot and never re-pull, so the new image just
+# sits unused until the tenant is reprovisioned.
+#
+# This workflow closes the gap by calling staging-CP's
+# /cp/admin/tenants/redeploy-fleet, which performs a canary-first,
+# batched, health-gated SSM redeploy across every live staging tenant.
+# Same endpoint shape as prod CP — only the host differs.
+#
+# Runtime ordering:
+#   1. publish-workspace-server-image completes on staging branch →
+#      new :staging-latest in GHCR.
+#   2. This workflow fires via workflow_run, waits 30s for GHCR's CDN
+#      to propagate the new tag.
+#   3. Calls redeploy-fleet with no canary (staging IS canary; we don't
+#      need a sub-canary inside it). Soak still applies to the first
+#      tenant in case of bad-deploy detection.
+#   4. Any failure aborts the rollout and leaves older tenants on the
+#      prior image — safer default than half-and-half state.
+#
+# Rollback path: re-run with workflow_dispatch + target_tag=staging-<sha>
+# of a known-good build.
+
+on:
+  workflow_run:
+    workflows: ['publish-workspace-server-image']
+    types: [completed]
+    branches: [main]
+permissions:
+  contents: read
+  # No write scopes needed — the workflow hits an external CP endpoint,
+  # not the GitHub API.
+
+# Serialize per-branch so two rapid staging pushes' redeploys don't
+# overlap and cause confusing per-tenant SSM state. cancel-in-progress
+# is false because aborting a half-rolled-out fleet leaves tenants
+# stuck on whatever image they happened to be on when cancelled.
+concurrency:
+  group: redeploy-tenants-on-staging
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  redeploy:
+    # Skip the auto-trigger if publish-workspace-server-image didn't
+    # actually succeed. workflow_run fires on any completion state; we
+    # don't want to redeploy against a half-built image.
+    # NOTE (Gitea port): workflow_dispatch trigger dropped; only the
+    # workflow_run path remains.
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 25
+    steps:
+      - name: Wait for GHCR tag propagation
+        # GHCR's edge cache takes ~15-30s to consistently serve the new
+        # :staging-latest manifest after the registry accepts the push.
+        # Same rationale as redeploy-tenants-on-main.yml.
+        run: sleep 30
+
+      - name: Call staging-CP redeploy-fleet
+        # CP_STAGING_ADMIN_API_TOKEN must be set as a repo/org secret
+        # on molecule-ai/molecule-core, matching staging-CP's
+        # CP_ADMIN_API_TOKEN env var (visible in Railway controlplane
+        # / staging environment). Stored separately from the prod
+        # CP_ADMIN_API_TOKEN so a leak of one doesn't auth the other.
+        env:
+          CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
+          CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+          TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
+          CANARY_SLUG: ${{ inputs.canary_slug || '' }}
+          SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
+          BATCH_SIZE: ${{ inputs.batch_size || '3' }}
+          DRY_RUN: ${{ inputs.dry_run || false }}
+        run: |
+          set -euo pipefail
+
+          # Schedule-vs-dispatch hardening (mirrors sweep-cf-orphans
+          # and sweep-cf-tunnels): hard-fail on auto-trigger when the
+          # secret is missing so a misconfigured-repo doesn't silently
+          # serve stale staging tenants. Soft-skip on operator dispatch.
+          if [ -z "${CP_STAGING_ADMIN_API_TOKEN:-}" ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::CP_STAGING_ADMIN_API_TOKEN secret not set — skipping redeploy"
+              echo "::warning::Set CP_STAGING_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
+              echo "::notice::Pull the value from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+              exit 0
+            fi
+            echo "::error::staging redeploy cannot run — CP_STAGING_ADMIN_API_TOKEN secret missing"
+            echo "::error::set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+            exit 1
+          fi
+
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --arg canary "$CANARY_SLUG" \
+            --argjson soak "$SOAK_SECONDS" \
+            --argjson batch "$BATCH_SIZE" \
+            --argjson dry "$DRY_RUN" \
+            '{
+              target_tag: $tag,
+              canary_slug: $canary,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          # Route -w into its own tempfile so curl's exit code (e.g. 56
+          # on connection-reset) can't pollute the captured stdout. The
+          # previous inline-substitution shape produced "000000" on
+          # connection reset — caught on main variant 2026-05-04
+          # redeploying sha 2b862f6. Same fix shape as the synth-E2E
+          # §9c gate (PR #2797). See lint-curl-status-capture.yml for
+          # the CI gate that pins this fix shape.
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          set -e
+          # Stderr from curl (-sS shows dial errors etc.) goes to the
+          # runner log so operators can see WHY a connection failed.
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          {
+            echo "## Staging tenant redeploy fleet"
+            echo ""
+            echo "**Target tag:** \`$TARGET_TAG\`"
+            echo "**Canary:** \`${CANARY_SLUG:-(none — staging is itself the canary)}\` (soak ${SOAK_SECONDS}s)"
+            echo "**Batch size:** $BATCH_SIZE"
+            echo "**Dry run:** $DRY_RUN"
+            echo "**HTTP:** $HTTP_CODE"
+            echo ""
+            echo "### Per-tenant result"
+            echo ""
+            echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
+            echo '|------|-------|------------|------|---------|-------|'
+            jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          # Distinguish "real fleet failure" from "E2E teardown race".
+          #
+          # CP returns HTTP 500 + ok=false whenever ANY tenant in the
+          # fleet failed SSM or healthz. In practice the recurring source
+          # of these is ephemeral test tenants being torn down by their
+          # parent E2E run mid-redeploy: the EC2 dies → SSM exit=2 or
+          # healthz timeout → CP marks the fleet failed → this workflow
+          # goes red even though every operator-facing tenant rolled fine.
+          #
+          # Ephemeral slug prefixes (kept in sync with sweep-stale-e2e-orgs.yml
+          # — see that file for the source-of-truth list and rationale):
+          #   - e2e-*       — canvas/saas/ext E2E suites
+          #   - rt-e2e-*    — runtime-test harness fixtures (RFC #2251)
+          # Long-lived prefixes that are NOT ephemeral and MUST hard-fail:
+          # demo-prep, dryrun-*, dryrun2-*, plus all human tenant slugs.
+          #
+          # Filter: if HTTP=500/ok=false AND every failed slug matches an
+          # ephemeral prefix, treat as soft-warn and let the verify step
+          # downstream handle unreachable-vs-stale (#2402). Any non-ephemeral
+          # failure or a non-500 HTTP response remains a hard failure.
+          OK=$(jq -r '.ok // "false"' "$HTTP_RESPONSE")
+          FAILED_SLUGS=$(jq -r '
+            .results[]?
+            | select((.healthz_ok != true) or (.ssm_status != "Success"))
+            | .slug' "$HTTP_RESPONSE" 2>/dev/null || true)
+          EPHEMERAL_PREFIX_RE='^(e2e-|rt-e2e-)'
+          NON_EPHEMERAL_FAILED=$(printf '%s\n' "$FAILED_SLUGS" | grep -v '^$' | grep -Ev "$EPHEMERAL_PREFIX_RE" || true)
+
+          if [ "$HTTP_CODE" = "200" ] && [ "$OK" = "true" ]; then
+            : # happy path — fall through to verification
+          elif [ "$HTTP_CODE" = "500" ] && [ -z "$NON_EPHEMERAL_FAILED" ] && [ -n "$FAILED_SLUGS" ]; then
+            COUNT=$(printf '%s\n' "$FAILED_SLUGS" | grep -Ec "$EPHEMERAL_PREFIX_RE" || true)
+            echo "::warning::redeploy-fleet returned HTTP 500 but every failed tenant ($COUNT) is ephemeral (e2e-*/rt-e2e-*) — treating as teardown race, soft-warning."
+            printf '%s\n' "$FAILED_SLUGS" | sed 's/^/::warning::  failed: /'
+          elif [ "$HTTP_CODE" != "200" ]; then
+            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
+            if [ -n "$NON_EPHEMERAL_FAILED" ]; then
+              echo "::error::non-ephemeral tenant(s) failed:"
+              printf '%s\n' "$NON_EPHEMERAL_FAILED" | sed 's/^/::error::  /'
+            fi
+            exit 1
+          else
+            # HTTP=200 but ok=false (shouldn't happen with current CP
+            # but keep the gate for completeness).
+            echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
+            exit 1
+          fi
+          echo "::notice::Staging tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
+
+          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
+
+      - name: Verify each staging tenant /buildinfo matches published SHA
+        # Mirror of the verify step in redeploy-tenants-on-main.yml — see
+        # there for the rationale (#2395 root fix). Staging has the same
+        # ssm_status-success-but-stale-image hazard and benefits from the
+        # same gate. Diff: TENANT_DOMAIN includes the `staging.` infix.
+        env:
+          EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
+          TENANT_DOMAIN: 'staging.moleculesai.app'
+        run: |
+          set -euo pipefail
+
+          # staging-latest is the staging-side moving tag; treat it the
+          # same way main treats `latest`. Operator-pinned SHAs skip
+          # verification (see main variant for why).
+          if [ "$TARGET_TAG" != "staging-latest" ] && [ "$TARGET_TAG" != "latest" ] && [ "$TARGET_TAG" != "$EXPECTED_SHA" ]; then
+            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
+            exit 0
+          fi
+
+          RESP="$RUNNER_TEMP/redeploy-response.json"
+          if [ ! -s "$RESP" ]; then
+            echo "::error::redeploy-response.json missing or empty"
+            exit 1
+          fi
+
+          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
+          if [ ${#SLUGS[@]} -eq 0 ]; then
+            echo "::warning::No staging tenants reported healthz_ok — nothing to verify"
+            exit 0
+          fi
+
+          echo "Verifying ${#SLUGS[@]} staging tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
+
+          # Two distinct failure modes here:
+          #   STALE_COUNT      — tenant returned a SHA that doesn't match. THIS is
+          #                      the #2395 bug class: tenant up + serving old code.
+          #                      Always hard-fail the workflow.
+          #   UNREACHABLE_COUNT — tenant didn't respond. Almost always a benign
+          #                      teardown race: redeploy-fleet snapshot says
+          #                      healthz_ok=true, then the E2E suite tears the
+          #                      ephemeral tenant down before this step runs (the
+          #                      e2e-* fixtures churn 5-10/hour on staging). Soft-
+          #                      warn so we don't block staging→main on cleanup.
+          #                      Real "tenant up but unreachable" is caught by CP's
+          #                      own healthz monitor + the post-redeploy alert; we
+          #                      don't need to double-count it here.
+          STALE_COUNT=0
+          UNREACHABLE_COUNT=0
+          STALE_LINES=()
+          UNREACHABLE_LINES=()
+          for slug in "${SLUGS[@]}"; do
+            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
+            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
+            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
+            if [ -z "$ACTUAL_SHA" ]; then
+              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
+              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
+              continue
+            fi
+            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
+              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
+            else
+              STALE_COUNT=$((STALE_COUNT + 1))
+              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
+            fi
+          done
+
+          {
+            echo ""
+            echo "### Per-tenant /buildinfo verification (staging)"
+            echo ""
+            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
+            echo ""
+            if [ $STALE_COUNT -gt 0 ]; then
+              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${STALE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $UNREACHABLE_COUNT -gt 0 ]; then
+              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely E2E teardown race (soft-warn, not failing):**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
+              echo "All ${#SLUGS[@]} staging tenants returned matching SHA. ✓"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ $UNREACHABLE_COUNT -gt 0 ]; then
+            echo "::warning::$UNREACHABLE_COUNT staging tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
+          fi
+
+          # Belt-and-suspenders sanity floor: if MORE than half the fleet is
+          # unreachable AND the fleet is large enough that "half down" is
+          # statistically meaningful, this is a real outage (e.g. new image
+          # crashes on startup), not a teardown race. Hard-fail.
+          #
+          # Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
+          # canary-verify step is the actual gate for "all tenants down"
+          # detection (it runs against the canary first and aborts the
+          # rollout if the canary fails to come up). Without the >=4 gate,
+          # a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
+          # quiet staging push) would re-flake on the exact teardown-race
+          # condition #2402 fixed: 1 of 1 unreachable = 100% > 50% → fail.
+          TOTAL_VERIFIED=${#SLUGS[@]}
+          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
+            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED staging tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
+            exit 1
+          fi
+
+          if [ $STALE_COUNT -gt 0 ]; then
+            echo "::error::$STALE_COUNT staging tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
+            exit 1
+          fi
+
+          echo "::notice::Staging tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."

.gitea/workflows/runtime-pin-compat.yml

@@ -0,0 +1,100 @@
+name: Runtime Pin Compatibility
+
+# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
+# RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `merge_group:` (no Gitea merge queue) and
+#     `workflow_dispatch:` (no inputs, but the trigger itself is
+#     parser-rejected when inputs are absent in some Gitea 1.22.x
+#     builds; safest to drop entirely — manual runs go via cron-trigger
+#     bump or push-with-paths-filter).
+#   - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
+#     file) instead of the .github/ one.
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
+# recurring (controlplane#253). The original failure mode:
+#   1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
+#      requires_dist metadata (incorrect — it actually imports
+#      a2a.server.routes which only exists in a2a-sdk 1.0+)
+#   2. `pip install molecule-ai-workspace-runtime` resolved cleanly
+#   3. `from molecule_runtime.main import main_sync` raised ImportError
+#   4. Every tenant workspace crashed; the canary tenant caught it but
+#      only after 5 hours of degraded staging
+#
+# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
+# top of `workspace/requirements.txt` and smoke-imports. Catches:
+#   - Upstream PyPI yanks
+#   - Bad re-releases of molecule-ai-workspace-runtime
+#   - Already-shipped wheels that stop importing because a transitive
+#     dep moved underneath
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      # Narrow filter: pypi-latest is sensitive only to changes that
+      # affect what we're INSTALLING (requirements.txt) or WHAT THE
+      # CHECK ITSELF DOES (this workflow file). Edits to workspace/
+      # source code don't change what's on PyPI right now, so they
+      # don't change this gate's verdict.
+      - 'workspace/requirements.txt'
+      - '.gitea/workflows/runtime-pin-compat.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace/requirements.txt'
+      - '.gitea/workflows/runtime-pin-compat.yml'
+  # Daily catch for upstream PyPI publishes that break the pin combo
+  # without any change in our repo (e.g. someone re-yanks an a2a-sdk
+  # release or molecule-ai-workspace-runtime publishes a bad bump).
+  schedule:
+    - cron: '0 13 * * *'  # 06:00 PT
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pypi-latest-install:
+    name: PyPI-latest install + import smoke
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install runtime + workspace requirements
+        # Install order is load-bearing: install the runtime FIRST so pip
+        # honors whatever a2a-sdk constraint the runtime metadata declares
+        # (this is the surface that broke in 2026-04-24 — runtime declared
+        # `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
+        # of workspace/requirements.txt then upgrades a2a-sdk to the
+        # constraint our runtime image actually pins. The import smoke
+        # below verifies the upgraded combination is consistent.
+        run: |
+          python -m venv /tmp/venv
+          /tmp/venv/bin/pip install --upgrade pip
+          /tmp/venv/bin/pip install molecule-ai-workspace-runtime
+          /tmp/venv/bin/pip install -r workspace/requirements.txt
+          /tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import — fail if metadata declares deps that don't satisfy real imports
+        # WORKSPACE_ID is validated at import time by platform_auth.py — EC2
+        # user-data sets it from the cloud-init template; set a placeholder
+        # here so the import smoke doesn't trip on the env-var guard.
+        env:
+          WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
+        run: |
+          /tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"

.gitea/workflows/runtime-prbuild-compat.yml

@@ -0,0 +1,139 @@
+name: Runtime PR-Built Compatibility
+
+# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
+#     (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
+#     finicky without them).
+#   - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
+#     pattern for ci.yml port).
+#   - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on every job (RFC §1 contract).
+#
+# Companion to `runtime-pin-compat.yml`. That workflow tests what's
+# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
+# PUBLISHED if THIS PR merges.
+#
+# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
+# wheel" job to the original runtime-pin-compat.yml, but both jobs
+# shared a `paths:` filter that was the union of their needs
+# (`workspace/**`). That meant the PyPI-latest job ran on every doc
+# edit even though the upstream PyPI artifact can't change with our
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter that matches the inputs it actually depends on.
+#
+# Catches the failure mode where a PR adds an import requiring a newer
+# SDK than `workspace/requirements.txt` pins:
+#   1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
+#      passes (it imports the OLD main.py from the wheel, not the PR's
+#      new main.py).
+#   2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
+#   3. Tenant images redeploy -> all crash on first boot with ImportError.
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  # event_name + sha keeps PR sync and the subsequent staging push on the
+  # same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    outputs:
+      wheel: ${{ steps.decide.outputs.wheel }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: decide
+        run: |
+          # Inline replacement for dorny/paths-filter — same pattern
+          # PR#372's ci.yml port used. Diffs against the PR base or the
+          # previous push SHA, then matches against the wheel-relevant
+          # path set.
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            # New branch or no previous SHA: treat as wheel-relevant.
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "wheel=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `PR-built wheel + import smoke`. Real work is
+  # gated per-step on `needs.detect-changes.outputs.wheel`.
+  local-build-install:
+    needs: detect-changes
+    name: PR-built wheel + import smoke
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.wheel != 'true'
+        run: |
+          echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
+          echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install build tooling
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: pip install build
+      - name: Build wheel from PR source (mirrors publish-runtime.yml)
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Use a fixed test version so the wheel filename is predictable.
+        # Doesn't reach PyPI — this build is local-only for the smoke.
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "0.0.0.dev0+pin-compat" \
+            --out /tmp/runtime-build
+          cd /tmp/runtime-build && python -m build
+      - name: Install built wheel + workspace requirements
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: |
+          python -m venv /tmp/venv-built
+          /tmp/venv-built/bin/pip install --upgrade pip
+          /tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
+          /tmp/venv-built/bin/pip install -r workspace/requirements.txt
+          /tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import the PR-built wheel
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Same script publish-runtime.yml runs against the to-be-PyPI wheel.
+        run: |
+          /tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"

.gitea/workflows/secret-pattern-drift.yml

@@ -0,0 +1,70 @@
+name: SECRET_PATTERNS drift lint
+
+# Ported from .github/workflows/secret-pattern-drift.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - on.paths references the new canonical .gitea/workflows/secret-scan.yml
+#     (the .github/ copy is removed by Cat A of this sweep).
+#   - CANONICAL_FILE inside scripts/lint_secret_pattern_drift.py was
+#     updated in the same Cat C-1 PR to point at .gitea/workflows/secret-scan.yml.
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# Detects when the canonical SECRET_PATTERNS array in
+# .gitea/workflows/secret-scan.yml diverges from known consumer
+# mirrors (workspace-runtime's bundled pre-commit hook today; more
+# can be added as the consumer set grows).
+#
+# Why this exists: every side that scans for credentials has its own
+# copy of the pattern list. They drift — most recently the runtime
+# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
+# so a developer's local pre-commit would let a sk-cp- token through
+# while the org-wide CI scan would refuse it. The cost of that drift
+# is dev confusion + delayed feedback; the fix is automated detection.
+#
+# Triggers:
+#   - schedule: daily 05:00 UTC. Catches drift introduced by edits
+#     to a consumer copy that didn't update canonical here.
+#   - push to main/staging where the canonical or this lint changed:
+#     catches the inverse — canonical updated but consumers not yet
+#     bumped. The lint will fail the push; that's intentional.
+
+on:
+  schedule:
+    # 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
+    # email lands when humans are starting their day, not
+    # interrupting it.
+    - cron: "0 5 * * *"
+  push:
+    branches: [main, staging]
+    paths:
+      - ".gitea/workflows/secret-scan.yml"
+      - ".gitea/workflows/secret-pattern-drift.yml"
+      - ".github/scripts/lint_secret_pattern_drift.py"
+      - ".githooks/pre-commit"
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+# Auto-injected GITHUB_TOKEN scoped to read-only. The lint only does git
+# checkout + HTTPS GETs to public consumer files; no writes to anything.
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Detect SECRET_PATTERNS drift
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Run drift lint
+        run: python3 .github/scripts/lint_secret_pattern_drift.py

.gitea/workflows/secret-scan.yml

@@ -7,40 +7,31 @@ name: Secret scan
 # slurping the URL from a token-embedded origin remote. We can't fix
 # upstream's clone hygiene, so we gate here.
 #
-# Also the canonical reusable workflow for the rest of the org. Other
-# Molecule-AI repos enroll with a single 3-line workflow:
-#
-#   jobs:
-#     secret-scan:
-#       uses: Molecule-AI/molecule-core/.github/workflows/secret-scan.yml@staging
-#
-# Pin to @staging not @main — staging is the active default branch,
-# main lags via the staging-promotion workflow. Updates ride along
-# automatically on the next consumer workflow run.
-#
 # Same regex set as the runtime's bundled pre-commit hook
 # (molecule-ai-workspace-runtime: molecule_runtime/scripts/pre-commit-checks.sh).
 # Keep the two sides aligned when adding patterns.
+#
+# Ported from .github/workflows/secret-scan.yml so the gate actually
+# fires on Gitea Actions. Differences from the GitHub version:
+#   - drops `merge_group` event (Gitea has no merge queue)
+#   - drops `workflow_call` (no cross-repo reusable invocation on Gitea)
+#   - SELF path updated to .gitea/workflows/secret-scan.yml
+# The job name + step name are identical to the GitHub workflow so the
+# status-check context (`Secret scan / Scan diff for credential-shaped
+# strings (pull_request)`) matches branch protection on molecule-core/main.
 
 on:
   pull_request:
     types: [opened, synchronize, reopened]
   push:
     branches: [main, staging]
-  # Required for GitHub merge queue: the queue's pre-merge CI run on
-  # `gh-readonly-queue/...` refs needs this check to fire so the queue
-  # gets a real result instead of stalling forever AWAITING_CHECKS.
-  merge_group:
-    types: [checks_requested]
-  # Reusable workflow entry point for other Molecule-AI repos.
-  workflow_call:
 
 jobs:
   scan:
     name: Scan diff for credential-shaped strings
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 2  # need previous commit to diff against on push events
 
@@ -50,27 +41,14 @@ jobs:
         if: github.event_name == 'pull_request'
         run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
 
-      # For merge_group events the queue's pre-merge ref is a commit on
-      # `gh-readonly-queue/...` whose parent is the queue's base_sha.
-      # That parent isn't part of the queue branch's shallow clone, so
-      # we fetch it explicitly. Without this the diff falls through to
-      # "no BASE → scan entire tree" mode and false-positives on legit
-      # test fixtures (e.g. canvas/src/lib/validation/__tests__/secret-formats.test.ts).
-      - name: Fetch merge_group base SHA (merge_group events only)
-        if: github.event_name == 'merge_group'
-        run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
-
       - name: Refuse if credential-shaped strings appear in diff additions
         env:
           # Plumb event-specific SHAs through env so the script doesn't
           # need conditional `${{ ... }}` interpolation per event type.
           # github.event.before/after only exist on push events;
-          # merge_group has its own base_sha/head_sha; pull_request has
-          # pull_request.base.sha / pull_request.head.sha.
+          # pull_request has pull_request.base.sha / pull_request.head.sha.
           PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
-          MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
           PUSH_BEFORE: ${{ github.event.before }}
           PUSH_AFTER: ${{ github.event.after }}
         run: |
@@ -102,10 +80,6 @@ jobs:
               BASE="$PR_BASE_SHA"
               HEAD="$PR_HEAD_SHA"
               ;;
-            merge_group)
-              BASE="$MG_BASE_SHA"
-              HEAD="$MG_HEAD_SHA"
-              ;;
             *)
               BASE="$PUSH_BEFORE"
               HEAD="$PUSH_AFTER"
@@ -144,12 +118,21 @@ jobs:
 
           # Self-exclude: this workflow file legitimately contains the
           # pattern strings as regex literals. Without an exclude it would
-          # block its own merge.
-          SELF=".github/workflows/secret-scan.yml"
+          # block its own merge. Both the .github/ original and this
+          # .gitea/ port are excluded so a sync between them stays clean.
+          SELF_GITHUB=".github/workflows/secret-scan.yml"
+          SELF_GITEA=".gitea/workflows/secret-scan.yml"
 
           OFFENDING=""
-          for f in $CHANGED; do
-            [ "$f" = "$SELF" ] && continue
+          # `while IFS= read -r` (not `for f in $CHANGED`) so filenames
+          # containing whitespace don't word-split silently — a path
+          # with a space would otherwise produce two iterations on
+          # tokens that aren't real filenames, breaking the
+          # self-exclude + diff lookup.
+          while IFS= read -r f; do
+            [ -z "$f" ] && continue
+            [ "$f" = "$SELF_GITHUB" ] && continue
+            [ "$f" = "$SELF_GITEA" ] && continue
             if [ -n "$DIFF_RANGE" ]; then
               ADDED=$(git diff --no-color --unified=0 "$BASE" "$HEAD" -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
             else
@@ -164,11 +147,18 @@ jobs:
                 break
               fi
             done
-          done
+          done <<< "$CHANGED"
 
           if [ -n "$OFFENDING" ]; then
             echo "::error::Credential-shaped strings detected in diff additions:"
-            printf "$OFFENDING"
+            # `printf '%b' "$OFFENDING"` interprets backslash escapes
+            # (the literal `\n` we appended above becomes a newline)
+            # WITHOUT treating OFFENDING as a format string. Plain
+            # `printf "$OFFENDING"` is a format-string sink: a filename
+            # containing `%` would be interpreted as a conversion
+            # specifier, corrupting the error message (or printing
+            # `%(missing)` artifacts).
+            printf '%b' "$OFFENDING"
             echo ""
             echo "The actual matched values are NOT echoed here, deliberately —"
             echo "round-tripping a leaked credential into CI logs widens the blast"

.gitea/workflows/sop-tier-check.yml

@@ -0,0 +1,126 @@
+# sop-tier-check — canonical Gitea Actions workflow for §SOP-6 enforcement.
+#
+# Logic lives in `.gitea/scripts/sop-tier-check.sh` (extracted 2026-05-09
+# from the previous inline-bash version). The script is the single source
+# of truth; this workflow file just sets env + invokes it.
+#
+# Copy BOTH files (`.gitea/workflows/sop-tier-check.yml` +
+# `.gitea/scripts/sop-tier-check.sh`) into any repo that wants the
+# §SOP-6 PR gate enforced. Pair with branch protection on the protected
+# branch:
+#   required_status_checks:    ["sop-tier-check / tier-check (pull_request)"]
+#   required_approving_reviews: 1
+#   approving_review_teams:    ["ceo", "managers", "engineers"]
+#
+# Tier → required-team expression (internal#189 AND-composition):
+#   tier:low    → engineers,managers,ceo        (OR: any one suffices)
+#   tier:medium → managers AND engineers AND qa???,security???  (AND: all required)
+#   tier:high   → ceo                           (OR: single team, wired for AND)
+#
+# "???" = teams not yet created in Gitea. When qa + security teams are
+# added, update TIER_EXPR["tier:medium"] in the script to remove the
+# markers. PRs already in-flight when qa/security are created continue
+# to work because their authors explicitly requested those reviews.
+#
+# Force-merge: Owners-team override remains available out-of-band via
+# the Gitea merge API; force-merge writes `incident.force_merge` to
+# `structure_events` per §Persistent structured logging gate (Phase 3).
+#
+# Environment variables:
+#   SOP_DEBUG=1          — per-API-call diagnostic lines. Default: off.
+#   SOP_LEGACY_CHECK=1   — revert to OR-gate for this run. Grace window
+#                           for PRs in-flight when AND-composition deployed.
+#                           Burn-in: remove after 2026-05-17 (7-day window).
+#
+# BURN-IN NOTE (internal#189 Phase 1): continue-on-error: true is set on
+# the tier-check job below. This prevents AND-composition from blocking
+# PRs during the 7-day burn-in. After 2026-05-17:
+#   1. Remove `continue-on-error: true` from this job block.
+#   2. Update this BURN-IN NOTE comment to mark the window closed.
+
+name: sop-tier-check
+
+# SECURITY: triggers MUST use `pull_request_target`, not `pull_request`.
+# `pull_request_target` loads the workflow definition from the BASE
+# branch (i.e. `main`), not the PR's HEAD. With `pull_request`, anyone
+# with write access to a feature branch could rewrite this file in
+# their PR to dump SOP_TIER_CHECK_TOKEN (org-read scope) to logs and
+# exfiltrate it. Verified 2026-05-09 against Gitea 1.22.6 —
+# `pull_request_target` (added in Gitea 1.21 via go-gitea/gitea#25229)
+# is the documented mitigation.
+#
+# This workflow does NOT call `actions/checkout` of PR HEAD code, so no
+# untrusted code is ever executed in the runner — we only HTTP-call the
+# Gitea API. If a future change adds a checkout step, it MUST pin to
+# `${{ github.event.pull_request.base.sha }}` (NOT `head.sha`) to keep
+# the trust boundary.
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize, reopened, labeled, unlabeled]
+  pull_request_review:
+    types: [submitted, dismissed, edited]
+
+jobs:
+  tier-check:
+    runs-on: ubuntu-latest
+    # BURN-IN: continue-on-error prevents AND-composition from blocking
+    # PRs during the 7-day window. Remove after 2026-05-17 (internal#189).
+    continue-on-error: true
+    permissions:
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Check out base branch (for the script)
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          # Pin to base.sha — pull_request_target's protection only
+          # works if we never check out PR HEAD. Same SHA the workflow
+          # itself was loaded from.
+          ref: ${{ github.event.pull_request.base.sha }}
+      - name: Install jq
+        # Gitea Actions runners (ubuntu-latest label) do not bundle jq.
+        # The sop-tier-check script uses jq for all JSON API parsing.
+        # Install jq before the script runs so sop-tier-check can pass.
+        #
+        # Method: apt-get first (reliable for Ubuntu runners with internet
+        # access to package mirrors). Falls back to GitHub binary download.
+        # GitHub releases may be unreachable from some runner networks
+        # (infra#241 follow-up: GitHub timeout after 3s on 5.78.80.188
+        # runners). The sop-tier-check script has its own fallback as a
+        # third line of defense. continue-on-error: true ensures this step
+        # failing does not block the job.
+        continue-on-error: true
+        run: |
+          # apt-get is the primary method — Ubuntu package mirrors are reliably
+          # reachable from runner containers. GitHub releases may be blocked
+          # or slow on some networks (infra#241 follow-up).
+          if apt-get update -qq && apt-get install -y -qq jq; then
+            echo "::notice::jq installed via apt-get: $(jq --version)"
+          elif timeout 120 curl -sSL \
+            "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
+            -o /usr/local/bin/jq && chmod +x /usr/local/bin/jq; then
+            echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
+          else
+            echo "::warning::jq install failed — apt-get and GitHub download both failed."
+          fi
+          jq --version 2>/dev/null || echo "::notice::jq not yet available — script fallback will retry"
+
+      - name: Verify tier label + reviewer team membership
+        # continue-on-error: true at step level — job-level is ignored by Gitea
+        # Actions (quirk #10, internal runbooks). Belt-and-suspenders with
+        # SOP_FAIL_OPEN=1 + || true below.
+        continue-on-error: true
+        env:
+          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+          SOP_DEBUG: '0'
+          SOP_LEGACY_CHECK: '0'
+          # SOP_FAIL_OPEN=1 makes the script always exit 0. The UI enforces
+          # the actual merge gate. Combined with continue-on-error: true
+          # above, this step never fails the job regardless of script exit.
+          SOP_FAIL_OPEN: '1'
+        run: |
+          bash .gitea/scripts/sop-tier-check.sh || true

.gitea/workflows/sweep-aws-secrets.yml

@@ -0,0 +1,129 @@
+name: Sweep stale AWS Secrets Manager secrets
+
+# Ported from .github/workflows/sweep-aws-secrets.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Janitor for per-tenant AWS Secrets Manager secrets
+# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
+# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
+# sweep-cf-orphans.yml — different cloud, same justification.
+#
+# Why this exists separately from a long-term reconciler integration:
+#   - molecule-controlplane's tenant_resources audit table (mig 024)
+#     currently tracks four resource kinds: CloudflareTunnel,
+#     CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
+#     not in the list, so the existing reconciler doesn't catch
+#     orphan secrets.
+#   - At ~$0.40/secret/month the cost grew to ~$19/month before this
+#     sweeper was written, indicating ~45+ orphan secrets from
+#     crashed provisions and incomplete deprovision flows.
+#   - The proper fix (KindSecretsManagerSecret + recorder hook +
+#     reconciler enumerator) is filed as a separate controlplane
+#     issue. This sweeper is the immediate cost-relief stopgap.
+#
+# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
+# This is a DEDICATED principal — the production `molecule-cp` IAM
+# user lacks `secretsmanager:ListSecrets` (it only has
+# Get/Create/Update/Delete on specific resources, scoped to its
+# operational needs). The janitor needs ListSecrets across the
+# `molecule/tenant/*` prefix, which warrants a separate principal so
+# we don't broaden the prod-CP policy.
+#
+# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
+# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
+# the mostly-orphan tunnels) refuses to nuke past the threshold.
+
+on:
+  schedule:
+    # Hourly at :30 — offsets from sweep-cf-orphans (:15) and
+    # sweep-cf-tunnels (:45) so the three janitors don't burst the
+    # CP admin endpoints at the same minute.
+    - cron: '30 * * * *'
+# Don't let two sweeps race the same AWS account.
+concurrency:
+  group: sweep-aws-secrets
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  sweep:
+    name: Sweep AWS Secrets Manager
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    # 30 min cap, mirroring the other janitors. AWS DeleteSecret is
+    # fast (~0.3s/call) so even a 100+ backlog drains in seconds
+    # under the 8-way xargs parallelism, but the cap is set generously
+    # to leave headroom for any actual API hang.
+    timeout-minutes: 30
+    env:
+      AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
+      CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+      CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
+      GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
+        # and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
+        #   - schedule → exit 1 on missing secrets (red CI surfaces it)
+        #   - workflow_dispatch → exit 0 with warning (operator-driven,
+        #     they already accepted the repo state)
+        run: |
+          missing=()
+          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-aws-secrets.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-aws-secrets.sh --execute
+          fi

.gitea/workflows/sweep-cf-orphans.yml

@@ -0,0 +1,151 @@
+name: Sweep stale Cloudflare DNS records
+
+# Ported from .github/workflows/sweep-cf-orphans.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Janitor for Cloudflare DNS records whose backing tenant/workspace no
+# longer exists. Without this loop, every short-lived E2E or canary
+# leaves a CF record on the moleculesai.app zone — the zone has a
+# 200-record quota (controlplane#239 hit it 2026-04-23+) and provisions
+# start failing with code 81045 once exhausted.
+#
+# Why a separate workflow vs sweep-stale-e2e-orgs.yml:
+#   - That workflow operates at the CP layer (DELETE /cp/admin/tenants/:slug
+#     drives the cascade). It assumes CP has the org row to drive the
+#     deprovision from. It doesn't catch records left behind when CP
+#     itself never knew about the tenant (canary scratch, manual ops
+#     experiments) or when the cascade's CF-delete branch failed.
+#   - sweep-cf-orphans.sh enumerates the CF zone directly and matches
+#     each record against live CP slugs + AWS EC2 names. It catches
+#     leaks the CP-driven sweep can't.
+#
+# Safety: the script's own MAX_DELETE_PCT gate refuses to nuke more
+# than 50% of records in a single run. If something has gone weird
+# (CP admin endpoint returns no orgs → every tenant looks orphan) the
+# gate halts before damage. Decision-function unit tests in
+# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
+# classifier.
+
+on:
+  schedule:
+    # Hourly. Mirrors sweep-stale-e2e-orgs cadence so the two janitors
+    # converge on the same tick. CF API rate budget is generous (1200
+    # req/5min); a single sweep makes ~1 list + N deletes (N<=quota/2).
+    - cron: '15 * * * *'  # offset from sweep-stale-e2e-orgs (top of hour)
+  # No `merge_group:` trigger on purpose. This is a janitor — it doesn't
+  # need to gate merges, and including it as written before #2088 fired
+  # the full sweep job (or its secret-check) on every PR going through
+  # the merge queue, generating one red CI run per merge-queue eval. If
+  # this workflow is ever wired up as a required check, re-add
+  #   merge_group: { types: [checks_requested] }
+  # AND gate the sweep step with `if: github.event_name != 'merge_group'`
+  # so merge-queue evals report success without actually running.
+
+# Don't let two sweeps race the same zone. workflow_dispatch during a
+# scheduled run would otherwise issue duplicate DELETE calls.
+concurrency:
+  group: sweep-cf-orphans
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  sweep:
+    name: Sweep CF orphans
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    # 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
+    # within one cron interval instead of burning a full tick. Realistic
+    # worst case is ~2 min: 4 sequential curls + 1 aws + N×CF-DELETE
+    # each individually capped at 10s by the script's curl -m flag.
+    timeout-minutes: 3
+    env:
+      CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+      CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
+      CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+      CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split (hardened 2026-04-28
+        # after the silent-no-op incident below):
+        #
+        # The earlier soft-skip-on-schedule policy hid a real leak. All
+        # six secrets were unset on this repo for an unknown duration;
+        # every hourly run printed a yellow ::warning:: and exited 0,
+        # so the workflow registered as "passing" while doing nothing.
+        # CF orphans accumulated to 152/200 (~76% of the zone quota
+        # gone) before a manual `dig`-driven audit caught it. Anything
+        # that runs as a janitor and reports green while idle is
+        # indistinguishable from "the janitor is healthy" — so we now
+        # treat schedule (and any future workflow_run/push triggers)
+        # as a hard-fail when secrets are missing.
+        #
+        #   - schedule / workflow_run / push → exit 1 (red CI run
+        #     surfaces the misconfiguration the next tick)
+        #   - workflow_dispatch              → exit 0 with a warning
+        #     (an operator ran this ad-hoc; they already accepted the
+        #     state of the repo and want the workflow to short-circuit
+        #     so they can rerun after fixing the secret)
+        run: |
+          missing=()
+          for var in CF_API_TOKEN CF_ZONE_ID CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry (intentional):
+        #   - Scheduled runs: github.event.inputs.dry_run is empty →
+        #     defaults to "false" below → script runs with --execute
+        #     (the whole point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default is true (line 38)
+        #     so an ad-hoc operator-triggered run is dry-run by default;
+        #     they have to flip the toggle to actually delete.
+        # The script's MAX_DELETE_PCT gate (default 50%) is the second
+        # line of defense regardless of mode.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-cf-orphans.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-cf-orphans.sh --execute
+          fi

.gitea/workflows/sweep-cf-tunnels.yml

@@ -0,0 +1,128 @@
+name: Sweep stale Cloudflare Tunnels
+
+# Ported from .github/workflows/sweep-cf-tunnels.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Janitor for Cloudflare Tunnels whose backing tenant no longer
+# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
+# records); same justification, different CF resource.
+#
+# Why this exists separately from sweep-cf-orphans:
+#   - DNS records live on the zone (`/zones/<id>/dns_records`).
+#   - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
+#   - Different CF API surface, different scopes; the existing CF
+#     token might not have `account:cloudflare_tunnel:edit`. Splitting
+#     the workflows keeps each one's secret-presence gate independent
+#     so neither silent-skips when the other's secret is missing.
+#   - Cleaner blast radius — operators can disable one without the
+#     other if a regression surfaces.
+#
+# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
+# the DNS sweep's 50% because tenant-shaped tunnels are mostly
+# orphans by design) refuses to nuke past the threshold.
+
+on:
+  schedule:
+    # Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
+    # janitors don't issue parallel CF API bursts at the same minute.
+    - cron: '45 * * * *'
+# Don't let two sweeps race the same account.
+concurrency:
+  group: sweep-cf-tunnels
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  sweep:
+    name: Sweep CF tunnels
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    # 30 min cap. Was 5 min on the theory that the only thing that
+    # could take >5min is a CF-API hang — but on 2026-05-02 a backlog
+    # of 672 stale tunnels accumulated (large staging E2E run + delayed
+    # sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
+    # ~7-8min to drain. The 5-min cap killed the run mid-sweep
+    # (cancelled at 424/672, see run 25248788312); a manual rerun
+    # finished the remainder fine.
+    #
+    # The fix is two-part: parallelize the delete loop (8-way xargs in
+    # the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
+    # cap so a one-off backlog doesn't trip a hangs-detector that
+    # turned out to be a real-job-too-slow detector. With 8-way
+    # parallelism, 600+ tunnels drains in ~60s; 30 min is generous
+    # headroom for actual hangs to still surface (and is in line with
+    # the sweep-cf-orphans companion job).
+    timeout-minutes: 30
+    env:
+      CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+      CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+      CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+      CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
+        # (hardened 2026-04-28 after the silent-no-op incident: the
+        # janitor reported green while doing nothing because secrets
+        # were unset, masking a 152/200 zone-record leak). Same
+        # principle applies here:
+        #   - schedule → exit 1 on missing secrets (red CI surfaces it)
+        #   - workflow_dispatch → exit 0 with warning (operator-driven,
+        #     they already accepted the repo state)
+        run: |
+          missing=()
+          for var in CF_API_TOKEN CF_ACCOUNT_ID CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-cf-tunnels.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-cf-tunnels.sh --execute
+          fi

.gitea/workflows/sweep-stale-e2e-orgs.yml

@@ -0,0 +1,243 @@
+name: Sweep stale e2e-* orgs (staging)
+
+# Ported from .github/workflows/sweep-stale-e2e-orgs.yml on 2026-05-11 per RFC
+# internal#219 §1 sweep. Differences from the GitHub version:
+#   - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
+#     per feedback_gitea_workflow_dispatch_inputs_unsupported).
+#   - Dropped `merge_group:` (no Gitea merge queue).
+#   - Dropped `environment:` blocks (Gitea has no environments).
+#   - Workflow-level env.GITHUB_SERVER_URL pinned per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on each job (RFC §1 contract).
+#
+
+# Janitor for staging tenants left behind when E2E cleanup didn't run:
+# CI cancellations, runner crashes, transient AWS errors mid-cascade,
+# bash trap missed (signal 9), etc. Without this loop, every failed
+# teardown leaks an EC2 + DNS + DB row until manual ops cleanup —
+# 2026-04-23 staging hit the 64 vCPU AWS quota from ~27 such orphans.
+#
+# Why not rely on per-test-run teardown:
+#   - Per-run teardown is best-effort by definition. Any process death
+#     after the test starts but before the trap fires leaves debris.
+#   - GH Actions cancellation kills the runner without grace period.
+#     The workflow's `if: always()` step usually catches this, but it
+#     too can fail (CP transient 5xx, runner network issue at the
+#     wrong moment).
+#   - Even when teardown runs, the CP cascade is best-effort in places
+#     (cascadeTerminateWorkspaces logs+continues; DNS deletion same).
+#   - This sweep is the catch-all that converges staging back to clean
+#     regardless of which specific path leaked.
+#
+# The PROPER fix is making CP cleanup transactional + verify-after-
+# terminate (filed separately as cleanup-correctness work). This
+# workflow is the safety net that catches everything else AND any
+# future leak source we haven't yet identified.
+
+on:
+  schedule:
+    # Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
+    # create to teardown — canary is ~8 min, full SaaS ~25 min). The
+    # previous hourly + 120-min stale threshold meant a leaked tenant
+    # could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
+    # leak. Tightening the cadence + threshold reduces the worst-case
+    # leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
+    # threshold) without risk of catching in-progress runs (the longest
+    # e2e run is the 25-min canary, well under the 30-min threshold).
+    # See molecule-controlplane#420 for the leak-class accounting that
+    # motivated this tightening.
+    - cron: '*/15 * * * *'
+# Don't let two sweeps fight. Cron + workflow_dispatch could overlap
+# on a manual trigger; queue rather than parallel-delete.
+concurrency:
+  group: sweep-stale-e2e-orgs
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  sweep:
+    name: Sweep e2e orgs
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    timeout-minutes: 15
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
+      DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
+      # Refuse to delete more than this many orgs in one tick. If the
+      # CP DB is briefly empty (or the admin endpoint goes weird and
+      # returns no created_at), every e2e- org would look stale.
+      # Bailing protects against runaway nukes.
+      SAFETY_CAP: 50
+
+    steps:
+      - name: Verify admin token present
+        run: |
+          if [ -z "$ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: Identify stale e2e orgs
+        id: identify
+        run: |
+          set -euo pipefail
+          # Fetch into a file so the python step reads it via stdin —
+          # cleaner than embedding $(curl ...) into a heredoc.
+          curl -sS --fail-with-body --max-time 30 \
+            "$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" \
+            > orgs.json
+
+          # Filter:
+          #   1. slug starts with one of the ephemeral test prefixes:
+          #        - 'e2e-'    — covers e2e-canary-, e2e-canvas-*, etc.
+          #        - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
+          #                      missing this prefix left two such tenants
+          #                      orphaned 8h on staging (2026-05-03), then
+          #                      hard-failed redeploy-tenants-on-staging
+          #                      and broke the staging→main auto-promote
+          #                      chain. Kept in sync with the EPHEMERAL_PREFIX_RE
+          #                      regex in redeploy-tenants-on-staging.yml.
+          #   2. created_at is older than MAX_AGE_MINUTES ago
+          # Output one slug per line to a file the next step reads.
+          python3 > stale_slugs.txt <<'PY'
+          import json, os
+          from datetime import datetime, timezone, timedelta
+          # SSOT for this list lives in the controlplane Go code:
+          # molecule-controlplane/internal/slugs/ephemeral.go
+          # (var EphemeralPrefixes). The redeploy-fleet auto-rollout
+          # also reads from there to SKIP these slugs — without that
+          # filter, fleet redeploy SSM-failed in-flight E2E tenants
+          # whose containers were still booting, breaking the test
+          # that just spun them up (molecule-controlplane#493).
+          # Update both files together.
+          EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
+          with open("orgs.json") as f:
+              data = json.load(f)
+          max_age = int(os.environ["MAX_AGE_MINUTES"])
+          cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
+          for o in data.get("orgs", []):
+              slug = o.get("slug", "")
+              if not slug.startswith(EPHEMERAL_PREFIXES):
+                  continue
+              created = o.get("created_at")
+              if not created:
+                  # Defensively skip rows without created_at — better
+                  # to leave one orphan than nuke a brand-new row
+                  # whose timestamp didn't render.
+                  continue
+              # Python 3.11+ handles RFC3339 with Z directly via
+              # fromisoformat; older runners need the trailing Z swap.
+              created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
+              if created_dt < cutoff:
+                  print(slug)
+          PY
+
+          count=$(wc -l < stale_slugs.txt | tr -d ' ')
+          echo "Found $count stale e2e org(s) older than ${MAX_AGE_MINUTES}m"
+          if [ "$count" -gt 0 ]; then
+            echo "First 20:"
+            head -20 stale_slugs.txt | sed 's/^/  /'
+          fi
+          echo "count=$count" >> "$GITHUB_OUTPUT"
+
+      - name: Safety gate
+        if: steps.identify.outputs.count != '0'
+        run: |
+          count="${{ steps.identify.outputs.count }}"
+          if [ "$count" -gt "$SAFETY_CAP" ]; then
+            echo "::error::Refusing to delete $count orgs in one sweep (cap=$SAFETY_CAP). Investigate manually — this usually means the CP admin API returned no created_at or returned a degraded result. Re-run with workflow_dispatch + max_age_minutes if intentional."
+            exit 1
+          fi
+          echo "Within safety cap ($count ≤ $SAFETY_CAP) ✓"
+
+      - name: Delete stale orgs
+        if: steps.identify.outputs.count != '0' && env.DRY_RUN != 'true'
+        run: |
+          set -uo pipefail
+          deleted=0
+          failed=0
+          while IFS= read -r slug; do
+            [ -z "$slug" ] && continue
+            # The DELETE handler requires {"confirm": "<slug>"} matching
+            # the URL slug — fat-finger guard. Idempotent: re-issuing
+            # picks up via org_purges.last_step.
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/del_resp -w "%{http_code}" \
+              --max-time 60 \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/del_code
+            set -e
+            # Stderr from curl (-sS shows dial errors etc.) goes to runner log.
+            http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
+            if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
+              deleted=$((deleted+1))
+              echo "  deleted: $slug"
+            else
+              failed=$((failed+1))
+              echo "  FAILED ($http_code): $slug — $(cat /tmp/del_resp 2>/dev/null | head -c 200)"
+            fi
+          done < stale_slugs.txt
+          echo ""
+          echo "Sweep summary: deleted=$deleted failed=$failed"
+          # Don't fail the workflow on per-org delete errors — the
+          # sweeper is best-effort. Next hourly tick re-attempts. We
+          # only fail loud at the safety-cap gate above.
+
+      - name: Sweep orphan tunnels
+        # Stale-org cleanup deletes the org (which cascades to tunnel
+        # delete inside the CP). But when that cascade fails partway —
+        # CP transient 5xx after the org row is deleted but before the
+        # CF tunnel delete completes — the tunnel persists with no
+        # matching org row. The reconciler in internal/sweep flags this
+        # as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
+        #
+        # `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
+        # reaper. Calling it here at the end of every sweep tick
+        # converges the staging CF account to clean even when CP
+        # cascades half-fail.
+        #
+        # PR #492 made the underlying DeleteTunnel actually check
+        # status — pre-fix it silent-succeeded on CF code 1022
+        # ("active connections"), so this step would have been a no-op
+        # against stuck connectors. Post-fix the cleanup invokes
+        # CleanupTunnelConnections + retry, which actually clears the
+        # 1022 case. (#2987)
+        #
+        # Best-effort. Failure here doesn't fail the workflow — next
+        # tick re-attempts. Errors flow to step output for ops review.
+        if: env.DRY_RUN != 'true'
+        run: |
+          set +e
+          curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
+            --max-time 60 \
+            -X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
+          set -e
+          http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
+          body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
+          if [ "$http_code" = "200" ]; then
+            count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
+            failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
+            echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
+          else
+            echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
+          fi
+
+      - name: Dry-run summary
+        if: env.DRY_RUN == 'true'
+        run: |
+          echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."

.gitea/workflows/test-ops-scripts.yml

@@ -0,0 +1,65 @@
+name: Ops Scripts Tests
+
+# Ported from .github/workflows/test-ops-scripts.yml on 2026-05-11 per
+# RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `merge_group:` trigger (no Gitea merge queue).
+#   - on.paths references .gitea/workflows/test-ops-scripts.yml (this
+#     file) instead of the .github/ one.
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# Runs the unittest suite for scripts/ on every PR + push that touches
+# anything under scripts/. Kept separate from the main CI so a script-only
+# change doesn't trigger the heavier Go/Canvas/Python pipelines.
+#
+# Discovery layout: tests sit alongside the code they test (see
+# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
+# test_build_runtime_package.py for the rewriter coverage). The job
+# below runs `unittest discover` TWICE — once from `scripts/`, once
+# from `scripts/ops/` — because neither dir has an `__init__.py`, so
+# a single discover from `scripts/` doesn't recurse into the ops
+# subdir. Two passes is simpler than retrofitting namespace packages.
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      - 'scripts/**'
+      - '.gitea/workflows/test-ops-scripts.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'scripts/**'
+      - '.gitea/workflows/test-ops-scripts.yml'
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Ops scripts (unittest)
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+      - name: Run scripts/ unittests (build_runtime_package, ...)
+        # Top-level scripts/ tests live alongside their target file
+        # (e.g. scripts/test_build_runtime_package.py exercises
+        # scripts/build_runtime_package.py). discover from scripts/
+        # picks up only top-level test_*.py because scripts/ops/ has
+        # no __init__.py — that's intentional, so we run two passes.
+        working-directory: scripts
+        run: python -m unittest discover -t . -p 'test_*.py' -v
+      - name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
+        working-directory: scripts/ops
+        run: python -m unittest discover -p 'test_*.py' -v

.githooks/pre-commit

@@ -95,21 +95,91 @@ if [ -n "$STAGED_GO" ]; then
 fi
 
 # ──────────────────────────────────────────────────────────
-# 5. Secrets: No tokens/keys in staged files
+# 5. Go: build check — catches bot-generated structurally-invalid Go (#1770)
 # ──────────────────────────────────────────────────────────
+#
+# Background: bot agents have produced syntactically-broken Go that the
+# patch tool happily applied (e.g. PR #1769 commit 66ea0b64 — function
+# declaration nested inside another function's body). Compilation failed,
+# staging Platform(Go) was red for hours. CI catches this AT PR-time but
+# by then the malformed commit is already shared.
+#
+# Pre-commit guard: when ANY .go file in workspace-server/ is staged, run
+# `go build ./...` from workspace-server. If it fails, reject the commit.
+# Cost: ~5-10s on a warm cache; acceptable for the class of bug it
+# catches. Skip when go isn't available (CI runners that need to bypass).
+
+if [ -n "$STAGED_GO" ]; then
+  if command -v go >/dev/null 2>&1; then
+    if ! (cd workspace-server && go build ./... >/tmp/precommit-go-build.log 2>&1); then
+      echo "❌ GO BUILD FAILED — staged Go changes don't compile (workspace-server/)."
+      echo "   Output:"
+      sed 's/^/     /' /tmp/precommit-go-build.log | head -20
+      echo "   Fix the build error before committing. See #1770 for context."
+      ERRORS=$((ERRORS + 1))
+    fi
+  else
+    # Bots and CI runners may bypass when go isn't installed — surface a
+    # warning so the absence is visible, but don't block. Humans hit this
+    # only if they didn't run setup.sh.
+    echo "⚠️  go not installed — skipping go-build pre-commit check (#1770)"
+  fi
+fi
+
+# ──────────────────────────────────────────────────────────
+# 6. Secrets: No tokens/keys in staged files
+# ──────────────────────────────────────────────────────────
+#
+# Pattern set MUST match .github/workflows/secret-scan.yml SECRET_PATTERNS
+# and molecule-ai-workspace-runtime/molecule_runtime/scripts/pre-commit-checks.sh —
+# .github/workflows/secret-pattern-drift.yml lints this invariant. Rebuilt
+# against canonical 2026-05-02 after #1569 Phase 1 discovery surfaced
+# real ghs_*/github_pat_* leaks that the prior pattern set
+# ('sk-ant-|sk-proj-|ghp_|gho_|AKIA|mol_pk_|cfut_') would have missed:
+# (a) it lacked ghs_ / ghu_ / ghr_ / github_pat_ / sk-svcacct- / sk-cp- /
+# xox[baprs]- / ASIA prefixes, (b) it skipped *.md and docs/* — but the
+# actual leaks lived in tick-reflections-temp.md, qa-audit-2026-04-21.md,
+# docs/incidents/INCIDENT_LOG.md.
+SECRET_PATTERNS=(
+  'ghp_[A-Za-z0-9]{36,}'           # GitHub PAT (classic)
+  'ghs_[A-Za-z0-9]{36,}'           # GitHub App installation token
+  'gho_[A-Za-z0-9]{36,}'           # GitHub OAuth user-to-server
+  'ghu_[A-Za-z0-9]{36,}'           # GitHub OAuth user
+  'ghr_[A-Za-z0-9]{36,}'           # GitHub OAuth refresh
+  'github_pat_[A-Za-z0-9_]{82,}'   # GitHub fine-grained PAT
+  'sk-ant-[A-Za-z0-9_-]{40,}'      # Anthropic API key
+  'sk-proj-[A-Za-z0-9_-]{40,}'     # OpenAI project key
+  'sk-svcacct-[A-Za-z0-9_-]{40,}'  # OpenAI service-account key
+  'sk-cp-[A-Za-z0-9_-]{60,}'       # MiniMax API key (F1088 vector — caught only after the fact)
+  'xox[baprs]-[A-Za-z0-9-]{20,}'   # Slack tokens (bot/app/user/refresh)
+  'AKIA[0-9A-Z]{16}'               # AWS access key ID
+  'ASIA[0-9A-Z]{16}'               # AWS STS temp access key ID
+)
 
 ALL_STAGED=$(git diff --cached --name-only --diff-filter=ACM || true)
 if [ -n "$ALL_STAGED" ]; then
   for f in $ALL_STAGED; do
-    # Skip binary, known safe files, hooks, docs, and markdown
-    if echo "$f" | grep -qE '\.png$|\.jpg$|\.ico$|\.woff|node_modules|\.lock$|\.githooks/|\.md$|docs/'; then
+    # Skip ONLY binary + lockfiles + the hook itself. Markdown +
+    # docs/* are NOT skipped — that was the bug (#1569 leaks were
+    # all in *.md). If a doc legitimately needs a token-shaped
+    # placeholder, use ghs_EXAMPLE_TOKEN_DO_NOT_USE — short enough
+    # to dodge the {36,} length suffix.
+    if echo "$f" | grep -qE '\.png$|\.jpg$|\.ico$|\.woff|node_modules|\.lock$|\.githooks/'; then
       continue
     fi
-    DIFF=$(git diff --cached "$f" 2>/dev/null | grep '^+' | grep -v '^+++' || true)
-    if echo "$DIFF" | grep -qE 'sk-ant-|sk-proj-|ghp_|gho_|AKIA[A-Z0-9]|mol_pk_|cfut_' 2>/dev/null; then
-      echo "❌ POSSIBLE SECRET in $f — do not commit API keys or tokens"
-      ERRORS=$((ERRORS + 1))
-    fi
+    DIFF=$(git diff --cached --no-color --unified=0 -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
+    [ -z "$DIFF" ] && continue
+    for pattern in "${SECRET_PATTERNS[@]}"; do
+      if echo "$DIFF" | grep -qE "$pattern"; then
+        echo "❌ POSSIBLE SECRET in $f (matched: ${pattern})"
+        echo "   The actual matched value is NOT echoed here — round-tripping a"
+        echo "   leaked credential into scrollback widens the blast radius."
+        echo "   If false positive (test/docs example), use a short placeholder"
+        echo "   like ghs_EXAMPLE_TOKEN_DO_NOT_USE that doesn't satisfy the length."
+        ERRORS=$((ERRORS + 1))
+        break
+      fi
+    done
   done
 fi
 

.github/dependabot.yml

@@ -0,0 +1,80 @@
+# Dependabot — auto-bump pinned dependencies.
+#
+# Why this exists:
+#
+# All `uses:` references in .github/workflows/*.yml are pinned to commit
+# SHAs (with `# v<N>` comments for human readability) instead of mutable
+# tags like `@v4`. Tag pinning is a known supply-chain risk: a maintainer
+# (or compromised maintainer account) can repoint `@v4` to malicious code
+# and our pipelines silently pull it. SHA pinning closes that risk.
+#
+# But SHA pinning has a maintenance cost: each upstream legitimate fix
+# requires manually finding + bumping the SHA. Dependabot for Actions
+# closes that gap by opening PRs to bump pinned SHAs whenever upstream
+# tags a new version. Reviewer evaluates the bump like any other
+# dependency PR.
+#
+# Combined: SHA pinning gives us security, Dependabot keeps us current.
+
+version: 2
+updates:
+  # GitHub Actions — every workflow file under .github/workflows/.
+  # Weekly cadence is enough for a CI surface this size; the supply-
+  # chain attack window is "minutes between repoint and pull," and
+  # weekly auto-bumps don't help with zero-days regardless. The point
+  # is to pull in non-zero-day fixes without operator effort, not to
+  # be real-time.
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    labels:
+      - dependencies
+      - github-actions
+    commit-message:
+      prefix: chore(deps)
+      include: scope
+
+  # Go module — workspace-server. Bumps go.mod deps via PR weekly.
+  - package-ecosystem: gomod
+    directory: "/workspace-server"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    labels:
+      - dependencies
+      - go
+    commit-message:
+      prefix: chore(deps)
+      include: scope
+
+  # npm — canvas (Next.js bundle). Largest dep tree in this repo;
+  # weekly cadence keeps the security surface fresh without flooding
+  # the queue. open-pull-requests-limit: 10 because npm churns more
+  # than the others.
+  - package-ecosystem: npm
+    directory: "/canvas"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 10
+    labels:
+      - dependencies
+      - npm
+    commit-message:
+      prefix: chore(deps)
+      include: scope
+
+  # Python — workspace runtime requirements. Pip/requirements.txt-
+  # backed rather than pyproject.toml; Dependabot supports both.
+  - package-ecosystem: pip
+    directory: "/workspace"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    labels:
+      - dependencies
+      - python
+    commit-message:
+      prefix: chore(deps)
+      include: scope

.github/scripts/lint_secret_pattern_drift.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Lint SECRET_PATTERNS drift across known consumers of molecule-core's canonical.
+
+The canonical SECRET_PATTERNS array in
+.github/workflows/secret-scan.yml is mirrored by every other side
+that scans for credentials: the workspace-runtime's bundled
+pre-commit hook, the molecule-controlplane inlined copy, etc. The
+mirror is enforced socially today — when someone adds a new pattern
+to canonical (e.g. the sk-cp- MiniMax token after F1088), the other
+sides are supposed to be updated in lockstep.
+
+This script automates the check. Diffs the canonical's pattern set
+against each known public consumer and exits non-zero on any
+mismatch. Wired into a daily cron + on-push gate via
+.github/workflows/secret-pattern-drift.yml.
+
+Private-repo consumers (currently molecule-controlplane's inlined
+copy) are out of scope here because the molecule-core workflow's
+GITHUB_TOKEN can't read other private repos in the org. They're
+expected to self-monitor via their own copy of this script — not a
+hard barrier, just a future expansion.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+import urllib.request
+from pathlib import Path
+
+CANONICAL_FILE = Path(".gitea/workflows/secret-scan.yml")
+
+# Public consumer mirrors. Each entry is (label, raw_url) — raw_url
+# points at the file's RAW content on the consumer's default branch
+# (or staging where applicable). Add an entry here when a new public
+# repo starts shipping its own SECRET_PATTERNS array.
+CONSUMERS: list[tuple[str, str]] = [
+    (
+        "molecule-ai-workspace-runtime/molecule_runtime/scripts/pre-commit-checks.sh",
+        "https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime/raw/branch/main/molecule_runtime/scripts/pre-commit-checks.sh",
+    ),
+]
+
+# In-repo consumers — paths read locally from the workflow checkout.
+# Read-from-disk avoids the staging→main lag that the URL fetcher
+# would hit (a freshly-edited canonical wouldn't yet be on the
+# consumer's default branch). Same drift semantics, no network.
+LOCAL_CONSUMERS: list[tuple[str, Path]] = [
+    (
+        ".githooks/pre-commit (molecule-core local hook)",
+        Path(".githooks/pre-commit"),
+    ),
+]
+
+# Matches the SECRET_PATTERNS=( ... ) array in either yaml-indented
+# (the canonical workflow's `run:` block) or shell-flat (runtime
+# hook) format. Patterns inside are single-quoted Bash strings; we
+# pull each via _PATTERN_RE.
+#
+# Closing `)` is anchored to the start of a line (possibly indented)
+# because pattern comments like `# GitHub PAT (classic)` contain
+# their own `)` mid-line — a non-anchored regex would match through
+# the comment's paren and capture only the first pattern.
+_ARRAY_RE = re.compile(r"SECRET_PATTERNS=\((.*?)^\s*\)", re.DOTALL | re.MULTILINE)
+_PATTERN_RE = re.compile(r"'([^']+)'")
+
+
+def extract_patterns(content: str, source_label: str) -> list[str]:
+    """Pull the SECRET_PATTERNS list out of either format. Raises if missing."""
+    m = _ARRAY_RE.search(content)
+    if not m:
+        raise SystemExit(f"::error::{source_label}: SECRET_PATTERNS=(...) array not found")
+    return _PATTERN_RE.findall(m.group(1))
+
+
+def fetch(url: str) -> str:
+    req = urllib.request.Request(
+        url, headers={"User-Agent": "secret-pattern-drift-lint/1"}
+    )
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return resp.read().decode("utf-8")
+
+
+def diff_patterns(canonical: list[str], consumer: list[str]) -> tuple[list[str], list[str]]:
+    """Return (missing_from_consumer, extra_in_consumer) — both sorted."""
+    canonical_set = set(canonical)
+    consumer_set = set(consumer)
+    return (
+        sorted(canonical_set - consumer_set),
+        sorted(consumer_set - canonical_set),
+    )
+
+
+def main() -> int:
+    if not CANONICAL_FILE.exists():
+        print(f"::error::canonical not found at {CANONICAL_FILE}")
+        return 1
+
+    canonical = extract_patterns(CANONICAL_FILE.read_text(), str(CANONICAL_FILE))
+    print(f"canonical ({CANONICAL_FILE}): {len(canonical)} patterns")
+
+    drift = False
+
+    # In-repo consumers first — these are read from the workflow's own
+    # checkout, so they never lag behind the canonical and a missing
+    # file IS a real error (not a fetch warning).
+    for label, path in LOCAL_CONSUMERS:
+        if not path.exists():
+            print(f"::error::{label}: file not found at {path}")
+            drift = True
+            continue
+        consumer = extract_patterns(path.read_text(), label)
+        missing, extra = diff_patterns(canonical, consumer)
+        if not missing and not extra:
+            print(f"  ✓ {label}: aligned ({len(consumer)} patterns)")
+            continue
+        drift = True
+        print(f"::error::DRIFT in {label}:")
+        for p in missing:
+            print(f"  -  missing from consumer: {p!r}")
+        for p in extra:
+            print(f"  -  extra in consumer (not in canonical): {p!r}")
+
+    for label, url in CONSUMERS:
+        try:
+            content = fetch(url)
+        except Exception as e:
+            # Fetch failures are warnings, not errors. A consumer
+            # whose default branch was just renamed (or whose file
+            # moved) shouldn't fail the lint until someone updates
+            # the URL above. Real drift is the failure mode this
+            # gate exists to catch — fetch reliability isn't.
+            print(f"::warning::{label}: fetch failed ({e}) — skipping")
+            continue
+
+        consumer = extract_patterns(content, label)
+        missing, extra = diff_patterns(canonical, consumer)
+        if not missing and not extra:
+            print(f"  ✓ {label}: aligned ({len(consumer)} patterns)")
+            continue
+
+        drift = True
+        print(f"::error::DRIFT in {label}:")
+        for p in missing:
+            print(f"  -  missing from consumer: {p!r}")
+        for p in extra:
+            print(f"  -  extra in consumer (not in canonical): {p!r}")
+
+    if drift:
+        print()
+        print("::error::SECRET_PATTERNS drift detected. Bring consumer(s) into")
+        print("alignment with the canonical SECRET_PATTERNS array in")
+        print(f"{CANONICAL_FILE} by adding the missing patterns and removing")
+        print("any extras. The two sides must stay byte-aligned on the pattern")
+        print("list — the runtime hook is the developer's local pre-commit,")
+        print("the canonical is the org-wide CI gate, divergence means a token")
+        print("can pass one but get rejected by the other.")
+        return 1
+
+    print()
+    print("✓ All known consumers aligned with canonical SECRET_PATTERNS.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/auto-promote-staging.yml

@@ -1,182 +0,0 @@
-name: Auto-promote staging → main
-
-# Fires after any of the staging-branch quality gates complete. When ALL
-# required gates are green on the same staging SHA, fast-forwards `main`
-# to that SHA automatically — closing the gap that historically let
-# features sit on staging for weeks waiting for a bulk promotion PR
-# (see molecule-core#1496 for the 1172-commit example).
-#
-# Safety model:
-# - Runs ONLY on workflow_run events for the staging branch.
-# - Requires EVERY named gate workflow to have the same head_sha and
-#   all be `conclusion == success`. If any of them is red, skipped,
-#   cancelled, or pending, we abort (stay on the current main).
-# - Uses --ff-only: refuses to advance main if main has diverged from
-#   the staging history (e.g. a hotfix landed directly on main). In
-#   that case a human resolves the fork.
-# - Writes a commit summary so the promote shows up in git log as a
-#   deliberate act, not a stealth move.
-#
-# **Initial rollout:** ship this file but leave the `enabled` input set
-# such that nothing auto-promotes until staging CI has been reliably
-# green for a few days. Toggle via repo variable `AUTO_PROMOTE_ENABLED`.
-
-on:
-  workflow_run:
-    workflows:
-      - CI
-      - E2E Staging Canvas (Playwright)
-      - E2E API Smoke Test
-      - CodeQL
-    types: [completed]
-  workflow_dispatch:
-    inputs:
-      force:
-        description: "Force promote even when AUTO_PROMOTE_ENABLED is unset (manual override)"
-        required: false
-        default: "false"
-
-permissions:
-  contents: write
-
-jobs:
-  check-all-gates-green:
-    # Only consider staging pushes. PRs into staging don't promote.
-    if: >
-      (github.event_name == 'workflow_run' &&
-       github.event.workflow_run.head_branch == 'staging' &&
-       github.event.workflow_run.event == 'push')
-      || github.event_name == 'workflow_dispatch'
-    runs-on: ubuntu-latest
-    outputs:
-      all_green: ${{ steps.gates.outputs.all_green }}
-      head_sha: ${{ steps.gates.outputs.head_sha }}
-    steps:
-      - name: Check all required gates on this SHA
-        id: gates
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
-          REPO: ${{ github.repository }}
-        run: |
-          set -euo pipefail
-
-          # Required gate workflow names. Must match the `name:` field
-          # in the respective .github/workflows/*.yml files.
-          GATES=(
-            "CI"
-            "E2E Staging Canvas (Playwright)"
-            "E2E API Smoke Test"
-            "CodeQL"
-          )
-
-          echo "head_sha=${HEAD_SHA}" >> "$GITHUB_OUTPUT"
-          echo "Checking gates on SHA ${HEAD_SHA}"
-
-          ALL_GREEN=true
-          for gate in "${GATES[@]}"; do
-            # Query the most recent run of this workflow on this SHA.
-            # event=push to avoid picking up PR runs. branch=staging to
-            # guard against someone dispatching the gate on a non-staging
-            # branch at the same SHA.
-            RESULT=$(gh run list \
-              --repo "$REPO" \
-              --workflow "$gate" \
-              --branch staging \
-              --event push \
-              --commit "$HEAD_SHA" \
-              --limit 1 \
-              --json status,conclusion \
-              --jq '.[0] | "\(.status)/\(.conclusion // "none")"' \
-              2>/dev/null || echo "missing/none")
-
-            echo "  $gate → $RESULT"
-
-            # Only completed/success counts. completed/failure or
-            # in_progress/anything or no record at all = abort.
-            if [ "$RESULT" != "completed/success" ]; then
-              ALL_GREEN=false
-            fi
-          done
-
-          echo "all_green=${ALL_GREEN}" >> "$GITHUB_OUTPUT"
-          if [ "$ALL_GREEN" != "true" ]; then
-            echo "::notice::auto-promote: not all gates are green on ${HEAD_SHA} — staying on current main"
-          fi
-
-  promote:
-    needs: check-all-gates-green
-    if: needs.check-all-gates-green.outputs.all_green == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check rollout gate
-        env:
-          AUTO_PROMOTE_ENABLED: ${{ vars.AUTO_PROMOTE_ENABLED }}
-          FORCE_INPUT: ${{ github.event.inputs.force }}
-        run: |
-          set -eu
-          # Repo variable AUTO_PROMOTE_ENABLED=true flips this on. While
-          # it's unset, the workflow dry-runs (logs what it would have
-          # done) but doesn't actually push to main. Set the variable in
-          # Settings → Secrets and variables → Actions → Variables.
-          if [ "${AUTO_PROMOTE_ENABLED:-}" != "true" ] && [ "${FORCE_INPUT:-false}" != "true" ]; then
-            {
-              echo "## ⏸ Auto-promote disabled"
-              echo
-              echo "Repo variable \`AUTO_PROMOTE_ENABLED\` is not set to \`true\`."
-              echo "All gates are green on staging; would have promoted to \`main\`."
-              echo
-              echo "To enable: Settings → Secrets and variables → Actions → Variables → \`AUTO_PROMOTE_ENABLED=true\`."
-              echo "To test once manually: workflow_dispatch with \`force=true\`."
-            } >> "$GITHUB_STEP_SUMMARY"
-            echo "::notice::auto-promote disabled — dry run only"
-            exit 0
-          fi
-
-      - name: Checkout main
-        if: ${{ vars.AUTO_PROMOTE_ENABLED == 'true' || github.event.inputs.force == 'true' }}
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Fast-forward main → staging HEAD
-        if: ${{ vars.AUTO_PROMOTE_ENABLED == 'true' || github.event.inputs.force == 'true' }}
-        env:
-          TARGET_SHA: ${{ needs.check-all-gates-green.outputs.head_sha }}
-        run: |
-          set -eu
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-          git fetch origin staging
-          git fetch origin main
-
-          # Refuse to advance main if it's diverged from staging history.
-          # Someone landed a commit directly on main that's not on
-          # staging → human needs to decide how to reconcile.
-          if ! git merge-base --is-ancestor "$(git rev-parse origin/main)" "$TARGET_SHA"; then
-            {
-              echo "## ❌ Auto-promote refused — main has diverged"
-              echo
-              echo "\`main\` (\`$(git rev-parse --short origin/main)\`) is not an ancestor of staging (\`${TARGET_SHA:0:7}\`)."
-              echo "Someone committed directly to main or the histories forked."
-              echo
-              echo "Resolve manually: merge main into staging, get CI green on the merged commit,"
-              echo "then the auto-promote will succeed on the next run."
-            } >> "$GITHUB_STEP_SUMMARY"
-            exit 1
-          fi
-
-          # Fast-forward main to the target SHA.
-          git checkout main
-          git merge --ff-only "$TARGET_SHA"
-          git push origin main
-
-          {
-            echo "## ✅ Auto-promoted main → ${TARGET_SHA:0:7}"
-            echo
-            echo "All gate workflows green on staging at this SHA."
-            echo "\`main\` fast-forwarded to match."
-          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/auto-tag-runtime.yml

@@ -1,113 +0,0 @@
-name: auto-tag-runtime
-
-# Auto-tag runtime releases on every merge to main that touches workspace/.
-# This is the entry point of the runtime CD chain:
-#
-#   merge PR → auto-tag-runtime (this) → publish-runtime → cascade → template
-#   image rebuilds → repull on hosts.
-#
-# Default bump is patch. Override via PR label `release:minor` or
-# `release:major` BEFORE merging — the label is read off the merged PR
-# associated with the push commit.
-#
-# Skips when:
-#   - The push isn't to main (other branches don't auto-release).
-#   - The merge commit message contains `[skip-release]` (escape hatch
-#     for cleanup PRs that touch workspace/ but shouldn't ship).
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "workspace/**"
-      - "scripts/build_runtime_package.py"
-      - ".github/workflows/auto-tag-runtime.yml"
-      - ".github/workflows/publish-runtime.yml"
-
-permissions:
-  contents: write    # to push the new tag
-  pull-requests: read # to read labels off the merged PR
-
-concurrency:
-  # Serialize tag bumps so two near-simultaneous merges can't both think
-  # they're 0.1.6 and race to push the same tag.
-  group: auto-tag-runtime
-  cancel-in-progress: false
-
-jobs:
-  tag:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0    # need full tag history for `git describe` / sort
-
-      - name: Skip when commit asks
-        id: skip
-        run: |
-          MSG=$(git log -1 --format=%B "${{ github.sha }}")
-          if echo "$MSG" | grep -qiE '\[skip-release\]|\[no-release\]'; then
-            echo "skip=true" >> "$GITHUB_OUTPUT"
-            echo "Commit message contains [skip-release] — no tag will be created."
-          else
-            echo "skip=false" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Determine bump kind from PR label
-        id: bump
-        if: steps.skip.outputs.skip != 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          # The merged PR for this push commit. `gh pr list --search` finds
-          # closed PRs whose merge commit matches; we take the first.
-          PR=$(gh pr list --state merged --search "${{ github.sha }}" --json number,labels --jq '.[0]' 2>/dev/null || echo "")
-          if [ -z "$PR" ] || [ "$PR" = "null" ]; then
-            echo "No merged PR found for ${{ github.sha }} — defaulting to patch bump."
-            echo "kind=patch" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          LABELS=$(echo "$PR" | jq -r '.labels[].name')
-          if echo "$LABELS" | grep -qx 'release:major'; then
-            echo "kind=major" >> "$GITHUB_OUTPUT"
-          elif echo "$LABELS" | grep -qx 'release:minor'; then
-            echo "kind=minor" >> "$GITHUB_OUTPUT"
-          else
-            echo "kind=patch" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Compute next version from latest runtime-v* tag
-        id: version
-        if: steps.skip.outputs.skip != 'true'
-        run: |
-          # Find the highest runtime-vX.Y.Z tag. `sort -V` handles semver
-          # ordering; `grep` filters to the right tag prefix.
-          LATEST=$(git tag --list 'runtime-v*' | sort -V | tail -1)
-          if [ -z "$LATEST" ]; then
-            # No prior tag — start the runtime line at 0.1.0.
-            CURRENT="0.0.0"
-          else
-            CURRENT="${LATEST#runtime-v}"
-          fi
-          MAJOR=$(echo "$CURRENT" | cut -d. -f1)
-          MINOR=$(echo "$CURRENT" | cut -d. -f2)
-          PATCH=$(echo "$CURRENT" | cut -d. -f3)
-          case "${{ steps.bump.outputs.kind }}" in
-            major) MAJOR=$((MAJOR+1)); MINOR=0; PATCH=0;;
-            minor) MINOR=$((MINOR+1)); PATCH=0;;
-            patch) PATCH=$((PATCH+1));;
-          esac
-          NEW="$MAJOR.$MINOR.$PATCH"
-          echo "current=$CURRENT" >> "$GITHUB_OUTPUT"
-          echo "new=$NEW" >> "$GITHUB_OUTPUT"
-          echo "Bumping runtime $CURRENT → $NEW (${{ steps.bump.outputs.kind }})"
-
-      - name: Push new tag
-        if: steps.skip.outputs.skip != 'true'
-        run: |
-          NEW_TAG="runtime-v${{ steps.version.outputs.new }}"
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git tag -a "$NEW_TAG" -m "runtime $NEW_TAG (auto-bump from ${{ steps.bump.outputs.kind }})"
-          git push origin "$NEW_TAG"
-          echo "Pushed $NEW_TAG — publish-runtime workflow will fire on the tag."

.github/workflows/block-internal-paths.yml

@@ -1,7 +1,7 @@
 name: Block internal-flavored paths
 
 # Hard CI gate. Internal content (positioning, competitive briefs, sales
-# playbooks, PMM/press drip, draft campaigns) lives in Molecule-AI/internal —
+# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
 # this public monorepo must never re-acquire those paths. CEO directive
 # 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
 #
@@ -26,7 +26,7 @@ jobs:
     name: Block forbidden paths
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 2  # need previous commit to diff against on push events
 
@@ -135,7 +135,7 @@ jobs:
             echo "::error::Forbidden internal-flavored paths detected:"
             printf "$OFFENDING"
             echo ""
-            echo "These paths belong in Molecule-AI/internal, not this public repo."
+            echo "These paths belong in molecule-ai/internal, not this public repo."
             echo "See docs/internal-content-policy.md for canonical locations."
             echo ""
             echo "If your file is genuinely public-facing (e.g. a blog post"

.github/workflows/canary-staging.yml

@@ -20,6 +20,19 @@ on:
     # a few minutes under load — that's fine for a canary.
     - cron: '*/30 * * * *'
   workflow_dispatch:
+    inputs:
+      keep_on_failure:
+        description: >-
+          Skip teardown when the canary fails (debugging only). The
+          tenant org + EC2 + CF tunnel + DNS stay alive so an operator
+          can SSM into the workspace EC2 and capture docker logs of the
+          failing claude-code container. REMEMBER to manually delete
+          via DELETE /cp/admin/tenants/<slug> when done so the org
+          doesn't accumulate cost. Only honored on workflow_dispatch;
+          cron runs always tear down (we don't want unattended cron
+          to leak resources).
+        type: boolean
+        default: false
 
 # Serialise with the full-SaaS workflow so they don't contend for the
 # same org-create quota on staging. Different group key from
@@ -50,23 +63,47 @@ jobs:
     env:
       MOLECULE_CP_URL: https://staging-api.moleculesai.app
       MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
-      # Without an LLM key the test_staging_full_saas.sh script provisions
-      # the workspace with empty secrets, hermes derive-provider.sh resolves
-      # `openai/gpt-4o` to PROVIDER=openrouter, no OPENROUTER_API_KEY is
-      # found in env, and A2A returns "No LLM provider configured" at
-      # request time (canary step 8/11). The full-lifecycle workflow
-      # (e2e-staging-saas.yml) has carried this secret since launch — the
-      # canary regressed when it was first split out and lost the env
-      # block. Issue #1500 had ~30 consecutive failures before this was
-      # spotted; do NOT remove without re-reading the script's secrets-
-      # injection block.
+      # MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
+      # Switched from hermes+OpenAI after #2578 (the staging OpenAI key
+      # account went over quota and stayed dead for 36+ hours, taking
+      # the canary red the entire time). claude-code template's
+      # `minimax` provider routes ANTHROPIC_BASE_URL to
+      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
+      # ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
+      # billing account, so OpenAI quota collapse no longer wedges the
+      # canary. Mirrors the migration continuous-synth-e2e.yml made on
+      # 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
+      # full_saas.sh branches SECRETS_JSON on which key is present —
+      # MiniMax wins when set.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so an operator-dispatched run with
+      # E2E_RUNTIME=hermes overridden via workflow_dispatch can still
+      # exercise the OpenAI path without re-editing the workflow.
       E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
       E2E_MODE: canary
-      E2E_RUNTIME: hermes
+      E2E_RUNTIME: claude-code
+      # Pin the canary to a specific MiniMax model rather than relying
+      # on the per-runtime default (which could resolve to "sonnet" →
+      # direct Anthropic and defeat the cost saving). M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
       E2E_RUN_ID: "canary-${{ github.run_id }}"
+      # Debug-only: when an operator dispatches with keep_on_failure=true,
+      # the canary script's E2E_KEEP_ORG=1 path skips teardown so the
+      # tenant org + EC2 stay alive for SSM-based log capture. Cron runs
+      # never set this (the input only exists on workflow_dispatch) so
+      # unattended cron always tears down. See molecule-core#129
+      # failure mode #1 — capturing the actual exception requires
+      # docker logs from the live container.
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Verify admin token present
         run: |
@@ -75,39 +112,74 @@ jobs:
             exit 2
           fi
 
-      - name: Verify OpenAI key present
+      - name: Verify LLM key present
         run: |
-          if [ -z "$E2E_OPENAI_API_KEY" ]; then
-            echo "::error::MOLECULE_STAGING_OPENAI_KEY secret not set — A2A will fail at request time with 'No LLM provider configured'"
+          # Per-runtime key check — claude-code uses MiniMax; hermes /
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
+          # rather than soft-skip per the lesson from synth E2E #2578:
+          # an empty key silently falls through to the wrong
+          # SECRETS_JSON branch and the canary fails 5 min later with
+          # a confusing auth error instead of the clean "secret
+          # missing" message at the top.
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              # Either MiniMax OR direct-Anthropic works — first
+              # non-empty wins in the test script's secrets-injection
+              # priority chain. Operators only need to set ONE of these
+              # secrets; we don't force a choice between them.
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
             exit 2
           fi
-          echo "OpenAI key present ✓ (len=${#E2E_OPENAI_API_KEY})"
+          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
 
       - name: Canary run
         id: canary
         run: bash tests/e2e/test_staging_full_saas.sh
 
-      # Alerting: open an issue only after THREE consecutive failures so
-      # transient flakes (Cloudflare DNS hiccup, AWS API blip) don't spam
-      # the issue list. If an issue is already open, we still comment on
-      # every failure so ops sees the streak. Auto-close on next green.
+      # Alerting: open a sticky issue on the FIRST failure; comment on
+      # subsequent failures; auto-close on next green. Comment-on-existing
+      # de-duplicates so a single open issue accumulates the streak —
+      # ops sees one issue with N comments rather than N issues.
       #
-      # Threshold rationale: canary fires every 30 min, so 3 failures =
-      # ~90 min of consecutive red — well past any single-run flake but
-      # still tight enough that a real outage gets surfaced before the
-      # next deploy window.
+      # Why no consecutive-failures threshold (e.g., wait 3 runs before
+      # filing): the prior threshold check used
+      # `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
+      # not expose (returns 404). On Gitea Actions the threshold call
+      # ALWAYS failed, breaking the entire alerting step and going days
+      # silent on real regressions (38h+ chronic red on 2026-05-07/08
+      # before this fix; tracked in molecule-core#129). Filing on first
+      # failure is also better UX — we want to know about the first red,
+      # not wait 90 min for it to "count." Real flakes get one issue +
+      # a quick close-on-green; persistent reds accumulate comments.
       - name: Open issue on failure
         if: failure()
-        uses: actions/github-script@v7
-        env:
-          # Inject the workflow path explicitly — context.workflow is
-          # the *name*, not the file path the actions API needs.
-          WORKFLOW_PATH: '.github/workflows/canary-staging.yml'
-          CONSECUTIVE_THRESHOLD: '3'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             const title = '🔴 Canary failing: staging SaaS smoke';
-            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const runURL = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
 
             // Find an existing open canary issue (stable title match).
             // If one exists, this isn't a "first failure" — comment and exit.
@@ -127,32 +199,12 @@ jobs:
               return;
             }
 
-            // No open issue yet — check the last N-1 runs' conclusions.
-            // We open the issue only if the last (THRESHOLD-1) runs ALSO
-            // failed (so this is the 3rd consecutive red).
-            const threshold = parseInt(process.env.CONSECUTIVE_THRESHOLD, 10);
-            const { data: runs } = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner, repo: context.repo.repo,
-              workflow_id: process.env.WORKFLOW_PATH,
-              status: 'completed',
-              per_page: threshold,
-              // Skip the current in-progress run; it isn't 'completed' yet.
-            });
-            // listWorkflowRuns returns recent first. We need (threshold-1)
-            // prior failures (current run is the threshold-th).
-            const priorFailures = (runs.workflow_runs || [])
-              .slice(0, threshold - 1)
-              .filter(r => r.id !== context.runId)
-              .filter(r => r.conclusion === 'failure')
-              .length;
-            if (priorFailures < threshold - 1) {
-              core.info(`Below threshold: ${priorFailures + 1}/${threshold} consecutive failures — not filing yet`);
-              return;
-            }
-
+            // No open issue yet — file one on this first failure. The
+            // comment-on-existing branch above means subsequent failures
+            // accumulate as comments on this same issue, so we don't
+            // spam new issues per run.
             const body =
-              `Canary run failed at ${new Date().toISOString()}, ` +
-              `${threshold} consecutive runs red.\n\n` +
+              `Canary run failed at ${new Date().toISOString()}.\n\n` +
               `Run: ${runURL}\n\n` +
               `This issue auto-closes on the next green canary run. ` +
               `Consecutive failures add a comment here rather than a new issue.`;
@@ -161,11 +213,11 @@ jobs:
               title, body,
               labels: ['canary-staging', 'bug'],
             });
-            core.info(`Opened canary failure issue (${threshold} consecutive reds)`);
+            core.info('Opened canary failure issue (first red)');
 
       - name: Auto-close canary issue on success
         if: success()
-        uses: actions/github-script@v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             const title = '🔴 Canary failing: staging SaaS smoke';
@@ -231,10 +283,38 @@ jobs:
                         and o.get('status') not in ('purged',)]
           print('\n'.join(candidates))
           " 2>/dev/null)
+          # Per-slug DELETE with HTTP-code verification. The previous
+          # `... >/dev/null || true` swallowed every failure, so a 5xx
+          # or timeout from CP looked identical to "successfully cleaned
+          # up" and the tenant kept eating ~2 vCPU until the hourly
+          # stale sweep caught it (up to 2h later). Now we capture the
+          # response code and surface non-2xx as a workflow warning, so
+          # the run page shows which slug leaked. We still don't `exit 1`
+          # on cleanup failure — a single-canary cleanup miss shouldn't
+          # fail-flag the canary itself when the actual smoke check
+          # passed. The sweep-stale-e2e-orgs cron (now every 15 min,
+          # 30-min threshold) is the safety net for whatever slips past.
+          # See molecule-controlplane#420.
+          leaks=()
           for slug in $orgs; do
-            curl -sS -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
               -H "Authorization: Bearer $ADMIN_TOKEN" \
               -H "Content-Type: application/json" \
-              -d "{\"confirm\":\"$slug\"}" >/dev/null || true
+              -d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
+            set -e
+            code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
           done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
           exit 0

.github/workflows/canary-verify.yml

@@ -1,19 +1,34 @@
 name: canary-verify
 
 # Runs the canary smoke suite against the staging canary tenant fleet
-# after a new :staging-<sha> image lands in GHCR. On green, promotes
-# :staging-<sha> → :latest so the prod tenant fleet's 5-minute
-# auto-updater picks up the verified digest. On red, :latest stays
-# on the prior known-good digest and prod is untouched.
+# after a new :staging-<sha> image lands in ECR. On green, calls the
+# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
+# the prod tenant fleet's 5-minute auto-updater picks up the verified
+# digest. On red, :latest stays on the prior known-good digest and
+# prod is untouched.
+#
+# Registry note (2026-05-10): This workflow previously used GHCR
+# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
+# during the 2026-05-06 Gitea suspension migration when publish-
+# workspace-server-image.yml switched to the operator's ECR org
+# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
+# platform-tenant). The GHCR → ECR migration was never applied to
+# this file, so canary-verify was silently smoke-testing the stale
+# GHCR image while the actual staging/prod tenants ran the ECR image.
+# Result: smoke tests could not catch a broken ECR build. Fix:
+#   - Wait step: reads SHA from running canary /health (tenant-
+#     agnostic, works regardless of registry).
+#   - Promote step: calls CP redeploy-fleet endpoint with target_tag=
+#     staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
+#     No longer attempts GHCR crane ops.
 #
 # Dependencies:
 #   - publish-workspace-server-image.yml publishes :staging-<sha>
-#     (NOT :latest) on main merge
-#   - canary tenants are configured to pull :staging-<sha> as their
-#     tenant image (set TENANT_IMAGE=ghcr.io/…:staging-<sha> on the
-#     canary provisioner code path OR rotate via an admin endpoint)
+#     to ECR on staging and main merges.
+#   - Canary tenants are configured to pull :staging-<sha> from ECR
+#     (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
 #   - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
-#     CANARY_CP_SHARED_SECRET are populated
+#     CANARY_CP_SHARED_SECRET are populated.
 
 on:
   workflow_run:
@@ -27,8 +42,12 @@ permissions:
   actions: read
 
 env:
-  IMAGE_NAME: ghcr.io/molecule-ai/platform
-  TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
+  # ECR registry (post-2026-05-06 SSOT for tenant images).
+  # publish-workspace-server-image.yml pushes here.
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
+  TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
+  # CP endpoint for redeploy-fleet (used in promote step below).
+  CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
 
 jobs:
   canary-smoke:
@@ -40,7 +59,7 @@ jobs:
       smoke_ran: ${{ steps.smoke.outputs.ran }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Compute sha
         id: compute
@@ -52,6 +71,12 @@ jobs:
         # the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
         # proceeding after 7 min even if not all canaries responded —
         # the smoke suite will catch any that didn't update.
+        #
+        # NOTE: The SHA is read from the running tenant's /health response,
+        # NOT from a registry lookup. This is registry-agnostic and works
+        # regardless of whether the tenant pulls from ECR, GHCR, or any
+        # other registry — the canary is telling us what it's actually
+        # running, which is the ground truth for smoke testing.
         env:
           CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
           EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
@@ -108,7 +133,7 @@ jobs:
               echo
               echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
               echo "Phase 2 canary fleet has not been stood up yet —"
-              echo "see [canary-tenants.md](https://github.com/Molecule-AI/molecule-controlplane/blob/main/docs/canary-tenants.md)."
+              echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
               echo
               echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
             } >> "$GITHUB_STEP_SUMMARY"
@@ -133,42 +158,98 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
 
   promote-to-latest:
-    # On green, retag :staging-<sha> → :latest for BOTH images.
-    # crane is a lightweight registry client (no Docker daemon needed on
-    # the runner) that can retag remotely with a single API call each.
-    # Gated on smoke_ran=true — without a real canary fleet the smoke
-    # step no-ops with success, and we don't want that to silently
-    # auto-promote every main merge.
+    # On green, calls the CP redeploy-fleet endpoint with target_tag=
+    # staging-<sha> to promote the verified ECR image. This is the same
+    # mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
+    #
+    # Pre-fix history: the old GHCR promote step used `crane tag` against
+    # ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
+    # image.yml had already migrated to ECR on 2026-05-07 (commit
+    # 10e510f5). The GHCR tags were never updated, so this step was
+    # silently promoting a stale GHCR image while actual prod tenants
+    # pulled from ECR. Canary smoke tests were GHCR-targeted and could
+    # not catch a broken ECR build.
     needs: canary-smoke
     if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
     runs-on: ubuntu-latest
+    env:
+      SHA: ${{ needs.canary-smoke.outputs.sha }}
+      CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
+      # CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
+      # Stored at the repo level so all workflows pick it up automatically.
+      CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+      # canary_slug pin: deploy the verified :staging-<sha> to the canary
+      # first (soak 120s), then fan out to the rest of the fleet.
+      CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
+      SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
+      BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
     steps:
-      - uses: imjasonh/setup-crane@v0.4
-
-      - name: GHCR login
+      - name: Check CP credentials
         run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | \
-            crane auth login ghcr.io -u "${{ github.actor }}" --password-stdin
+          if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
+            echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
+            echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
+            exit 1
+          fi
 
-      - name: Retag platform :staging-<sha> → :latest
+      - name: Promote verified ECR image to :latest
         run: |
-          crane tag \
-            "${IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}" \
-            latest
+          set -euo pipefail
 
-      - name: Retag tenant :staging-<sha> → :latest
-        run: |
-          crane tag \
-            "${TENANT_IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}" \
-            latest
+          TARGET_TAG="staging-${SHA}"
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --argjson soak "${SOAK_SECONDS:-120}" \
+            --argjson batch "${BATCH_SIZE:-3}" \
+            --argjson dry false \
+            '{
+              target_tag: $tag,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          if [ -n "${CANARY_SLUG:-}" ]; then
+            BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
+          fi
+
+          echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  target_tag: $TARGET_TAG"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          CURL_EXIT=$?
+          set -e
+
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
+            exit 1
+          fi
 
       - name: Summary
         run: |
           {
-            echo "## Canary verified — :latest promoted"
-            echo
-            echo "- \`${IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}\` → \`${IMAGE_NAME}:latest\`"
-            echo "- \`${TENANT_IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}\` → \`${TENANT_IMAGE_NAME}:latest\`"
-            echo
-            echo "Prod tenant fleet will pick up the new digest on its next 5-min auto-update cycle."
+            echo "## Canary verified — :latest promoted via CP redeploy-fleet"
+            echo ""
+            echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
+            echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
+            echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
+            echo "- **Batch size:** ${BATCH_SIZE:-3}"
+            echo ""
+            echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
+            echo "The fleet's 5-minute health-check loop will pick up the update automatically."
           } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/cascade-list-drift-gate.yml

@@ -0,0 +1,39 @@
+name: cascade-list-drift-gate
+
+# Structural gate: TEMPLATES list in publish-runtime.yml must match
+# manifest.json's workspace_templates exactly. Closes the recurrence
+# path of PR #2556 (the data fix) and is the first concrete deliverable
+# of RFC #388 PR-3.
+#
+# Why a gate, not just discipline: PR #2536 pruned the manifest, but the
+# cascade list wasn't updated for ~weeks before someone (PR #2556)
+# noticed during an unrelated audit. During that window, codex never
+# rebuilt on a runtime publish. A structural gate catches the drift
+# the same day either file changes.
+#
+# Triggers narrowly to keep CI quiet: only on PRs that actually change
+# one of the two files. The path-filtered split + always-emit-result
+# pattern (memory: "Required check names need a job that always runs")
+# is unnecessary here because the workflow IS the check name and PR
+# branch protection should require it directly. Future-proof: if this
+# becomes a required check, add a no-op aggregator with always() so the
+# name still emits when paths don't match.
+
+on:
+  pull_request:
+    branches: [staging, main]
+    paths:
+      - manifest.json
+      - .github/workflows/publish-runtime.yml
+      - scripts/check-cascade-list-vs-manifest.sh
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Check cascade list matches manifest
+        run: bash scripts/check-cascade-list-vs-manifest.sh

.github/workflows/check-merge-group-trigger.yml

@@ -1,123 +0,0 @@
-name: Check merge_group trigger on required workflows
-
-# Pre-merge guard against the deadlock pattern where a workflow whose
-# check is in `required_status_checks` lacks a `merge_group:` trigger.
-# Without it, GitHub merge queue stalls forever in AWAITING_CHECKS
-# because the required check can't fire on `gh-readonly-queue/...` refs.
-#
-# This workflow:
-#   1. Lists required status checks on the branch protection rule for `staging`
-#   2. For each required check, finds the workflow that produces it (by job
-#      name match)
-#   3. Fails if any such workflow lacks `merge_group:` in its triggers
-#
-# Reasoning for staging-only: main has its own CI gating model (PR review),
-# but staging is what the merge queue runs on, so it's the trigger that
-# matters.
-
-on:
-  pull_request:
-    paths:
-      - '.github/workflows/**.yml'
-      - '.github/workflows/**.yaml'
-  push:
-    branches: [staging, main]
-    paths:
-      - '.github/workflows/**.yml'
-      - '.github/workflows/**.yaml'
-  # Self-listen on merge_group so the linter passes its own queue run.
-  merge_group:
-    types: [checks_requested]
-
-jobs:
-  check:
-    name: Required workflows have merge_group trigger
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@v4
-      - name: Verify merge_group trigger on required-check workflows
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO: ${{ github.repository }}
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          # Branch we care about — the one merge queue runs on.
-          BRANCH=staging
-
-          # Pull the list of required status check contexts. If the branch
-          # has no protection or no required checks, exit clean — nothing
-          # to lint.
-          REQUIRED=$(gh api "repos/${REPO}/branches/${BRANCH}/protection/required_status_checks" \
-            --jq '.contexts[]' 2>/dev/null || true)
-          if [ -z "$REQUIRED" ]; then
-            echo "No required status checks on ${BRANCH} — nothing to verify."
-            exit 0
-          fi
-
-          echo "Required checks on ${BRANCH}:"
-          echo "${REQUIRED}" | sed 's/^/  - /'
-          echo
-
-          # Build a map: workflow file -> set of job names declared in it.
-          # We use yq if available, otherwise grep the `name:` lines under
-          # `jobs:`. Stick with grep for portability — runner image always
-          # has it; yq isn't in the default image as of 2026-04.
-          declare -A workflow_jobs
-          shopt -s nullglob
-          for wf in .github/workflows/*.yml .github/workflows/*.yaml; do
-            [ -f "$wf" ] || continue
-            # Extract the workflow name (the `name:` at file root).
-            wf_name=$(awk '/^name:[[:space:]]/ {sub(/^name:[[:space:]]+/,""); gsub(/^"|"$/,""); print; exit}' "$wf")
-            # Extract job step names from the `jobs:` block. A job step is:
-            #   - id under `jobs:` (key with 2-space indent followed by colon)
-            #   - the `name:` field inside that job (4-space indent)
-            # We collect both because required_status_checks contexts can
-            # match either, depending on how the workflow was authored.
-            jobs_block=$(awk '/^jobs:/{flag=1; next} flag' "$wf")
-            job_names=$(echo "$jobs_block" | awk '/^[[:space:]]{4}name:[[:space:]]/ {sub(/^[[:space:]]+name:[[:space:]]+/,""); gsub(/^["'"'"']|["'"'"']$/,""); print}')
-            workflow_jobs["$wf"]="${wf_name}"$'\n'"${job_names}"
-          done
-
-          # For each required check, find the workflow that produces it.
-          # Then verify that workflow lists merge_group as a trigger.
-          FAILED=0
-          while IFS= read -r check; do
-            [ -z "$check" ] && continue
-            owning_wf=""
-            for wf in "${!workflow_jobs[@]}"; do
-              if echo "${workflow_jobs[$wf]}" | grep -Fxq "$check"; then
-                owning_wf="$wf"
-                break
-              fi
-            done
-
-            if [ -z "$owning_wf" ]; then
-              echo "::warning::Required check '${check}' has no matching workflow in this repo. Skipping (may be from an external app)."
-              continue
-            fi
-
-            # Does the workflow's trigger list include merge_group?
-            # Match either bare `merge_group:` line or merge_group with
-            # subsequent indented config (types: [checks_requested]).
-            if grep -qE '^[[:space:]]*merge_group:' "$owning_wf"; then
-              echo "OK: '${check}' (in $owning_wf) — has merge_group trigger"
-            else
-              echo "::error file=${owning_wf}::Required check '${check}' is produced by ${owning_wf}, but the workflow does not declare a 'merge_group:' trigger. With merge queue enabled on ${BRANCH}, this will deadlock the queue (every PR sits AWAITING_CHECKS forever). Add this to the workflow's 'on:' block:"
-              echo "::error file=${owning_wf}::  merge_group:"
-              echo "::error file=${owning_wf}::    types: [checks_requested]"
-              FAILED=1
-            fi
-          done <<< "$REQUIRED"
-
-          if [ "$FAILED" -ne 0 ]; then
-            echo
-            echo "::error::Block. See errors above. Reference: $(grep -l 'reference_merge_queue' /dev/null 2>/dev/null || echo 'memory: reference_merge_queue_enablement.md')."
-            exit 1
-          fi
-
-          echo
-          echo "All required workflows on ${BRANCH} declare merge_group triggers."

.github/workflows/check-migration-collisions.yml

@@ -0,0 +1,58 @@
+name: Check migration collisions
+
+# Hard gate (#2341): fails a PR that adds a migration prefix already
+# claimed by the base branch or another open PR. Caught manually 2026-04-30
+# during PR #2276 rebase: 044_runtime_image_pins collided with
+# 044_platform_inbound_secret from RFC #2312. This workflow makes that
+# check automatic.
+#
+# Trigger model: pull_request only — there's no value running this on
+# pushes to staging or main (those are post-merge; the gate must fire
+# pre-merge to be useful). Path filter scopes to PRs that actually touch
+# migrations.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'workspace-server/migrations/**'
+      - 'scripts/ops/check_migration_collisions.py'
+      - '.github/workflows/check-migration-collisions.yml'
+
+permissions:
+  contents: read
+  # gh pr list/diff need read access to other PRs
+  pull-requests: read
+
+jobs:
+  check:
+    name: Migration version collision check
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Need history to diff against base ref
+          fetch-depth: 0
+
+      - name: Detect collisions
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: origin/${{ github.event.pull_request.base.ref }}
+          HEAD_REF: ${{ github.event.pull_request.head.sha }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          # gh CLI uses GH_TOKEN from env
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Ensure the named base ref exists locally. checkout@v4 with
+          # fetch-depth=0 pulls full history, but the explicit fetch is
+          # cheap insurance against form-of-ref differences across runs.
+          #
+          # IMPORTANT: do NOT pass --depth=1 here. The script below uses
+          # `git diff origin/<base>...<head>` (three-dot, merge-base form),
+          # which fails with "fatal: no merge base" if the base ref is
+          # shallow. The auto-promote staging→main PR (#2361) was blocked
+          # by exactly this for ~5h on 2026-04-30 — the depth=1 fetch
+          # overwrote checkout@v4's full-history clone with a shallow tip.
+          git fetch origin "${{ github.event.pull_request.base.ref }}" || true
+          python3 scripts/ops/check_migration_collisions.py

.github/workflows/ci.yml

@@ -32,7 +32,7 @@ jobs:
       python: ${{ steps.check.outputs.python }}
       scripts: ${{ steps.check.outputs.scripts }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
       - id: check
@@ -63,29 +63,42 @@ jobs:
           echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
           echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
 
+  # Platform (Go) is a required check on staging. Always-run + per-step
+  # gating (see Canvas (Next.js) for the rationale and the failure mode
+  # this avoids).
   platform-build:
     name: Platform (Go)
     needs: changes
-    if: needs.changes.outputs.platform == 'true'
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: workspace-server
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - if: needs.changes.outputs.platform != 'true'
+        working-directory: .
+        run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.platform == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.platform == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
         with:
           go-version: 'stable'
-      - run: go mod download
-      - run: go build ./cmd/server
-      # CLI (molecli) moved to standalone repo: github.com/Molecule-AI/molecule-cli
-      - run: go vet ./... || true
-      - name: Run golangci-lint
+      - if: needs.changes.outputs.platform == 'true'
+        run: go mod download
+      - if: needs.changes.outputs.platform == 'true'
+        run: go build ./cmd/server
+      # CLI (molecli) moved to standalone repo: github.com/molecule-ai/molecule-cli
+      - if: needs.changes.outputs.platform == 'true'
+        run: go vet ./... || true
+      - if: needs.changes.outputs.platform == 'true'
+        name: Run golangci-lint
         run: golangci-lint run --timeout 3m ./... || true
-      - name: Run tests with race detection and coverage
+      - if: needs.changes.outputs.platform == 'true'
+        name: Run tests with race detection and coverage
         run: go test -race -coverprofile=coverage.out ./...
 
-      - name: Per-file coverage report
+      - if: needs.changes.outputs.platform == 'true'
+        name: Per-file coverage report
         # Advisory — lists every source file with its coverage so reviewers
         # can see at-a-glance where gaps are. Sorted ascending so the worst
         # offenders float to the top. Does NOT fail the build; the hard
@@ -98,7 +111,8 @@ jobs:
                    END {for (f in s) printf "%6.1f%%  %s\n", s[f]/c[f], f}' \
             | sort -n
 
-      - name: Check coverage thresholds
+      - if: needs.changes.outputs.platform == 'true'
+        name: Check coverage thresholds
         # Enforces two gates from #1823 Layer 1:
         #   1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
         #   2. Per-file floor — non-test .go files in security-critical
@@ -151,7 +165,7 @@ jobs:
               # Strip the package-import prefix so we can match .coverage-allowlist.txt
               # entries written as paths relative to workspace-server/.
               # Handle both module paths: platform/workspace-server/... and platform/...
-              rel=$(echo "$file" | sed 's|^github.com/Molecule-AI/molecule-monorepo/platform/workspace-server/||; s|^github.com/Molecule-AI/molecule-monorepo/platform/||')
+              rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
 
               if echo "$ALLOWLIST" | grep -qxF "$rel"; then
                 echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
@@ -178,40 +192,83 @@ jobs:
             exit 1
           fi
 
+  # Canvas (Next.js) — required check, always runs. See platform-build
+  # comment above for the rationale.
+  #
+  # Supersedes the canvas-build-noop pattern attempted in PR #2321: two
+  # jobs sharing `name:` doesn't actually satisfy branch protection
+  # because the SKIPPED check run sibling is treated as not-passed
+  # regardless of how many SUCCESS siblings it has. Verified empirically
+  # on PR #2314 — mergeStateStatus stayed BLOCKED until I collapsed to
+  # a single-job-with-conditional-steps shape.
   canvas-build:
     name: Canvas (Next.js)
     needs: changes
-    if: needs.changes.outputs.canvas == 'true'
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: canvas
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - if: needs.changes.outputs.canvas != 'true'
+        working-directory: .
+        run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.canvas == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.canvas == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
           node-version: '22'
-      - run: rm -f package-lock.json && npm install
-      - run: npm run build
-      - name: Run tests
-        run: npx vitest run
+      - if: needs.changes.outputs.canvas == 'true'
+        run: rm -f package-lock.json && npm install
+      - if: needs.changes.outputs.canvas == 'true'
+        run: npm run build
+      - if: needs.changes.outputs.canvas == 'true'
+        name: Run tests with coverage
+        # Coverage instrumentation is configured in canvas/vitest.config.ts
+        # (provider: v8, reporters: text + html + json-summary). Step 2 of
+        # #1815 — wires coverage into CI so we get a baseline visible on
+        # every PR. No threshold gate yet; thresholds dial in (Step 3, also
+        # tracked in #1815) after the team sees what current coverage is.
+        # Per the inline comment in vitest.config.ts: "first land
+        # observability so we can see the baseline, then dial in
+        # thresholds + a hard gate" — this PR ships the observability half.
+        run: npx vitest run --coverage
+      - name: Upload coverage summary as artifact
+        if: needs.changes.outputs.canvas == 'true' && always()
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
+        # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
+        # implement, surfacing as `GHESNotSupportedError: @actions/artifact
+        # v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
+        # currently supported on GHES`. Drop this pin when Gitea ships
+        # the v4 protocol (tracked: post-Gitea-1.23 followup).
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: canvas-coverage-${{ github.run_id }}
+          path: canvas/coverage/
+          retention-days: 7
+          if-no-files-found: warn
 
   # MCP Server + SDK removed from CI — now in standalone repos:
-  # - github.com/Molecule-AI/molecule-mcp-server (npm CI)
-  # - github.com/Molecule-AI/molecule-sdk-python (PyPI CI)
+  # - github.com/molecule-ai/molecule-mcp-server (npm CI)
+  # - github.com/molecule-ai/molecule-sdk-python (PyPI CI)
 
   # e2e-api job moved to .github/workflows/e2e-api.yml (issue #458).
   # It now has workflow-level concurrency (cancel-in-progress: false) so
   # new pushes queue the E2E run rather than cancelling it at the run level.
 
+  # Shellcheck (E2E scripts) — required check, always runs. See
+  # platform-build for the rationale.
   shellcheck:
     name: Shellcheck (E2E scripts)
     needs: changes
-    if: needs.changes.outputs.scripts == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
+      - if: needs.changes.outputs.scripts != 'true'
+        run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.scripts == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
         # shellcheck is pre-installed on ubuntu-latest runners (via apt).
         # infra/scripts/ is included because setup.sh + nuke.sh gate the
         # README quickstart — a shellcheck regression there silently breaks
@@ -221,19 +278,35 @@ jobs:
           find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
             | xargs -0 shellcheck --severity=warning
 
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Lint cleanup-trap hygiene (RFC #2873)
+        # Asserts every shell E2E test that calls `mktemp` also installs
+        # an EXIT trap. Catches the /tmp-leak class — a missing trap
+        # silently leaks scratch into CI runners (~10-100KB per run).
+        # See tests/e2e/lint_cleanup_traps.sh for the rule + fix pattern.
+        run: bash tests/e2e/lint_cleanup_traps.sh
+
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Run E2E bash unit tests (no live infra)
+        # Pure-bash unit tests for E2E helper libs (lib/*.sh). These pin
+        # behavior of dispatch logic that — when broken — silently masks as
+        # "Could not resolve authentication method" only after a successful
+        # tenant + workspace provision (PR #2571 incident, 2026-05-03). Add
+        # new self-contained unit tests here as the lib/ directory grows;
+        # tests requiring live CP/tenant credentials belong in the dedicated
+        # e2e-staging-* workflows, not this job.
+        run: |
+          bash tests/e2e/test_model_slug.sh
+
   canvas-deploy-reminder:
     name: Canvas Deploy Reminder
     runs-on: ubuntu-latest
     needs: [changes, canvas-build]
     # Only fires on direct pushes to main (i.e. after staging→main promotion).
     if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
-    permissions:
-      # Required to post commit comments via the GitHub API.
-      contents: write
     steps:
-      - name: Post deploy reminder as commit comment
+      - name: Write deploy reminder to step summary
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMMIT_SHA: ${{ github.sha }}
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
@@ -260,15 +333,19 @@ jobs:
           printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
             "$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
 
-          gh api \
-            --method POST \
-            "repos/${{ github.repository }}/commits/${{ github.sha }}/comments" \
-            --field "body=@/tmp/deploy-reminder.md"
+          # Gitea has no commit-comments API (no equivalent of
+          # POST /repos/{owner}/{repo}/commits/{commit_sha}/comments).
+          # Write to GITHUB_STEP_SUMMARY instead — both GitHub Actions and
+          # Gitea Actions render this as the workflow run's summary page,
+          # which is where operators look for post-deploy action items.
+          # (#75 / PR-D)
+          cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
 
+  # Python Lint & Test — required check, always runs. See platform-build
+  # for the rationale.
   python-lint:
     name: Python Lint & Test
     needs: changes
-    if: needs.changes.outputs.python == 'true'
     runs-on: ubuntu-latest
     env:
       WORKSPACE_ID: test
@@ -276,17 +353,91 @@ jobs:
       run:
         working-directory: workspace
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - if: needs.changes.outputs.python != 'true'
+        working-directory: .
+        run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.python == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.python == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: '3.11'
           cache: pip
           cache-dependency-path: workspace/requirements.txt
-      - run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov
+      - if: needs.changes.outputs.python == 'true'
+        run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
       # Coverage flags + fail-under floor moved into workspace/pytest.ini
       # (issue #1817) so local `pytest` and CI use identical config.
-      - run: python -m pytest --tb=short
+      - if: needs.changes.outputs.python == 'true'
+        run: python -m pytest --tb=short
+
+      - if: needs.changes.outputs.python == 'true'
+        name: Per-file critical-path coverage (MCP / inbox / auth)
+        # MCP-critical Python files have a per-file floor on top of the
+        # 86% total floor in pytest.ini. Rationale (issue #2790, after
+        # the PR #2766 → PR #2771 cycle): the total floor averages ~6000
+        # lines, so a single MCP file could regress to ~50% with no
+        # complaint as long as other modules compensate. These five
+        # files handle multi-tenant routing + auth + inbox dispatch —
+        # a coverage drop here is the same risk shape as a Go-side
+        # workspace-server token/secrets file dropping below 10%.
+        #
+        # Floor 75% sits below current actuals (80-96%) so this gate is
+        # strictly additive — no existing PR fails. Ratchet plan in
+        # COVERAGE_FLOOR.md.
+        run: |
+          set -e
+          PER_FILE_FLOOR=75
+          CRITICAL_FILES=(
+            "a2a_mcp_server.py"
+            "mcp_cli.py"
+            "a2a_tools.py"
+            "a2a_tools_inbox.py"
+            "inbox.py"
+            "platform_auth.py"
+          )
+
+          # pytest already wrote .coverage; emit a JSON view scoped to
+          # the critical files so jq/python can read the per-file pct
+          # without parsing tabular text. --include uses fnmatch, and
+          # the leading "*" allows the file to live anywhere under the
+          # workspace root (today they sit at workspace/<name>.py).
+          INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
+          INCLUDES="${INCLUDES%,}"
+          python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
+
+          FAILED=0
+          for f in "${CRITICAL_FILES[@]}"; do
+            # Match by top-level path key (e.g. "a2a_tools.py", not
+            # "builtin_tools/a2a_tools.py" — different file at 100%).
+            # The keys in coverage.json are paths relative to the run
+            # cwd (workspace/), so the critical-path entry sits at the
+            # bare basename.
+            pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
+            if [ "$pct" = "MISSING" ]; then
+              echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
+              FAILED=$((FAILED+1))
+              continue
+            fi
+            echo "$f: ${pct}%"
+            if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
+              echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
+              FAILED=$((FAILED+1))
+            fi
+          done
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
+            echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
+            echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
+            echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
+            echo "  (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
+            echo "  (b) if this is unavoidable historical debt, file an issue and propose"
+            echo "      adjusting the floor with rationale in COVERAGE_FLOOR.md."
+            exit 1
+          fi
 
       # SDK + plugin validation moved to standalone repo:
-      # github.com/Molecule-AI/molecule-sdk-python
+      # github.com/molecule-ai/molecule-sdk-python
 

.github/workflows/codeql.yml

@@ -1,128 +0,0 @@
-name: CodeQL
-
-# Controls CodeQL scan triggers for this repo.
-#
-# GitHub's "Code quality" default setup (the UI-configured one) is
-# hardcoded to only scan the default branch — on this repo that's
-# `staging`, so PRs promoting staging→main would otherwise never be
-# scanned. This workflow fills that gap by explicitly scanning both
-# branches on push and PR.
-#
-# Runs on ubuntu-latest (GHA-hosted — public repo, free). GHAS is NOT
-# enabled on this repo, so results are not uploaded to the Security
-# tab — the scan fails the PR check on findings, and the SARIF is
-# kept as a workflow artifact for triage.
-
-on:
-  push:
-    branches: [main, staging]
-  pull_request:
-    branches: [main, staging]
-  # GitHub merge queue fires `merge_group` for the queue's pre-merge CI run.
-  # Required so CodeQL Analyze checks get a real result on the queued
-  # commit instead of a false-green. Event only fires once merge queue is
-  # enabled on the target branch — safe to add unconditionally.
-  merge_group:
-    types: [checks_requested]
-  schedule:
-    # Weekly run picks up findings in code that hasn't been touched.
-    - cron: '30 1 * * 0'
-
-# Workflow-level concurrency: only one CodeQL run per branch/PR at a time.
-# `cancel-in-progress: false` queues new runs so a quick follow-up push
-# doesn't nuke a 45-min analysis mid-flight.
-concurrency:
-  group: codeql-${{ github.ref }}
-  cancel-in-progress: false
-
-permissions:
-  actions: read
-  contents: read
-  # No security-events: write — we don't call the upload API.
-
-jobs:
-  analyze:
-    name: Analyze (${{ matrix.language }})
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [go, javascript-typescript, python]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Checkout sibling plugin repo
-        # Same reasoning as publish-workspace-server-image.yml — the Go
-        # module's replace directive needs the plugin source so
-        # CodeQL's "go build" phase can resolve.
-        if: matrix.language == 'go'
-        uses: actions/checkout@v4
-        with:
-          repository: Molecule-AI/molecule-ai-plugin-github-app-auth
-          path: molecule-ai-plugin-github-app-auth
-          token: ${{ secrets.PLUGIN_REPO_PAT || secrets.GITHUB_TOKEN }}
-
-      # jq is pre-installed on ubuntu-latest — no setup step needed.
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-          # security-extended widens past the default to include the
-          # full security-query set for a public SaaS surface.
-          queries: security-extended
-
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
-
-      - name: Perform CodeQL Analysis
-        id: analyze
-        uses: github/codeql-action/analyze@v3
-        with:
-          category: "/language:${{ matrix.language }}"
-          # upload: never — GHAS isn't enabled on this repo, so the
-          # upload API 403s. Write SARIF locally instead.
-          upload: never
-          output: sarif-results/${{ matrix.language }}
-
-      - name: Parse SARIF + fail on findings
-        # The analyze step writes <database>.sarif into the output
-        # directory — database name is the short CodeQL lang id, not
-        # the matrix value (e.g. "javascript-typescript" →
-        # javascript.sarif), so glob rather than hardcode.
-        # Filter to error/warning severity: security-extended emits
-        # "note" rows for informational findings we don't want to fail
-        # the build over.
-        shell: bash
-        run: |
-          set -euo pipefail
-          dir="sarif-results/${{ matrix.language }}"
-          sarif=$(ls "$dir"/*.sarif 2>/dev/null | head -1 || true)
-          if [ -z "$sarif" ] || [ ! -f "$sarif" ]; then
-            echo "::error::No SARIF file found under $dir"
-            ls -la "$dir" 2>/dev/null || true
-            exit 1
-          fi
-          echo "Parsing $sarif"
-          count=$(jq '[.runs[].results[] | select(.level == "error" or .level == "warning")] | length' "$sarif")
-          echo "CodeQL findings (error+warning) for ${{ matrix.language }}: $count"
-          if [ "$count" -gt 0 ]; then
-            echo "::error::CodeQL found $count issues. Details below; full SARIF in the artifact."
-            jq -r '.runs[].results[] | select(.level == "error" or .level == "warning") | "  - [\(.level)] \(.ruleId // "?"): \(.message.text // "(no message)") @ \(.locations[0].physicalLocation.artifactLocation.uri // "?"):\(.locations[0].physicalLocation.region.startLine // "?")"' "$sarif"
-            exit 1
-          fi
-
-      - name: Upload SARIF artifact
-        # Keep SARIF around on success + failure so triagers can diff.
-        # 14-day retention — longer than default 3, short enough not
-        # to bloat quota.
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: codeql-sarif-${{ matrix.language }}
-          path: sarif-results/${{ matrix.language }}/
-          retention-days: 14

.github/workflows/continuous-synth-e2e.yml

@@ -0,0 +1,257 @@
+name: Continuous synthetic E2E (staging)
+
+# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
+# regressions visible only at runtime — schema drift, deployment-pipeline
+# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
+#
+# Why this gate exists:
+#   PR-time CI catches code-level regressions but not deployment-time or
+#   integration-time ones. Today's empirical data:
+#     • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
+#       JSON-RPC parse layer between sender and receiver. Visible only
+#       to a sender exercising the full path.
+#     • RFC #2312 chat upload — landed on staging-branch but never
+#       reached staging tenants because publish-workspace-server-image
+#       was main-only. Caught by manual dogfooding hours after deploy.
+#   Both would have surfaced within 15-20 min of regression if a
+#   continuous synth-E2E was running.
+#
+# Cadence: every 20 min (3x/hour). The script is conservatively
+# bounded at 10 min wall-clock; even on degraded staging it should
+# finish before the next firing. cron-overlap is guarded by the
+# concurrency group below.
+#
+# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
+# Plus a fresh tenant provisioned + torn down each run (Railway +
+# AWS pennies). Negligible.
+#
+# Failure handling: when the run fails, the workflow exits non-zero
+# and GitHub's standard email/notification path fires. Operators
+# can subscribe to this workflow's failure channel for paging-grade
+# alerting.
+
+on:
+  schedule:
+    # Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
+    #   1. Stay off the top-of-hour. GitHub Actions scheduler drops
+    #      :00 firings under high load (own docs:
+    #      https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
+    #      Prior history: cron was '0,20,40' (2026-05-02) — only :00
+    #      ever survived. Bumped to '10,30,50' (2026-05-03) on the
+    #      theory that further-from-:00 wins. Empirically 2026-05-04
+    #      that ALSO dropped to ~60 min effective cadence (only ~1
+    #      schedule fire per hour — see molecule-core#2726). Detection
+    #      latency was claimed 20 min, actual 60 min.
+    #   2. Avoid colliding with the existing :15 sweep-cf-orphans
+    #      and :45 sweep-cf-tunnels — both hit the CF API and we
+    #      don't want to fight for rate-limit tokens.
+    #   3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
+    #      secrets, sweep-stale-e2e-orgs every :15) — multiple
+    #      overlapping cron registrations on the same minute is part
+    #      of what GH drops under load.
+    # Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
+    # lanes (1-3 min away from any other cron). Even with empirically-
+    # observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
+    # fires = ~30 min cadence; closer to the 20-min target than the
+    # current shape and provides a real degradation alarm if drops
+    # get worse.
+    - cron: '2,12,22,32,42,52 * * * *'
+  workflow_dispatch:
+    inputs:
+      runtime:
+        description: "Runtime to provision (claude-code = default + cheapest via MiniMax; langgraph = OpenAI-only; hermes = SDK-native path, slower)"
+        required: false
+        default: "claude-code"
+        type: string
+      model_slug:
+        description: "Model id to provision the workspace with (default MiniMax-M2.7-highspeed; e.g. 'sonnet' to test direct Anthropic, 'openai/gpt-4o' for hermes)"
+        required: false
+        default: "MiniMax-M2.7-highspeed"
+        type: string
+      keep_org:
+        description: "Skip teardown for post-mortem debugging (only manual dispatch — never set this for cron runs)"
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  # No issue-write here — failures surface as red runs in the workflow
+  # history. If you want auto-issue-on-fail, add a follow-up step that
+  # uses gh issue create gated on `if: failure()`. Keeping the surface
+  # minimal until that's actually wanted.
+
+# Serialize so two firings can never overlap. Cron firing every 20 min
+# but scripts conservatively bounded at 10 min — overlap shouldn't
+# happen in steady state, but if a run hangs we don't want N more
+# stacking up.
+concurrency:
+  group: continuous-synth-e2e
+  cancel-in-progress: false
+
+jobs:
+  synth:
+    name: Synthetic E2E against staging
+    runs-on: ubuntu-latest
+    # Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
+    # (apt-get update + install docker.io/jq/awscli/caddy + snap install
+    # ssm-agent) runs from raw Ubuntu on every boot — none of it is
+    # pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
+    # across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
+    # latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
+    # A 12min budget leaves only ~2min for the workspace (which needs
+    # ~3.5min for claude-code cold boot) on slow-apt days, blowing the
+    # budget. 20min absorbs the worst tenant tail so the workspace probe
+    # gets the full ~7min it needs even on a slow apt day. Real fix:
+    # pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
+    timeout-minutes: 20
+    env:
+      # claude-code default: cold-start ~5 min (comparable to langgraph),
+      # but uses MiniMax-M2.7-highspeed via the template's third-party-
+      # Anthropic-compat path (workspace-configs-templates/claude-code-
+      # default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
+      # gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
+      # exhaustion class that took the canary down 2026-05-03 (#265).
+      # Operators can pick langgraph / hermes via workflow_dispatch
+      # when they specifically need to exercise the OpenAI or SDK-
+      # native paths.
+      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
+      # Pin the canary to a specific MiniMax model rather than relying
+      # on the per-runtime default ("sonnet" → routes to direct
+      # Anthropic, defeats the cost saving). Operators can override
+      # via workflow_dispatch by setting a different E2E_MODEL_SLUG
+      # input if they need to exercise a specific model. M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
+      # Bound to 10 min so a stuck provision fails the run instead of
+      # holding up the next cron firing. 15-min default in the script
+      # is for the on-PR full lifecycle where we have more headroom.
+      E2E_PROVISION_TIMEOUT_SECS: '600'
+      # Slug suffix — namespaced "synth-" so these runs are
+      # distinguishable from PR-driven runs in CP admin.
+      E2E_RUN_ID: synth-${{ github.run_id }}
+      # Forced false for cron; respected for manual dispatch
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
+      MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      # MiniMax key is the canary's PRIMARY auth path. claude-code
+      # template's `minimax` provider routes ANTHROPIC_BASE_URL to
+      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
+      # tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
+      # which key is present — MiniMax wins when set.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so operators can dispatch with
+      # E2E_RUNTIME=langgraph or =hermes and still have a working
+      # canary path. The script picks the right blob shape based on
+      # which key is non-empty.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        run: |
+          # Hard-fail on missing secret REGARDLESS of trigger. Previously
+          # this step soft-skipped on workflow_dispatch via `exit 0`, but
+          # `exit 0` only ends the STEP — subsequent steps still ran with
+          # the empty secret, the synth script fell through to the wrong
+          # SECRETS_JSON branch, and the canary failed 5 min later with a
+          # confusing "Agent error (Exception)" instead of the clean
+          # "secret missing" message at the top. Caught 2026-05-04 by
+          # dispatched run 25296530706: claude-code + missing MINIMAX
+          # silently used OpenAI keys but kept model=MiniMax-M2.7, then
+          # the workspace 401'd against MiniMax once it tried to call.
+          # Fix: exit 1 in both cron and dispatch paths. Operators who
+          # want to verify a YAML change without setting up the secret
+          # can read the verify-secrets step's stderr — the failure is
+          # itself the verification signal.
+          if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
+            echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
+            echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+            exit 1
+          fi
+
+          # LLM-key requirement is per-runtime: claude-code accepts
+          # EITHER MiniMax OR direct-Anthropic (whichever is set first),
+          # langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_KEY).
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
+            echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
+            exit 1
+          fi
+
+      - name: Install required tools
+        run: |
+          # The script depends on jq + curl (already on ubuntu-latest)
+          # and python3 (likewise). Verify they're all present so we
+          # fail fast on a runner image regression rather than mid-script.
+          for cmd in jq curl python3; do
+            command -v "$cmd" >/dev/null 2>&1 || {
+              echo "::error::required tool '$cmd' not on PATH — runner image regression?"
+              exit 1
+            }
+          done
+
+      - name: Run synthetic E2E
+        # The script handles its own teardown via EXIT trap; even on
+        # failure (timeout, assertion), the org is deprovisioned and
+        # leaks are reported. Exit code propagates from the script.
+        run: |
+          bash tests/e2e/test_staging_full_saas.sh
+
+      - name: Failure summary
+        # Runs only on failure. Adds a job summary so the workflow run
+        # page shows a quick "what happened" instead of forcing readers
+        # to scroll through script output.
+        if: failure()
+        run: |
+          {
+            echo "## Continuous synth E2E failed"
+            echo ""
+            echo "**Run ID:** ${{ github.run_id }}"
+            echo "**Trigger:** ${{ github.event_name }}"
+            echo "**Runtime:** ${E2E_RUNTIME}"
+            echo "**Slug:** synth-${{ github.run_id }}"
+            echo ""
+            echo "### What this means"
+            echo ""
+            echo "Staging just regressed on a path that previously worked. Likely classes:"
+            echo "- Schema mismatch between sender and receiver (#2345 class)"
+            echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
+            echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
+            echo "- Staging-CP env var rotation"
+            echo ""
+            echo "### Next steps"
+            echo ""
+            echo "1. Check the script output above for the assertion that failed"
+            echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
+            echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
+            echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/e2e-api.yml

@@ -1,47 +1,205 @@
 name: E2E API Smoke Test
 # Extracted from ci.yml so workflow-level concurrency can protect this job
 # from run-level cancellation (issue #458).
+#
+# Trigger model (revised 2026-04-29):
+#
+# Always FIRES on push/pull_request to staging+main. Real work is gated
+# per-step on `needs.detect-changes.outputs.api` — when paths under
+# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
+# changed, the no-op step alone runs and emits SUCCESS for the
+# `E2E API Smoke Test` check, satisfying branch protection without
+# spending CI cycles. See the in-job comment on the `e2e-api` job for
+# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
+# PR #2264 incident that drove the consolidation.
+#
+# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
+# -------------------------------------------------------------------
+# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
+# Gitea act_runner runs with `container.network: host` (operator host
+# `/opt/molecule/runners/config.yaml`), which means:
+#
+#   * Two concurrent runs both try to bind their `-p 15432:5432` /
+#     `-p 16379:6379` host ports — the second postgres/redis FATALs
+#     with `Address in use` and `docker run` returns exit 125 with
+#     `Conflict. The container name "/molecule-ci-postgres" is already
+#     in use by container ...`. Verified in run a7/2727 on 2026-05-07.
+#   * The fixed container names `molecule-ci-postgres` / `-redis` (the
+#     pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
+#     `docker rm -f` at the start of the second job KILLS the first
+#     job's still-running postgres/redis.
+#
+# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
+# platform-server is a Go binary on the host, not a containerised
+# step):
+#
+#   1. Unique container names per run:
+#         pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
+#         redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
+#      `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
+#      same run_id.
+#   2. Ephemeral host port per run (`-p 0:5432`), then read the actual
+#      bound port via `docker port` and export DATABASE_URL/REDIS_URL
+#      pointing at it. No fixed host-port → no port collision.
+#   3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
+#      the original flake fixed in #92 and the script's still IPv6-
+#      enabled.
+#   4. `if: always()` cleanup so containers don't leak when test steps
+#      fail.
+#
+# Issue #94 items #2 + #3 (also fixed here):
+#   * Pre-pull `alpine:latest` so the platform-server's provisioner
+#     (`internal/handlers/container_files.go`) can stand up its
+#     ephemeral token-write helper without a daemon.io round-trip.
+#   * Create `molecule-core-net` bridge network if missing so the
+#     provisioner's container.HostConfig {NetworkMode: ...} attach
+#     succeeds.
+# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
+# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
+# they DO come up. Timeouts are not the bottleneck; not bumped.
+#
+# Item explicitly NOT fixed here: failing test `Status back online`
+# fails because the platform's langgraph workspace template image
+# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
+# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
+# template-registry resolution issue (ADR-002 / local-build mode) and
+# belongs in a separate change that touches workspace-server, not
+# this workflow file.
 
 on:
   push:
     branches: [main, staging]
-    paths:
-      - 'workspace-server/**'
-      - 'tests/e2e/**'
-      - '.github/workflows/e2e-api.yml'
   pull_request:
     branches: [main, staging]
-    paths:
-      - 'workspace-server/**'
-      - 'tests/e2e/**'
-      - '.github/workflows/e2e-api.yml'
+  workflow_dispatch:
 
 concurrency:
-  group: e2e-api-${{ github.ref }}
+  # Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
+  # same auto-promote-staging brittleness as e2e-staging-canvas — back-
+  # to-back staging pushes share refs/heads/staging, so the older push's
+  # queued run gets cancelled when a newer push lands. Auto-promote-
+  # staging then sees `completed/cancelled` for the older SHA and stays
+  # put; the newer SHA's gates may eventually save the day, but if the
+  # newer push gets cancelled too, we deadlock.
+  #
+  # See e2e-staging-canvas.yml's identical concurrency block for the full
+  # rationale and the 2026-04-28 incident reference.
+  group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
   cancel-in-progress: false
 
 jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      api: ${{ steps.decide.outputs.api }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            api:
+              - 'workspace-server/**'
+              - 'tests/e2e/**'
+              - '.github/workflows/e2e-api.yml'
+      - id: decide
+        # Always run real work for manual dispatch — no diff context to
+        # filter against and ops dispatching this expects the suite to
+        # actually exercise the platform.
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "api=${{ steps.filter.outputs.api }}" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `E2E API Smoke Test`. Real work is gated per-step
+  # on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
+  # check run for every job that matches `name:`, and a job-level
+  # `if: false` produces a SKIPPED check run. Branch protection treats
+  # all check runs with a matching context name on the latest commit as a
+  # SET — any SKIPPED in the set fails the required-check eval, even with
+  # SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
+  # 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
+  # promotion despite all real work succeeding. Collapsing to a single
+  # always-running job with conditional steps emits exactly one SUCCESS
+  # check run regardless of paths filter — branch-protection-clean.
   e2e-api:
+    needs: detect-changes
     name: E2E API Smoke Test
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
-      DATABASE_URL: postgres://dev:dev@localhost:15432/molecule?sslmode=disable
-      REDIS_URL: redis://localhost:16379
+      # Unique per-run container names so concurrent runs on the host-
+      # network act_runner don't collide on name OR port.
+      # `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
+      # same run_id. PORT is set later (after docker port lookup) since
+      # we let Docker assign an ephemeral host port.
+      PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
+      REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
       PORT: "8080"
-      PG_CONTAINER: molecule-ci-postgres
-      REDIS_CONTAINER: molecule-ci-redis
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.api != 'true'
+        run: |
+          echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
+          echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.api == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.api == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
         with:
           go-version: 'stable'
           cache: true
           cache-dependency-path: workspace-server/go.sum
-      - name: Start Postgres (docker)
+      - name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
+        if: needs.detect-changes.outputs.api == 'true'
         run: |
+          # Provisioner uses alpine:latest for ephemeral token-write
+          # containers (workspace-server/internal/handlers/container_files.go).
+          # Pre-pull so the first provision in test_api.sh doesn't race
+          # the daemon's pull cache. Idempotent — `docker pull` is a no-op
+          # when the image is already present.
+          docker pull alpine:latest >/dev/null
+          # Provisioner attaches workspace containers to
+          # molecule-core-net (workspace-server/internal/provisioner/
+          # provisioner.go::DefaultNetwork). The bridge already exists on
+          # the operator host's docker daemon — `network create` is
+          # idempotent via `|| true`.
+          docker network create molecule-core-net >/dev/null 2>&1 || true
+          echo "alpine:latest pre-pulled; molecule-core-net ensured."
+      - name: Start Postgres (docker)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          # Defensive cleanup — only matches THIS run's container name,
+          # so it cannot kill a sibling run's postgres. (Pre-fix the
+          # name was static and this rm hit other runs' containers.)
           docker rm -f "$PG_CONTAINER" 2>/dev/null || true
-          docker run -d --name "$PG_CONTAINER" -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule -p 15432:5432 postgres:16
+          # `-p 0:5432` requests an ephemeral host port; we read it back
+          # below and export DATABASE_URL.
+          docker run -d --name "$PG_CONTAINER" \
+            -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
+            -p 0:5432 postgres:16 >/dev/null
+          # Resolve the host-side port assignment. `docker port` prints
+          # `0.0.0.0:NNNN` (and on host-net runners may also print an
+          # IPv6 line — take the first IPv4 line).
+          PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
+          if [ -z "$PG_PORT" ]; then
+            # Fallback: any first line. Some Docker versions print only
+            # one line.
+            PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
+          fi
+          if [ -z "$PG_PORT" ]; then
+            echo "::error::Could not resolve host port for $PG_CONTAINER"
+            docker port "$PG_CONTAINER" 5432/tcp || true
+            docker logs "$PG_CONTAINER" || true
+            exit 1
+          fi
+          # 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
+          echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
+          echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
+          echo "Postgres host port: ${PG_PORT}"
           for i in $(seq 1 30); do
             if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
               echo "Postgres ready after ${i}s"
@@ -53,9 +211,23 @@ jobs:
           docker logs "$PG_CONTAINER" || true
           exit 1
       - name: Start Redis (docker)
+        if: needs.detect-changes.outputs.api == 'true'
         run: |
           docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
-          docker run -d --name "$REDIS_CONTAINER" -p 16379:6379 redis:7
+          docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
+          REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
+          if [ -z "$REDIS_PORT" ]; then
+            REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
+          fi
+          if [ -z "$REDIS_PORT" ]; then
+            echo "::error::Could not resolve host port for $REDIS_CONTAINER"
+            docker port "$REDIS_CONTAINER" 6379/tcp || true
+            docker logs "$REDIS_CONTAINER" || true
+            exit 1
+          fi
+          echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
+          echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
+          echo "Redis host port: ${REDIS_PORT}"
           for i in $(seq 1 15); do
             if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
               echo "Redis ready after ${i}s"
@@ -67,17 +239,22 @@ jobs:
           docker logs "$REDIS_CONTAINER" || true
           exit 1
       - name: Build platform
+        if: needs.detect-changes.outputs.api == 'true'
         working-directory: workspace-server
         run: go build -o platform-server ./cmd/server
       - name: Start platform (background)
+        if: needs.detect-changes.outputs.api == 'true'
         working-directory: workspace-server
         run: |
+          # DATABASE_URL + REDIS_URL exported by the start-postgres /
+          # start-redis steps point at this run's per-run host ports.
           ./platform-server > platform.log 2>&1 &
           echo $! > platform.pid
       - name: Wait for /health
+        if: needs.detect-changes.outputs.api == 'true'
         run: |
           for i in $(seq 1 30); do
-            if curl -sf http://localhost:8080/health > /dev/null; then
+            if curl -sf http://127.0.0.1:8080/health > /dev/null; then
               echo "Platform up after ${i}s"
               exit 0
             fi
@@ -87,6 +264,7 @@ jobs:
           cat workspace-server/platform.log || true
           exit 1
       - name: Assert migrations applied
+        if: needs.detect-changes.outputs.api == 'true'
         run: |
           tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
           if [ "$tables" != "1" ]; then
@@ -96,25 +274,34 @@ jobs:
           fi
           echo "Migrations OK"
       - name: Run E2E API tests
+        if: needs.detect-changes.outputs.api == 'true'
         run: bash tests/e2e/test_api.sh
       - name: Run notify-with-attachments E2E
+        if: needs.detect-changes.outputs.api == 'true'
         run: bash tests/e2e/test_notify_attachments_e2e.sh
       - name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
-        # Validates the test script itself runs cleanly even with no LLM
-        # keys (both phases skip gracefully). The wire-real coverage with
-        # actual keys runs in canary-staging.yml + e2e-staging-saas.yml.
+        if: needs.detect-changes.outputs.api == 'true'
         run: bash tests/e2e/test_priority_runtimes_e2e.sh
+      - name: Run poll-mode + since_id cursor E2E (#2339)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_poll_mode_e2e.sh
+      - name: Run poll-mode chat upload E2E (RFC #2891)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
       - name: Dump platform log on failure
-        if: failure()
+        if: failure() && needs.detect-changes.outputs.api == 'true'
         run: cat workspace-server/platform.log || true
       - name: Stop platform
-        if: always()
+        if: always() && needs.detect-changes.outputs.api == 'true'
         run: |
           if [ -f workspace-server/platform.pid ]; then
             kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
           fi
       - name: Stop service containers
-        if: always()
+        # always() so containers don't leak when test steps fail. The
+        # cleanup is best-effort: if the container is already gone
+        # (e.g. concurrent rerun race), don't fail the job.
+        if: always() && needs.detect-changes.outputs.api == 'true'
         run: |
           docker rm -f "$PG_CONTAINER" 2>/dev/null || true
           docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true

.github/workflows/e2e-staging-canvas.yml

@@ -13,16 +13,18 @@ name: E2E Staging Canvas (Playwright)
 # workflow — mirrors what PR #1891 does for e2e-api.yml.
 
 on:
+  # Trigger model (revised 2026-04-29):
+  #
+  # Always fires on push/pull_request; real work is gated per-step on
+  # `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
+  # changed, the no-op step alone runs and emits SUCCESS for the
+  # `Canvas tabs E2E` check, satisfying branch protection without
+  # spending CI cycles. See e2e-api.yml for the rationale on why this
+  # is a single job rather than two-jobs-sharing-name.
   push:
-    branches: [main, staging]
-    paths:
-      - 'canvas/**'
-      - '.github/workflows/e2e-staging-canvas.yml'
+    branches: [main]
   pull_request:
-    branches: [main, staging]
-    paths:
-      - 'canvas/**'
-      - '.github/workflows/e2e-staging-canvas.yml'
+    branches: [main]
   workflow_dispatch:
   schedule:
     # Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
@@ -30,11 +32,59 @@ on:
     - cron: '0 8 * * 0'
 
 concurrency:
-  group: e2e-staging-canvas
+  # Per-SHA grouping (changed 2026-04-28 from a single global group). The
+  # global group made auto-promote-staging brittle: when a staging push
+  # queued behind an in-flight run and a third entrant (a PR run, a
+  # follow-on push) entered the group, the staging push got cancelled —
+  # leaving auto-promote-staging looking at `completed/cancelled` for a
+  # required gate and refusing to advance main. Observed 2026-04-28
+  # 23:51-23:53 on staging tip 3f99fede.
+  #
+  # The original intent of the global group was to throttle parallel
+  # E2E provisions (each spins a fresh EC2). At our scale that throttle
+  # isn't worth the correctness cost — fresh-org-per-run isolates the
+  # state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
+  # is rounding error vs. the cost of a stuck pipeline.
+  #
+  # Per-SHA still dedupes accidental double-triggers for the SAME SHA.
+  # It does NOT cancel obsolete-PR-version runs on force-push; that
+  # wasted CI is acceptable given the alternative is losing staging-tip
+  # data that auto-promote-staging needs.
+  group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
   cancel-in-progress: false
 
 jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      canvas: ${{ steps.decide.outputs.canvas }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            canvas:
+              - 'canvas/**'
+              - '.github/workflows/e2e-staging-canvas.yml'
+      - id: decide
+        # Always run real tests for manual dispatch and the weekly cron —
+        # both exist precisely to exercise the suite, regardless of diff.
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "canvas=${{ steps.filter.outputs.canvas }}" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `Canvas tabs E2E`. Real work is gated per-step on
+  # `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
+  # rationale — same path-filter check-name parity issue blocked PR #2264
+  # (staging→main) on 2026-04-29 because branch protection treats matching-
+  # name check runs as a SET, and any SKIPPED member fails the eval.
   playwright:
+    needs: detect-changes
     name: Canvas tabs E2E
     runs-on: ubuntu-latest
     timeout-minutes: 40
@@ -49,9 +99,18 @@ jobs:
         working-directory: canvas
 
     steps:
-      - uses: actions/checkout@v4
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.canvas != 'true'
+        working-directory: .
+        run: |
+          echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
+          echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
+
+      - if: needs.detect-changes.outputs.canvas == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Verify admin token present
+        if: needs.detect-changes.outputs.canvas == 'true'
         run: |
           if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
             echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
@@ -59,74 +118,98 @@ jobs:
           fi
 
       - name: Set up Node
-        uses: actions/setup-node@v4
+        if: needs.detect-changes.outputs.canvas == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
           node-version: '20'
           cache: 'npm'
           cache-dependency-path: canvas/package-lock.json
 
       - name: Install canvas deps
+        if: needs.detect-changes.outputs.canvas == 'true'
         run: npm ci
 
       - name: Install Playwright browsers
+        if: needs.detect-changes.outputs.canvas == 'true'
         run: npx playwright install --with-deps chromium
 
       - name: Run staging canvas E2E
+        if: needs.detect-changes.outputs.canvas == 'true'
         run: npx playwright test --config=playwright.staging.config.ts
 
       - name: Upload Playwright report on failure
-        if: failure()
-        uses: actions/upload-artifact@v4
+        if: failure() && needs.detect-changes.outputs.canvas == 'true'
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
+        # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
+        # implement (see ci.yml upload step for the canonical error
+        # cite). Drop this pin when Gitea ships the v4 protocol.
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
         with:
           name: playwright-report-staging
           path: canvas/playwright-report-staging/
           retention-days: 14
 
       - name: Upload screenshots on failure
-        if: failure()
-        uses: actions/upload-artifact@v4
+        if: failure() && needs.detect-changes.outputs.canvas == 'true'
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
         with:
           name: playwright-screenshots
           path: canvas/test-results/
           retention-days: 14
 
-      # Safety-net teardown mirrors the bash-harness workflow — if
-      # globalTeardown didn't run (worker crash, runner cancel), this
-      # step sweeps any e2e-canvas-* org tagged with today's date.
+      # Safety-net teardown — fires only when Playwright's globalTeardown
+      # didn't (worker crash, runner cancel). Reads the slug from
+      # canvas/.playwright-staging-state.json (written by staging-setup
+      # as its first action, before any CP call) and deletes only that
+      # slug.
+      #
+      # Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
+      # orgs to compensate for setup-crash-before-state-file-write. That
+      # over-aggressive cleanup raced concurrent canvas-E2E runs and
+      # poisoned each other's tenants — observed 2026-04-30 when three
+      # real-test runs killed each other mid-test, surfacing as
+      # `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
+      # DNS record. Pattern-sweep removed; setup now writes the state
+      # file before any CP work, so the slug is always recoverable.
       - name: Teardown safety net
-        if: always()
+        if: always() && needs.detect-changes.outputs.canvas == 'true'
         env:
           ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
         run: |
           set +e
-          # Midnight-UTC rollover guard: a single-date filter misses
-          # orgs created on the prior UTC day when the run crosses
-          # midnight (incident 2026-04-26 23:46Z → 2026-04-27 00:12Z:
-          # slug `e2e-canvas-20260426-1u8nz3` survived because the
-          # safety-net step ran on the 27th, computed `today=20260427`,
-          # and the filter `e2e-canvas-20260427-` never matched). Sweep
-          # both today AND yesterday's dates so a cross-midnight run
-          # still cleans up its own slug.
-          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
-            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
-            | python3 -c "
-          import json, sys, datetime
-          d = json.load(sys.stdin)
-          today = datetime.date.today()
-          yesterday = today - datetime.timedelta(days=1)
-          prefixes = (
-              f'e2e-canvas-{today.strftime(\"%Y%m%d\")}-',
-              f'e2e-canvas-{yesterday.strftime(\"%Y%m%d\")}-',
-          )
-          candidates = [o['slug'] for o in d.get('orgs', [])
-                        if any(o.get('slug','').startswith(p) for p in prefixes)
-                        and o.get('status') not in ('purged',)]
-          print('\n'.join(candidates))
-          " 2>/dev/null)
-          for slug in $orgs; do
-            curl -sS -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
-              -H "Authorization: Bearer $ADMIN_TOKEN" \
-              -H "Content-Type: application/json" \
-              -d "{\"confirm\":\"$slug\"}" >/dev/null || true
-          done
+          STATE_FILE=".playwright-staging-state.json"
+          if [ ! -f "$STATE_FILE" ]; then
+            echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
+            exit 0
+          fi
+          slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
+          if [ -z "$slug" ]; then
+            echo "::warning::State file present but slug missing; nothing to clean up."
+            exit 0
+          fi
+          echo "Deleting orphan tenant: $slug"
+          # Verify HTTP 2xx instead of `>/dev/null || true` swallowing
+          # failures. A 5xx or timeout previously looked identical to
+          # success, leaving the tenant alive for up to ~45 min until
+          # sweep-stale-e2e-orgs caught it. Surface failures as
+          # workflow warnings naming the slug. Don't `exit 1` — a single
+          # cleanup miss shouldn't fail-flag the canvas test when the
+          # actual smoke check passed; the sweeper is the safety net.
+          # See molecule-controlplane#420.
+          # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+          # pollution of the captured status (lint-curl-status-capture.yml).
+          set +e
+          curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
+            -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
+          set -e
+          code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
+          if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+            echo "[teardown] deleted $slug (HTTP $code)"
+          else
+            echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
+          fi
           exit 0

.github/workflows/e2e-staging-external.yml

@@ -0,0 +1,184 @@
+name: E2E Staging External Runtime
+
+# Regression for the four/five workspaces.status=awaiting_agent transitions
+# that silently failed in production for five days before migration 046
+# extended the workspace_status enum (see
+# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
+#
+# Why this is its own workflow (not folded into e2e-staging-saas.yml):
+#   - The full-saas harness defaults to runtime=hermes, never exercises
+#     external-runtime. Adding an `external` parameter to that script
+#     would force every push to staging through both lifecycles in
+#     series, doubling the EC2 cold-start budget.
+#   - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
+#     window, 90s default + sweep interval), which we wait through
+#     deliberately. Folding it into hermes would make the long path
+#     even longer.
+#   - It can run in parallel with the hermes E2E since both create
+#     fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
+#     `e2e-...`).
+#
+# Triggers:
+#   - Push to staging when any source affecting external runtime,
+#     hibernation, or the migration set changes.
+#   - PR review for the same set.
+#   - Manual workflow_dispatch.
+#   - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
+#     30 min after e2e-staging-saas.yml's 07:00 UTC cron).
+#
+# Concurrency: serialized so two staging pushes don't fight for the
+# same EC2 quota window. cancel-in-progress=false so a half-rolled
+# tenant always finishes its teardown.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_restart.go'
+      - 'workspace-server/internal/registry/healthsweep.go'
+      - 'workspace-server/internal/registry/liveness.go'
+      - 'workspace-server/migrations/**'
+      - 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
+      - 'tests/e2e/test_staging_external_runtime.sh'
+      - '.github/workflows/e2e-staging-external.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_restart.go'
+      - 'workspace-server/internal/registry/healthsweep.go'
+      - 'workspace-server/internal/registry/liveness.go'
+      - 'workspace-server/migrations/**'
+      - 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
+      - 'tests/e2e/test_staging_external_runtime.sh'
+      - '.github/workflows/e2e-staging-external.yml'
+  workflow_dispatch:
+    inputs:
+      keep_org:
+        description: "Skip teardown for debugging (only via manual dispatch)"
+        required: false
+        type: boolean
+        default: false
+      stale_wait_secs:
+        description: "Seconds to wait for the heartbeat-staleness sweep (default 180 = 90s window + 90s buffer)"
+        required: false
+        default: "180"
+  schedule:
+    - cron: '30 7 * * *'
+
+concurrency:
+  group: e2e-staging-external
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-staging-external:
+    name: E2E Staging External Runtime
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
+      E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            # Schedule + push triggers must hard-fail when the token is
+            # missing — silent skip would mask infra rot. Manual dispatch
+            # gets the same hard-fail; an operator running this on a fork
+            # without secrets configured needs to know up-front.
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: CP staging health preflight
+        run: |
+          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
+          if [ "$code" != "200" ]; then
+            echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
+            exit 1
+          fi
+          echo "Staging CP healthy ✓"
+
+      - name: Run external-runtime E2E
+        id: e2e
+        run: bash tests/e2e/test_staging_external_runtime.sh
+
+      # Mirror the e2e-staging-saas.yml safety net: if the runner is
+      # cancelled (e.g. concurrent staging push), the test script's
+      # EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
+      # *this* run id.
+      - name: Teardown safety net (runs on cancel/failure)
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
+          # so concurrent runs and unrelated dev probes are not touched.
+          # Sweep today AND yesterday so a midnight-crossing run still
+          # cleans up its own slug.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if not run_id:
+              # Without a run id we cannot scope safely; bail rather
+              # than risk deleting unrelated tenants.
+              sys.exit(0)
+          prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
+          for o in d.get('orgs', []):
+              s = o.get('slug', '')
+              if s.startswith(prefixes) and o.get('status') != 'purged':
+                  print(s)
+          " 2>/dev/null)
+          if [ -n "$orgs" ]; then
+            echo "Safety-net sweep: deleting leftover orgs:"
+            echo "$orgs"
+            # Per-slug verified DELETE — see molecule-controlplane#420.
+            # `>/dev/null 2>&1` previously hid every failure; surface
+            # non-2xx as workflow warnings so the run page names what
+            # leaked. Sweeper catches the rest within ~45 min.
+            leaks=()
+            for slug in $orgs; do
+              # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+              # pollution of the captured status (lint-curl-status-capture.yml).
+              set +e
+              curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
+                -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+                -H "Authorization: Bearer $ADMIN_TOKEN" \
+                -H "Content-Type: application/json" \
+                -d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
+              set -e
+              code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
+              if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+                echo "[teardown] deleted $slug (HTTP $code)"
+              else
+                echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
+                leaks+=("$slug")
+              fi
+            done
+            if [ ${#leaks[@]} -gt 0 ]; then
+              echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+            fi
+          else
+            echo "Safety-net sweep: no leftover orgs to clean."
+          fi

.github/workflows/e2e-staging-saas.yml

@@ -20,13 +20,12 @@ name: E2E Staging SaaS (full lifecycle)
 #     via the same paths watcher that e2e-api.yml uses)
 
 on:
-  # Fire on staging push too — previously this only ran on main, which
-  # meant the most thorough end-to-end test caught regressions AFTER
-  # they shipped to staging (and then to the auto-promote PR). Running
-  # on staging push catches them BEFORE the staging→main promotion
-  # opens, so a green canary into auto-promote is more meaningful.
+  # Trunk-based (Phase 3 of internal#81): main is the only branch.
+  # Previously this fired on staging push too because staging was a
+  # superset of main and ran the gate ahead of auto-promote; with no
+  # staging branch, main is where E2E gates the deploy.
   push:
-    branches: [staging, main]
+    branches: [main]
     paths:
       - 'workspace-server/internal/handlers/registry.go'
       - 'workspace-server/internal/handlers/workspace_provision.go'
@@ -36,7 +35,7 @@ on:
       - 'tests/e2e/test_staging_full_saas.sh'
       - '.github/workflows/e2e-staging-saas.yml'
   pull_request:
-    branches: [staging, main]
+    branches: [main]
     paths:
       - 'workspace-server/internal/handlers/registry.go'
       - 'workspace-server/internal/handlers/workspace_provision.go'
@@ -48,9 +47,9 @@ on:
   workflow_dispatch:
     inputs:
       runtime:
-        description: "Runtime to test (hermes | claude-code | langgraph)"
+        description: "Runtime to test (claude-code [default, MiniMax] | hermes [OpenAI] | langgraph [OpenAI])"
         required: false
-        default: "hermes"
+        default: "claude-code"
       keep_org:
         description: "Skip teardown for debugging (only use via manual dispatch!)"
         required: false
@@ -83,16 +82,37 @@ jobs:
       # retrieval + teardown. Configure in
       # Settings → Secrets and variables → Actions → Repository secrets.
       MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
-      # OpenAI key for workspace LLM calls (section 8 A2A). Without it,
-      # Hermes runtime crashes at boot with "No provider API key found".
-      # Configure at Settings → Secrets → Actions → MOLECULE_STAGING_OPENAI_KEY.
+      # MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
+      # from hermes+OpenAI default after #2578 (the staging OpenAI key
+      # account went over quota and stayed dead for 36+ hours, taking
+      # the full-lifecycle E2E red on every provisioning-critical push).
+      # claude-code template's `minimax` provider routes
+      # ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
+      # MINIMAX_API_KEY at boot — separate billing account so an
+      # OpenAI quota collapse no longer wedges the gate. Mirrors the
+      # canary-staging.yml + continuous-synth-e2e.yml migrations.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so an operator-dispatched run with
+      # E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
+      # exercise the OpenAI path.
       E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
-      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'hermes' }}
+      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
+      # Pin the model when running on the default claude-code path —
+      # the per-runtime default ("sonnet") routes to direct Anthropic
+      # and defeats the cost saving. Operators can override via the
+      # workflow_dispatch flow (no input wired here yet — runtime
+      # override is enough for ad-hoc).
+      E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
       E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
       E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Verify admin token present
         run: |
@@ -102,13 +122,45 @@ jobs:
           fi
           echo "Admin token present ✓"
 
-      - name: Verify OpenAI key present
+      - name: Verify LLM key present
         run: |
-          if [ -z "$E2E_OPENAI_API_KEY" ]; then
-            echo "::error::MOLECULE_STAGING_OPENAI_KEY secret not set — workspaces will fail at boot with 'No provider API key found'"
+          # Per-runtime key check — claude-code uses MiniMax; hermes /
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
+          # rather than soft-skip per #2578's lesson — empty key
+          # silently falls through to the wrong SECRETS_JSON branch and
+          # produces a confusing auth error 5 min later instead of the
+          # clean "secret missing" message at the top.
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              # Either MiniMax OR direct-Anthropic works — first
+              # non-empty wins in the test script's secrets-injection
+              # priority chain.
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
             exit 2
           fi
-          echo "OpenAI key present ✓ (len=${#E2E_OPENAI_API_KEY})"
+          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
 
       - name: CP staging health preflight
         run: |
@@ -164,11 +216,31 @@ jobs:
                         and o.get('instance_status') not in ('purged',)]
           print('\n'.join(candidates))
           " 2>/dev/null)
+          # Per-slug verified DELETE (was `>/dev/null || true` — see
+          # molecule-controlplane#420). Surface non-2xx as a workflow
+          # warning naming the leaked slug; don't exit 1 (sweeper is
+          # the safety net within ~45 min).
+          leaks=()
           for slug in $orgs; do
             echo "Safety-net teardown: $slug"
-            curl -sS -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
               -H "Authorization: Bearer $ADMIN_TOKEN" \
               -H "Content-Type: application/json" \
-              -d "{\"confirm\":\"$slug\"}" >/dev/null || true
+              -d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
+            set -e
+            code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
           done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
           exit 0

.github/workflows/e2e-staging-sanity.yml

@@ -50,7 +50,7 @@ jobs:
       E2E_INTENTIONAL_FAILURE: "1"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Verify admin token present
         run: |
@@ -89,7 +89,7 @@ jobs:
 
       - name: Open issue if safety net is broken
         if: failure()
-        uses: actions/github-script@v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             const title = "🚨 E2E teardown safety net broken";
@@ -143,10 +143,29 @@ jobs:
                         and o.get('status') not in ('purged',)]
           print('\n'.join(candidates))
           " 2>/dev/null)
+          # Per-slug verified DELETE — see molecule-controlplane#420.
+          # Failures surface as workflow warnings; the sweeper is the
+          # safety net within ~45 min.
+          leaks=()
           for slug in $orgs; do
-            curl -sS -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
               -H "Authorization: Bearer $ADMIN_TOKEN" \
               -H "Content-Type: application/json" \
-              -d "{\"confirm\":\"$slug\"}" >/dev/null || true
+              -d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
+            set -e
+            code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
           done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
           exit 0

.github/workflows/handlers-postgres-integration.yml

@@ -0,0 +1,252 @@
+name: Handlers Postgres Integration
+
+# Real-Postgres integration tests for workspace-server/internal/handlers/.
+# Triggered on every PR/push that touches the handlers package.
+#
+# Why this workflow exists
+# ------------------------
+# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
+# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
+# depend on the row state AFTER the SQL runs. The result_preview-lost
+# bug shipped to staging in PR #2854 because every unit test was
+# satisfied with "an UPDATE statement fired" — none verified the row's
+# preview field actually landed. The local-postgres E2E that retrofit
+# self-review caught it took 2 minutes to set up and would have caught
+# the bug at PR-time.
+#
+# Why this workflow does NOT use `services: postgres:` (Class B fix)
+# ------------------------------------------------------------------
+# Our act_runner config has `container.network: host` (operator host
+# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
+# the job container AND every service container. With host-net, two
+# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
+# second postgres FATALs with `could not create any TCP/IP sockets:
+# Address in use`, and Docker auto-removes it (act_runner sets
+# AutoRemove:true on service containers). By the time the migrations
+# step runs `psql`, the postgres container is gone, hence
+# `Connection refused` then `failed to remove container: No such
+# container` at cleanup time.
+#
+# Per-job `container.network` override is silently ignored by
+# act_runner — `--network and --net in the options will be ignored.`
+# appears in the runner log. Documented constraint.
+#
+# So we sidestep `services:` entirely. The job container still uses
+# host-net (inherited from runner config; required for cache server
+# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
+# postgres on the existing `molecule-core-net` bridge with a
+# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
+# read its bridge IP via `docker inspect`. A host-net job container
+# can reach a bridge-net container directly via the bridge IP (verified
+# manually on operator host 2026-05-08).
+#
+# Trade-offs vs. the original `services:` shape:
+#   + No host-port collision; N parallel runs share the bridge cleanly
+#   + `if: always()` cleanup runs even on test-step failure
+#   - One more step in the workflow (+~3 lines)
+#   - Requires `molecule-core-net` to exist on the operator host
+#     (it does; declared in docker-compose.yml + docker-compose.infra.yml)
+#
+# Class B Hongming-owned CICD red sweep, 2026-05-08.
+#
+# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  merge_group:
+    types: [checks_requested]
+  workflow_dispatch:
+
+concurrency:
+  group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  detect-changes:
+    name: detect-changes
+    runs-on: ubuntu-latest
+    outputs:
+      handlers: ${{ steps.filter.outputs.handlers }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            handlers:
+              - 'workspace-server/internal/handlers/**'
+              - 'workspace-server/internal/wsauth/**'
+              - 'workspace-server/migrations/**'
+              - '.github/workflows/handlers-postgres-integration.yml'
+
+  # Single-job-with-per-step-if pattern: always runs to satisfy the
+  # required-check name on branch protection; real work gates on the
+  # paths filter. See ci.yml's Platform (Go) for the same shape.
+  integration:
+    name: Handlers Postgres Integration
+    needs: detect-changes
+    runs-on: ubuntu-latest
+    env:
+      # Unique name per run so concurrent jobs don't collide on the
+      # bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
+      # workflow_dispatch reruns of the same run_id.
+      PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
+      # Bridge network already exists on the operator host (declared
+      # in docker-compose.yml + docker-compose.infra.yml).
+      PG_NETWORK: molecule-core-net
+    defaults:
+      run:
+        working-directory: workspace-server
+    steps:
+      - if: needs.detect-changes.outputs.handlers != 'true'
+        working-directory: .
+        run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Start sibling Postgres on bridge network
+        working-directory: .
+        run: |
+          # Sanity: the bridge network must exist on the operator host.
+          # Hard-fail loud if it doesn't — easier to spot than a silent
+          # auto-create that diverges from the rest of the stack.
+          if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
+            echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
+            exit 1
+          fi
+
+          # If a stale container with the same name exists (rerun on
+          # the same run_id), wipe it first.
+          docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
+
+          docker run -d \
+            --name "${PG_NAME}" \
+            --network "${PG_NETWORK}" \
+            --health-cmd "pg_isready -U postgres" \
+            --health-interval 5s \
+            --health-timeout 5s \
+            --health-retries 10 \
+            -e POSTGRES_PASSWORD=test \
+            -e POSTGRES_DB=molecule \
+            postgres:15-alpine >/dev/null
+
+          # Read back the bridge IP. Always present immediately after
+          # `docker run -d` for bridge networks.
+          PG_HOST=$(docker inspect "${PG_NAME}" \
+            --format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
+          if [ -z "${PG_HOST}" ]; then
+            echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
+            docker logs "${PG_NAME}" || true
+            exit 1
+          fi
+          echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
+          echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
+          echo "Started ${PG_NAME} at ${PG_HOST}:5432"
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Apply migrations to Postgres service
+        env:
+          PGPASSWORD: test
+        run: |
+          # Wait for postgres to actually accept connections. Docker's
+          # health-cmd handles container-side readiness, but the wire
+          # to the bridge IP is best-tested with pg_isready directly.
+          for i in {1..15}; do
+            if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
+            echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
+          done
+
+          # Apply every .up.sql in lexicographic order with
+          # ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
+          # blocking the suite. This handles the current schema state
+          # where a few historical migrations (e.g. 017_memories_fts_*)
+          # depend on tables that were later renamed/dropped and so
+          # cannot replay from scratch. The migrations that DO succeed
+          # land their tables, which is sufficient for the integration
+          # tests in handlers/.
+          #
+          # Why not maintain a curated allowlist: every new migration
+          # touching a handlers/-tested table would have to update this
+          # workflow. With apply-all-or-skip, a future migration that
+          # adds a column to delegations runs automatically (its base
+          # table 049_delegations.up.sql already succeeded above it in
+          # the order). Operators only need to revisit this if the
+          # migration chain becomes legitimately replayable end-to-end.
+          #
+          # Per-migration result is logged so a failed migration that
+          # SHOULD have been replayable surfaces in the CI log instead
+          # of silently failing.
+          # Apply both *.sql (legacy, lives next to its module) and
+          # *.up.sql (newer up/down convention) in a single
+          # lexicographically-sorted pass. Excluding *.down.sql so the
+          # newest-naming-convention pairs don't undo themselves mid-run.
+          # Pre-#149-followup this loop only globbed *.up.sql, which
+          # silently skipped 001_workspaces.sql + 009_activity_logs.sql
+          # — fine while no integration test depended on those tables,
+          # not fine once a cross-table atomicity test came in.
+          set +e
+          for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
+            if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
+                  -f "$migration" >/dev/null 2>&1; then
+              echo "✓ $(basename "$migration")"
+            else
+              echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
+            fi
+          done
+          set -e
+
+          # Sanity: the delegations + workspaces + activity_logs tables
+          # MUST exist for the integration tests to be meaningful. Hard-
+          # fail if any didn't land — that would be a real regression we
+          # want loud.
+          for tbl in delegations workspaces activity_logs pending_uploads; do
+            if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
+                -c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
+                | grep -q 1; then
+              echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
+              exit 1
+            fi
+            echo "✓ $tbl table present"
+          done
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Run integration tests
+        run: |
+          # INTEGRATION_DB_URL is exported by the start-postgres step;
+          # points at the per-run bridge IP, not 127.0.0.1, so concurrent
+          # workflow runs don't fight over a host-net 5432 port.
+          go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
+
+      - if: failure() && needs.detect-changes.outputs.handlers == 'true'
+        name: Diagnostic dump on failure
+        env:
+          PGPASSWORD: test
+        run: |
+          echo "::group::postgres container status"
+          docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
+          docker logs "${PG_NAME}" 2>&1 | tail -50 || true
+          echo "::endgroup::"
+          echo "::group::delegations table state"
+          psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
+          echo "::endgroup::"
+
+      - if: always() && needs.detect-changes.outputs.handlers == 'true'
+        name: Stop sibling Postgres
+        working-directory: .
+        run: |
+          # always() so containers don't leak when migrations or tests
+          # fail. The cleanup is best-effort: if the container is
+          # already gone (e.g. concurrent rerun race), don't fail the job.
+          docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
+          echo "Cleaned up ${PG_NAME}"
+

.github/workflows/harness-replays.yml

@@ -0,0 +1,248 @@
+name: Harness Replays
+
+# Boots tests/harness (production-shape compose topology with TenantGuard,
+# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
+# every replay under tests/harness/replays/. Fails the PR if any replay
+# fails.
+#
+# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
+# a public route in router.go but forgot to add it to TenantGuard's
+# allowlist. The handler-level test in buildinfo_test.go constructed a
+# minimal gin engine without TenantGuard — green. The harness's
+# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
+# inject X-Molecule-Org-Id, so the curl path is identical to production's
+# redeploy verifier), but no one ran the harness pre-merge. The bug
+# shipped; the redeploy verifier silently soft-warned every tenant as
+# "unreachable" for ~1 day before being noticed.
+#
+# This gate makes "did you actually run the harness?" a CI invariant
+# instead of a memory-discipline thing.
+#
+# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
+# to staging+main, real work is gated per-step on detect-changes output.
+# One job → one check run → branch-protection-clean (the SKIPPED-in-set
+# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'tests/harness/**'
+      - '.github/workflows/harness-replays.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'tests/harness/**'
+      - '.github/workflows/harness-replays.yml'
+  workflow_dispatch:
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  # Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
+  # cancellation deadlock — see e2e-api.yml's concurrency block for
+  # the 2026-04-28 incident that codified this pattern.
+  group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      run: ${{ steps.decide.outputs.run }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - id: decide
+        run: |
+          # workflow_dispatch: always run (manual trigger)
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Determine the base commit to diff against.
+          # For pull_request: use base.sha (the merge-base with main/staging).
+          # For push: use github.event.before (the previous tip of the branch).
+          # Fallback for new branches (all-zeros SHA): run everything.
+          if [ "${{ github.event_name }}" = "pull_request" ] && \
+             [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          elif [ -n "${{ github.event.before }}" ] && \
+               ! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
+            BASE="${{ github.event.before }}"
+          else
+            # New branch or github.event.before unavailable — run everything.
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # GitHub Actions and Gitea Actions both expose github.sha for HEAD.
+          DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
+          echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
+
+          if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.github/workflows/harness-replays\.yml$'; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job that always runs. Real work is gated per-step on
+  # detect-changes.outputs.run so an unrelated PR (e.g. doc-only
+  # change to molecule-controlplane wired here later) emits the
+  # required check without spending CI cycles. Single-job pattern
+  # matches e2e-api.yml — see that workflow's comment for why a
+  # job-level `if: false` would block branch protection via the
+  # SKIPPED-in-set bug.
+  harness-replays:
+    needs: detect-changes
+    name: Harness Replays
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.run != 'true'
+        run: |
+          echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
+          echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
+          echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
+
+      - if: needs.detect-changes.outputs.run == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # Log what files were detected so future failures include the diff.
+      - name: Log detected changes
+        if: needs.detect-changes.outputs.run == 'true'
+        run: |
+          echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
+
+      # github-app-auth sibling-checkout removed 2026-05-07 (#157):
+      # the plugin was dropped + Dockerfile.tenant no longer COPYs it.
+
+      # Pre-clone manifest deps before docker compose builds the tenant
+      # image (Task #173 followup — same pattern as
+      # publish-workspace-server-image.yml's "Pre-clone manifest deps"
+      # step).
+      #
+      # Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
+      # and tenant-beta from workspace-server/Dockerfile.tenant with
+      # context=../.. (repo root). That Dockerfile expects
+      # .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
+      # to be present at build context root (post-#173 it COPYs from there
+      # instead of running an in-image clone — the in-image clone failed
+      # with "could not read Username for https://git.moleculesai.app"
+      # because there's no auth path inside the build sandbox).
+      #
+      # Without this step harness-replays fails before any replay runs,
+      # with `failed to calculate checksum of ref ...
+      # "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
+      # (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
+      # symptom, different root cause: staging still has the in-image
+      # clone path, hits the auth error directly).
+      #
+      # 2026-05-08 sub-finding (#192): the clone step ALSO fails when
+      # any referenced workspace-template repo is private and the
+      # AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
+      # access. Root cause: 5 of 9 workspace-template repos
+      # (openclaw, codex, crewai, deepagents, gemini-cli) had been
+      # marked private with no team grant. Resolution: flipped them
+      # to public per `feedback_oss_first_repo_visibility_default`
+      # (the OSS surface should be public). Layer-3 (customer-private +
+      # marketplace third-party repos) tracked separately in
+      # internal#102.
+      #
+      # Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
+      # is the devops-engineer persona PAT, NOT the founder PAT (per
+      # `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
+      # embeds it as basic-auth for the duration of the clones and strips
+      # .git directories — the token never enters the resulting image.
+      - name: Pre-clone manifest deps
+        if: needs.detect-changes.outputs.run == 'true'
+        env:
+          MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
+            echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
+            exit 1
+          fi
+          mkdir -p .tenant-bundle-deps
+          bash scripts/clone-manifest.sh \
+            manifest.json \
+            .tenant-bundle-deps/workspace-configs-templates \
+            .tenant-bundle-deps/org-templates \
+            .tenant-bundle-deps/plugins
+          # Sanity-check counts so a silent partial clone fails fast
+          # instead of producing a half-empty image.
+          ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
+          echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
+
+      - name: Install Python deps for replays
+        # peer-discovery-404 (and future replays) eval Python against the
+        # running tenant — importing workspace/a2a_client.py pulls in
+        # httpx. tests/harness/requirements.txt holds just the HTTP-client
+        # surface to keep CI install fast (~3s) vs the full
+        # workspace/requirements.txt (~30s).
+        if: needs.detect-changes.outputs.run == 'true'
+        run: pip install -r tests/harness/requirements.txt
+
+      - name: Run all replays against the harness
+        # run-all-replays.sh: boot via up.sh → seed via seed.sh → run
+        # every replays/*.sh → tear down via down.sh on EXIT (trap).
+        # Non-zero exit on any replay failure.
+        #
+        # KEEP_UP=1: without this, the script's trap-on-EXIT tears
+        # down containers immediately on failure, leaving the dump
+        # step below with nothing to dump (verified on PR #2410's
+        # first run — tenant became unhealthy, trap fired, dump
+        # step saw empty containers). Keeping them up lets the
+        # failure path collect tenant/cp-stub/cf-proxy logs. The
+        # always-run "Force teardown" step does the actual cleanup.
+        if: needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        env:
+          KEEP_UP: "1"
+        run: ./run-all-replays.sh
+
+      - name: Dump compose logs on failure
+        # SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
+        # file even for read-only `logs` calls. up.sh generates a per-run key
+        # and exports it to its OWN shell — this step runs in a fresh shell
+        # that wouldn't see it, so without a placeholder the validate step
+        # errors before logs print (verified against PR #2492's first run:
+        # "required variable SECRETS_ENCRYPTION_KEY is missing a value").
+        # A placeholder is fine — we're only reading log streams, not booting.
+        if: failure() && needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        env:
+          SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
+        run: |
+          echo "=== docker compose ps ==="
+          docker compose -f compose.yml ps || true
+          echo "=== tenant-alpha logs ==="
+          docker compose -f compose.yml logs tenant-alpha || true
+          echo "=== tenant-beta logs ==="
+          docker compose -f compose.yml logs tenant-beta || true
+          echo "=== cp-stub logs ==="
+          docker compose -f compose.yml logs cp-stub || true
+          echo "=== cf-proxy logs ==="
+          docker compose -f compose.yml logs cf-proxy || true
+          echo "=== postgres-alpha logs (last 100) ==="
+          docker compose -f compose.yml logs --tail 100 postgres-alpha || true
+          echo "=== postgres-beta logs (last 100) ==="
+          docker compose -f compose.yml logs --tail 100 postgres-beta || true
+
+      - name: Force teardown
+        # We pass KEEP_UP=1 to run-all-replays.sh so the dump step
+        # above sees real containers — that means we own teardown
+        # explicitly here. Always run.
+        if: always() && needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        run: ./down.sh || true

.github/workflows/lint-curl-status-capture.yml

@@ -0,0 +1,94 @@
+name: Lint curl status-code capture
+
+# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
+# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
+#
+#   HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
+#
+# When curl exits non-zero (connection reset → 56, --fail-with-body 4xx/5xx
+# → 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
+# "000" for connection failures or the actual code for HTTP errors. The
+# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
+# stdout, producing values like "000000" or "409000" that fail string
+# comparisons against "200" while looking superficially right.
+#
+# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
+# #2797). Memory: feedback_curl_status_capture_pollution.md.
+#
+# Fix shape (route -w into a tempfile so curl's exit code can't pollute):
+#
+#   set +e
+#   curl ... -w '%{http_code}' >code.txt 2>/dev/null
+#   set -e
+#   HTTP_CODE=$(cat code.txt 2>/dev/null)
+#   [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+on:
+  pull_request:
+    paths: ['.github/workflows/**']
+  push:
+    branches: [main, staging]
+    paths: ['.github/workflows/**']
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  scan:
+    name: Scan workflows for curl status-capture pollution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
+        run: |
+          set -uo pipefail
+          # Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
+          # subshell where the entire command-substitution wraps a curl that
+          # ends with `|| echo "000"`. Must distinguish from the SAFE shape
+          # `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
+          # tempfile produces empty stdout, no pollution.
+          python3 <<'PY'
+          import os, re, sys, glob
+
+          BAD_FILES = []
+
+          # Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
+          # The `\\n` is the bash line-continuation that lets curl flags span lines.
+          # We collapse continuation lines first, then look for the single-line bad pattern.
+          PATTERN = re.compile(
+              r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
+              re.DOTALL,
+          )
+
+          # Self-skip: this lint workflow contains the literal anti-pattern in
+          # its own docstring — that's intentional, not a bug.
+          SELF = ".github/workflows/lint-curl-status-capture.yml"
+
+          for f in sorted(glob.glob(".github/workflows/*.yml")):
+              if f == SELF:
+                  continue
+              with open(f) as fh:
+                  content = fh.read()
+              # Collapse bash line-continuations (\\\n + leading whitespace)
+              # into a single logical line so the regex can see the full
+              # curl invocation as one chunk.
+              flat = re.sub(r'\\\s*\n\s*', ' ', content)
+              for m in PATTERN.finditer(flat):
+                  BAD_FILES.append((f, m.group(0)[:120]))
+
+          if not BAD_FILES:
+              print("✓ No curl-status-capture pollution patterns detected")
+              sys.exit(0)
+
+          print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
+          for f, snippet in BAD_FILES:
+              print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
+              print(f"   matched: {snippet}…")
+          print()
+          print("Fix template:")
+          print('  set +e')
+          print('  curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
+          print('  set -e')
+          print('  HTTP_CODE=$(cat code.txt 2>/dev/null)')
+          print('  [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
+          sys.exit(1)
+          PY

.github/workflows/pr-guards.yml

@@ -1,22 +0,0 @@
-name: pr-guards
-
-# Thin caller that delegates to the molecule-ci reusable guard. Today
-# the guard is just "disable auto-merge when a new commit is pushed
-# after auto-merge was enabled" — added 2026-04-27 after PR #2174
-# auto-merged with only its first commit because the second commit
-# was pushed after the merge queue had locked the PR's SHA.
-#
-# When more PR-time guards land in molecule-ci, add them here as
-# additional jobs that share the same pull_request:synchronize
-# trigger.
-
-on:
-  pull_request:
-    types: [synchronize]
-
-permissions:
-  pull-requests: write
-
-jobs:
-  disable-auto-merge-on-push:
-    uses: Molecule-AI/molecule-ci/.github/workflows/disable-auto-merge-on-push.yml@main

.github/workflows/promote-latest.yml

@@ -1,85 +0,0 @@
-name: promote-latest
-
-# Manually retag ghcr.io/molecule-ai/platform:staging-<sha> →  :latest
-# (and the same for the tenant image). Use this to:
-#
-#   1. Promote a :staging-<sha> to prod before the canary fleet is live
-#      (one-off during the initial rollout).
-#   2. Roll back :latest to a prior known-good digest after a bad
-#      promotion slipped past canary (use scripts/rollback-latest.sh
-#      for a local / emergency path; this workflow is for scheduled
-#      or from-browser promotions).
-#
-# Running this workflow needs no extra secrets — GitHub's default
-# GITHUB_TOKEN has write:packages for repo-owned GHCR images, which
-# is all we need for a remote retag via `crane tag`.
-
-on:
-  workflow_dispatch:
-    inputs:
-      sha:
-        description: 'Short sha to promote (e.g. 4c1d56e). Must match an existing :staging-<sha> tag.'
-        required: true
-        type: string
-
-permissions:
-  contents: read
-  packages: write
-
-env:
-  IMAGE_NAME: ghcr.io/molecule-ai/platform
-  TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
-
-jobs:
-  promote:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: imjasonh/setup-crane@v0.4
-
-      - name: GHCR login
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" \
-            | crane auth login ghcr.io -u "${{ github.actor }}" --password-stdin
-
-      - name: Retag platform image
-        run: |
-          set -eu
-          SRC="${IMAGE_NAME}:staging-${{ inputs.sha }}"
-          if ! crane digest "$SRC" >/dev/null 2>&1; then
-            echo "::error::$SRC not found in registry — double-check the sha."
-            exit 1
-          fi
-          EXPECTED=$(crane digest "$SRC")
-          crane tag "$SRC" latest
-          ACTUAL=$(crane digest "${IMAGE_NAME}:latest")
-          if [ "$ACTUAL" != "$EXPECTED" ]; then
-            echo "::error::retag digest mismatch (expected $EXPECTED, got $ACTUAL)"
-            exit 1
-          fi
-          echo "OK  ${IMAGE_NAME}:latest → $ACTUAL"
-
-      - name: Retag tenant image
-        run: |
-          set -eu
-          SRC="${TENANT_IMAGE_NAME}:staging-${{ inputs.sha }}"
-          if ! crane digest "$SRC" >/dev/null 2>&1; then
-            echo "::error::$SRC not found — tenant image may not have built for this sha."
-            exit 1
-          fi
-          EXPECTED=$(crane digest "$SRC")
-          crane tag "$SRC" latest
-          ACTUAL=$(crane digest "${TENANT_IMAGE_NAME}:latest")
-          if [ "$ACTUAL" != "$EXPECTED" ]; then
-            echo "::error::tenant retag digest mismatch"
-            exit 1
-          fi
-          echo "OK  ${TENANT_IMAGE_NAME}:latest → $ACTUAL"
-
-      - name: Summary
-        run: |
-          {
-            echo "## :latest promoted to staging-${{ inputs.sha }}"
-            echo
-            echo "Both platform + tenant images retagged. Prod tenants"
-            echo "will auto-pull within their 5-min update cycle."
-          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/publish-canvas-image.yml

@@ -42,17 +42,33 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Log in to GHCR
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+
+      # Health check: verify Docker daemon is accessible before attempting any
+      # build steps. This fails loudly at step 1 when the runner's docker.sock
+      # is inaccessible rather than silently continuing to the build step
+      # where docker build fails deep in ECR auth with a cryptic error.
+      - name: Verify Docker daemon access
+        run: |
+          set -euo pipefail
+          echo "::group::Docker daemon health check"
+          docker info 2>&1 | head -5 || {
+            echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
+            echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
+            exit 1
+          }
+          echo "Docker daemon OK"
+          echo "::endgroup::"
 
       - name: Compute tags
         id: tags
@@ -85,7 +101,7 @@ jobs:
           echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
 
       - name: Build & push canvas image to GHCR
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
         with:
           context: ./canvas
           file: ./canvas/Dockerfile

.github/workflows/publish-runtime.yml

@@ -1,309 +0,0 @@
-name: publish-runtime
-
-# Publishes molecule-ai-workspace-runtime to PyPI from monorepo workspace/.
-# Monorepo workspace/ is the only source-of-truth for runtime code; this
-# workflow is the bridge from monorepo edits to the PyPI artifact that
-# the 8 workspace-template-* repos depend on.
-#
-# Triggered by:
-#   - Pushing a tag matching `runtime-vX.Y.Z` (the version is derived from
-#     the tag — `runtime-v0.1.6` publishes `0.1.6`).
-#   - Manual workflow_dispatch with an explicit `version` input (useful for
-#     dev/test releases without tagging the repo).
-#   - Auto: any push to `staging` that touches `workspace/**`. The version
-#     is derived by querying PyPI for the current latest and bumping the
-#     patch component. This closes the human-in-loop gap that caused the
-#     2026-04-27 RuntimeCapabilities ImportError outage — adapter symbol
-#     additions in workspace/adapters/base.py used to require an operator
-#     to remember to publish; now the merge itself triggers the publish.
-#
-# The workflow:
-#   1. Runs scripts/build_runtime_package.py to copy workspace/ →
-#      build/molecule_runtime/ with imports rewritten (`a2a_client` →
-#      `molecule_runtime.a2a_client`).
-#   2. Builds wheel + sdist with `python -m build`.
-#   3. Publishes to PyPI via the PyPA Trusted Publisher action (OIDC).
-#      No static API token is stored — PyPI verifies the workflow's
-#      OIDC claim against the trusted-publisher config registered for
-#      molecule-ai-workspace-runtime (Molecule-AI/molecule-core,
-#      publish-runtime.yml, environment pypi-publish).
-#
-# After publish: the 8 template repos pick up the new version on their
-# next image rebuild (their requirements.txt pin
-# `molecule-ai-workspace-runtime>=0.1.0`, so any new release is eligible).
-# To force-pull immediately, bump the pin in each template repo's
-# requirements.txt and merge — that triggers their own publish-image.yml.
-
-on:
-  push:
-    tags:
-      - "runtime-v*"
-    branches:
-      - staging
-    paths:
-      # Auto-publish when staging gets changes that affect what gets
-      # published. Path filter ONLY applies to branch pushes — tag pushes
-      # still fire regardless.
-      #
-      # workspace/** is the source-of-truth for runtime code.
-      # scripts/build_runtime_package.py is the build script — changes to
-      # it (e.g. a fix to the import rewriter or a manifest emit) directly
-      # affect what ships in the wheel even if no workspace/ file changes.
-      # The 2026-04-27 lib/ subpackage incident missed an auto-publish for
-      # exactly this reason — PR #2174 only changed scripts/ and the
-      # operator had to remember a manual dispatch.
-      - "workspace/**"
-      - "scripts/build_runtime_package.py"
-  workflow_dispatch:
-    inputs:
-      version:
-        description: "Version to publish (e.g. 0.1.6). Required for manual dispatch."
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-# Serialize publishes so two staging merges landing seconds apart don't
-# both compute "latest+1" and race on PyPI upload. The second one waits.
-concurrency:
-  group: publish-runtime
-  cancel-in-progress: false
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    environment: pypi-publish
-    permissions:
-      contents: read
-      id-token: write   # PyPI Trusted Publisher (OIDC) — no PYPI_TOKEN needed
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: pip
-
-      - name: Derive version (tag, manual input, or PyPI auto-bump)
-        id: version
-        run: |
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            VERSION="${{ inputs.version }}"
-          elif echo "$GITHUB_REF_NAME" | grep -q "^runtime-v"; then
-            # Tag is `runtime-vX.Y.Z` — strip the prefix.
-            VERSION="${GITHUB_REF_NAME#runtime-v}"
-          else
-            # Auto-publish from staging push. Query PyPI for the current
-            # latest and bump the patch component. concurrency: group above
-            # serializes parallel staging merges so we don't race on the
-            # bump. If PyPI is unreachable, fail loud — better to skip a
-            # publish than to overwrite an existing version.
-            LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
-              | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
-            MAJOR=$(echo "$LATEST" | cut -d. -f1)
-            MINOR=$(echo "$LATEST" | cut -d. -f2)
-            PATCH=$(echo "$LATEST" | cut -d. -f3)
-            VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
-            echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
-          fi
-          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
-            echo "::error::version $VERSION does not match PEP 440"
-            exit 1
-          fi
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "Publishing molecule-ai-workspace-runtime $VERSION"
-
-      - name: Install build tooling
-        run: pip install build twine
-
-      - name: Build package from workspace/
-        run: |
-          python scripts/build_runtime_package.py \
-            --version "${{ steps.version.outputs.version }}" \
-            --out "${{ runner.temp }}/runtime-build"
-
-      - name: Build wheel + sdist
-        working-directory: ${{ runner.temp }}/runtime-build
-        run: python -m build
-
-      - name: Verify package contents (sanity)
-        working-directory: ${{ runner.temp }}/runtime-build
-        run: |
-          python -m twine check dist/*
-          # Smoke-import the built wheel to catch import-rewrite mistakes
-          # before they hit PyPI. Asserts on STABLE INVARIANTS only —
-          # symbols + classes that are part of the package's public
-          # contract (BaseAdapter interface, the canonical a2a sentinel,
-          # core submodules). Don't add feature-flag-style assertions
-          # here — they fire false-positive every time staging is mid-
-          # release of that feature.
-          python -m venv /tmp/smoke
-          /tmp/smoke/bin/pip install --quiet dist/*.whl
-          WORKSPACE_ID=00000000-0000-0000-0000-000000000000 \
-          PLATFORM_URL=http://localhost:8080 \
-            /tmp/smoke/bin/python -c "
-          # Importing main is the strongest smoke test we can do here:
-          # main.py is the entry point and pulls every other module
-          # transitively. If the build script missed an import rewrite
-          # (e.g. left a bare \`from transcript_auth import ...\` instead
-          # of \`from molecule_runtime.transcript_auth import ...\` — the
-          # 0.1.16 incident), this fails with ModuleNotFoundError instead
-          # of shipping to PyPI and breaking every workspace startup.
-          # Import the entry-point target by NAME — not just the module.
-          # The wheel's pyproject.toml declares
-          # `molecule-runtime = molecule_runtime.main:main_sync` so if
-          # main_sync goes missing (it did in 0.1.16-0.1.18), every
-          # workspace startup fails with `ImportError: cannot import name
-          # 'main_sync'`. Plain `import molecule_runtime.main` doesn't
-          # catch that because the module loads fine.
-          from molecule_runtime.main import main_sync  # noqa: F401
-          from molecule_runtime import a2a_client, a2a_tools
-          from molecule_runtime.builtin_tools import memory
-          from molecule_runtime.adapters import get_adapter, BaseAdapter, AdapterConfig
-          # Stable invariants: package exports + BaseAdapter shape.
-          assert a2a_client._A2A_ERROR_PREFIX, 'a2a_client missing error sentinel'
-          assert callable(get_adapter), 'adapters.get_adapter must be callable'
-          assert hasattr(BaseAdapter, 'name'), 'BaseAdapter interface broken'
-          assert hasattr(AdapterConfig, '__init__'), 'AdapterConfig dataclass missing'
-
-          # Call-shape smoke for AgentCard. Pure imports don't catch
-          # field-shape regressions in upstream SDKs that only surface
-          # at construction time. Two bugs of this exact class shipped
-          # since the a2a-sdk 1.0 migration:
-          #   - state_transition_history=True (fixed in #2179)
-          #   - supported_protocols=[...] (the protobuf field is
-          #     supported_interfaces — caused every workspace boot
-          #     to crash with `ValueError: Protocol message AgentCard
-          #     has no "supported_protocols" field`; fixed alongside
-          #     this smoke)
-          #
-          # This block instantiates the EXACT classes main.py uses,
-          # with the EXACT keyword arguments. If a future a2a-sdk
-          # upgrade renames any of supported_interfaces / streaming /
-          # push_notifications / etc., the publish fails here instead
-          # of breaking every workspace startup. main.py and this
-          # smoke MUST stay in lockstep — adding a kwarg to one
-          # without mirroring it here is the regression vector.
-          from a2a.types import AgentCard, AgentCapabilities, AgentSkill, AgentInterface
-          AgentCard(
-              name='smoke-agent',
-              description='publish-runtime smoke test',
-              version='0.0.0-smoke',
-              supported_interfaces=[
-                  AgentInterface(protocol_binding='https://a2a.g/v1', url='http://localhost:8080'),
-              ],
-              capabilities=AgentCapabilities(
-                  streaming=True,
-                  push_notifications=False,
-              ),
-              skills=[
-                  AgentSkill(
-                      id='smoke-skill',
-                      name='Smoke',
-                      description='no-op',
-                      tags=['smoke'],
-                      examples=['noop'],
-                  ),
-              ],
-              default_input_modes=['text/plain', 'application/json'],
-              default_output_modes=['text/plain', 'application/json'],
-          )
-          print('✓ AgentCard call-shape smoke passed')
-          print('✓ smoke import passed')
-          "
-
-      - name: Publish to PyPI (Trusted Publisher / OIDC)
-        # PyPI side is configured: project molecule-ai-workspace-runtime →
-        # publisher Molecule-AI/molecule-core, workflow publish-runtime.yml,
-        # environment pypi-publish. The action mints a short-lived OIDC
-        # token and exchanges it for a PyPI upload credential — no static
-        # API token in this repo's secrets.
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: ${{ runner.temp }}/runtime-build/dist/
-
-  cascade:
-    # After PyPI accepts the upload, fan out a repository_dispatch to each
-    # template repo so they rebuild their image against the new runtime.
-    # Each template's `runtime-published.yml` receiver picks up the event,
-    # pulls the new PyPI version (their requirements.txt pin is `>=`), and
-    # republishes ghcr.io/molecule-ai/workspace-template-<runtime>:latest.
-    #
-    # Soft-fail per repo: if one template's dispatch fails (perms missing,
-    # repo archived, etc.) we still try the others and surface the failures
-    # in the workflow summary instead of aborting the whole cascade.
-    needs: publish
-    runs-on: ubuntu-latest
-    steps:
-      - name: Wait for PyPI to propagate the new version
-        # PyPI accepts the upload, then takes a few seconds to make it
-        # available via the package index. If the cascade fires too
-        # fast, downstream template builds run `pip install` against
-        # an index that hasn't seen the new version yet — they resolve
-        # to the previous one, and docker layer cache then locks that
-        # in for subsequent rebuilds (the cache trap that bit us five
-        # times tonight).
-        #
-        # Poll PyPI's JSON API for up to 60s. Cheap (~50ms per poll),
-        # avoids over-trusting "publish job said success."
-        env:
-          RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
-        run: |
-          set -eu
-          for i in $(seq 1 30); do
-            if curl -fsS "https://pypi.org/pypi/molecule-ai-workspace-runtime/${RUNTIME_VERSION}/json" >/dev/null 2>&1; then
-              echo "::notice::✓ PyPI serving ${RUNTIME_VERSION} after ${i} polls"
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "::error::PyPI never propagated ${RUNTIME_VERSION} within 60s — refusing to fan out cascade against stale index"
-          exit 1
-
-      - name: Fan out repository_dispatch
-        env:
-          # Fine-grained PAT with `actions:write` on the 8 template repos.
-          # GITHUB_TOKEN can't fire dispatches across repos — needs an explicit
-          # token. Stored as a repo secret; rotate per the standard schedule.
-          DISPATCH_TOKEN: ${{ secrets.TEMPLATE_DISPATCH_TOKEN }}
-          # Single source of truth: the publish job's output, which handles
-          # tag/manual-input/auto-bump uniformly. The previous fallback
-          # (`steps.version.outputs.version` from inside the cascade job)
-          # was a dead reference — different job, no shared step scope.
-          RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
-        run: |
-          set +e   # don't abort on a single repo failure — collect them all
-          if [ -z "$DISPATCH_TOKEN" ]; then
-            echo "::warning::TEMPLATE_DISPATCH_TOKEN secret not set — skipping cascade. PyPI was published; templates will pick up the new version on their own next rebuild."
-            exit 0
-          fi
-          VERSION="$RUNTIME_VERSION"
-          if [ -z "$VERSION" ]; then
-            echo "::error::publish job did not expose a version output — cascade cannot fan out"
-            exit 1
-          fi
-          TEMPLATES="claude-code langgraph crewai autogen deepagents hermes gemini-cli openclaw"
-          FAILED=""
-          for tpl in $TEMPLATES; do
-            REPO="Molecule-AI/molecule-ai-workspace-template-$tpl"
-            STATUS=$(curl -sS -o /tmp/dispatch.out -w "%{http_code}" \
-              -X POST "https://api.github.com/repos/$REPO/dispatches" \
-              -H "Authorization: Bearer $DISPATCH_TOKEN" \
-              -H "Accept: application/vnd.github+json" \
-              -H "X-GitHub-Api-Version: 2022-11-28" \
-              -d "{\"event_type\":\"runtime-published\",\"client_payload\":{\"runtime_version\":\"$VERSION\"}}")
-            if [ "$STATUS" = "204" ]; then
-              echo "✓ dispatched $tpl ($VERSION)"
-            else
-              echo "::warning::✗ failed to dispatch $tpl: HTTP $STATUS — $(cat /tmp/dispatch.out)"
-              FAILED="$FAILED $tpl"
-            fi
-          done
-          if [ -n "$FAILED" ]; then
-            echo "::warning::Cascade incomplete. Failed templates:$FAILED"
-            # Don't fail the whole job — PyPI publish already succeeded;
-            # operators can retry the failed templates manually.
-          fi

.github/workflows/publish-workspace-server-image.yml

@@ -1,134 +0,0 @@
-name: publish-workspace-server-image
-
-# Builds and pushes Docker images to GHCR when staging is promoted to main.
-# PRs target staging (default branch). Only main push triggers production builds.
-# EC2 tenant instances pull the tenant image from GHCR.
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - 'workspace-server/**'
-      - 'canvas/**'
-      - 'manifest.json'
-      - '.github/workflows/publish-platform-image.yml'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  packages: write
-
-env:
-  IMAGE_NAME: ghcr.io/molecule-ai/platform
-  TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
-
-jobs:
-  build-and-push:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Checkout sibling plugin repo
-        # workspace-server/Dockerfile expects
-        # ./molecule-ai-plugin-github-app-auth at build-context root because
-        # the Go module has a `replace` directive pointing at /plugin inside
-        # the image. Pre-repo-split the plugin lived in the monorepo; the
-        # 2026-04-18 restructure moved it out but didn't add this clone step
-        # — which is why publish was failing after that restructure.
-        #
-        # Uses a fine-grained PAT (PLUGIN_REPO_PAT) because the plugin repo
-        # is private and the default GITHUB_TOKEN is scoped to THIS repo.
-        # The PAT needs Contents:Read on Molecule-AI/molecule-ai-plugin-
-        # github-app-auth. Falls back to the default token for the (rare)
-        # case where an operator made the plugin repo public.
-        uses: actions/checkout@v4
-        with:
-          repository: Molecule-AI/molecule-ai-plugin-github-app-auth
-          path: molecule-ai-plugin-github-app-auth
-          token: ${{ secrets.PLUGIN_REPO_PAT || secrets.GITHUB_TOKEN }}
-
-      - name: Log in to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Compute tags
-        id: tags
-        run: |
-          echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
-
-      # Canary-gated release: we publish :staging-<sha> ONLY here. The
-      # :latest tag (which existing prod tenants auto-pull every 5 min)
-      # is promoted by .github/workflows/canary-verify.yml after the
-      # staging canary fleet green-lights this digest.
-      # That means:
-      #   - Every main merge produces a :staging-<sha> image
-      #   - Canary tenants (configured to pull :staging-<sha>) pick it up
-      #   - canary-verify.yml runs smoke tests against them
-      #   - On green → canary-verify retags :staging-<sha> → :latest
-      #   - On red → :latest stays on the prior good digest, prod is safe
-      # Every push of :staging-<sha> also retags the same digest as
-      # :staging-latest so staging CP (which pins TENANT_IMAGE at
-      # :staging-latest) picks up new builds automatically — no more manual
-      # Railway env-var edits. Prod's :latest retag still happens in
-      # canary-verify.yml after the canary fleet greenlights this digest;
-      # :staging-latest is strictly the "most recent main build," not a
-      # canary-verified promotion.
-      #
-      # Before this, TENANT_IMAGE on Railway staging was pinned to a static
-      # :staging-<sha> and drifted months behind (2026-04-24 incident:
-      # canary tenant ran :staging-a14cf86, 10 days stale, which lacked
-      # applyRuntimeModelEnv and caused every E2E to route hermes+openai
-      # through openrouter → 401). See issue filed with this PR.
-      - name: Build & push platform image to GHCR (staging-<sha> + staging-latest)
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ./workspace-server/Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: |
-            ${{ env.IMAGE_NAME }}:staging-${{ steps.tags.outputs.sha }}
-            ${{ env.IMAGE_NAME }}:staging-latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          labels: |
-            org.opencontainers.image.source=https://github.com/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.description=Molecule AI platform (Go API server) — pending canary verify
-
-      - name: Build & push tenant image to GHCR (staging-<sha> + staging-latest)
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ./workspace-server/Dockerfile.tenant
-          platforms: linux/amd64
-          push: true
-          tags: |
-            ${{ env.TENANT_IMAGE_NAME }}:staging-${{ steps.tags.outputs.sha }}
-            ${{ env.TENANT_IMAGE_NAME }}:staging-latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          # Canvas uses same-origin fetches. The tenant Go platform
-          # reverse-proxies /cp/* to the SaaS CP via its CP_UPSTREAM_URL
-          # env; the tenant's /canvas/viewport, /approvals/pending,
-          # /org/templates etc. live on the tenant platform itself.
-          # Both legs share one origin (the tenant subdomain) so
-          # PLATFORM_URL="" forces canvas to fetch paths as relative,
-          # which land same-origin.
-          #
-          # Self-hosted / private-label deployments override this at
-          # build time with a specific backend (e.g. local dev:
-          # NEXT_PUBLIC_PLATFORM_URL=http://localhost:8080).
-          build-args: |
-            NEXT_PUBLIC_PLATFORM_URL=
-          labels: |
-            org.opencontainers.image.source=https://github.com/${{ github.repository }}
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.description=Molecule AI tenant platform + canvas — pending canary verify

.github/workflows/railway-pin-audit.yml

@@ -0,0 +1,207 @@
+name: Railway pin audit (drift detection)
+
+# Daily audit of Railway env vars for drift-prone image-tag pins —
+# automation-cadence layer over the detection script + regression test
+# shipped in PR #2168 (#2001 closure).
+#
+# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
+# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
+# "every fix didn't propagate" — really the tenant image was so old it
+# didn't read the env vars those fixes produced. The audit script
+# (scripts/ops/audit-railway-sha-pins.sh) flags drift; this workflow
+# runs the same check unattended on a daily cron.
+#
+# Cadence: once a day, 13:00 UTC (06:00 PT). Daily is the right
+# cadence for variables-tier config — Railway env var changes are
+# deliberate operator actions, low-frequency. Hourly would risk
+# Railway API rate-limit surprises and is overkill for the change rate.
+#
+# Issue-on-failure: drift triggers a priority-high issue, mirroring
+# .github/workflows/e2e-staging-sanity.yml's pattern. Drift is
+# medium-priority "config slipped, fix at next ops window," not
+# active-outage paging.
+#
+# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
+# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN
+# (silent-success on schedule was the failure-mode class that bit the
+# team before; cron firing without checking anything is worse than no
+# cron). The workflow_dispatch trigger SOFT-SKIPS on missing secret so
+# an operator can dry-run the workflow shape during initial provisioning
+# without tripping a fake red.
+
+on:
+  schedule:
+    - cron: '0 13 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: railway-pin-audit
+  cancel-in-progress: false
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  audit:
+    name: Audit Railway env vars for drift-prone pins
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify RAILWAY_AUDIT_TOKEN present
+        # Schedule trigger: hard-fail when the secret is missing —
+        # otherwise the cron silently runs against the wrong scope (or
+        # exits 2 from the script and we issue-spam) without anyone
+        # noticing the token rot.
+        # Dispatch trigger: soft-skip — operator may be dry-running the
+        # workflow shape before provisioning the secret. Logged as a
+        # workflow notice, not a failure.
+        env:
+          RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+          EVENT_NAME: ${{ github.event_name }}
+        id: secret_check
+        run: |
+          set -euo pipefail
+          if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
+            echo "have_secret=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "have_secret=false" >> "$GITHUB_OUTPUT"
+          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
+            echo "::notice::RAILWAY_AUDIT_TOKEN not configured — soft-skipping (manual dispatch)"
+            exit 0
+          fi
+          echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
+          exit 1
+
+      - name: Install Railway CLI
+        if: steps.secret_check.outputs.have_secret == 'true'
+        # Pinned hash matching the public install instructions; bump in
+        # tandem with the audit-script's documented Railway CLI version.
+        run: |
+          set -euo pipefail
+          curl -fsSL https://railway.com/install.sh | sh
+          # The installer drops the binary in ~/.railway/bin
+          echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
+
+      - name: Verify Railway CLI authenticated
+        if: steps.secret_check.outputs.have_secret == 'true'
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set -euo pipefail
+          # `railway whoami` exits non-zero when the token is
+          # unauthenticated or doesn't have any project access.
+          if ! railway whoami >/dev/null 2>&1; then
+            echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
+            exit 2
+          fi
+
+      - name: Link molecule-platform project
+        if: steps.secret_check.outputs.have_secret == 'true'
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        # Project ID from reference_production_stack: molecule-platform
+        # / 7ccc8c68-61f4-42ab-9be5-586eeee11768. Linking is per-process,
+        # so we re-link in this CI shell (the audit script comment says
+        # it deliberately doesn't chdir for you because the linked
+        # project's identity matters).
+        run: |
+          set -euo pipefail
+          railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
+
+      - name: Run drift audit
+        if: steps.secret_check.outputs.have_secret == 'true'
+        id: audit
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set +e
+          bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
+          rc=${PIPESTATUS[0]}
+          echo "rc=$rc" >> "$GITHUB_OUTPUT"
+          # Capture the audit log for the issue body.
+          {
+            echo 'log<<AUDIT_EOF'
+            cat /tmp/audit.log
+            echo 'AUDIT_EOF'
+          } >> "$GITHUB_OUTPUT"
+          # Exit codes from the script:
+          #   0 — no drift; workflow goes green
+          #   1 — drift detected; we'll file an issue and fail the run
+          #   2 — railway CLI unauthenticated / project unlinked; fail
+          # Anything else: also fail.
+          case "$rc" in
+            0) exit 0 ;;
+            1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
+            2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
+            *) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
+          esac
+
+      - name: Open / update drift issue
+        if: failure() && steps.audit.outputs.rc == '1'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          AUDIT_LOG: ${{ steps.audit.outputs.log }}
+        with:
+          script: |
+            const title = "🚨 Railway env-var drift detected";
+            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const body =
+              `Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n` +
+              `**What this means:** an env var (likely on \`controlplane\`) is pinned to a SHA-shaped or semver tag instead of a floating tag. ` +
+              `Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service doesn't pick them up.\n\n` +
+              `**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (\`:staging-latest\`, \`:main\`) unless the pin is intentional and documented in the ops runbook.\n\n` +
+              `**Audit output:**\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\`\n\n` +
+              `Run: ${runURL}\n\n` +
+              `Closes automatically when a subsequent daily run reports clean.`;
+
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'railway-drift',
+            });
+            const match = existing.find(i => i.title === title);
+            if (match) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: match.number,
+                body: `Still drifting. ${runURL}\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\``,
+              });
+            } else {
+              await github.rest.issues.create({
+                owner: context.repo.owner, repo: context.repo.repo,
+                title, body,
+                labels: ['railway-drift', 'bug', 'priority-high'],
+              });
+            }
+
+      - name: Close stale drift issue on clean run
+        # When a previously-flagged drift gets fixed by an operator,
+        # the next daily run goes green. Close any open `railway-drift`
+        # issue with a confirmation comment so the queue doesn't carry
+        # stale ones.
+        if: success() && steps.audit.outputs.rc == '0'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'railway-drift',
+            });
+            for (const issue of existing) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: issue.number,
+                body: `Daily audit clean — drift resolved. ${runURL}`,
+              });
+              await github.rest.issues.update({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: issue.number,
+                state: 'closed',
+                state_reason: 'completed',
+              });
+            }

.github/workflows/redeploy-tenants-on-main.yml

@@ -3,22 +3,28 @@ name: redeploy-tenants-on-main
 # Auto-refresh prod tenant EC2s after every main merge.
 #
 # Why this workflow exists: publish-workspace-server-image builds and
-# pushes a new platform-tenant:latest + :<sha> to GHCR on every merge
-# to main, but running tenants pulled their image once at boot and
-# never re-pull. Users see stale code indefinitely.
+# pushes a new platform-tenant :<sha> to ECR on every merge to main,
+# but running tenants pulled their image once at boot and never re-pull.
+# Users see stale code indefinitely.
 #
 # This workflow closes the gap by calling the control-plane admin
 # endpoint that performs a canary-first, batched, health-gated rolling
-# redeploy across every live tenant. Implemented in Molecule-AI/
+# redeploy across every live tenant. Implemented in molecule-ai/
 # molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
 # (feat/tenant-auto-redeploy, landing alongside this workflow).
 #
+# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
+# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
+# Gitea suspension migration. The canary-verify.yml promote step now
+# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
+#
 # Runtime ordering:
-#   1. publish-workspace-server-image completes → new :latest in GHCR.
-#   2. This workflow fires via workflow_run, waits 30s for GHCR's
-#      CDN to propagate the new tag to the region the tenants pull from.
-#   3. Calls redeploy-fleet with canary_slug=hongmingwang and a 60s
-#      soak. Canary proves the image boots; batches follow.
+#   1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
+#   2. This workflow fires via workflow_run, calls redeploy-fleet with
+#      target_tag=staging-<sha>. No CDN propagation wait needed —
+#      ECR image manifest is consistent immediately after push.
+#   3. Calls redeploy-fleet with canary_slug (if set) and a soak
+#      period. Canary proves the image boots; batches follow.
 #   4. Any failure aborts the rollout and leaves older tenants on the
 #      prior image — safer default than half-and-half state.
 #
@@ -34,15 +40,34 @@ on:
   workflow_dispatch:
     inputs:
       target_tag:
-        description: 'Tenant image tag to deploy (e.g. "latest" or "a59f1a6c"). Defaults to latest when empty.'
+        # Empty default → auto-trigger and dispatch-without-input both
+        # resolve to `staging-<short_head_sha>` (the digest publish-image
+        # just pushed). Pre-fix this defaulted to 'latest', which only
+        # gets retagged by canary-verify's promote-to-latest job — and
+        # that job soft-skips when CANARY_TENANT_URLS is unset (the
+        # current state, until Phase 2 canary fleet is live). Result:
+        # `:latest` had been pinned to a 4-day-old digest (2026-04-28)
+        # while every main push pushed fresh `staging-<sha>` images;
+        # every prod redeploy pulled the stale `:latest` and the verify
+        # step correctly flagged 3/3 tenants STALE. Pulling the
+        # just-published `staging-<sha>` directly skips the dead retag
+        # path. When canary fleet is real, this workflow should chain
+        # on canary-verify completion (workflow_run from canary-verify),
+        # not publish-image — separate, smaller PR.
+        description: 'Tenant image tag to deploy (e.g. "latest", "staging-a59f1a6c"). Empty = auto staging-<head_sha>.'
         required: false
         type: string
-        default: 'latest'
+        default: ''
       canary_slug:
         description: 'Tenant slug to deploy first + soak (empty = skip canary, fan out immediately).'
         required: false
         type: string
-        default: 'hongmingwang'
+        # Must be an actual prod tenant slug (current: hongming,
+        # chloe-dong, reno-stars). The previous default 'hongmingwang'
+        # didn't match any tenant — CP soft-skipped the missing canary
+        # and the fleet rolled out without the soak gate, defeating the
+        # whole point of canary-first.
+        default: 'hongming'
       soak_seconds:
         description: 'Seconds to wait after canary before fanning out.'
         required: false
@@ -64,6 +89,20 @@ permissions:
   # No write scopes needed — the workflow hits an external CP endpoint,
   # not the GitHub API.
 
+# Serialize redeploys so two rapid main pushes' redeploys don't overlap
+# and cause confusing per-tenant SSM state. Without this, GitHub's
+# implicit workflow_run queueing would *probably* serialize them, but
+# the explicit block makes the invariant defensible. Mirrors the
+# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
+#
+# cancel-in-progress: false → aborting a half-rolled-out fleet would
+# leave tenants stuck on whatever image they happened to be on when
+# cancelled. Better to finish the in-flight rollout before starting
+# the next one.
+concurrency:
+  group: redeploy-tenants-on-main
+  cancel-in-progress: false
+
 jobs:
   redeploy:
     # Skip the auto-trigger if publish-workspace-server-image didn't
@@ -75,24 +114,50 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 25
     steps:
-      - name: Wait for GHCR tag propagation
-        # GHCR's edge cache takes ~15-30s to consistently serve the new
-        # :latest manifest after the registry accepts the push. Without
-        # this sleep, the first tenant's docker pull sometimes races
-        # and fetches the previous digest; sleeping is the cheapest
-        # way to reduce that without polling GHCR for the new digest.
-        run: sleep 30
+      - name: Note on ECR propagation
+        # ECR image manifests are consistent immediately after push — no
+        # CDN cache to wait for. The old GHCR-based workflow had a 30s
+        # sleep to avoid race conditions; ECR makes that unnecessary.
+        run: echo "ECR image available immediately after push — proceeding."
+
+      - name: Compute target tag
+        id: tag
+        # Resolution order:
+        #   1. Operator-supplied input (workflow_dispatch with explicit
+        #      tag) → used verbatim. Lets ops pin `latest` for emergency
+        #      rollback to last canary-verified digest, or pin a specific
+        #      `staging-<sha>` to roll back to a known-good build.
+        #   2. Default → `staging-<short_head_sha>`. The just-published
+        #      digest. Bypasses the `:latest` retag path that's currently
+        #      dead (canary-verify soft-skips without canary fleet, so
+        #      the only thing retagging `:latest` today is the manual
+        #      promote-latest.yml — last run 2026-04-28). Auto-trigger
+        #      from workflow_run uses workflow_run.head_sha; manual
+        #      dispatch with no input falls through to github.sha.
+        env:
+          INPUT_TAG: ${{ inputs.target_tag }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+        run: |
+          set -euo pipefail
+          if [ -n "${INPUT_TAG:-}" ]; then
+            echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
+            echo "Using operator-pinned tag: $INPUT_TAG"
+          else
+            SHORT="${HEAD_SHA:0:7}"
+            echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
+            echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
+          fi
 
       - name: Call CP redeploy-fleet
         # CP_ADMIN_API_TOKEN must be set as a repo/org secret on
-        # Molecule-AI/molecule-core, matching the staging/prod CP's
+        # molecule-ai/molecule-core, matching the staging/prod CP's
         # CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
         # repo's secrets for CI.
         env:
           CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
           CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
-          TARGET_TAG: ${{ inputs.target_tag || 'latest' }}
-          CANARY_SLUG: ${{ inputs.canary_slug || 'hongmingwang' }}
+          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
+          CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
           SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
           BATCH_SIZE: ${{ inputs.batch_size || '3' }}
           DRY_RUN: ${{ inputs.dry_run || false }}
@@ -123,12 +188,29 @@ jobs:
           echo "  body: $BODY"
 
           HTTP_RESPONSE=$(mktemp)
-          HTTP_CODE=$(curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+          HTTP_CODE_FILE=$(mktemp)
+          # Route -w into its own tempfile so curl's exit code (e.g. 56
+          # on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
+          # pollute the captured stdout. The previous inline-substitution
+          # shape produced "000000" on connection reset (curl wrote
+          # "000" via -w, then the inline echo-fallback appended another
+          # "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
+          # set +e/-e keeps the non-zero curl exit from tripping the
+          # outer pipeline. See lint-curl-status-capture.yml for the
+          # CI gate that pins this fix shape.
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
             -m 1200 \
             -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
             -H "Content-Type: application/json" \
             -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
-            -d "$BODY" || echo "000")
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          set -e
+          # Stderr from curl (e.g. dial errors with -sS) goes to the runner
+          # log so operators can see WHY a connection failed. Stdout is
+          # captured to $HTTP_CODE_FILE because that's where -w writes.
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
 
           echo "HTTP $HTTP_CODE"
           cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
@@ -161,4 +243,158 @@ jobs:
             echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
             exit 1
           fi
-          echo "::notice::Tenant fleet redeploy complete."
+          echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
+
+          # Stash the response for the verify step. $RUNNER_TEMP outlasts
+          # the step boundary; $HTTP_RESPONSE doesn't.
+          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
+
+      - name: Verify each tenant /buildinfo matches published SHA
+        # ROOT FIX FOR #2395.
+        #
+        # `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
+        # didn't error" — NOT "the new image is running on the tenant."
+        # `:latest` lives in the local Docker daemon's image cache; if
+        # the SSM document does `docker compose up -d` without an
+        # explicit `docker pull`, the daemon serves the previously-
+        # cached digest and the container restarts on stale code.
+        # 2026-04-30 incident: hongmingwang's tenant reported
+        # ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
+        # chat_files for 30+ min — the lazy-heal fix never reached the
+        # user despite green deploy + green redeploy.
+        #
+        # This step closes the gap by curling each tenant's /buildinfo
+        # endpoint (added in workspace-server/internal/buildinfo +
+        # /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
+        # returned git_sha to the SHA the workflow expects. Mismatches
+        # fail the workflow, which is what `ok=true` should have
+        # guaranteed all along.
+        #
+        # When the redeploy was triggered by workflow_dispatch with a
+        # specific tag (target_tag != "latest"), the expected SHA may
+        # not equal ${{ github.sha }} — in that case we resolve via
+        # GHCR's manifest. For workflow_run (default :latest) the
+        # workflow_run.head_sha is the SHA that just published.
+        env:
+          EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
+          # Tenant subdomain template — slugs from the response are
+          # appended. Production CP issues `<slug>.moleculesai.app`;
+          # staging CP issues `<slug>.staging.moleculesai.app`. This
+          # workflow runs on main → prod CP → no `staging.` infix.
+          TENANT_DOMAIN: 'moleculesai.app'
+        run: |
+          set -euo pipefail
+
+          EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
+          if [ "$TARGET_TAG" != "latest" ] \
+             && [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
+             && [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
+            # workflow_dispatch with a pinned tag that isn't the head
+            # SHA — operator is rolling back / pinning. Skip the
+            # verification because we don't have the expected SHA in
+            # this context (would need to crane-inspect the GHCR
+            # manifest, which is a follow-up). Failing-open here is
+            # safe: the operator chose the tag deliberately.
+            #
+            # `staging-<short_head_sha>` IS verified — it's the new
+            # auto-trigger default (see Compute target tag step) and
+            # the digest under that tag SHOULD match EXPECTED_SHA.
+            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
+            exit 0
+          fi
+
+          RESP="$RUNNER_TEMP/redeploy-response.json"
+          if [ ! -s "$RESP" ]; then
+            echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
+            exit 1
+          fi
+
+          # Pull only successfully-redeployed tenants. Any tenant that
+          # halted the rollout already failed the previous step, so we
+          # don't double-count them here.
+          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
+          if [ ${#SLUGS[@]} -eq 0 ]; then
+            echo "::warning::No tenants reported healthz_ok — nothing to verify"
+            exit 0
+          fi
+
+          echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
+
+          # Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
+          # vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
+          # comment for the full rationale; same logic applies on prod even
+          # though prod has fewer ephemeral tenants — the asymmetry would be a
+          # gratuitous fork.
+          STALE_COUNT=0
+          UNREACHABLE_COUNT=0
+          STALE_LINES=()
+          UNREACHABLE_LINES=()
+          for slug in "${SLUGS[@]}"; do
+            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
+            # 30s total: tenant just SSM-restarted, may still be coming
+            # up. Retry-on-empty rather than retry-on-status — we want
+            # to fail fast on "responded with wrong SHA", not "still
+            # warming up".
+            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
+            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
+            if [ -z "$ACTUAL_SHA" ]; then
+              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
+              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
+              continue
+            fi
+            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
+              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
+            else
+              STALE_COUNT=$((STALE_COUNT + 1))
+              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
+            fi
+          done
+
+          {
+            echo ""
+            echo "### Per-tenant /buildinfo verification"
+            echo ""
+            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
+            echo ""
+            if [ $STALE_COUNT -gt 0 ]; then
+              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${STALE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $UNREACHABLE_COUNT -gt 0 ]; then
+              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
+              echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ $UNREACHABLE_COUNT -gt 0 ]; then
+            echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
+          fi
+
+          # Belt-and-suspenders sanity floor: same logic as the staging
+          # variant — see that file's comment for the full rationale.
+          # Floor only applies when fleet >= 4; below that, canary-verify
+          # is the actual gate.
+          TOTAL_VERIFIED=${#SLUGS[@]}
+          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
+            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
+            exit 1
+          fi
+
+          if [ $STALE_COUNT -gt 0 ]; then
+            echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
+            exit 1
+          fi
+
+          echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."

.github/workflows/redeploy-tenants-on-staging.yml

@@ -0,0 +1,362 @@
+name: redeploy-tenants-on-staging
+
+# Auto-refresh staging tenant EC2s after every staging-branch merge.
+#
+# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
+# the :staging-latest tag. Sister workflow exists for prod (rolls
+# :latest after canary-verify). Both share the same shape — just
+# different CP_URL + target_tag + admin token secret.
+#
+# Why this workflow exists: publish-workspace-server-image now builds
+# on every staging-branch push (PR #2335), pushing
+# platform-tenant:staging-latest to GHCR. Existing tenants pulled
+# their image once at boot and never re-pull, so the new image just
+# sits unused until the tenant is reprovisioned.
+#
+# This workflow closes the gap by calling staging-CP's
+# /cp/admin/tenants/redeploy-fleet, which performs a canary-first,
+# batched, health-gated SSM redeploy across every live staging tenant.
+# Same endpoint shape as prod CP — only the host differs.
+#
+# Runtime ordering:
+#   1. publish-workspace-server-image completes on staging branch →
+#      new :staging-latest in GHCR.
+#   2. This workflow fires via workflow_run, waits 30s for GHCR's CDN
+#      to propagate the new tag.
+#   3. Calls redeploy-fleet with no canary (staging IS canary; we don't
+#      need a sub-canary inside it). Soak still applies to the first
+#      tenant in case of bad-deploy detection.
+#   4. Any failure aborts the rollout and leaves older tenants on the
+#      prior image — safer default than half-and-half state.
+#
+# Rollback path: re-run with workflow_dispatch + target_tag=staging-<sha>
+# of a known-good build.
+
+on:
+  workflow_run:
+    workflows: ['publish-workspace-server-image']
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      target_tag:
+        description: 'Tenant image tag to deploy (e.g. "staging-latest" or "staging-a59f1a6c"). Defaults to staging-latest when empty.'
+        required: false
+        type: string
+        default: 'staging-latest'
+      canary_slug:
+        description: 'Tenant slug to deploy first + soak (empty = skip canary, fan out immediately). Default empty for staging since staging itself is the canary.'
+        required: false
+        type: string
+        default: ''
+      soak_seconds:
+        description: 'Seconds to wait after canary before fanning out. Only meaningful if canary_slug is set.'
+        required: false
+        type: string
+        default: '60'
+      batch_size:
+        description: 'How many tenants SSM redeploys in parallel per batch.'
+        required: false
+        type: string
+        default: '3'
+      dry_run:
+        description: 'Plan only — do not actually redeploy.'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  # No write scopes needed — the workflow hits an external CP endpoint,
+  # not the GitHub API.
+
+# Serialize per-branch so two rapid staging pushes' redeploys don't
+# overlap and cause confusing per-tenant SSM state. cancel-in-progress
+# is false because aborting a half-rolled-out fleet leaves tenants
+# stuck on whatever image they happened to be on when cancelled.
+concurrency:
+  group: redeploy-tenants-on-staging
+  cancel-in-progress: false
+
+jobs:
+  redeploy:
+    # Skip the auto-trigger if publish-workspace-server-image didn't
+    # actually succeed. workflow_run fires on any completion state; we
+    # don't want to redeploy against a half-built image.
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Wait for GHCR tag propagation
+        # GHCR's edge cache takes ~15-30s to consistently serve the new
+        # :staging-latest manifest after the registry accepts the push.
+        # Same rationale as redeploy-tenants-on-main.yml.
+        run: sleep 30
+
+      - name: Call staging-CP redeploy-fleet
+        # CP_STAGING_ADMIN_API_TOKEN must be set as a repo/org secret
+        # on molecule-ai/molecule-core, matching staging-CP's
+        # CP_ADMIN_API_TOKEN env var (visible in Railway controlplane
+        # / staging environment). Stored separately from the prod
+        # CP_ADMIN_API_TOKEN so a leak of one doesn't auth the other.
+        env:
+          CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
+          CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+          TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
+          CANARY_SLUG: ${{ inputs.canary_slug || '' }}
+          SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
+          BATCH_SIZE: ${{ inputs.batch_size || '3' }}
+          DRY_RUN: ${{ inputs.dry_run || false }}
+        run: |
+          set -euo pipefail
+
+          # Schedule-vs-dispatch hardening (mirrors sweep-cf-orphans
+          # and sweep-cf-tunnels): hard-fail on auto-trigger when the
+          # secret is missing so a misconfigured-repo doesn't silently
+          # serve stale staging tenants. Soft-skip on operator dispatch.
+          if [ -z "${CP_STAGING_ADMIN_API_TOKEN:-}" ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::CP_STAGING_ADMIN_API_TOKEN secret not set — skipping redeploy"
+              echo "::warning::Set CP_STAGING_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
+              echo "::notice::Pull the value from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+              exit 0
+            fi
+            echo "::error::staging redeploy cannot run — CP_STAGING_ADMIN_API_TOKEN secret missing"
+            echo "::error::set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+            exit 1
+          fi
+
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --arg canary "$CANARY_SLUG" \
+            --argjson soak "$SOAK_SECONDS" \
+            --argjson batch "$BATCH_SIZE" \
+            --argjson dry "$DRY_RUN" \
+            '{
+              target_tag: $tag,
+              canary_slug: $canary,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          # Route -w into its own tempfile so curl's exit code (e.g. 56
+          # on connection-reset) can't pollute the captured stdout. The
+          # previous inline-substitution shape produced "000000" on
+          # connection reset — caught on main variant 2026-05-04
+          # redeploying sha 2b862f6. Same fix shape as the synth-E2E
+          # §9c gate (PR #2797). See lint-curl-status-capture.yml for
+          # the CI gate that pins this fix shape.
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          set -e
+          # Stderr from curl (-sS shows dial errors etc.) goes to the
+          # runner log so operators can see WHY a connection failed.
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          {
+            echo "## Staging tenant redeploy fleet"
+            echo ""
+            echo "**Target tag:** \`$TARGET_TAG\`"
+            echo "**Canary:** \`${CANARY_SLUG:-(none — staging is itself the canary)}\` (soak ${SOAK_SECONDS}s)"
+            echo "**Batch size:** $BATCH_SIZE"
+            echo "**Dry run:** $DRY_RUN"
+            echo "**HTTP:** $HTTP_CODE"
+            echo ""
+            echo "### Per-tenant result"
+            echo ""
+            echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
+            echo '|------|-------|------------|------|---------|-------|'
+            jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          # Distinguish "real fleet failure" from "E2E teardown race".
+          #
+          # CP returns HTTP 500 + ok=false whenever ANY tenant in the
+          # fleet failed SSM or healthz. In practice the recurring source
+          # of these is ephemeral test tenants being torn down by their
+          # parent E2E run mid-redeploy: the EC2 dies → SSM exit=2 or
+          # healthz timeout → CP marks the fleet failed → this workflow
+          # goes red even though every operator-facing tenant rolled fine.
+          #
+          # Ephemeral slug prefixes (kept in sync with sweep-stale-e2e-orgs.yml
+          # — see that file for the source-of-truth list and rationale):
+          #   - e2e-*       — canvas/saas/ext E2E suites
+          #   - rt-e2e-*    — runtime-test harness fixtures (RFC #2251)
+          # Long-lived prefixes that are NOT ephemeral and MUST hard-fail:
+          # demo-prep, dryrun-*, dryrun2-*, plus all human tenant slugs.
+          #
+          # Filter: if HTTP=500/ok=false AND every failed slug matches an
+          # ephemeral prefix, treat as soft-warn and let the verify step
+          # downstream handle unreachable-vs-stale (#2402). Any non-ephemeral
+          # failure or a non-500 HTTP response remains a hard failure.
+          OK=$(jq -r '.ok // "false"' "$HTTP_RESPONSE")
+          FAILED_SLUGS=$(jq -r '
+            .results[]?
+            | select((.healthz_ok != true) or (.ssm_status != "Success"))
+            | .slug' "$HTTP_RESPONSE" 2>/dev/null || true)
+          EPHEMERAL_PREFIX_RE='^(e2e-|rt-e2e-)'
+          NON_EPHEMERAL_FAILED=$(printf '%s\n' "$FAILED_SLUGS" | grep -v '^$' | grep -Ev "$EPHEMERAL_PREFIX_RE" || true)
+
+          if [ "$HTTP_CODE" = "200" ] && [ "$OK" = "true" ]; then
+            : # happy path — fall through to verification
+          elif [ "$HTTP_CODE" = "500" ] && [ -z "$NON_EPHEMERAL_FAILED" ] && [ -n "$FAILED_SLUGS" ]; then
+            COUNT=$(printf '%s\n' "$FAILED_SLUGS" | grep -Ec "$EPHEMERAL_PREFIX_RE" || true)
+            echo "::warning::redeploy-fleet returned HTTP 500 but every failed tenant ($COUNT) is ephemeral (e2e-*/rt-e2e-*) — treating as teardown race, soft-warning."
+            printf '%s\n' "$FAILED_SLUGS" | sed 's/^/::warning::  failed: /'
+          elif [ "$HTTP_CODE" != "200" ]; then
+            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
+            if [ -n "$NON_EPHEMERAL_FAILED" ]; then
+              echo "::error::non-ephemeral tenant(s) failed:"
+              printf '%s\n' "$NON_EPHEMERAL_FAILED" | sed 's/^/::error::  /'
+            fi
+            exit 1
+          else
+            # HTTP=200 but ok=false (shouldn't happen with current CP
+            # but keep the gate for completeness).
+            echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
+            exit 1
+          fi
+          echo "::notice::Staging tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
+
+          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
+
+      - name: Verify each staging tenant /buildinfo matches published SHA
+        # Mirror of the verify step in redeploy-tenants-on-main.yml — see
+        # there for the rationale (#2395 root fix). Staging has the same
+        # ssm_status-success-but-stale-image hazard and benefits from the
+        # same gate. Diff: TENANT_DOMAIN includes the `staging.` infix.
+        env:
+          EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
+          TENANT_DOMAIN: 'staging.moleculesai.app'
+        run: |
+          set -euo pipefail
+
+          # staging-latest is the staging-side moving tag; treat it the
+          # same way main treats `latest`. Operator-pinned SHAs skip
+          # verification (see main variant for why).
+          if [ "$TARGET_TAG" != "staging-latest" ] && [ "$TARGET_TAG" != "latest" ] && [ "$TARGET_TAG" != "$EXPECTED_SHA" ]; then
+            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
+            exit 0
+          fi
+
+          RESP="$RUNNER_TEMP/redeploy-response.json"
+          if [ ! -s "$RESP" ]; then
+            echo "::error::redeploy-response.json missing or empty"
+            exit 1
+          fi
+
+          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
+          if [ ${#SLUGS[@]} -eq 0 ]; then
+            echo "::warning::No staging tenants reported healthz_ok — nothing to verify"
+            exit 0
+          fi
+
+          echo "Verifying ${#SLUGS[@]} staging tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
+
+          # Two distinct failure modes here:
+          #   STALE_COUNT      — tenant returned a SHA that doesn't match. THIS is
+          #                      the #2395 bug class: tenant up + serving old code.
+          #                      Always hard-fail the workflow.
+          #   UNREACHABLE_COUNT — tenant didn't respond. Almost always a benign
+          #                      teardown race: redeploy-fleet snapshot says
+          #                      healthz_ok=true, then the E2E suite tears the
+          #                      ephemeral tenant down before this step runs (the
+          #                      e2e-* fixtures churn 5-10/hour on staging). Soft-
+          #                      warn so we don't block staging→main on cleanup.
+          #                      Real "tenant up but unreachable" is caught by CP's
+          #                      own healthz monitor + the post-redeploy alert; we
+          #                      don't need to double-count it here.
+          STALE_COUNT=0
+          UNREACHABLE_COUNT=0
+          STALE_LINES=()
+          UNREACHABLE_LINES=()
+          for slug in "${SLUGS[@]}"; do
+            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
+            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
+            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
+            if [ -z "$ACTUAL_SHA" ]; then
+              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
+              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
+              continue
+            fi
+            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
+              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
+            else
+              STALE_COUNT=$((STALE_COUNT + 1))
+              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
+            fi
+          done
+
+          {
+            echo ""
+            echo "### Per-tenant /buildinfo verification (staging)"
+            echo ""
+            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
+            echo ""
+            if [ $STALE_COUNT -gt 0 ]; then
+              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${STALE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $UNREACHABLE_COUNT -gt 0 ]; then
+              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely E2E teardown race (soft-warn, not failing):**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
+              echo "All ${#SLUGS[@]} staging tenants returned matching SHA. ✓"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ $UNREACHABLE_COUNT -gt 0 ]; then
+            echo "::warning::$UNREACHABLE_COUNT staging tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
+          fi
+
+          # Belt-and-suspenders sanity floor: if MORE than half the fleet is
+          # unreachable AND the fleet is large enough that "half down" is
+          # statistically meaningful, this is a real outage (e.g. new image
+          # crashes on startup), not a teardown race. Hard-fail.
+          #
+          # Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
+          # canary-verify step is the actual gate for "all tenants down"
+          # detection (it runs against the canary first and aborts the
+          # rollout if the canary fails to come up). Without the >=4 gate,
+          # a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
+          # quiet staging push) would re-flake on the exact teardown-race
+          # condition #2402 fixed: 1 of 1 unreachable = 100% > 50% → fail.
+          TOTAL_VERIFIED=${#SLUGS[@]}
+          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
+            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED staging tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
+            exit 1
+          fi
+
+          if [ $STALE_COUNT -gt 0 ]; then
+            echo "::error::$STALE_COUNT staging tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
+            exit 1
+          fi
+
+          echo "::notice::Staging tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."

.github/workflows/retarget-main-to-staging.yml

@@ -1,94 +0,0 @@
-name: Retarget main PRs to staging
-
-# Mechanical enforcement of SHARED_RULES rule 8 ("Staging-first workflow, no
-# exceptions"). When a bot opens a PR against main, retarget it to staging
-# automatically and leave an explanatory comment. Human CEO-authored PRs (the
-# staging→main promotion PR, etc.) are left alone — they're the authorised
-# exception to the rule.
-#
-# Why an Action instead of only a prompt rule: prompt rules depend on every
-# role's system-prompt.md staying in sync. Today 5 of 8 engineer roles
-# (core-be, core-fe, app-fe, app-qa, devops-engineer) don't have the
-# staging-first section — the bot keeps opening PRs to main. An Action
-# enforces the invariant regardless of prompt drift.
-
-on:
-  pull_request_target:
-    types: [opened, reopened]
-    branches: [main]
-
-permissions:
-  pull-requests: write
-
-jobs:
-  retarget:
-    name: Retarget to staging
-    runs-on: ubuntu-latest
-    # Only fire for bot-authored PRs. Human CEO PRs (staging→main promotion)
-    # are intentional and pass through.
-    if: >-
-      github.event.pull_request.user.type == 'Bot'
-      || endsWith(github.event.pull_request.user.login, '[bot]')
-      || github.event.pull_request.user.login == 'app/molecule-ai'
-      || github.event.pull_request.user.login == 'molecule-ai[bot]'
-    steps:
-      - name: Retarget PR base to staging
-        id: retarget
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
-        # Issue #1884: when the bot opens a PR against main and there's
-        # already another PR on the same head branch targeting staging,
-        # GitHub's PATCH /pulls returns 422 with
-        # "A pull request already exists for base branch 'staging' …".
-        # The retarget can't proceed — but the right response is to
-        # close the now-redundant main-PR, not to fail the workflow
-        # noisily. Detect that specific 422 and close instead.
-        run: |
-          set +e
-          echo "Retargeting PR #${PR_NUMBER} (author: ${PR_AUTHOR}) from main → staging"
-          PATCH_OUTPUT=$(gh api -X PATCH \
-            "repos/${{ github.repository }}/pulls/${PR_NUMBER}" \
-            -f base=staging \
-            --jq '.base.ref' 2>&1)
-          PATCH_EXIT=$?
-          set -e
-          if [ "$PATCH_EXIT" -eq 0 ]; then
-            echo "::notice::Retargeted PR #${PR_NUMBER} → staging"
-            echo "outcome=retargeted" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          # Specifically match the 422 duplicate-base/head error so
-          # any OTHER PATCH failure (auth, deleted PR, etc.) still
-          # surfaces as a real workflow failure.
-          if echo "$PATCH_OUTPUT" | grep -q "pull request already exists for base branch 'staging'"; then
-            echo "::notice::PR #${PR_NUMBER}: duplicate target-staging PR exists on same head — closing this main-PR as redundant."
-            gh pr close "$PR_NUMBER" \
-              --repo "${{ github.repository }}" \
-              --comment "[retarget-bot] Closing — another PR on the same head branch already targets \`staging\`. This PR is redundant. See issue #1884 for the rationale."
-            echo "outcome=closed-as-duplicate" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          echo "::error::Retarget PATCH failed and was NOT a duplicate-base error:"
-          echo "$PATCH_OUTPUT" >&2
-          exit 1
-
-      - name: Post explainer comment
-        if: steps.retarget.outputs.outcome == 'retargeted'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: |
-          gh pr comment "$PR_NUMBER" \
-            --repo "${{ github.repository }}" \
-            --body "$(cat <<'BODY'
-          [retarget-bot] This PR was opened against `main` and has been retargeted to `staging` automatically.
-
-          **Why:** per [SHARED_RULES rule 8](https://github.com/Molecule-AI/molecule-ai-org-template-molecule-dev/blob/main/SHARED_RULES.md), all feature work targets `staging` first; the CEO promotes `staging → main` separately.
-
-          **What changed:** just the base branch — no code change. CI will re-run against `staging`. If you get merge conflicts, rebase on `staging`.
-
-          **If this PR is the CEO's staging→main promotion:** the Action skipped you (only bot-authored PRs are retargeted). If you see this comment on your CEO PR, that's a bug — please tag @HongmingWang-Rabbit.
-          BODY
-          )"

.github/workflows/runtime-pin-compat.yml

@@ -10,15 +10,29 @@ name: Runtime Pin Compatibility
 #   4. Every tenant workspace crashed; the canary tenant caught it but
 #      only after 5 hours of degraded staging
 #
-# This workflow installs the runtime in a fresh Python venv from PyPI
-# and tries the same import the EC2 user-data does. If pip resolution
-# silently produces a broken combo, this gate fails before the tenant
-# image gets published.
+# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
+# top of `workspace/requirements.txt` and smoke-imports. Catches:
+#   - Upstream PyPI yanks
+#   - Bad re-releases of molecule-ai-workspace-runtime
+#   - Already-shipped wheels that stop importing because a transitive
+#     dep moved underneath
+#
+# This is the "PyPI artifact health" half of pin compatibility. The
+# companion workflow `runtime-prbuild-compat.yml` covers the
+# "PR-introduced breakage" half by building the wheel from THIS PR's
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter — the pypi-latest job no longer fires on doc-only
+# workspace/ edits whose content can't change what's currently on PyPI.
 
 on:
   push:
     branches: [main, staging]
     paths:
+      # Narrow filter: pypi-latest is sensitive only to changes that
+      # affect what we're INSTALLING (requirements.txt) or WHAT THE
+      # CHECK ITSELF DOES (this workflow file). Edits to workspace/
+      # source code don't change what's on PyPI right now, so they
+      # don't change this gate's verdict.
       - 'workspace/requirements.txt'
       - '.github/workflows/runtime-pin-compat.yml'
   pull_request:
@@ -42,12 +56,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  default-install:
-    name: Default install + import smoke
+  pypi-latest-install:
+    name: PyPI-latest install + import smoke
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: '3.11'
           cache: pip

.github/workflows/runtime-prbuild-compat.yml

@@ -0,0 +1,152 @@
+name: Runtime PR-Built Compatibility
+
+# Companion to `runtime-pin-compat.yml`. That workflow tests what's
+# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
+# PUBLISHED if THIS PR merges.
+#
+# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
+# wheel" job to the original runtime-pin-compat.yml, but both jobs
+# shared a `paths:` filter that was the union of their needs
+# (`workspace/**`). That meant the PyPI-latest job ran on every doc
+# edit even though the upstream PyPI artifact can't change with our
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter that matches the inputs it actually depends on.
+#
+# Catches the failure mode where a PR adds an import requiring a newer
+# SDK than `workspace/requirements.txt` pins:
+#   1. Pip resolves the existing PyPI wheel + the old SDK pin → smoke
+#      passes (it imports the OLD main.py from the wheel, not the PR's
+#      new main.py).
+#   2. Merge → publish-runtime.yml ships a wheel WITH the new import.
+#   3. Tenant images redeploy → all crash on first boot with
+#      ImportError.
+#
+# By building from the PR's source and smoke-importing THAT wheel, we
+# fail at PR-time instead of after publish.
+#
+# Required-check shape (2026-05-01): the workflow runs on EVERY push +
+# PR + merge_group event with no top-level `paths:` filter, then uses a
+# detect-changes job + per-step `if:` gates inside ONE always-running
+# job named `PR-built wheel + import smoke`. PRs that don't touch
+# wheel-relevant paths get a no-op SUCCESS check run, satisfying branch
+# protection without re-running the heavy build. Same pattern as
+# e2e-api.yml — see its comment for the full rationale + the 2026-04-29
+# PR #2264 incident that motivated the always-run-with-if-gates shape.
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  workflow_dispatch:
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  # Include event_name so a PR sync (event=pull_request) and the
+  # subsequent staging push (event=push) on the SAME merge SHA don't
+  # collide in one group. Without event_name, both runs hashed to
+  # the same key and cancel-in-progress=true cancelled whichever
+  # arrived second — usually the push run, which staging branch-
+  # protection then sees as a CANCELLED required check and refuses
+  # to mark merged. Caught 2026-05-05 across PR #2869's runs (run
+  # ids 25371863455 / 25371811486 / 25371078157 / 25370403142 — every
+  # staging push run cancelled, every matching PR run green).
+  #
+  # Per memory `feedback_concurrency_group_per_sha.md` — same drift
+  # class that broke auto-promote-staging on 2026-04-28. Pin invariant:
+  # event_name + sha is the minimum unique key for these workflows.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      wheel: ${{ steps.decide.outputs.wheel }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            wheel:
+              - 'workspace/**'
+              - 'scripts/build_runtime_package.py'
+              - 'scripts/wheel_smoke.py'
+              - '.github/workflows/runtime-prbuild-compat.yml'
+      - id: decide
+        # Always run real work for manual dispatch + merge_group — no
+        # diff-against-base in those contexts, and the gate exists to
+        # validate the to-be-merged state regardless of which paths it
+        # touched (paths-filter would default to "no changes" which is
+        # the wrong answer when the queue is composing many PRs).
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "merge_group" ]; then
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "wheel=${{ steps.filter.outputs.wheel }}" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `PR-built wheel + import smoke`. Real work is
+  # gated per-step on `needs.detect-changes.outputs.wheel`. Same shape
+  # as e2e-api.yml's e2e-api job — see its comment block for the full
+  # rationale (SKIPPED check runs block branch protection even with
+  # SUCCESS siblings; collapsing to one always-run job emits exactly
+  # one SUCCESS check run).
+  local-build-install:
+    needs: detect-changes
+    name: PR-built wheel + import smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.wheel != 'true'
+        run: |
+          echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
+          echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install build tooling
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: pip install build
+      - name: Build wheel from PR source (mirrors publish-runtime.yml)
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Use a fixed test version so the wheel filename is predictable.
+        # Doesn't reach PyPI — this build is local-only for the smoke.
+        # Use the SAME build script with the SAME args as
+        # publish-runtime.yml's build step. The temp dir path differs
+        # (`/tmp/runtime-build` here vs `${{ runner.temp }}/runtime-build`
+        # in publish-runtime.yml — they coincide on ubuntu-latest but
+        # the call sites are not byte-identical). The smoke import is
+        # also intentionally narrower than publish's: this gate exists
+        # to catch SDK-version-import drift specifically; full invariant
+        # coverage lives in publish-runtime.yml's own pre-PyPI smoke.
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "0.0.0.dev0+pin-compat" \
+            --out /tmp/runtime-build
+          cd /tmp/runtime-build && python -m build
+      - name: Install built wheel + workspace requirements
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: |
+          python -m venv /tmp/venv-built
+          /tmp/venv-built/bin/pip install --upgrade pip
+          /tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
+          /tmp/venv-built/bin/pip install -r workspace/requirements.txt
+          /tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import the PR-built wheel
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Same script publish-runtime.yml runs against the to-be-PyPI wheel.
+        # Closes the PR-time vs publish-time gap: a PR adding a new SDK
+        # call-shape no longer passes here (narrow `import main_sync`) only
+        # to fail post-merge in publish-runtime's broader smoke.
+        run: |
+          /tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"

.github/workflows/secret-pattern-drift.yml

@@ -0,0 +1,58 @@
+name: SECRET_PATTERNS drift lint
+
+# Detects when the canonical SECRET_PATTERNS array in
+# .github/workflows/secret-scan.yml diverges from known consumer
+# mirrors (workspace-runtime's bundled pre-commit hook today; more
+# can be added as the consumer set grows).
+#
+# Why this exists: every side that scans for credentials has its own
+# copy of the pattern list. They drift — most recently the runtime
+# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
+# so a developer's local pre-commit would let a sk-cp- token through
+# while the org-wide CI scan would refuse it. The cost of that drift
+# is dev confusion + delayed feedback; the fix is automated detection.
+#
+# Triggers:
+#   - schedule: daily 05:00 UTC. Catches drift introduced by edits
+#     to a consumer copy that didn't update canonical here.
+#   - push to main/staging where the canonical or this lint changed:
+#     catches the inverse — canonical updated but consumers not yet
+#     bumped. The lint will fail the push; that's intentional, the
+#     person editing canonical is the right person to also update
+#     the consumer.
+#   - workflow_dispatch: ad-hoc operator runs.
+
+on:
+  schedule:
+    # 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
+    # email lands when humans are starting their day, not
+    # interrupting it.
+    - cron: "0 5 * * *"
+  push:
+    branches: [main, staging]
+    paths:
+      - ".github/workflows/secret-scan.yml"
+      - ".github/workflows/secret-pattern-drift.yml"
+      - ".github/scripts/lint_secret_pattern_drift.py"
+      - ".githooks/pre-commit"
+  workflow_dispatch:
+
+# GITHUB_TOKEN scoped to read-only. The lint only does git checkout
+# + HTTPS GETs to public consumer files; no writes to anything.
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Detect SECRET_PATTERNS drift
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Run drift lint
+        run: python3 .github/scripts/lint_secret_pattern_drift.py

.github/workflows/sweep-aws-secrets.yml

@@ -0,0 +1,129 @@
+name: Sweep stale AWS Secrets Manager secrets
+
+# Janitor for per-tenant AWS Secrets Manager secrets
+# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
+# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
+# sweep-cf-orphans.yml — different cloud, same justification.
+#
+# Why this exists separately from a long-term reconciler integration:
+#   - molecule-controlplane's tenant_resources audit table (mig 024)
+#     currently tracks four resource kinds: CloudflareTunnel,
+#     CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
+#     not in the list, so the existing reconciler doesn't catch
+#     orphan secrets.
+#   - At ~$0.40/secret/month the cost grew to ~$19/month before this
+#     sweeper was written, indicating ~45+ orphan secrets from
+#     crashed provisions and incomplete deprovision flows.
+#   - The proper fix (KindSecretsManagerSecret + recorder hook +
+#     reconciler enumerator) is filed as a separate controlplane
+#     issue. This sweeper is the immediate cost-relief stopgap.
+#
+# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
+# This is a DEDICATED principal — the production `molecule-cp` IAM
+# user lacks `secretsmanager:ListSecrets` (it only has
+# Get/Create/Update/Delete on specific resources, scoped to its
+# operational needs). The janitor needs ListSecrets across the
+# `molecule/tenant/*` prefix, which warrants a separate principal so
+# we don't broaden the prod-CP policy.
+#
+# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
+# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
+# the mostly-orphan tunnels) refuses to nuke past the threshold.
+
+on:
+  schedule:
+    # Hourly at :30 — offsets from sweep-cf-orphans (:15) and
+    # sweep-cf-tunnels (:45) so the three janitors don't burst the
+    # CP admin endpoints at the same minute.
+    - cron: '30 * * * *'
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run only — list what would be deleted, no deletion"
+        required: false
+        type: boolean
+        default: true
+      max_delete_pct:
+        description: "Override safety gate (default 50, set higher only for major cleanup)"
+        required: false
+        default: "50"
+      grace_hours:
+        description: "Skip secrets created within this many hours (default 24)"
+        required: false
+        default: "24"
+
+# Don't let two sweeps race the same AWS account.
+concurrency:
+  group: sweep-aws-secrets
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    name: Sweep AWS Secrets Manager
+    runs-on: ubuntu-latest
+    # 30 min cap, mirroring the other janitors. AWS DeleteSecret is
+    # fast (~0.3s/call) so even a 100+ backlog drains in seconds
+    # under the 8-way xargs parallelism, but the cap is set generously
+    # to leave headroom for any actual API hang.
+    timeout-minutes: 30
+    env:
+      AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
+      CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
+      CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
+      GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
+        # and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
+        #   - schedule → exit 1 on missing secrets (red CI surfaces it)
+        #   - workflow_dispatch → exit 0 with warning (operator-driven,
+        #     they already accepted the repo state)
+        run: |
+          missing=()
+          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-aws-secrets.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-aws-secrets.sh --execute
+          fi

.github/workflows/sweep-cf-orphans.yml

@@ -78,15 +78,30 @@ jobs:
       MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Verify required secrets present
         id: verify
-        # Soft skip when secrets aren't configured. The 6 secrets have
-        # to be set on the repo manually before this workflow can do
-        # real work; until they are, the schedule is a no-op rather
-        # than a recurring red CI run. workflow_dispatch surfaces a
-        # warning so an operator running it ad-hoc sees the gap.
+        # Schedule-vs-dispatch behaviour split (hardened 2026-04-28
+        # after the silent-no-op incident below):
+        #
+        # The earlier soft-skip-on-schedule policy hid a real leak. All
+        # six secrets were unset on this repo for an unknown duration;
+        # every hourly run printed a yellow ::warning:: and exited 0,
+        # so the workflow registered as "passing" while doing nothing.
+        # CF orphans accumulated to 152/200 (~76% of the zone quota
+        # gone) before a manual `dig`-driven audit caught it. Anything
+        # that runs as a janitor and reports green while idle is
+        # indistinguishable from "the janitor is healthy" — so we now
+        # treat schedule (and any future workflow_run/push triggers)
+        # as a hard-fail when secrets are missing.
+        #
+        #   - schedule / workflow_run / push → exit 1 (red CI run
+        #     surfaces the misconfiguration the next tick)
+        #   - workflow_dispatch              → exit 0 with a warning
+        #     (an operator ran this ad-hoc; they already accepted the
+        #     state of the repo and want the workflow to short-circuit
+        #     so they can rerun after fixing the secret)
         run: |
           missing=()
           for var in CF_API_TOKEN CF_ZONE_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
@@ -95,9 +110,16 @@ jobs:
             fi
           done
           if [ ${#missing[@]} -gt 0 ]; then
-            echo "::warning::skipping sweep — secrets not yet configured: ${missing[*]}"
-            echo "skip=true" >> "$GITHUB_OUTPUT"
-            exit 0
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
+            exit 1
           fi
           echo "All required secrets present ✓"
           echo "skip=false" >> "$GITHUB_OUTPUT"

.github/workflows/sweep-cf-tunnels.yml

@@ -0,0 +1,124 @@
+name: Sweep stale Cloudflare Tunnels
+
+# Janitor for Cloudflare Tunnels whose backing tenant no longer
+# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
+# records); same justification, different CF resource.
+#
+# Why this exists separately from sweep-cf-orphans:
+#   - DNS records live on the zone (`/zones/<id>/dns_records`).
+#   - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
+#   - Different CF API surface, different scopes; the existing CF
+#     token might not have `account:cloudflare_tunnel:edit`. Splitting
+#     the workflows keeps each one's secret-presence gate independent
+#     so neither silent-skips when the other's secret is missing.
+#   - Cleaner blast radius — operators can disable one without the
+#     other if a regression surfaces.
+#
+# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
+# the DNS sweep's 50% because tenant-shaped tunnels are mostly
+# orphans by design) refuses to nuke past the threshold.
+
+on:
+  schedule:
+    # Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
+    # janitors don't issue parallel CF API bursts at the same minute.
+    - cron: '45 * * * *'
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run only — list what would be deleted, no deletion"
+        required: false
+        type: boolean
+        default: true
+      max_delete_pct:
+        description: "Override safety gate (default 90, set higher only for major cleanup)"
+        required: false
+        default: "90"
+
+# Don't let two sweeps race the same account.
+concurrency:
+  group: sweep-cf-tunnels
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    name: Sweep CF tunnels
+    runs-on: ubuntu-latest
+    # 30 min cap. Was 5 min on the theory that the only thing that
+    # could take >5min is a CF-API hang — but on 2026-05-02 a backlog
+    # of 672 stale tunnels accumulated (large staging E2E run + delayed
+    # sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
+    # ~7-8min to drain. The 5-min cap killed the run mid-sweep
+    # (cancelled at 424/672, see run 25248788312); a manual rerun
+    # finished the remainder fine.
+    #
+    # The fix is two-part: parallelize the delete loop (8-way xargs in
+    # the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
+    # cap so a one-off backlog doesn't trip a hangs-detector that
+    # turned out to be a real-job-too-slow detector. With 8-way
+    # parallelism, 600+ tunnels drains in ~60s; 30 min is generous
+    # headroom for actual hangs to still surface (and is in line with
+    # the sweep-cf-orphans companion job).
+    timeout-minutes: 30
+    env:
+      CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+      CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+      CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
+      CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
+        # (hardened 2026-04-28 after the silent-no-op incident: the
+        # janitor reported green while doing nothing because secrets
+        # were unset, masking a 152/200 zone-record leak). Same
+        # principle applies here:
+        #   - schedule → exit 1 on missing secrets (red CI surfaces it)
+        #   - workflow_dispatch → exit 0 with warning (operator-driven,
+        #     they already accepted the repo state)
+        run: |
+          missing=()
+          for var in CF_API_TOKEN CF_ACCOUNT_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-cf-tunnels.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-cf-tunnels.sh --execute
+          fi

.github/workflows/sweep-stale-e2e-orgs.yml

@@ -25,16 +25,23 @@ name: Sweep stale e2e-* orgs (staging)
 
 on:
   schedule:
-    # Every hour on the hour. E2E orgs are short-lived (~10-25 min wall
-    # clock from create to teardown). Anything older than the
-    # MAX_AGE_MINUTES threshold below is presumed dead.
-    - cron: '0 * * * *'
+    # Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
+    # create to teardown — canary is ~8 min, full SaaS ~25 min). The
+    # previous hourly + 120-min stale threshold meant a leaked tenant
+    # could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
+    # leak. Tightening the cadence + threshold reduces the worst-case
+    # leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
+    # threshold) without risk of catching in-progress runs (the longest
+    # e2e run is the 25-min canary, well under the 30-min threshold).
+    # See molecule-controlplane#420 for the leak-class accounting that
+    # motivated this tightening.
+    - cron: '*/15 * * * *'
   workflow_dispatch:
     inputs:
       max_age_minutes:
-        description: "Delete e2e-* orgs older than N minutes (default 120)"
+        description: "Delete e2e-* orgs older than N minutes (default 30)"
         required: false
-        default: "120"
+        default: "30"
       dry_run:
         description: "Dry run only — list what would be deleted"
         required: false
@@ -58,7 +65,7 @@ jobs:
     env:
       MOLECULE_CP_URL: https://staging-api.moleculesai.app
       ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
-      MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '120' }}
+      MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
       DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
       # Refuse to delete more than this many orgs in one tick. If the
       # CP DB is briefly empty (or the admin endpoint goes weird and
@@ -87,20 +94,36 @@ jobs:
             > orgs.json
 
           # Filter:
-          #   1. slug starts with 'e2e-' (covers e2e-, e2e-canary-,
-          #      e2e-canvas-* — all variants the test scripts mint)
+          #   1. slug starts with one of the ephemeral test prefixes:
+          #        - 'e2e-'    — covers e2e-canary-, e2e-canvas-*, etc.
+          #        - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
+          #                      missing this prefix left two such tenants
+          #                      orphaned 8h on staging (2026-05-03), then
+          #                      hard-failed redeploy-tenants-on-staging
+          #                      and broke the staging→main auto-promote
+          #                      chain. Kept in sync with the EPHEMERAL_PREFIX_RE
+          #                      regex in redeploy-tenants-on-staging.yml.
           #   2. created_at is older than MAX_AGE_MINUTES ago
           # Output one slug per line to a file the next step reads.
           python3 > stale_slugs.txt <<'PY'
           import json, os
           from datetime import datetime, timezone, timedelta
+          # SSOT for this list lives in the controlplane Go code:
+          # molecule-controlplane/internal/slugs/ephemeral.go
+          # (var EphemeralPrefixes). The redeploy-fleet auto-rollout
+          # also reads from there to SKIP these slugs — without that
+          # filter, fleet redeploy SSM-failed in-flight E2E tenants
+          # whose containers were still booting, breaking the test
+          # that just spun them up (molecule-controlplane#493).
+          # Update both files together.
+          EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
           with open("orgs.json") as f:
               data = json.load(f)
           max_age = int(os.environ["MAX_AGE_MINUTES"])
           cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
           for o in data.get("orgs", []):
               slug = o.get("slug", "")
-              if not slug.startswith("e2e-"):
+              if not slug.startswith(EPHEMERAL_PREFIXES):
                   continue
               created = o.get("created_at")
               if not created:
@@ -144,12 +167,18 @@ jobs:
             # The DELETE handler requires {"confirm": "<slug>"} matching
             # the URL slug — fat-finger guard. Idempotent: re-issuing
             # picks up via org_purges.last_step.
-            http_code=$(curl -sS -o /tmp/del_resp -w "%{http_code}" \
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/del_resp -w "%{http_code}" \
               --max-time 60 \
               -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
               -H "Authorization: Bearer $ADMIN_TOKEN" \
               -H "Content-Type: application/json" \
-              -d "{\"confirm\":\"$slug\"}" || echo "000")
+              -d "{\"confirm\":\"$slug\"}" >/tmp/del_code
+            set -e
+            # Stderr from curl (-sS shows dial errors etc.) goes to runner log.
+            http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
             if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
               deleted=$((deleted+1))
               echo "  deleted: $slug"
@@ -164,7 +193,47 @@ jobs:
           # sweeper is best-effort. Next hourly tick re-attempts. We
           # only fail loud at the safety-cap gate above.
 
+      - name: Sweep orphan tunnels
+        # Stale-org cleanup deletes the org (which cascades to tunnel
+        # delete inside the CP). But when that cascade fails partway —
+        # CP transient 5xx after the org row is deleted but before the
+        # CF tunnel delete completes — the tunnel persists with no
+        # matching org row. The reconciler in internal/sweep flags this
+        # as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
+        #
+        # `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
+        # reaper. Calling it here at the end of every sweep tick
+        # converges the staging CF account to clean even when CP
+        # cascades half-fail.
+        #
+        # PR #492 made the underlying DeleteTunnel actually check
+        # status — pre-fix it silent-succeeded on CF code 1022
+        # ("active connections"), so this step would have been a no-op
+        # against stuck connectors. Post-fix the cleanup invokes
+        # CleanupTunnelConnections + retry, which actually clears the
+        # 1022 case. (#2987)
+        #
+        # Best-effort. Failure here doesn't fail the workflow — next
+        # tick re-attempts. Errors flow to step output for ops review.
+        if: env.DRY_RUN != 'true'
+        run: |
+          set +e
+          curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
+            --max-time 60 \
+            -X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
+          set -e
+          http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
+          body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
+          if [ "$http_code" = "200" ]; then
+            count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
+            failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
+            echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
+          else
+            echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
+          fi
+
       - name: Dry-run summary
         if: env.DRY_RUN == 'true'
         run: |
-          echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s). Re-run with dry_run=false to actually delete."
+          echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."

.github/workflows/test-ops-scripts.yml

@@ -1,19 +1,27 @@
 name: Ops Scripts Tests
 
-# Runs the unittest suite for scripts/ops/ on every PR + push that touches
-# the directory. Kept separate from the main CI so a script-only change
-# doesn't trigger the heavier Go/Canvas/Python pipelines.
+# Runs the unittest suite for scripts/ on every PR + push that touches
+# anything under scripts/. Kept separate from the main CI so a script-only
+# change doesn't trigger the heavier Go/Canvas/Python pipelines.
+#
+# Discovery layout: tests sit alongside the code they test (see
+# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
+# test_build_runtime_package.py for the rewriter coverage). The job
+# below runs `unittest discover` TWICE — once from `scripts/`, once
+# from `scripts/ops/` — because neither dir has an `__init__.py`, so
+# a single discover from `scripts/` doesn't recurse into the ops
+# subdir. Two passes is simpler than retrofitting namespace packages.
 
 on:
   push:
     branches: [main, staging]
     paths:
-      - 'scripts/ops/**'
+      - 'scripts/**'
       - '.github/workflows/test-ops-scripts.yml'
   pull_request:
     branches: [main, staging]
     paths:
-      - 'scripts/ops/**'
+      - 'scripts/**'
       - '.github/workflows/test-ops-scripts.yml'
   merge_group:
     types: [checks_requested]
@@ -27,10 +35,18 @@ jobs:
     name: Ops scripts (unittest)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: '3.11'
-      - name: Run unittest
+      - name: Run scripts/ unittests (build_runtime_package, …)
+        # Top-level scripts/ tests live alongside their target file
+        # (e.g. scripts/test_build_runtime_package.py exercises
+        # scripts/build_runtime_package.py). discover from scripts/
+        # picks up only top-level test_*.py because scripts/ops/ has
+        # no __init__.py — that's intentional, so we run two passes.
+        working-directory: scripts
+        run: python -m unittest discover -t . -p 'test_*.py' -v
+      - name: Run scripts/ops/ unittests (sweep_cf_decide, …)
         working-directory: scripts/ops
         run: python -m unittest discover -p 'test_*.py' -v

.gitignore

@@ -131,6 +131,13 @@ backups/
 # Cloned by publish-workspace-server-image.yml so the Dockerfile's
 # replace-directive path resolves. Lives in its own repo.
 /molecule-ai-plugin-github-app-auth/
+# Tenant-image build context — populated by the workflow's
+# "Pre-clone manifest deps" step. Mirrors the public manifest, holds the
+# same content as the three /<>/ dirs above but namespaced under one
+# parent so the Docker build context is a single COPY-friendly tree.
+# Each entry is a transient working-dir, never source-of-truth, never
+# committed.
+/.tenant-bundle-deps/
 
 # Internal-flavored content lives in Molecule-AI/internal — NEVER in this
 # public monorepo. Migrated 2026-04-23 (CEO directive). The CI workflow
@@ -146,3 +153,4 @@ backups/
 *-temp.txt
 /test-pmm-*.txt
 /tick-reflections-*.md
+tests/harness/cp-stub/cp-stub

CONTRIBUTING.md

@@ -22,7 +22,7 @@ development workflow, conventions, and how to get your changes merged.
 
 ```bash
 # Clone the repo
-git clone https://github.com/Molecule-AI/molecule-core.git
+git clone https://git.moleculesai.app/molecule-ai/molecule-core.git
 cd molecule-core
 
 # Install git hooks
@@ -53,6 +53,29 @@ cp .env.example .env
 
 See `CLAUDE.md` for a full list of environment variables and their purposes.
 
+## What goes where (content vs code)
+
+This repo is scoped to **code** (canvas, workspace, workspace-server, related
+infra). Public content (blog posts, marketing copy, OG images, SEO briefs,
+DevRel demos) lives in [`Molecule-AI/docs`](https://git.moleculesai.app/molecule-ai/docs).
+The `Block forbidden paths` CI gate fails any PR that writes to `marketing/`
+or other removed paths — open against `Molecule-AI/docs` instead.
+
+| Content type | Target |
+|---|---|
+| Blog posts | `Molecule-AI/docs` → `content/blog/<YYYY-MM-DD-slug>/` |
+| Doc pages | `Molecule-AI/docs` → `content/docs/` |
+| Marketing copy / PMM positioning | `Molecule-AI/docs` → `marketing/` |
+| OG images, visual assets | `Molecule-AI/docs` → `app/` or `marketing/` |
+| SEO briefs | `Molecule-AI/docs` → `marketing/` |
+| DevRel demos (runnable code) | Standalone repo under `Molecule-AI/`, OR embedded in `Molecule-AI/docs` |
+| Launch checklists, internal tracking | GitHub Issues — **not** committed files |
+| Engineering docs (`docs/adr/`, `docs/architecture/`, `docs/incidents/`) | This repo (internal, not published) |
+| Live product pages (e.g. `canvas/src/app/pricing/page.tsx`) | This repo (these are app code, not marketing copy) |
+
+If a PR fails the `Block forbidden paths` check, the contents belong in
+`Molecule-AI/docs`. No CI drag, no Canvas E2E, content lands in minutes.
+
 ## Development Workflow
 
 ### Branch Naming
@@ -87,7 +110,7 @@ causing a render loop when any node position changed.
 
 1. **Repo-wide:** "Automatically delete head branches" is on. Once a PR merges, the branch is deleted server-side. Any subsequent `git push` to that branch fails with `remote rejected — no such branch`.
 
-2. **CI:** the `pr-guards` workflow (calling [molecule-ci `disable-auto-merge-on-push`](https://github.com/Molecule-AI/molecule-ci/blob/main/.github/workflows/disable-auto-merge-on-push.yml)) fires on every push to an open PR. If auto-merge was already enabled, it's disabled and a comment is posted. You must explicitly re-enable after verifying the new commit.
+2. **CI:** the `pr-guards` workflow (calling [molecule-ci `disable-auto-merge-on-push`](https://git.moleculesai.app/molecule-ai/molecule-ci/src/branch/main/.github/workflows/disable-auto-merge-on-push.yml)) fires on every push to an open PR. If auto-merge was already enabled, it's disabled and a comment is posted. You must explicitly re-enable after verifying the new commit.
 
 **Workflow rules that follow from the guards:**
 - Push **all** commits before running `gh pr merge --auto`.
@@ -152,6 +175,17 @@ and run CI manually.
 - Type hints on public functions
 - pytest for all tests
 
+## External integrations
+
+Code in this repo lands in molecule-core. Some related runtime artifacts
+live in their own repos:
+
+- [`Molecule-AI/molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime) — Python adapter SDK (`molecule_runtime`) that runs inside containerized Molecule workspaces. Bridges Claude Code SDK / hermes / langgraph / etc. → A2A queue.
+- [`Molecule-AI/molecule-sdk-python`](https://git.moleculesai.app/molecule-ai/molecule-sdk-python) — `A2AServer` + `RemoteAgentClient` for external agents that register over the public `/registry/register` flow.
+- [`Molecule-AI/molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel) — Claude Code channel plugin. Bridges A2A traffic into a running Claude Code session via MCP `notifications/claude/channel`. Polling-based (no tunnel required); install with `claude --channels plugin:molecule@Molecule-AI/molecule-mcp-claude-channel`.
+
+When extending the **A2A surface** in molecule-core (`workspace-server/internal/handlers/a2a_proxy.go` etc.), consider whether the change has a downstream impact on the runtime SDK or the channel plugin — they're versioned independently but share the wire shape.
+
 ## Architecture Overview
 
 See `CLAUDE.md` for detailed architecture documentation, including:

COVERAGE_FLOOR.md

@@ -1,7 +1,7 @@
 # Coverage Floor
 
-CI enforces three coverage gates on `workspace-server` (Go). All defined in
-`.github/workflows/ci.yml` → `platform-build` job.
+CI enforces coverage gates on two surfaces — `workspace-server` (Go) and
+`workspace/` (Python). All defined in `.github/workflows/ci.yml`.
 
 ## Current floors (2026-04-23)
 
@@ -76,3 +76,51 @@ This gate makes "no untested critical paths merged" a mechanical property of
 the CI, not a behavioural property of QA agents or individual reviewers —
 which is the only way to make it survive fleet outages, agent rotations, or
 QA process changes.
+
+## Python (workspace/) — added 2026-05-04 from #2790
+
+The Python side has its own gates in the `python-lint` job:
+
+| Gate | Threshold | Where |
+|---|---|---|
+| **Total floor** | `86%` | `workspace/pytest.ini` `--cov-fail-under=86` (issue #1817) |
+| **Critical-path per-file floor** | `75%` | Inline shell step after the pytest run |
+
+### Critical-path Python files
+
+These handle multi-tenant routing, auth tokens, and inbox dispatch. A
+coverage drop here is the same risk shape as a Go-side `tokens*` /
+`secrets*` file regressing below 10%.
+
+- `workspace/a2a_mcp_server.py` — MCP dispatcher (PR #2766 / #2771)
+- `workspace/mcp_cli.py` — molecule-mcp standalone CLI entry
+- `workspace/a2a_tools.py` — workspace-scoped tool implementations
+- `workspace/inbox.py` — multi-workspace inbox + per-workspace cursors
+- `workspace/platform_auth.py` — per-workspace token resolver
+
+### Why 75% (vs 86% total)
+
+The total floor averages ~6000 lines across `workspace/`. A single MCP
+file could drop to ~50% with no CI complaint as long as other modules
+compensate. The per-file floor closes that distribution gap. 75% sits
+below current actuals (80–96% as of 2026-05-04) — strictly additive,
+no existing PR fails.
+
+### Python ratchet plan
+
+| Date | Total | Per-file critical | Notes |
+|---|---|---|---|
+| 2026-05-04 | 86% | 75% | Initial gate (this file). |
+| 2026-06-04 | 86% | 80% | First ratchet — at-floor files must catch up. |
+| 2026-07-04 | 88% | 85% | |
+| 2026-08-04 | 90% | 90% | Target steady-state. |
+
+### Why this Python gate exists
+
+Issue #2790, after the PR #2766 → PR #2771 cycle. PR #2766 added
+multi-workspace routing through `a2a_tools.py` + `a2a_mcp_server.py`,
+shipped to main with green CI, but the dispatcher silently dropped a
+load-bearing kwarg for 4 of 9 tools — caught only by post-merge code
+review. The structural drift gate (`test_dispatcher_schema_drift.py`,
+PR #2791) catches the schema↔dispatcher mismatch class; this floor
+catches the broader "MCP-critical file regressed" class.

Makefile

@@ -0,0 +1,28 @@
+# Top-level Makefile — convenience wrappers around docker compose.
+#
+# Most molecule-core dev work happens via these shortcuts. CI doesn't
+# use this Makefile; CI calls docker compose / go test directly so the
+# Makefile can evolve without breaking the build.
+
+.PHONY: help dev up down logs build test
+
+help: ## Show this help.
+	@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-12s\033[0m %s\n", $$1, $$2}'
+
+dev: ## Start the full stack with air hot-reload for the platform service.
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml up
+
+up: ## Start the full stack in production-shape mode (no air, normal Dockerfile).
+	docker compose up
+
+down: ## Stop the stack and remove containers (volumes preserved).
+	docker compose down
+
+logs: ## Tail logs from all services (Ctrl-C to detach).
+	docker compose logs -f
+
+build: ## Force a fresh build of the platform image (no cache).
+	docker compose build --no-cache platform
+
+test: ## Run Go unit tests in workspace-server/.
+	cd workspace-server && go test -race ./...

README.md

@@ -1,7 +1,7 @@
 <div align="center">
 
 <p>
-  <img src="./docs/assets/branding/molecule-icon.png" alt="Molecule AI Icon Logo" width="160" />
+  <img src="./docs/assets/branding/molecule-icon.svg" alt="Molecule AI" width="160" />
 </p>
 
 <p>
@@ -39,8 +39,8 @@
   <a href="./docs/agent-runtime/workspace-runtime.md"><strong>Workspace Runtime</strong></a>
 </p>
 
-[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-core)
-[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-core)
+[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/new/template?template=https://git.moleculesai.app/molecule-ai/molecule-core)
+[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://git.moleculesai.app/molecule-ai/molecule-core)
 
 </div>
 
@@ -53,8 +53,8 @@ Molecule AI is the most powerful way to govern an AI agent organization in produ
 It combines the parts that are usually scattered across demos, internal glue code, and framework-specific tooling into one product:
 
 - one org-native control plane for teams, roles, hierarchy, and lifecycle
-- one runtime layer that lets LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, and OpenClaw run side by side
-- one memory model that keeps recall, sharing, and skill evolution aligned with organizational boundaries
+- one runtime layer that lets **eight** agent runtimes — LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, **Hermes**, **Gemini CLI**, and OpenClaw — run side by side behind one workspace contract
+- one memory model that keeps recall, sharing, and skill evolution aligned with organizational boundaries (Memory v2 backed by pgvector for semantic recall)
 - one operational surface for observing, pausing, restarting, inspecting, and improving live workspaces
 
 Most teams can build a workflow, a strong single agent, a coding agent, or a custom multi-agent graph.
@@ -75,7 +75,7 @@ You do not wire collaboration paths by hand. Hierarchy defines the default commu
 
 ### 3. Runtime choice stops being a dead-end decision
 
-LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, and OpenClaw can all plug into the same workspace abstraction. Teams can standardize governance without forcing every group onto one runtime.
+LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, Hermes, Gemini CLI, and OpenClaw can all plug into the same workspace abstraction. Teams can standardize governance without forcing every group onto one runtime.
 
 ### 4. Memory is treated like infrastructure
 
@@ -117,6 +117,8 @@ Molecule AI is not trying to replace the frameworks below. It is the system that
 | **Claude Code** | Shipping on `main` | Real coding workflows, CLI-native continuity | Secure workspace abstraction, A2A delegation, org boundaries, shared control plane |
 | **CrewAI** | Shipping on `main` | Role-based crews | Persistent workspace identity, policy consistency, shared canvas and registry |
 | **AutoGen** | Shipping on `main` | Assistant/tool orchestration | Standardized deployment, hierarchy-aware collaboration, shared ops plane |
+| **Hermes 4** | Shipping on `main` | Hybrid reasoning, native tools, json_schema (NousResearch/hermes-agent) | Option B upstream hook, A2A bridge to OpenAI-compat API, multi-provider provider derivation |
+| **Gemini CLI** | Shipping on `main` | Google Gemini CLI continuity | Workspace lifecycle, A2A, hierarchy-aware collaboration, shared ops plane |
 | **OpenClaw** | Shipping on `main` | CLI-native runtime with its own session model | Workspace lifecycle, templates, activity logs, topology-aware collaboration |
 | **NemoClaw** | WIP on `feat/nemoclaw-t4-docker` | NVIDIA-oriented runtime path | Planned to join the same abstraction once merged; not yet part of `main` |
 
@@ -182,9 +184,10 @@ The result is not just “an agent that learns.” It is **an organization that
 
 ## What Ships In `main`
 
-### Canvas
+### Canvas (v4)
 
 - Next.js 15 + React Flow + Zustand
+- **warm-paper theme system** — light / dark / follow-system, SSR cookie + nonce'd boot script + ThemeProvider; terminal + code surfaces stay dark unconditionally
 - drag-to-nest team building
 - empty-state deployment + onboarding wizard
 - template palette
@@ -193,8 +196,9 @@ The result is not just “an agent that learns.” It is **an organization that
 
 ### Platform
 
-- Go/Gin control plane
-- workspace CRUD and provisioning
+- Go 1.25 / Gin control plane (80+ HTTP endpoints + Gorilla WebSocket fanout)
+- workspace CRUD and provisioning (pluggable Provisioner — Docker locally, EC2 + SSM in production)
+- **A2A response path is a typed discriminated union (RFC #2967)** — frozen dataclasses + total parser; 100% unit + adversarial fuzz coverage
 - registry and heartbeats
 - browser-safe A2A proxy
 - team expansion/collapse
@@ -204,10 +208,10 @@ The result is not just “an agent that learns.” It is **an organization that
 
 ### Runtime
 
-- unified `workspace/` image
-- adapter-driven execution
+- unified `workspace/` image; thin AMI in production (us-east-2)
+- adapter-driven execution across **8 runtimes** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw)
 - Agent Card registration
-- awareness-backed memory integration
+- awareness-backed memory integration; **Memory v2 backed by pgvector** for semantic recall
 - plugin-mounted shared rules/skills
 - hot-reloadable local skills
 - coordinator-only delegation path
@@ -221,6 +225,21 @@ The result is not just “an agent that learns.” It is **an organization that
 - runtime tiers
 - direct workspace inspection through terminal and files
 
+### SaaS (via [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane))
+
+- multi-tenant on AWS EC2 + Neon (per-tenant Postgres branch) + Cloudflare Tunnels (per-tenant, no public ports)
+- WorkOS AuthKit + Stripe Checkout + Customer Portal
+- AWS KMS envelope encryption (DB / Redis connection strings); AWS Secrets Manager for tenant bootstrap
+- `tenant_resources` audit table + 30-min boot-event-aware reconciler — every CF / AWS lifecycle event recorded, claim vs live state diffed
+
+### Bring your own Claude Code session (via [`molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel))
+
+- Claude Code plugin that bridges Molecule A2A traffic into a local Claude Code session via MCP
+- subscribe to one or more workspaces; peer messages surface as conversation turns; replies route back through Molecule's A2A
+- no tunnel, no public endpoint — the plugin self-registers each watched workspace as `delivery_mode=poll` and long-polls `/activity?since_id=…`
+- multi-tenant friendly: one plugin install can watch workspaces across multiple Molecule tenants (`MOLECULE_PLATFORM_URLS` per-workspace)
+- install via the standard marketplace flow: `/plugin marketplace add Molecule-AI/molecule-mcp-claude-channel` → `/plugin install molecule-channel@molecule-mcp-claude-channel`
+
 ## Built For Teams That Need More Than A Demo
 
 Molecule AI is especially strong when you need to run:
@@ -233,23 +252,29 @@ Molecule AI is especially strong when you need to run:
 ## Architecture
 
 ```text
-Canvas (Next.js :3000)  <--HTTP / WS-->  Platform (Go :8080)  <---> Postgres + Redis
-         |                                          |
-         |                                          +--> Docker provisioner / bundles / templates / secrets
+Canvas (Next.js 15, warm-paper :3000)  <--HTTP / WS-->  Platform (Go 1.25 :8080)  <---> Postgres + Redis
+         |                                                           |
+         |                                                           +--> Provisioner: Docker (local) / EC2 + SSM (prod)
+         |                                                           +--> bundles · templates · secrets · KMS
          |
-         +-------------------- shows --------------------> workspaces, teams, tasks, traces, events
+         +------------------------- shows ------------------------> workspaces, teams, tasks, traces, events
 
-Workspace Runtime (Python image with adapters)
-  - LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / OpenClaw
-  - Agent Card + A2A server
-  - heartbeat + activity + awareness-backed memory
+Workspace Runtime (Python ≥3.11, image with adapters)
+  - 8 adapters: LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / Hermes / Gemini CLI / OpenClaw
+  - Agent Card + A2A server (typed-SSOT response path, RFC #2967)
+  - heartbeat + activity + awareness-backed memory (Memory v2 — pgvector semantic recall)
   - skills + plugins + hot reload
+
+SaaS Control Plane (molecule-controlplane, private)
+  - per-tenant EC2 + Neon (Postgres branch) + Cloudflare Tunnel
+  - WorkOS · Stripe · KMS · AWS Secrets Manager
+  - tenant_resources audit + 30-min reconciler
 ```
 
 ## Quick Start
 
 ```bash
-git clone https://github.com/Molecule-AI/molecule-core.git
+git clone https://git.moleculesai.app/molecule-ai/molecule-core.git
 cd molecule-core
 
 cp .env.example .env
@@ -259,7 +284,7 @@ cp .env.example .env
 ./infra/scripts/setup.sh
 # Boots Postgres (:5432), Redis (:6379), Langfuse (:3001),
 # and Temporal (:7233 gRPC, :8233 UI) on the shared
-# `molecule-monorepo-net` Docker network. Temporal runs with
+# `molecule-core-net` Docker network. Temporal runs with
 # no auth on localhost — dev-only; production must gate it.
 #
 # Also populates the template/plugin registry by cloning every repo
@@ -303,7 +328,11 @@ Then open `http://localhost:3000`:
 
 ## Current Scope
 
-The current `main` branch already includes the core platform, canvas, memory model, six production adapters, skill lifecycle, and operational surfaces. Adjacent runtime work such as **NemoClaw** remains branch-level until merged, and this README keeps that distinction explicit on purpose.
+The current `main` branch ships the core platform, Canvas v4 (warm-paper themed), Memory v2 (pgvector semantic recall), the typed-SSOT A2A response path (RFC #2967), **eight production adapters** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw), skill lifecycle, and operational surfaces.
+
+The companion private repo [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) provides the SaaS surface — multi-tenant orchestration on EC2 + Neon + Cloudflare Tunnels, KMS envelope encryption, WorkOS auth, Stripe billing, and a `tenant_resources` audit table with a 30-min reconciler.
+
+Adjacent runtime work such as **NemoClaw** remains branch-level until merged, and this README keeps that distinction explicit on purpose.
 
 ## License
 

README.zh-CN.md

@@ -1,7 +1,7 @@
 <div align="center">
 
 <p>
-  <img src="./docs/assets/branding/molecule-icon.png" alt="Molecule AI 图案 Logo" width="160" />
+  <img src="./docs/assets/branding/molecule-icon.svg" alt="Molecule AI" width="160" />
 </p>
 
 <p>
@@ -38,8 +38,8 @@
   <a href="./docs/agent-runtime/workspace-runtime.md"><strong>Workspace Runtime</strong></a>
 </p>
 
-[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-core)
-[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-core)
+[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/new/template?template=https://git.moleculesai.app/molecule-ai/molecule-core)
+[![Deploy to Render](https://render.com/images/deploy-to-render-button.svg)](https://render.com/deploy?repo=https://git.moleculesai.app/molecule-ai/molecule-core)
 
 </div>
 
@@ -52,8 +52,8 @@ Molecule AI 是目前最强的 AI Agent 组织治理方案之一，用来把 age
 它把过去分散在 demo、内部胶水代码和各类 framework 私有工具里的关键能力，收敛成一个产品：
 
 - 一套组织原生 control plane，管理团队、角色、层级和生命周期
-- 一套 runtime abstraction，让 LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、OpenClaw 并存运行
-- 一套与组织边界对齐的 memory 模型，把 recall、sharing 和 skill evolution 放进同一体系
+- 一套 runtime abstraction，让 **8 个** agent runtime —— LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、**Hermes**、**Gemini CLI**、OpenClaw —— 共用一套 workspace 契约
+- 一套与组织边界对齐的 memory 模型，把 recall、sharing 和 skill evolution 放进同一体系（Memory v2 由 pgvector 支撑语义召回）
 - 一套面向线上 workspace 的运维面，统一完成观测、暂停、重启、检查和持续改进
 
 今天很多团队能做好 workflow、单 agent、coding agent，或者自定义 multi-agent graph 中的一种。
@@ -74,7 +74,7 @@ Molecule AI 填的就是这个空白。
 
 ### 3. Runtime 选择不再是死路
 
-LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、OpenClaw 都可以挂到同一个 workspace abstraction 下。团队可以统一治理方式，而不必统一到底层 runtime。
+LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、Hermes、Gemini CLI、OpenClaw 都可以挂到同一个 workspace abstraction 下。团队可以统一治理方式，而不必统一到底层 runtime。
 
 ### 4. Memory 被当成基础设施来做
 
@@ -116,6 +116,8 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 | **Claude Code** | `main` 已支持 | 真实编码工作流、CLI-native continuity | 安全 workspace 抽象、A2A delegation、组织边界、共享 control plane |
 | **CrewAI** | `main` 已支持 | 角色型 crew 模式清晰 | 持久 workspace 身份、统一策略、共享 Canvas 和 registry |
 | **AutoGen** | `main` 已支持 | assistant/tool orchestration | 统一部署、层级协作、共享运维平面 |
+| **Hermes 4** | `main` 已支持 | 混合推理、原生工具调用、json_schema 输出（NousResearch/hermes-agent） | Option B 上游 hook、A2A 桥接 OpenAI 兼容 API、多 provider 自动派生 |
+| **Gemini CLI** | `main` 已支持 | Google Gemini CLI 持续会话 | workspace 生命周期、A2A、层级感知协作、共享运维平面 |
 | **OpenClaw** | `main` 已支持 | CLI-native runtime，自有 session 模型 | workspace 生命周期、templates、activity logs、拓扑感知协作 |
 | **NemoClaw** | `feat/nemoclaw-t4-docker` 分支 WIP | NVIDIA 方向 runtime 路线 | 计划并入同一抽象层，但当前还不是 `main` 已合并能力 |
 
@@ -181,9 +183,10 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 
 ## `main` 分支已经具备什么
 
-### Canvas
+### Canvas（v4）
 
 - Next.js 15 + React Flow + Zustand
+- **warm-paper 主题系统** —— light / dark / 跟随系统；SSR cookie + nonce'd boot 脚本 + ThemeProvider；终端与代码面板始终保持深色
 - drag-to-nest 团队构建
 - empty state + onboarding wizard
 - template palette
@@ -192,8 +195,9 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 
 ### Platform
 
-- Go/Gin control plane
-- workspace CRUD 和 provisioning
+- Go 1.25 / Gin control plane（80+ HTTP 端点 + Gorilla WebSocket fanout）
+- workspace CRUD 和 provisioning（可插拔 Provisioner —— 本地 Docker、生产 EC2 + SSM）
+- **A2A 响应路径已收敛为类型化的判别联合（RFC #2967）** —— 冻结 dataclass + 全量 parser；100% 单元测试 + 对抗性 fuzz 覆盖
 - registry 与 heartbeat
 - 浏览器安全的 A2A proxy
 - team expansion/collapse
@@ -203,10 +207,10 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 
 ### Runtime
 
-- 统一 `workspace/` 镜像
-- adapter 驱动执行
+- 统一 `workspace/` 镜像；生产环境采用 thin AMI（us-east-2）
+- adapter 驱动执行，覆盖 **8 个 runtime**（Claude Code、Hermes、Gemini CLI、LangGraph、DeepAgents、CrewAI、AutoGen、OpenClaw）
 - Agent Card 注册
-- awareness-backed memory
+- awareness-backed memory；**Memory v2 由 pgvector 支撑**语义召回
 - plugin 挂载共享 rules/skills
 - 本地 skills 热加载
 - coordinator-only delegation 路径
@@ -220,6 +224,21 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 - runtime tiers
 - 终端与文件层面的 workspace 直接排障
 
+### SaaS（由 [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) 提供）
+
+- 多租户运行在 AWS EC2 + Neon（每租户一个 Postgres branch）+ Cloudflare Tunnels（每租户一条隧道，对外不开任何端口）
+- WorkOS AuthKit + Stripe Checkout + Customer Portal
+- AWS KMS 信封加密（DB / Redis 连接串）；AWS Secrets Manager 负责租户 bootstrap
+- `tenant_resources` 审计表 + 30 分钟 boot-event-aware reconciler —— 每个 CF / AWS lifecycle 事件都有记录，每 30 分钟比对 claim 与实际状态
+
+### 在 Claude Code 里直接接入（由 [`molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel) 提供）
+
+- 把 Molecule A2A 流量桥接到本地 Claude Code 会话的 MCP 插件
+- 订阅一个或多个 workspace；peer 的消息会以 user-turn 出现，回复会经 Molecule A2A 路由出去
+- 无需公网隧道、无需公开端点 —— 插件启动时自动把每个 watched workspace 注册成 `delivery_mode=poll`，长轮询 `/activity?since_id=…`
+- 多租户友好：单次安装即可同时 watch 跨多个 Molecule 租户的 workspace（`MOLECULE_PLATFORM_URLS` 按 workspace 配置）
+- 通过标准 marketplace 流程安装：`/plugin marketplace add Molecule-AI/molecule-mcp-claude-channel` → `/plugin install molecule-channel@molecule-mcp-claude-channel`
+
 ## 适合什么团队
 
 Molecule AI 特别适合下面这些场景：
@@ -232,23 +251,29 @@ Molecule AI 特别适合下面这些场景：
 ## 架构总览
 
 ```text
-Canvas (Next.js :3000)  <--HTTP / WS-->  Platform (Go :8080)  <---> Postgres + Redis
-         |                                          |
-         |                                          +--> Docker provisioner / bundles / templates / secrets
+Canvas (Next.js 15, warm-paper :3000)  <--HTTP / WS-->  Platform (Go 1.25 :8080)  <---> Postgres + Redis
+         |                                                           |
+         |                                                           +--> Provisioner: Docker (本地) / EC2 + SSM (生产)
+         |                                                           +--> bundles · templates · secrets · KMS
          |
-         +-------------------- 展示 --------------------> workspaces, teams, tasks, traces, events
+         +------------------------- 展示 ------------------------> workspaces, teams, tasks, traces, events
 
-Workspace Runtime (Python image with adapters)
-  - LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / OpenClaw
-  - Agent Card + A2A server
-  - heartbeat + activity + awareness-backed memory
+Workspace Runtime (Python ≥3.11，含 adapter 集合的镜像)
+  - 8 个 adapter: LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / Hermes / Gemini CLI / OpenClaw
+  - Agent Card + A2A server（typed-SSOT 响应路径，RFC #2967）
+  - heartbeat + activity + awareness-backed memory（Memory v2 —— pgvector 语义召回）
   - skills + plugins + hot reload
+
+SaaS Control Plane (molecule-controlplane，私有)
+  - 每租户 EC2 + Neon (Postgres branch) + Cloudflare Tunnel
+  - WorkOS · Stripe · KMS · AWS Secrets Manager
+  - tenant_resources 审计 + 30 分钟 reconciler
 ```
 
 ## 快速开始
 
 ```bash
-git clone https://github.com/Molecule-AI/molecule-core.git
+git clone https://git.moleculesai.app/molecule-ai/molecule-core.git
 cd molecule-core
 
 cp .env.example .env
@@ -258,7 +283,7 @@ cp .env.example .env
 ./infra/scripts/setup.sh
 # 启动 Postgres (:5432)、Redis (:6379)、Langfuse (:3001)
 # 以及 Temporal (:7233 gRPC, :8233 UI)，全部挂在共享的
-# `molecule-monorepo-net` Docker 网络上。Temporal 默认无鉴权，
+# `molecule-core-net` Docker 网络上。Temporal 默认无鉴权，
 # 仅用于本地开发；生产环境必须加 mTLS / API Key。
 #
 # 同时会根据 manifest.json 拉取所有模板/插件仓库到
@@ -296,7 +321,11 @@ npm run dev
 
 ## 当前范围说明
 
-当前 `main` 已经包含核心平台、Canvas、memory model、6 个正式 adapter、skill lifecycle 和主要运维面。像 **NemoClaw** 这样的相邻 runtime 路线仍然属于分支级工作，只有合并后才会进入正式支持列表，这里会明确区分。
+当前 `main` 已经包含核心平台、Canvas v4（warm-paper 主题）、Memory v2（pgvector 语义召回）、typed-SSOT A2A 响应路径（RFC #2967）、**8 个正式 adapter**（Claude Code、Hermes、Gemini CLI、LangGraph、DeepAgents、CrewAI、AutoGen、OpenClaw）、skill lifecycle，以及主要运维面。
+
+配套的私有仓库 [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) 提供 SaaS 层 —— 多租户编排（EC2 + Neon + Cloudflare Tunnels）、KMS 信封加密、WorkOS 鉴权、Stripe 计费，以及 `tenant_resources` 审计表加 30 分钟 reconciler。
+
+像 **NemoClaw** 这样的相邻 runtime 路线仍然属于分支级工作，只有合并后才会进入正式支持列表，这里会明确区分。
 
 ## License
 

canvas/.dockerignore

@@ -0,0 +1,10 @@
+# Excluded from `docker build` context. Without this, the COPY . . step in
+# canvas/Dockerfile clobbers the freshly-installed node_modules with the
+# host's (potentially broken / wrong-arch) copy — the @tailwindcss/oxide
+# native binary disagreed and broke `next build`.
+node_modules
+.next
+.git
+*.log
+.env*
+!.env.example

canvas/Dockerfile

@@ -1,7 +1,11 @@
-FROM node:22-alpine AS builder
+FROM node:22-alpine@sha256:cb15fca92530d7ac113467696cf1001208dac49c3c64355fd1348c11a88ddf8f AS builder
 WORKDIR /app
 COPY package.json package-lock.json* ./
-RUN npm install
+# `npm ci` (not `install`) for lockfile-exact reproducibility.
+# `--include=optional` ensures the platform-specific @tailwindcss/oxide
+# native binary lands — without it, postcss fails with "Cannot read
+# properties of undefined (reading 'All')" at build time.
+RUN npm ci --include=optional
 COPY . .
 ARG NEXT_PUBLIC_PLATFORM_URL=http://localhost:8080
 ARG NEXT_PUBLIC_WS_URL=ws://localhost:8080/ws
@@ -11,7 +15,7 @@ ENV NEXT_PUBLIC_WS_URL=$NEXT_PUBLIC_WS_URL
 ENV NEXT_PUBLIC_ADMIN_TOKEN=$NEXT_PUBLIC_ADMIN_TOKEN
 RUN npm run build
 
-FROM node:22-alpine
+FROM node:22-alpine@sha256:cb15fca92530d7ac113467696cf1001208dac49c3c64355fd1348c11a88ddf8f
 WORKDIR /app
 COPY --from=builder /app/.next/standalone ./
 COPY --from=builder /app/.next/static ./.next/static

canvas/components.json

@@ -4,10 +4,9 @@
   "rsc": true,
   "tsx": true,
   "tailwind": {
-    "config": "tailwind.config.ts",
     "css": "src/app/globals.css",
-    "baseColor": "zinc",
-    "cssVariables": false
+    "baseColor": "neutral",
+    "cssVariables": true
   },
   "aliases": {
     "components": "@/components",

canvas/e2e/staging-setup.ts

@@ -111,6 +111,20 @@ export default async function globalSetup(_config: FullConfig): Promise<void> {
   const adminAuth = { Authorization: `Bearer ${ADMIN_TOKEN}` };
   console.log(`[staging-setup] Using slug=${slug}`);
 
+  // Write the state file FIRST, before any CP call. Teardown (both
+  // Playwright globalTeardown and the workflow safety-net) reads this
+  // file to identify the slug it must clean up. If we wait until the
+  // end of setup to write it (the previous behavior), a crash during
+  // any of steps 1-6 leaves the org orphaned in CP with no record on
+  // disk — forcing the workflow safety-net into a pattern-sweep over
+  // every `e2e-canvas-<date>-*` org, which races with concurrent
+  // canvas-E2E runs and deletes their live tenants. Race observed
+  // 2026-04-30 on PR #2264 staging→main: three real-test runs killed
+  // each other's tenants mid-test, surfacing as `getaddrinfo ENOTFOUND`
+  // when CP cleaned up the just-deleted DNS record.
+  const stateFile = join(process.cwd(), ".playwright-staging-state.json");
+  writeFileSync(stateFile, JSON.stringify({ slug }, null, 2));
+
   // 1. Create org via admin endpoint — no WorkOS session needed
   const create = await jsonFetch(`${CP_URL}/cp/admin/orgs`, {
     method: "POST",
@@ -155,7 +169,17 @@ export default async function globalSetup(_config: FullConfig): Promise<void> {
         orgID = row.id;
         return true;
       }
-      if (row.instance_status === "failed") throw new Error(`provision failed: ${slug}`);
+      if (row.instance_status === "failed") {
+        // Dump every diagnostic field the admin row carries — boot stage,
+        // last error, terraform/SSM state, etc. The bare slug message used
+        // to surface ZERO context, so triaging a failed provision meant
+        // re-running locally to repro. Now the failure log carries enough
+        // to point at the right subsystem (CP/AWS/SSM/runtime) without a
+        // second round-trip.
+        throw new Error(
+          `provision failed: ${slug} — admin-orgs row: ${JSON.stringify(row)}`,
+        );
+      }
       return null;
     },
     PROVISION_TIMEOUT_MS,
@@ -235,7 +259,17 @@ export default async function globalSetup(_config: FullConfig): Promise<void> {
       if (r.status !== 200) return null;
       if (r.body?.status === "online") return true;
       if (r.body?.status === "failed") {
-        throw new Error(`Workspace failed: ${r.body.last_sample_error || ""}`);
+        // last_sample_error is often empty when the failure happens before
+        // the agent emits a sample (e.g. boot crash, image pull error,
+        // missing PYTHONPATH, OpenAI quota at startup). Dumping the full
+        // body gives triage the boot_stage / last_error / image fields it
+        // needs without a second probe. Otherwise this propagates as a
+        // bare "Workspace failed: " — the exact useless message that
+        // sent #2632 to the issue tracker.
+        const detail = r.body.last_sample_error
+          ? r.body.last_sample_error
+          : `(no last_sample_error) full body: ${JSON.stringify(r.body)}`;
+        throw new Error(`Workspace failed: ${detail}`);
       }
       return null;
     },
@@ -245,8 +279,8 @@ export default async function globalSetup(_config: FullConfig): Promise<void> {
   );
   console.log(`[staging-setup] Workspace online`);
 
-  // 7. Hand state off to tests + teardown
-  const stateFile = join(process.cwd(), ".playwright-staging-state.json");
+  // 7. Hand state off to tests + teardown — overwrite the slug-only
+  // bootstrap state with the full state spec tests need.
   writeFileSync(
     stateFile,
     JSON.stringify({ slug, tenantURL, workspaceId, tenantToken }, null, 2),

canvas/e2e/staging-teardown.ts

@@ -24,7 +24,11 @@ export default async function globalTeardown(): Promise<void> {
 
   const stateFile = join(process.cwd(), ".playwright-staging-state.json");
   if (!existsSync(stateFile)) {
-    console.warn("[staging-teardown] no state file — setup must have failed before org create; nothing to tear down");
+    // staging-setup writes this file as its first action, before any
+    // CP call. Missing here means setup never ran (CANVAS_E2E_STAGING
+    // unset, or ran in a different cwd) — there's no slug we created
+    // that needs cleaning up.
+    console.warn("[staging-teardown] no state file — nothing to tear down");
     return;
   }
 

canvas/next.config.ts

@@ -17,6 +17,24 @@ import { dirname, join } from "node:path";
 // update one heuristic. Production is unaffected: `output: "standalone"`
 // bakes resolved env into the build, and the marker file isn't shipped.
 loadMonorepoEnv();
+// Boot-time matched-pair guard for ADMIN_TOKEN / NEXT_PUBLIC_ADMIN_TOKEN.
+// When ADMIN_TOKEN is set on the workspace-server (server-side bearer
+// gate, wsauth_middleware.go ~L245), the canvas MUST send the matching
+// NEXT_PUBLIC_ADMIN_TOKEN as `Authorization: Bearer ...` on every API
+// call. If only one is set, every workspace API call 401s silently —
+// the canvas hydrates with empty data and the user sees a broken page
+// with no console hint about the auth-config mismatch.
+//
+// Pre-fix the matched-pair contract was descriptive only (a comment in
+// .env): future devs/agents could re-misconfigure with one of the two
+// unset and silently 401. Closes the post-PR-#174 self-review gap.
+//
+// Warn-only (not exit) — production canvas Docker images bake these
+// vars into the build at image-build time, and a missed pair there
+// would still emit the warning at runtime via the standalone server's
+// startup. Killing the process on misconfiguration would turn a
+// recoverable auth issue into a hard crashloop.
+checkAdminTokenPair();
 
 const nextConfig: NextConfig = {
   output: "standalone",
@@ -57,6 +75,43 @@ function loadMonorepoEnv() {
   );
 }
 
+// Boot-time matched-pair guard. Runs after .env has been loaded so the
+// check sees the post-load state. The two env vars must be set or
+// unset together; one-without-the-other is the silent-401 footgun.
+//
+// Treats empty string ("") as unset. An explicitly-empty `KEY=` in
+// .env counts as set-to-empty in `process.env`, but for auth purposes
+// an empty bearer token is equivalent to no token — so both
+// `ADMIN_TOKEN=` and an unset ADMIN_TOKEN are equivalent relative to
+// the matched-pair invariant.
+//
+// Returns void; side effect is the console.error warning. Kept as a
+// separate function (exported) so a future test can reset env, call
+// this, and assert on captured stderr.
+export function checkAdminTokenPair(): void {
+  const serverSet = !!process.env.ADMIN_TOKEN;
+  const clientSet = !!process.env.NEXT_PUBLIC_ADMIN_TOKEN;
+  if (serverSet === clientSet) return;
+  // Distinct messages so the operator can tell which half is missing
+  // — the fix is symmetric (set the other one) but the diagnostic
+  // mentions which side is currently set so they don't have to grep.
+  if (serverSet && !clientSet) {
+    // eslint-disable-next-line no-console
+    console.error(
+      "[next.config] ADMIN_TOKEN is set but NEXT_PUBLIC_ADMIN_TOKEN is not — " +
+        "canvas will 401 against workspace-server because the bearer header " +
+        "is never attached. Set both to the same value, or unset both.",
+    );
+  } else {
+    // eslint-disable-next-line no-console
+    console.error(
+      "[next.config] NEXT_PUBLIC_ADMIN_TOKEN is set but ADMIN_TOKEN is not — " +
+        "workspace-server will reject the bearer because no AdminAuth gate " +
+        "is configured. Set both to the same value, or unset both.",
+    );
+  }
+}
+
 function findMonorepoRoot(start: string): string | null {
   let dir = start;
   for (let i = 0; i < 6; i++) {

canvas/package.json

@@ -32,15 +32,15 @@
     "@playwright/test": "^1.59.1",
     "@testing-library/jest-dom": "^6.6.0",
     "@testing-library/react": "^16.1.0",
-    "@types/node": "^22.0.0",
+    "@types/node": "^25.6.0",
     "@types/react": "^19.0.0",
     "@types/react-dom": "^19.0.0",
     "@vitejs/plugin-react": "^6.0.1",
     "@vitest/coverage-v8": "^4.1.5",
-    "autoprefixer": "^10.4.0",
-    "jsdom": "^25.0.0",
-    "postcss": "^8.5.12",
-    "tailwindcss": "^3.4.0",
+    "@tailwindcss/postcss": "^4.0.0",
+    "jsdom": "^29.1.1",
+    "postcss": "^8.5.13",
+    "tailwindcss": "^4.0.0",
     "typescript": "^5.7.0",
     "vitest": "^4.1.2"
   }

canvas/postcss.config.js

@@ -1,6 +1,5 @@
 module.exports = {
   plugins: {
-    tailwindcss: {},
-    autoprefixer: {},
+    "@tailwindcss/postcss": {},
   },
 };

canvas/src/app/api/buildinfo/__tests__/route.test.ts

@@ -0,0 +1,48 @@
+/**
+ * Canvas /api/buildinfo — version-display endpoint mirroring
+ * workspace-server's /buildinfo. Lets `curl <url>/api/buildinfo`
+ * confirm which git SHA is live on a canvas deployment.
+ */
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { GET } from "../route";
+
+const ENV_KEYS = ["VERCEL_GIT_COMMIT_SHA", "VERCEL_GIT_COMMIT_REF", "VERCEL_ENV"];
+
+describe("GET /api/buildinfo", () => {
+  let saved: Record<string, string | undefined>;
+
+  beforeEach(() => {
+    saved = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+    for (const k of ENV_KEYS) delete process.env[k];
+  });
+
+  afterEach(() => {
+    for (const k of ENV_KEYS) {
+      if (saved[k] === undefined) delete process.env[k];
+      else process.env[k] = saved[k];
+    }
+  });
+
+  it("returns dev sentinel when Vercel env vars are unset", async () => {
+    const res = await GET();
+    const body = await res.json();
+    expect(body).toEqual({ git_sha: "dev", git_ref: "", vercel_env: "local" });
+  });
+
+  it("reports the SHA Vercel injected at build time", async () => {
+    process.env.VERCEL_GIT_COMMIT_SHA = "abc1234567890";
+    process.env.VERCEL_GIT_COMMIT_REF = "main";
+    process.env.VERCEL_ENV = "production";
+    const res = await GET();
+    const body = await res.json();
+    expect(body.git_sha).toBe("abc1234567890");
+    expect(body.git_ref).toBe("main");
+    expect(body.vercel_env).toBe("production");
+  });
+
+  it("returns 200 status and JSON content type", async () => {
+    const res = await GET();
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toContain("application/json");
+  });
+});

canvas/src/app/api/buildinfo/route.ts

@@ -0,0 +1,18 @@
+import { NextResponse } from "next/server";
+
+// Mirror of workspace-server's GET /buildinfo (PR #2398). Lets a developer
+// confirm which git SHA is live on a canvas deployment with the same
+// `curl <url>/buildinfo` flow they use against tenant workspaces.
+//
+// Vercel injects VERCEL_GIT_COMMIT_SHA / _REF / VERCEL_ENV at build time
+// from the deploying commit; outside Vercel (local `next dev`, harness)
+// these are unset and the endpoint reports `git_sha: "dev"`. Same sentinel
+// the workspace-server uses pre-ldflags-injection so both surfaces speak
+// the same vocabulary.
+export async function GET() {
+  return NextResponse.json({
+    git_sha: process.env.VERCEL_GIT_COMMIT_SHA ?? "dev",
+    git_ref: process.env.VERCEL_GIT_COMMIT_REF ?? "",
+    vercel_env: process.env.VERCEL_ENV ?? "local",
+  });
+}

canvas/src/app/globals.css

@@ -1,28 +1,139 @@
+@import "tailwindcss";
+@plugin "@tailwindcss/typography";
+
+/*
+ * Tailwind v4 defaults the `dark:` variant to `prefers-color-scheme: dark`.
+ * Our theme switcher writes `data-theme="dark"` on <html> instead (so user
+ * choice via the toggle wins over OS preference). Re-bind `dark:` to that
+ * attribute so component classes like `dark:bg-zinc-800` track the same
+ * source of truth as the `[data-theme="dark"]` token overrides below.
+ */
+@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *));
+
+/*
+ * Load order:
+ *   1. Tailwind core (v4) — provides preflight + utility generation.
+ *   2. xterm — overrides preflight on its own .xterm-* class names; must
+ *      load AFTER tailwind so its specificity wins.
+ *   3. theme-tokens.css — canvas-only motion + deploy animation vars
+ *      (--mol-duration-*, --mol-easing-*, --mol-deploy-*). NOT colour
+ *      tokens; the warm-paper @theme block below owns those.
+ *   4. settings-panel.css / org-deploy.css — feature stylesheets that
+ *      reference the variables above.
+ */
 @import "xterm/css/xterm.css";
-/* Theme tokens MUST load before any feature stylesheet that
-   references them so custom properties are in scope. */
 @import "../styles/theme-tokens.css";
 @import "../styles/settings-panel.css";
 @import "../styles/org-deploy.css";
 
-@tailwind base;
-@tailwind components;
-@tailwind utilities;
+/*
+ * Warm-paper semantic tokens — light defaults via @theme, dark
+ * overrides via [data-theme="dark"]. Names are role-based
+ * (`bg-surface`, `text-ink`, `border-line`) not colour-based, so the
+ * same component classes work in either mode.
+ *
+ * Source of truth: molecule-app/app/globals.css. Keep aligned across
+ * surfaces (landing, market, app, canvas) so a token tweak ripples
+ * everywhere via a single PR per repo.
+ *
+ * Theme preference is persisted in the `mol_theme` cookie scoped to
+ * Domain=.moleculesai.app so the choice follows the user across
+ * subdomains. The inline boot script in app/layout.tsx applies it
+ * before paint to eliminate flash.
+ */
+@theme {
+  /* Surface — page / elevated card / sunken input / deep card */
+  --color-surface: #fafaf7;
+  --color-surface-elevated: #ffffff;
+  --color-surface-sunken: #f3f1ec;
+  --color-surface-card: #efece4;
+
+  /* Borders */
+  --color-line: #e6e2d8;
+  --color-line-soft: #efece4;
+
+  /* Text */
+  --color-ink: #15181c;
+  --color-ink-mid: #5a5e66;
+  --color-ink-soft: #8b8e95;
+
+  /* Brand + state */
+  --color-accent: #3b5bdb;
+  --color-accent-strong: #1a2f99;
+  --color-warm: #c0532b;
+  --color-good: #2f7a4d;
+  --color-bad: #b94e4a;
+}
+
+[data-theme="dark"] {
+  --color-surface: #0e1014;
+  --color-surface-elevated: #15181c;
+  --color-surface-sunken: #0a0b0e;
+  --color-surface-card: #1a1d23;
+
+  --color-line: #2a2f3a;
+  --color-line-soft: #1f2329;
+
+  --color-ink: #f4f1e9;
+  --color-ink-mid: #c8c2b4;
+  --color-ink-soft: #8d92a0;
+
+  /* Accents brighten slightly for AA contrast on dark backgrounds. */
+  --color-accent: #6883e8;
+  --color-accent-strong: #8aa1ee;
+  --color-warm: #d96f48;
+  --color-good: #4ca06e;
+  --color-bad: #d27773;
+}
+
+:root {
+  color-scheme: light;
+}
+[data-theme="dark"] {
+  color-scheme: dark;
+}
+
+/*
+ * Always-dark surface tokens. Terminals (xterm), the console modal,
+ * and log streams stay dark in both modes — readable green-on-black
+ * code surfaces don't translate cleanly to a light theme. Components
+ * that should not light-flip use `bg-bg`, `bg-bg-elev`, `bg-bg-card`,
+ * `text-ink-mute`, `text-ink-dim`, `border-line-strong` instead of
+ * the warm-paper utilities above.
+ *
+ * Distinct names (bg-* / ink-mute / ink-dim / line-strong) so they
+ * don't collide with the warm-paper namespace (surface / ink /
+ * line). Both palettes coexist; the choice between them is per
+ * component, not per theme.
+ */
+@theme {
+  --color-bg: rgb(9 9 11);            /* zinc-950 */
+  --color-bg-elev: rgb(24 24 27);     /* zinc-900 */
+  --color-bg-card: rgb(39 39 42);     /* zinc-800 */
+  --color-line-strong: rgb(63 63 70); /* zinc-700 */
+  --color-ink-mute: rgb(161 161 170); /* zinc-400 */
+  --color-ink-dim: rgb(113 113 122);  /* zinc-500 */
+  --color-accent-dim: rgb(96 165 250);/* blue-400 */
+  --color-plasma: rgb(59 130 246);    /* blue-500 */
+  --color-warn: rgb(251 191 36);      /* amber-400 */
+}
 
 body {
   margin: 0;
   padding: 0;
   overflow: hidden;
-  background: #09090b;
-  color: #e4e4e7;
+  background-color: var(--color-surface);
+  color: var(--color-ink);
   font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif;
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
 }
 
-/* React Flow overrides for dark theme */
+/* React Flow overrides for both themes. Edge stroke pulls from the
+   semantic line token so dark mode keeps its existing zinc-700 look
+   and light mode picks up the warm-paper line colour. */
 .react-flow__edge-path {
-  stroke: #3f3f46 !important;
+  stroke: var(--color-line) !important;
   stroke-width: 1.5 !important;
 }
 
@@ -58,7 +169,8 @@ body {
   transition: box-shadow var(--mol-duration-fast) ease;
 }
 
-/* Scrollbar styling */
+/* Scrollbar styling. Track + thumb pull from the surface tokens so
+   they feel native to either theme. */
 ::-webkit-scrollbar {
   width: 6px;
   height: 6px;
@@ -69,17 +181,17 @@ body {
 }
 
 ::-webkit-scrollbar-thumb {
-  background: #3f3f46;
+  background: var(--color-line);
   border-radius: 3px;
 }
 
 ::-webkit-scrollbar-thumb:hover {
-  background: #52525b;
+  background: var(--color-line-strong, var(--color-ink-soft));
 }
 
 /* Selection */
 ::selection {
-  background: rgba(59, 130, 246, 0.3);
+  background: color-mix(in srgb, var(--color-accent) 30%, transparent);
 }
 
 /* Panel slide animation */
@@ -162,4 +274,17 @@ body {
   .react-flow__node {
     animation: none !important;
   }
+
+  /* React Flow Controls toolbar buttons — WCAG 2.4.7 focus-visible */
+  .react-flow__controls button:focus-visible {
+    outline: 2px solid var(--accent, #3b5bdb);
+    outline-offset: 2px;
+  }
+
+  /* React Flow Minimap nodes — WCAG 2.4.7 focus-visible */
+  .react-flow__minimap:focus-visible,
+  .react-flow__minimap svg:focus-visible {
+    outline: 2px solid var(--accent, #3b5bdb);
+    outline-offset: 2px;
+  }
 }

canvas/src/app/layout.tsx

@@ -1,8 +1,31 @@
 import type { Metadata } from "next";
-import { headers } from "next/headers";
+import { Inter, JetBrains_Mono } from "next/font/google";
+import { cookies, headers } from "next/headers";
 import "./globals.css";
+
+// Self-hosted at build time → CSP-safe (font-src 'self' covers them
+// because Next.js serves the .woff2 from /_next/static). Exposed as
+// CSS variables so the mobile palette can reference them without
+// importing this module.
+const interFont = Inter({
+  subsets: ["latin"],
+  display: "swap",
+  variable: "--font-inter",
+});
+const monoFont = JetBrains_Mono({
+  subsets: ["latin"],
+  display: "swap",
+  variable: "--font-jetbrains",
+});
 import { AuthGate } from "@/components/AuthGate";
 import { CookieConsent } from "@/components/CookieConsent";
+import { PurchaseSuccessModal } from "@/components/PurchaseSuccessModal";
+import { ThemeProvider } from "@/lib/theme-provider";
+import {
+  THEME_COOKIE,
+  readThemeCookie,
+  themeBootScript,
+} from "@/lib/theme-cookie";
 
 export const metadata: Metadata = {
   title: "Molecule AI",
@@ -15,7 +38,7 @@ export default async function RootLayout({
   children: React.ReactNode;
 }) {
   // Read the per-request CSP nonce that middleware.ts sets via the
-  // `x-nonce` request header. This call is load-bearing for TWO
+  // `x-nonce` request header. This call is load-bearing for THREE
   // independent reasons:
   //
   //   1. It opts the root layout into dynamic rendering. Without a
@@ -31,22 +54,62 @@ export default async function RootLayout({
   //      is actually read via `headers()`. The header's existence on
   //      the request isn't enough — Next.js watches for the read.
   //
-  // Keeping the `nonce` variable unused is intentional: we don't need
-  // to pass it to any custom <Script nonce={...}> tags right now, the
-  // framework takes care of its own bootstrap scripts once the read
-  // happens. Destructuring via `await` + `.get()` is the minimum shape
-  // Next.js recognizes as "dynamic server-side access".
-  await headers();
+  //   3. We need the nonce to attach to the inline theme boot script
+  //      below, otherwise CSP rejects it in production where
+  //      script-src is `'self' 'nonce-{nonce}' 'strict-dynamic'`.
+  //      'strict-dynamic' propagates trust from a nonce'd script to
+  //      scripts it inserts, but does NOT forgive an un-nonce'd
+  //      sibling — the boot script must carry its own nonce.
+  const hdrs = await headers();
+  const nonce = hdrs.get("x-nonce") ?? undefined;
+
+  // SSR: read the user's saved preference. For light/dark we can stamp
+  // data-theme on <html> here so the very first paint matches; for
+  // "system" we leave the attribute off and let the inline boot script
+  // resolve from matchMedia before paint.
+  const cookieStore = await cookies();
+  const theme = readThemeCookie(cookieStore.get(THEME_COOKIE)?.value);
+  const initialDataTheme = theme === "system" ? undefined : theme;
 
   return (
-    <html lang="en">
-      <body className="bg-zinc-950 text-white">
-        {/* AuthGate is a client component; it checks the session on mount
-            and bounces anonymous users to the control plane's login page
-            when running on a tenant subdomain. Non-SaaS hosts (localhost,
-            vercel preview URL, apex) pass through unchanged. */}
-        <AuthGate>{children}</AuthGate>
-        <CookieConsent />
+    // suppressHydrationWarning on <html>: the inline boot script below
+    // mutates `data-theme` before React hydrates (system mode reads
+    // matchMedia + writes the attribute). That's the entire point of the
+    // script — eliminate the flash — and it's the documented escape hatch
+    // for "the server-rendered HTML is intentionally not what React would
+    // produce client-side at this exact attribute."
+    <html lang="en" data-theme={initialDataTheme} suppressHydrationWarning>
+      <head>
+        {/*
+         * Boot script: runs synchronously before the body paints, sets
+         * data-theme on <html> for "system" preference based on the OS
+         * media query. For explicit light/dark, SSR already set the
+         * attribute above and the script's write is a no-op.
+         *
+         * `nonce` comes from middleware's per-request CSP nonce — see
+         * the comment block above for why CSP requires this even though
+         * the page also has 'strict-dynamic'.
+         */}
+        <script
+          nonce={nonce}
+          dangerouslySetInnerHTML={{ __html: themeBootScript }}
+        />
+      </head>
+      <body className={`bg-surface text-ink ${interFont.variable} ${monoFont.variable}`}>
+        <ThemeProvider initialTheme={theme}>
+          {/* AuthGate is a client component; it checks the session on mount
+              and bounces anonymous users to the control plane's login page
+              when running on a tenant subdomain. Non-SaaS hosts (localhost,
+              vercel preview URL, apex) pass through unchanged. */}
+          <AuthGate>{children}</AuthGate>
+          <CookieConsent />
+          {/* Demo Mock #1: post-purchase success toast. Mounted at the
+              layout level so it persists across page state transitions
+              (loading → hydrated → error) without being unmounted and
+              losing its open-state. Reads ?purchase_success=1 from the
+              URL on first paint, then strips the param. */}
+          <PurchaseSuccessModal />
+        </ThemeProvider>
       </body>
     </html>
   );

canvas/src/app/orgs/page.tsx

@@ -18,7 +18,7 @@
 // quick bounce between signup and either Checkout or the tenant UI.
 
 import { useEffect, useState } from "react";
-import { fetchSession, redirectToLogin, type Session } from "@/lib/auth";
+import { fetchSession, redirectToLogin, signOut, type Session } from "@/lib/auth";
 import { PLATFORM_URL } from "@/lib/api";
 import { formatCredits, pillTone, bannerKind } from "@/lib/credits";
 import { TermsGate } from "@/components/TermsGate";
@@ -110,15 +110,15 @@ export default function OrgsPage() {
   }, []);
 
   if (session === "loading" || (orgs === null && error === null)) {
-    return <Shell><p className="text-zinc-400">Loading…</p></Shell>;
+    return <Shell><p className="text-ink-mid">Loading…</p></Shell>;
   }
   if (error) {
     return (
       <Shell>
-        <p role="alert" className="text-red-400">Error: {error}</p>
+        <p role="alert" className="text-bad">Error: {error}</p>
         <button
           onClick={() => window.location.reload()}
-          className="mt-4 rounded bg-zinc-800 px-4 py-2 text-sm text-zinc-200 hover:bg-zinc-700"
+          className="mt-4 rounded bg-surface-card px-4 py-2 text-sm text-ink hover:bg-surface-card"
         >
           Retry
         </button>
@@ -129,14 +129,14 @@ export default function OrgsPage() {
     return <EmptyState banner={justCheckedOut ? <CheckoutBanner /> : null} />;
   }
   return (
-    <Shell>
+    <Shell session={session}>
       {justCheckedOut && <CheckoutBanner />}
       <ul className="space-y-3">
         {orgs.map((o) => (
           <OrgRow key={o.id} org={o} />
         ))}
       </ul>
-      <div className="mt-8 border-t border-zinc-800 pt-6">
+      <div className="mt-8 border-t border-line pt-6">
         <CreateOrgForm
           onCreated={(slug) => {
             // Refresh the list so the new org appears + its CTA fires.
@@ -160,13 +160,23 @@ function CheckoutBanner() {
   );
 }
 
-function Shell({ children }: { children: React.ReactNode }) {
+function Shell({
+  children,
+  session,
+}: {
+  children: React.ReactNode;
+  // Optional: when present, the header renders the signed-in email +
+  // a Sign-out button. The empty-state Shell call doesn't have a
+  // session in scope, so accept null and skip the header chrome there.
+  session?: Session | null;
+}) {
   return (
-    <main className="min-h-screen bg-zinc-950 text-zinc-100">
+    <main className="min-h-screen bg-surface text-ink">
       <TermsGate>
         <div className="mx-auto max-w-2xl px-6 pt-20 pb-12">
-          <h1 className="text-3xl font-bold text-white">Your organizations</h1>
-          <p className="mt-2 text-zinc-400">
+          {session ? <AccountBar session={session} /> : null}
+          <h1 className="text-3xl font-bold text-ink">Your organizations</h1>
+          <p className="mt-2 text-ink-mid">
             Each org is an isolated Molecule workspace.
           </p>
           <DataResidencyNotice />
@@ -177,6 +187,40 @@ function Shell({ children }: { children: React.ReactNode }) {
   );
 }
 
+// AccountBar renders the signed-in email + a Sign-out button at the
+// top of the page. Without this the user has no way to log out — the
+// /cp/auth/signout endpoint exists on the control plane but no UI ever
+// called it. Reported externally on 2026-05-05; this is the fix.
+//
+// Click → calls signOut() which POSTs /cp/auth/signout (clears the
+// WorkOS session cookie + revokes at the provider) then bounces to
+// /cp/auth/login. The signOut helper is best-effort — even on a 5xx
+// or network failure the redirect fires so the user never gets stuck
+// on an authed-looking page after they clicked Sign out.
+function AccountBar({ session }: { session: Session }) {
+  const [signingOut, setSigningOut] = useState(false);
+  return (
+    <div className="mb-6 flex items-center justify-between text-sm text-ink-mid">
+      <span title="Signed-in user">{session.email}</span>
+      <button
+        type="button"
+        disabled={signingOut}
+        onClick={async () => {
+          setSigningOut(true);
+          await signOut();
+          // Redirect happens inside signOut; this line is for tests +
+          // edge cases (jsdom, blocked navigation) where it doesn't.
+          setSigningOut(false);
+        }}
+        className="rounded border border-line bg-surface-card px-3 py-1 text-xs text-ink hover:bg-surface-card disabled:opacity-50"
+        aria-label="Sign out"
+      >
+        {signingOut ? "Signing out…" : "Sign out"}
+      </button>
+    </div>
+  );
+}
+
 // DataResidencyNotice surfaces where workspace data lives so EU-based
 // signups can make an informed choice (GDPR Art. 13 disclosure
 // requirement). Plain text, no icon — the goal is clarity, not
@@ -184,7 +228,7 @@ function Shell({ children }: { children: React.ReactNode }) {
 // region dropdown.
 function DataResidencyNotice() {
   return (
-    <p className="mt-3 rounded border border-zinc-800 bg-zinc-900/60 px-3 py-2 text-xs text-zinc-400">
+    <p className="mt-3 rounded border border-line bg-surface-sunken/60 px-3 py-2 text-xs text-ink-mid">
       Workspaces run in AWS us-east-2 (Ohio, United States). EU region support is on the roadmap — reach out to
       {" "}
       <a href="mailto:support@moleculesai.app" className="underline">
@@ -197,11 +241,11 @@ function DataResidencyNotice() {
 
 function OrgRow({ org }: { org: Org }) {
   return (
-    <li className="rounded-lg border border-zinc-800 bg-zinc-900 p-4">
+    <li className="rounded-lg border border-line bg-surface-sunken p-4">
       <div className="flex items-center justify-between">
         <div>
-          <div className="font-medium text-white">{org.name}</div>
-          <div className="text-sm text-zinc-400">
+          <div className="font-medium text-ink">{org.name}</div>
+          <div className="text-sm text-ink-mid">
             {org.slug} · <StatusLabel status={org.status} /> · {org.plan || "free"}
           </div>
           <div className="mt-2 flex items-center gap-2">
@@ -237,21 +281,21 @@ function LowCreditsBanner({ org }: { org: Org }) {
   if (kind === "overage") {
     const used = (org.overage_used_credits ?? 0).toLocaleString();
     return (
-      <span className="text-xs text-amber-300">
+      <span className="text-xs text-warm">
         overage active · {used} used
       </span>
     );
   }
   if (kind === "out-of-credits") {
     return (
-      <a href={`/pricing?org=${encodeURIComponent(org.slug)}`} className="text-xs text-red-300 underline">
+      <a href={`/pricing?org=${encodeURIComponent(org.slug)}`} className="text-xs text-bad underline">
         out of credits — upgrade to keep running
       </a>
     );
   }
   // trial-tail
   return (
-    <a href={`/pricing?org=${encodeURIComponent(org.slug)}`} className="text-xs text-amber-300 underline">
+    <a href={`/pricing?org=${encodeURIComponent(org.slug)}`} className="text-xs text-warm underline">
       trial almost out
     </a>
   );
@@ -260,11 +304,11 @@ function LowCreditsBanner({ org }: { org: Org }) {
 function StatusLabel({ status }: { status: OrgStatus }) {
   const cls =
     status === "running"
-      ? "text-emerald-400"
+      ? "text-good"
       : status === "awaiting_payment"
-      ? "text-amber-400"
+      ? "text-warm"
       : status === "failed"
-      ? "text-red-400"
+      ? "text-bad"
       : "text-sky-400";
   const label =
     status === "awaiting_payment"
@@ -303,21 +347,21 @@ function OrgCTA({ org }: { org: Org }) {
     return (
       <a
         href="mailto:support@moleculesai.app"
-        className="rounded bg-zinc-700 px-4 py-2 text-sm font-medium text-zinc-200 hover:bg-zinc-600"
+        className="rounded bg-surface-card px-4 py-2 text-sm font-medium text-ink hover:bg-surface-card"
       >
         Contact support
       </a>
     );
   }
   // provisioning / unknown — non-interactive
-  return <span className="text-sm text-zinc-500">{org.status}…</span>;
+  return <span className="text-sm text-ink-mid">{org.status}…</span>;
 }
 
 function EmptyState({ banner }: { banner?: React.ReactNode }) {
   return (
     <Shell>
       {banner}
-      <p className="text-zinc-300">
+      <p className="text-ink-mid">
         You don't have any organizations yet. Create one to get started — your
         workspace spins up automatically once billing is set up.
       </p>
@@ -365,7 +409,7 @@ function CreateOrgForm({ onCreated }: { onCreated: (slug: string) => void }) {
   return (
     <form onSubmit={submit} className="space-y-3">
       <div>
-        <label htmlFor="org-slug" className="block text-sm text-zinc-300">Slug (URL)</label>
+        <label htmlFor="org-slug" className="block text-sm text-ink-mid">Slug (URL)</label>
         <input
           id="org-slug"
           value={slug}
@@ -374,28 +418,28 @@ function CreateOrgForm({ onCreated }: { onCreated: (slug: string) => void }) {
           placeholder="acme"
           required
           aria-describedby="org-slug-hint"
-          className="mt-1 w-full rounded border border-zinc-700 bg-zinc-800 px-3 py-2 text-sm text-zinc-100"
+          className="mt-1 w-full rounded border border-line bg-surface-card px-3 py-2 text-sm text-ink"
         />
-        <p id="org-slug-hint" className="mt-1 text-xs text-zinc-500">
+        <p id="org-slug-hint" className="mt-1 text-xs text-ink-mid">
           Lowercase letters, numbers, and hyphens only. Cannot be changed later.
         </p>
       </div>
       <div>
-        <label htmlFor="org-name" className="block text-sm text-zinc-300">Display name</label>
+        <label htmlFor="org-name" className="block text-sm text-ink-mid">Display name</label>
         <input
           id="org-name"
           value={name}
           onChange={(e) => setName(e.target.value)}
           placeholder="Acme Corp"
           required
-          className="mt-1 w-full rounded border border-zinc-700 bg-zinc-800 px-3 py-2 text-sm text-zinc-100"
+          className="mt-1 w-full rounded border border-line bg-surface-card px-3 py-2 text-sm text-ink"
         />
       </div>
-      {err && <p role="alert" className="text-sm text-red-400">{err}</p>}
+      {err && <p role="alert" className="text-sm text-bad">{err}</p>}
       <button
         type="submit"
         disabled={submitting}
-        className="rounded bg-blue-600 px-4 py-2 text-sm font-medium text-white hover:bg-blue-500 disabled:opacity-50"
+        className="rounded bg-accent-strong px-4 py-2 text-sm font-medium text-white hover:bg-accent disabled:opacity-50"
       >
         {submitting ? "Creating…" : "Create organization"}
       </button>