Merge pull request 'fix(handlers): sync-persist poll-mode A2A message in staging (internal#471)' (#1375) from fix/staging-sync-persist-fix into staging

Adds USER_MESSAGE WebSocket broadcast so a user's own outbound message
appears live on all their other sessions without requiring a manual
refresh.

**Server (Go)**
- New `EventUserMessage` constant in `events/types.go`
- New `extractCanvasUserMessage()` in `a2a_proxy_helpers.go` — parses
  A2A JSON-RPC request body, extracts text parts + file attachments
  from user-role message/send calls
- `logA2ASuccess()` now broadcasts `USER_MESSAGE` when canvas sends a
  message (callerID == "" + method == "message/send" + 2xx)

**Canvas (TypeScript)**
- `useChatSocket`: new `onUserMessage` callback + `USER_MESSAGE` handler
  in the socket event listener
- `ChatTab`: wires `onUserMessage` via `appendMessageDeduped`
- `MobileChat`: same wiring for mobile variant
- New test file: 9 tests covering text/file/text+file/wrong-workspace/
  empty/no-callback/other-events/replay/attachment-filtering cases

**Dedup contract**: the originating session's optimistic insert (useChatSend
step 2) and the incoming USER_MESSAGE broadcast share the same
`appendMessageDeduped` path — within the 3-second window the two collapse
into one bubble, so the user sees exactly one message bubble.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.env.example

@@ -50,6 +50,9 @@ MOLECULE_ENV=development                       # Environment label (development/
 # Container/runtime detection
 # MOLECULE_IN_DOCKER=                    # Set when running the platform inside Docker (accepts 1/0, true/false). Triggers A2A proxy to rewrite 127.0.0.1:<port> agent URLs to Docker bridge hostnames. Auto-detected via /.dockerenv; only set if detection fails or to force off.
 
+# Observability (Awareness)
+# AWARENESS_URL=                         # If set, injected into workspace containers along with a deterministic AWARENESS_NAMESPACE derived from workspace ID. Enables the cross-session memory MCP server.
+
 # GitHub
 # GITHUB_REPO=owner/repo                 # Target repo for agent initial_prompt clone (e.g. Molecule-AI/molecule-monorepo). Read inside workspace containers.
 # GITHUB_TOKEN=                          # Personal access token / installation token used by agents that clone private repos. Register as a global secret via POST /admin/secrets for propagation to workspace env. Token is used in-URL during clone and then scrubbed from .git/config via `git remote set-url`.

.gitea/scripts/ci-required-drift.py

@@ -274,8 +274,7 @@ def required_checks_env(audit_doc: dict) -> set[str]:
                     found.append(v)
     if not found:
         sys.stderr.write(
-            f"::error::REQUIRED_CHECKS env not found in any step of "
-            f"{AUDIT_WORKFLOW_PATH}\n"
+            f"::error::REQUIRED_CHECKS env not found in any step of {AUDIT_WORKFLOW_PATH}\n"
         )
         sys.exit(3)
     if len(found) > 1:
@@ -385,15 +384,10 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
     contexts = set(protection.get("status_check_contexts") or [])
 
     # ----- F1: job exists in CI but not under sentinel.needs -----
-    # Post-#1766 contract: the sentinel may deliberately have no `needs:`
-    # and instead poll path-relevant statuses dynamically. In that case
-    # F1 is a false positive — skip it. F1b (typos in existing needs)
-    # is naturally skipped when needs is empty.
     missing_from_needs = sorted(jobs - needs)
-    if missing_from_needs and needs:
+    if missing_from_needs:
         findings.append(
-            "F1 — jobs in ci.yml NOT under sentinel `needs:` "
-            "(sentinel doesn't gate them):\n"
+            "F1 — jobs in ci.yml NOT under sentinel `needs:` (sentinel doesn't gate them):\n"
             + "\n".join(f"  - {n}" for n in missing_from_needs)
         )
 
@@ -403,8 +397,7 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
     stale_needs = sorted(needs - jobs_all)
     if stale_needs:
         findings.append(
-            "F1b — sentinel `needs:` lists jobs NOT present in ci.yml "
-            "(typo or removed job):\n"
+            "F1b — sentinel `needs:` lists jobs NOT present in ci.yml (typo or removed job):\n"
             + "\n".join(f"  - {n}" for n in stale_needs)
         )
 
@@ -412,9 +405,7 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
     # Compute the contexts the CI YAML actually produces. The sentinel
     # is in (B) intentionally (`ci / all-required (pull_request)`); we
     # whitelist it explicitly.
-    emitted_contexts = {
-        expected_context(j) for j in jobs
-    } | {expected_context(SENTINEL_JOB)}
+    emitted_contexts = {expected_context(j) for j in jobs} | {expected_context(SENTINEL_JOB)}
     # Contexts NOT produced by ci.yml may still come from other
     # workflows in the repo (Secret scan etc). We can't enumerate
     # every workflow's emissions cheaply; instead, flag only contexts
@@ -427,9 +418,8 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
     )
     if stale_protection:
         findings.append(
-            "F2 — protection `status_check_contexts` entries with `ci / ` "
-            "prefix that NO job in ci.yml emits "
-            "(stale name → silent advisory gate):\n"
+            "F2 — protection `status_check_contexts` entries with `ci / ` prefix that NO "
+            "job in ci.yml emits (stale name → silent advisory gate):\n"
             + "\n".join(f"  - {c}" for c in stale_protection)
         )
 
@@ -504,8 +494,7 @@ def render_body(branch: str, findings: list[str], debug: dict) -> str:
         f"# Drift detected on `{REPO}/{branch}`",
         "",
         "Auto-filed by `.gitea/workflows/ci-required-drift.yml` "
-        "(RFC [internal#219]"
-        "(https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
+        "(RFC [internal#219](https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
         "",
         "## Findings",
         "",
@@ -516,11 +505,8 @@ def render_body(branch: str, findings: list[str], debug: dict) -> str:
             "",
             "## Resolution",
             "",
-            "- **F1 / F1b**: if the sentinel job has a `needs:` block, add "
-            "the missing job to it in `.gitea/workflows/ci.yml`, or remove "
-            "the stale entry. If the sentinel deliberately has no `needs:` "
-            "(path-aware polling sentinel per post-#1766 contract), this "
-            "finding is expected and F1 is skipped.",
+            "- **F1 / F1b**: add the missing job to `all-required.needs:` "
+            "in `.gitea/workflows/ci.yml`, or remove the stale entry.",
             "- **F2**: rename the protection context to match an emitter, "
             "or remove it from `status_check_contexts` "
             "(PATCH `/api/v1/repos/{owner}/{repo}/branch_protections/{branch}`).",
@@ -561,12 +547,12 @@ def file_or_update(
 
     if dry_run:
         print(f"::notice::[dry-run] would file/update drift issue for {branch}")
-        print("::group::[dry-run] title")
+        print(f"::group::[dry-run] title")
         print(title)
-        print("::endgroup::")
-        print("::group::[dry-run] body")
+        print(f"::endgroup::")
+        print(f"::group::[dry-run] body")
         print(body)
-        print("::endgroup::")
+        print(f"::endgroup::")
         return
 
     existing = find_open_issue(title)

.gitea/scripts/detect-changes.py

@@ -1,176 +0,0 @@
-#!/usr/bin/env python3
-"""Shared path-filter helper for Gitea Actions workflows.
-
-Computes changed files against the PR base SHA or push-before SHA and writes
-boolean outputs to GITHUB_OUTPUT. If the diff base is missing or untrusted, the
-helper fails open by setting every output in the selected profile to true.
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import re
-import subprocess
-import sys
-from pathlib import Path
-
-PROFILES: dict[str, dict[str, str]] = {
-    "ci": {
-        "platform": r"^workspace-server/",
-        "canvas": r"^canvas/",
-        "python": r"^workspace/",
-        "scripts": r"^tests/e2e/|^scripts/|^infra/scripts/",
-    },
-    "handlers-postgres": {
-        "handlers": (
-            r"^workspace-server/internal/handlers/"
-            r"|^workspace-server/internal/wsauth/"
-            r"|^workspace-server/migrations/"
-            r"|^\.gitea/workflows/handlers-postgres-integration\.yml$"
-        ),
-    },
-    "e2e-api": {
-        "api": r"^workspace-server/|^tests/e2e/|^\.gitea/workflows/e2e-api\.yml$",
-    },
-}
-
-
-def classify(profile: str, paths: list[str]) -> dict[str, bool]:
-    patterns = PROFILES[profile]
-    return {
-        name: any(re.search(pattern, path) for path in paths)
-        for name, pattern in patterns.items()
-    }
-
-
-def all_true(profile: str) -> dict[str, bool]:
-    return {name: True for name in PROFILES[profile]}
-
-
-def resolve_base(event_name: str, pr_base_sha: str, push_before: str) -> str:
-    if event_name == "pull_request" and pr_base_sha:
-        return pr_base_sha
-    return push_before
-
-
-def is_zero_sha(value: str) -> bool:
-    return not value or bool(re.fullmatch(r"0+", value))
-
-
-def run_git(args: list[str], *, timeout: int = 30) -> subprocess.CompletedProcess[str]:
-    return subprocess.run(
-        ["git", *args],
-        check=False,
-        text=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        timeout=timeout,
-    )
-
-
-def base_exists(base: str) -> bool:
-    return run_git(["cat-file", "-e", base]).returncode == 0
-
-
-def fetch_base(base: str, base_ref: str) -> None:
-    # Gitea may reject fetching an arbitrary unadvertised SHA from a shallow
-    # PR checkout. Fetch the advertised base branch first, then fall back to
-    # the SHA for hosts that allow it.
-    if base_ref:
-        run_git(["fetch", "--depth=1", "origin", base_ref])
-    if not base_exists(base):
-        run_git(["fetch", "--depth=1", "origin", base])
-
-
-def deepen_base_ref(base_ref: str) -> None:
-    if base_ref:
-        run_git(["fetch", "--deepen=200", "origin", base_ref], timeout=60)
-
-
-def merge_base(base: str) -> str | None:
-    proc = run_git(["merge-base", base, "HEAD"])
-    if proc.returncode != 0:
-        return None
-    value = proc.stdout.strip()
-    return value or None
-
-
-def changed_paths(base: str, *, use_merge_base: bool) -> list[str] | None:
-    compare_base = base
-    if use_merge_base:
-        compare_base = merge_base(base) or ""
-        if not compare_base:
-            return None
-
-    proc = run_git(["diff", "--name-only", compare_base, "HEAD"])
-    if proc.returncode != 0:
-        return None
-    return [line for line in proc.stdout.splitlines() if line]
-
-
-def write_outputs(values: dict[str, bool], output_path: str | None) -> None:
-    lines = [f"{name}={'true' if value else 'false'}" for name, value in values.items()]
-    if output_path:
-        with Path(output_path).open("a", encoding="utf-8") as fh:
-            for line in lines:
-                fh.write(line + "\n")
-    else:
-        for line in lines:
-            print(line)
-
-
-def detect(
-    profile: str,
-    event_name: str,
-    pr_base_sha: str,
-    push_before: str,
-    base_ref: str = "",
-) -> dict[str, bool]:
-    base = resolve_base(event_name, pr_base_sha, push_before)
-    if is_zero_sha(base):
-        return all_true(profile)
-
-    if not base_exists(base):
-        fetch_base(base, base_ref)
-    if not base_exists(base):
-        return all_true(profile)
-
-    use_merge_base = event_name == "pull_request"
-    if use_merge_base and base_ref and merge_base(base) is None:
-        deepen_base_ref(base_ref)
-
-    paths = changed_paths(base, use_merge_base=use_merge_base)
-    if paths is None:
-        return all_true(profile)
-    return classify(profile, paths)
-
-
-def parse_args(argv: list[str]) -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--profile", required=True, choices=sorted(PROFILES))
-    parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME", ""))
-    parser.add_argument("--pr-base-sha", default="")
-    parser.add_argument("--base-ref", default="")
-    parser.add_argument(
-        "--push-before",
-        default=os.environ.get("GITHUB_EVENT_BEFORE", ""),
-    )
-    return parser.parse_args(argv)
-
-
-def main(argv: list[str]) -> int:
-    args = parse_args(argv)
-    values = detect(
-        args.profile,
-        args.event_name,
-        args.pr_base_sha,
-        args.push_before,
-        args.base_ref,
-    )
-    write_outputs(values, os.environ.get("GITHUB_OUTPUT"))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))

.gitea/scripts/gitea-merge-queue.py

@@ -183,9 +183,7 @@ def required_contexts_green(
         status = latest_statuses.get(context)
         state = status_state(status or {})
         if state != "success":
-            if pr_labels and _is_tier_low_pending_ok(
-                latest_statuses, context, pr_labels
-            ):
+            if pr_labels and _is_tier_low_pending_ok(latest_statuses, context, pr_labels):
                 continue  # tier:low soft-fail: accept pending sop-checklist
             missing_or_bad.append(f"{context}={state or 'missing'}")
     return not missing_or_bad, missing_or_bad

.gitea/scripts/lint-curl-status-capture.py

@@ -13,9 +13,11 @@ from __future__ import annotations
 import argparse
 import glob
 import re
+import sys
 from pathlib import Path
 from typing import NamedTuple
 
+
 SELF = ".gitea/workflows/lint-curl-status-capture.yml"
 
 

.gitea/scripts/lint_bp_context_emit_match.py

@@ -283,7 +283,7 @@ def _ensure_labels(repo: str, names: list[str]) -> list[int]:
     if status != "ok" or not isinstance(labels, list):
         return []
     out: list[int] = []
-    by_name = {label["name"]: label["id"] for label in labels if isinstance(label, dict)}
+    by_name = {l["name"]: l["id"] for l in labels if isinstance(l, dict)}
     for n in names:
         if n in by_name:
             out.append(by_name[n])

.gitea/scripts/lint_continue_on_error_tracking.py

@@ -82,7 +82,7 @@ import sys
 import urllib.error
 import urllib.parse
 import urllib.request
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 

.gitea/scripts/lint_pre_flip_continue_on_error.py

@@ -641,15 +641,6 @@ def main(argv: list[str] | None = None) -> int:
 
     base_workflows = workflows_at_sha(BASE_SHA)
     head_workflows = workflows_at_sha(HEAD_SHA)
-    # Ignore workflow files that are identical on both sides — old branches
-    # that haven't rebased onto main carry stale copies of workflows that
-    # were updated later. Comparing those stale copies against the current
-    # base produces false-positive "flips".
-    base_workflows = {
-        p: t for p, t in base_workflows.items()
-        if p in head_workflows and head_workflows[p] != t
-    }
-    head_workflows = {p: t for p, t in head_workflows.items() if p in base_workflows}
     flips = detect_flips(base_workflows, head_workflows)
 
     if not flips:

.gitea/scripts/main-red-watchdog.py

@@ -61,7 +61,6 @@ import os
 import shutil
 import subprocess
 import sys
-import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -90,28 +89,6 @@ API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
 # match by exact title without parsing.
 TITLE_PREFIX = "[main-red]"
 
-# Contexts that are scheduled or non-required — their pending/failure
-# state should not block stale-issue closeout (mc#1789).
-SCHEDULED_CONTEXT_PATTERNS = (
-    "Staging SaaS smoke",
-    "Continuous synthetic E2E",
-    "main-red-watchdog",
-    "ci-arm64-advisory",
-)
-
-# Settling window (seconds) between initial red detection and the
-# pre-file recheck. The recheck filters out the two largest false-
-# positive classes seen in mc#1597..1630 (task #394, 2026-05-21):
-#   1. HEAD moved on (a new commit landed mid-tick) — the prior red SHA
-#      is no longer authoritative; let the next cron tick re-evaluate.
-#   2. Combined status recovered on the SAME SHA (transient
-#      cancel-cascade rolled forward to success on retry).
-# 90s is well below the hourly cron cadence; a real failure that
-# persists past it is the one we want surfaced.
-# Override with WATCHDOG_RECHECK_DELAY_SECS for tests / local probes
-# (the test suite stubs time.sleep to a no-op).
-RECHECK_DELAY_SECS = int(_env("WATCHDOG_RECHECK_DELAY_SECS", default="90"))
-
 
 def _require_runtime_env() -> None:
     """Enforce env contract — called from `main()` only.
@@ -195,49 +172,6 @@ def api(
         return status, {"_raw": raw.decode("utf-8", errors="replace")}
 
 
-# --------------------------------------------------------------------------
-# action_run.status resolver — extensibility hook for task #394.
-# --------------------------------------------------------------------------
-def _resolve_action_run_status(target_url: str) -> int | None:
-    """Resolve the underlying Gitea `action_run.status` integer for the
-    run referenced by `target_url`, returning None if the resolver
-    cannot reach an authoritative source from the runner.
-
-    Canonical Gitea 1.22.6 enum (per `models/actions/status.go` +
-    `reference_gitea_action_status_enum_corrected_2026_05_19`):
-        1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
-        5=Waiting,  6=Running, 7=Blocked
-    Only `status == 2` is a real defect; status=3 is cancel-cascade and
-    status=1 is an emission artifact (Gitea wrote a 'failure' commit_status
-    row for a run that actually succeeded — observed empirically on
-    `publish-canvas-image` jobs at SHAs in mc#1597..1630).
-
-    CURRENT STATE (2026-05-20, verified): Gitea 1.22.6 exposes NO REST
-    endpoint for `action_run.status`. Probed:
-        /api/v1/repos/{o}/{r}/actions/runs/{id}   → HTTP 404
-        /api/v1/repos/{o}/{r}/actions/jobs/{id}   → HTTP 404
-        /api/v1/repos/{o}/{r}/actions/tasks/{id}  → HTTP 404
-        /swagger.v1.json paths containing 'actions' → secrets+variables+runners only
-    The SPA backend (`/{repo}/actions/runs/{id}/jobs/{idx}` POST) requires
-    a session CSRF token, unreachable from a runner. The only authoritative
-    source today is direct DB access (`mol_action_status` on op-host,
-    `docker exec molecule-postgres-1 psql ...`), which the runner cannot
-    reach.
-
-    Therefore: this hook returns None on every call. Callers MUST fall
-    back to the description-string filter (existing) plus the HEAD
-    recheck (this PR). When a future Gitea release (>=1.23 expected) or
-    an op-host proxy exposes the endpoint, replace the body of this
-    function with an `api(...)` call — the caller contract is stable.
-
-    See also:
-        - `reference_chronic_red_sweep_cancelled_vs_failed_filter`
-        - `feedback_gitea_status_enum_use_helper_not_raw_int`
-    """
-    _ = target_url  # noqa: F841 — intentional placeholder
-    return None
-
-
 # --------------------------------------------------------------------------
 # Gitea reads
 # --------------------------------------------------------------------------
@@ -274,11 +208,6 @@ def get_combined_status(sha: str) -> dict:
     return body
 
 
-def _entry_state(s: dict) -> str:
-    """Per-entry status key in Gitea 1.22.6 is `status`; fall back to `state`."""
-    return s.get("status") or s.get("state") or ""
-
-
 def is_red(status: dict) -> tuple[bool, list[dict]]:
     """Return (is_red, failed_statuses).
 
@@ -289,31 +218,6 @@ def is_red(status: dict) -> tuple[bool, list[dict]]:
 
     `failed_statuses` is the list of per-context entries whose own
     `state` is in the red set; useful for the issue body.
-
-    Cancel-cascade filter (mc#1564, 2026-05-19):
-      Gitea maps BOTH `action_run.status=2 (Failure)` AND
-      `action_run.status=3 (Cancelled)` to commit-status string
-      `"failure"`. On a busy main with
-      `concurrency: cancel-in-progress: true`, every merge burst
-      cancels prior in-flight runs (status=3) — those bubble to the
-      combined-status `failure` and inflate the watchdog's red%,
-      generating phantom `[main-red]` issues (mc#1562/#1552/#1540/...).
-      Canonical Gitea 1.22.6 enum per `models/actions/status.go` +
-      `reference_gitea_action_status_enum_corrected_2026_05_19`:
-          1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
-          5=Waiting, 6=Running, 7=Blocked
-      We only want status=2 (real defects) to file. At the
-      commit-status layer we don't have the integer enum directly
-      (only the `failure` rollup string), so we use the description
-      string Gitea writes when a run is cancelled — empirically
-      `"Has been cancelled"` (verified 2026-05-19 via #1562 body).
-      Real failures show `"Failing after Ns"` and are unaffected.
-      This is option B from mc#1564 (description-string filter, no
-      extra API call). Description-string stability is a soft contract
-      with Gitea; if a future release renames it, the cancel-cascade
-      entries will simply leak back through (visible-not-silent), and
-      we'll either re-pin the string or upgrade to option A (resolve
-      the underlying action_run.status integer via target_url).
     """
     combined = status.get("state")
     statuses = status.get("statuses") or []
@@ -326,30 +230,14 @@ def is_red(status: dict) -> tuple[bool, list[dict]]:
     # "no per-context entries were in a red state" fallback even when
     # the combined-state correctly flagged red. See
     # `feedback_smoke_test_vendor_truth_not_shape_match`.
-    def _is_cancel_cascade(s: dict) -> bool:
-        """status=3 entry per Gitea 1.22.6 description-string contract.
-        Match exactly (after strip) — substring match would catch
-        legitimate test names like "Has been cancelled by the user
-        unexpectedly" in failure logs."""
-        desc = (s.get("description") or "").strip()
-        return desc == "Has been cancelled"
+    def _entry_state(s: dict) -> str:
+        return s.get("status") or s.get("state") or ""
 
     failed = [
         s for s in statuses
-        if isinstance(s, dict)
-        and _entry_state(s) in red_states
-        and not _is_cancel_cascade(s)
+        if isinstance(s, dict) and _entry_state(s) in red_states
     ]
-    # Combined state alone is no longer sufficient — combined=failure
-    # may be 100% cancel-cascade. Drive `red` off the FILTERED list:
-    # if every red-shaped per-entry was cancel-cascade, `failed` is
-    # empty and we report green. Combined-failure with no per-entry
-    # detail (empty `statuses[]`) still trips red — that's the
-    # "CI emitter set combined-status directly" edge case from
-    # render_body's fallback path; we keep filing on it so the
-    # operator sees the breadcrumb.
-    combined_red_no_detail = combined in red_states and not statuses
-    return (bool(failed) or combined_red_no_detail, failed)
+    return (combined in red_states or bool(failed), failed)
 
 
 # --------------------------------------------------------------------------
@@ -364,15 +252,6 @@ def title_for(sha: str) -> str:
     return f"{TITLE_PREFIX} {REPO}: {sha[:10]}"
 
 
-def _is_scheduled_context(context: str) -> bool:
-    """Return True if `context` is a known scheduled/non-required job.
-
-    These contexts run on a schedule and should not block stale-issue
-    closeout when main's required CI has recovered (mc#1789).
-    """
-    return any(pattern.lower() in context.lower() for pattern in SCHEDULED_CONTEXT_PATTERNS)
-
-
 def list_open_red_issues() -> list[dict]:
     """All open issues whose title starts with `[main-red] {repo}: `.
 
@@ -382,34 +261,23 @@ def list_open_red_issues() -> list[dict]:
     file-or-update path to POST a duplicate — exactly the regression
     class the helper-raises contract closes.
 
-    Pagination is exhausted (mc#1789). The old "by design ≤ 1" invariant
-    was false — backlog can exceed 50 open issues.
+    Gitea issue search returns at most 50/page; we only need open
+    `[main-red]` issues which are by design ≤ 1 at any time per repo,
+    so a single page is enough.
     """
-    prefix = f"{TITLE_PREFIX} {REPO}: "
-    all_issues: list[dict] = []
-    page = 1
-    limit = 50
-    while True:
-        _, results = api(
-            "GET",
-            f"/repos/{OWNER}/{NAME}/issues",
-            query={"state": "open", "type": "issues", "limit": str(limit), "page": str(page)},
+    _, results = api(
+        "GET",
+        f"/repos/{OWNER}/{NAME}/issues",
+        query={"state": "open", "type": "issues", "limit": "50"},
+    )
+    if not isinstance(results, list):
+        raise ApiError(
+            f"issue search returned non-list body (got {type(results).__name__})"
         )
-        if not isinstance(results, list):
-            raise ApiError(
-                f"issue search returned non-list body (got {type(results).__name__})"
-            )
-        matched = [
-            i for i in results
-            if isinstance(i, dict)
+    prefix = f"{TITLE_PREFIX} {REPO}: "
+    return [i for i in results if isinstance(i, dict)
             and isinstance(i.get("title"), str)
-            and i["title"].startswith(prefix)
-        ]
-        all_issues.extend(matched)
-        if len(results) < limit:
-            break
-        page += 1
-    return all_issues
+            and i["title"].startswith(prefix)]
 
 
 def find_open_issue_for_sha(sha: str) -> dict | None:
@@ -609,7 +477,6 @@ def close_open_red_issues_for_other_shas(
     current_sha: str,
     *,
     dry_run: bool = False,
-    close_same_sha: bool = False,
 ) -> int:
     """When main is green at current_sha, close any open `[main-red]`
     issues whose title references a different SHA. Returns the number
@@ -618,25 +485,15 @@ def close_open_red_issues_for_other_shas(
     Lineage note: we only close issues whose title prefix matches; if
     a human renamed the issue or added a suffix this won't touch it.
     That's intentional — manual editorial state takes precedence.
-
-    Args:
-        close_same_sha: set True when the caller already knows main is
-            green at current_sha (e.g. recovery block) and wants to close
-            the open issue for THIS SHA too. Defaults False so the
-            green-path callers never accidentally close an issue they just
-            filed on the same tick.
     """
     target_title = title_for(current_sha)
     open_red = list_open_red_issues()
     closed = 0
     for issue in open_red:
         if issue.get("title") == target_title:
-            if not close_same_sha:
-                # Same SHA — caller should not have invoked this if main is
-                # green. Skip defensively (guards against green-path callers
-                # that accidentally pass the SHA they just filed for).
-                continue
-            # close_same_sha=True: close even this SHA's issue (recovery path)
+            # Same SHA — caller should not have invoked this if main is
+            # green. Skip defensively.
+            continue
         num = issue.get("number")
         if not isinstance(num, int):
             continue
@@ -713,123 +570,29 @@ def run_once(*, dry_run: bool = False) -> int:
     }
 
     if red:
-        # HEAD recheck (task #394 — guards mc#1597..1630 false-positive
-        # cluster). After the initial detection, wait RECHECK_DELAY_SECS
-        # (default 90s; tests stub time.sleep) and re-evaluate:
-        #
-        #   1. Re-fetch HEAD SHA. If HEAD moved, a new commit landed
-        #      mid-tick — the prior red SHA is no longer authoritative
-        #      and the next cron run will re-evaluate against the new
-        #      HEAD. Skip-file.
-        #
-        #   2. If HEAD unchanged, re-fetch the combined status. If it
-        #      recovered (combined state no longer in {failure,error}
-        #      after the cancel-cascade filter), a transient retry
-        #      rolled the run forward. Skip-file.
-        #
-        # Both paths emit a Loki event distinguishable from the real
-        # `main_red_detected` so obs queries can track filter activity.
-        # The settling window is well below the hourly cron cadence —
-        # genuine failures persist past it and are surfaced normally.
-        time.sleep(RECHECK_DELAY_SECS)
-
-        recheck_sha = get_head_sha(WATCH_BRANCH)
-        if recheck_sha != sha:
-            emit_loki_event("main_red_skipped_head_drift", sha, [])
-            print(
-                f"::notice::skip-file (HEAD moved): initial red at "
-                f"{sha[:10]} but HEAD is now {recheck_sha[:10]} on "
-                f"{WATCH_BRANCH}; next cron tick will re-evaluate."
-            )
-            # HEAD drifted — close any stale main-red issue for the prior SHA
-            # before returning, so we don't leave stale open issues when main
-            # is no longer pointing at the red commit.
-            close_open_red_issues_for_other_shas(recheck_sha, dry_run=dry_run)
-            return 0
-
-        recheck_status = get_combined_status(sha)
-        recheck_red, recheck_failed = is_red(recheck_status)
-        if not recheck_red:
-            emit_loki_event("main_red_skipped_recovered", sha, [])
-            print(
-                f"::notice::skip-file (recovered after settling): "
-                f"combined state at {sha[:10]} flipped to "
-                f"{recheck_status.get('state')!r} on recheck; "
-                f"initial red was a transient cancel-cascade."
-            )
-            # CI recovered on the same SHA — close any stale main-red issue
-            # that was filed on a prior tick for this SHA.
-            close_open_red_issues_for_other_shas(sha, dry_run=dry_run, close_same_sha=True)
-            return 0
-
-        # Still red after settling — file/update. Use the recheck data
-        # as authoritative so the issue body reflects the latest state.
-        failed = recheck_failed
-        debug["recheck_combined_state"] = recheck_status.get("state")
-        debug["recheck_failed_contexts"] = [
-            s.get("context") for s in failed
-        ]
-
         failed_ctxs = [s.get("context") for s in failed if s.get("context")]
         emit_loki_event("main_red_detected", sha, failed_ctxs)
         print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
               f"{len(failed)} failed context(s)")
         file_or_update_red(sha, failed, debug, dry_run=dry_run)
     else:
-        # Green or pending-with-no-real-failures. Close stale issues
-        # from earlier SHAs when required CI has recovered.
-        #
-        # mc#1789: main often sits at combined `pending` because
-        # scheduled/non-required contexts (Staging SaaS smoke,
-        # Continuous synthetic E2E, main-red-watchdog itself,
-        # ci-arm64-advisory) are still running. We close stale issues
-        # as long as no *non-scheduled* context has failed and no
-        # *non-scheduled* context is still pending — i.e. required CI
-        # is effectively green.
-        #
-        # The success-only gate is preserved for the canonical green
-        # path; the extended check below only fires when combined is
-        # `pending` but all required work is done.
-        combined_state = status.get("state")
-        if combined_state == "success":
-            should_close = True
-            close_reason = "GREEN"
-        else:
-            statuses = status.get("statuses") or []
-            non_scheduled_pending = [
-                s for s in statuses
-                if isinstance(s, dict)
-                and (_entry_state(s) == "pending")
-                and not _is_scheduled_context(s.get("context", ""))
-            ]
-            non_scheduled_failed = [
-                s for s in statuses
-                if isinstance(s, dict)
-                and (_entry_state(s) in {"failure", "error"})
-                and not _is_scheduled_context(s.get("context", ""))
-            ]
-            # Cancel-cascade already filtered by is_red(); red=False
-            # here means no real failures. We additionally check that
-            # no non-scheduled context is still pending.
-            should_close = not non_scheduled_pending and not non_scheduled_failed
-            close_reason = "pending-but-required-green"
-
-        if should_close:
+        # Green (or pending — pending is treated as not-red so we don't
+        # spam during the post-merge CI window). Close any stale issues
+        # from earlier SHAs only when we're actually green; pending
+        # means CI hasn't finished and the prior issue might still be
+        # accurate.
+        if status.get("state") == "success":
             closed = close_open_red_issues_for_other_shas(sha, dry_run=dry_run)
             if closed:
                 emit_loki_event(
                     "main_returned_to_green", sha,
                     [],
                 )
-            print(
-                f"::notice::main is {close_reason} at {sha[:10]} on {WATCH_BRANCH} "
-                f"(closed {closed} stale issue(s))"
-            )
+            print(f"::notice::main is GREEN at {sha[:10]} on {WATCH_BRANCH} "
+                  f"(closed {closed} stale issue(s))")
         else:
-            print(
-                f"::notice::main has pending-or-failed required CI at {sha[:10]} "
-                f"on {WATCH_BRANCH} (combined state={combined_state!r}; no action)"
-            )
+            print(f"::notice::main is PENDING at {sha[:10]} on {WATCH_BRANCH} "
+                  f"(combined state={status.get('state')!r}; no action)")
     return 0
 
 

.gitea/scripts/prod-auto-deploy.py

@@ -17,14 +17,18 @@ import urllib.error
 import urllib.request
 from urllib.parse import quote
 
+
 TRUE_VALUES = {"1", "true", "yes", "on", "disabled", "disable"}
 PROD_CP_URL = "https://api.moleculesai.app"
 DEFAULT_REQUIRED_CONTEXTS = [
+    "CI / Platform (Go) (push)",
+    "CI / Canvas (Next.js) (push)",
+    "CI / Shellcheck (E2E scripts) (push)",
+    "CI / Python Lint & Test (push)",
     "CI / all-required (push)",
     "Secret scan / Scan diff for credential-shaped strings (push)",
 ]
 TERMINAL_FAILURE_STATES = {"failure", "error", "cancelled", "canceled", "skipped"}
-REDEPLOY_PATH = "/cp/admin/tenants/redeploy-fleet"
 
 
 def truthy_flag(value: str | None) -> bool:
@@ -67,12 +71,6 @@ def build_plan(env: dict[str, str]) -> dict:
         "soak_seconds": _int_env(env, "PROD_AUTO_DEPLOY_SOAK_SECONDS", 60, minimum=0),
         "batch_size": _int_env(env, "PROD_AUTO_DEPLOY_BATCH_SIZE", 3),
         "dry_run": truthy_flag(env.get("PROD_AUTO_DEPLOY_DRY_RUN", "")),
-        # confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
-        # contract (cp#228 / task #308) for fleet-wide intent. Empty body
-        # / {confirm:false} / {only_slugs:[]} → 400. This caller is the
-        # production auto-deploy step that rolls every live tenant (canary
-        # + fan-out), no slug scoping, so confirm:true is correct.
-        "confirm": True,
     }
     if canary_slug:
         body["canary_slug"] = canary_slug
@@ -130,154 +128,6 @@ def required_contexts(env: dict[str, str]) -> list[str]:
     return [line.strip() for line in raw.replace(",", "\n").splitlines() if line.strip()]
 
 
-def chunks(items: list[str], size: int) -> list[list[str]]:
-    return [items[i : i + size] for i in range(0, len(items), size)]
-
-
-class RolloutFailed(RuntimeError):
-    def __init__(self, message: str, response: dict):
-        super().__init__(message)
-        self.response = response
-
-
-def slugs_from_redeploy_response(body: dict) -> list[str]:
-    slugs: list[str] = []
-    for row in body.get("results") or []:
-        slug = str(row.get("slug") or "").strip()
-        if slug:
-            slugs.append(slug)
-    return slugs
-
-
-def scoped_redeploy_body(base: dict, slugs: list[str]) -> dict:
-    body = dict(base)
-    body.pop("canary_slug", None)
-    body["only_slugs"] = slugs
-    body["soak_seconds"] = 0
-    body["batch_size"] = max(1, len(slugs))
-    return body
-
-
-def cp_api_json(method: str, url: str, token: str, body: dict | None = None) -> tuple[int, dict]:
-    data = None
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/json",
-    }
-    if body is not None:
-        data = json.dumps(body).encode("utf-8")
-        headers["Content-Type"] = "application/json"
-    req = urllib.request.Request(url, data=data, headers=headers, method=method)
-    try:
-        with urllib.request.urlopen(req, timeout=120) as resp:
-            return resp.status, json.loads(resp.read())
-    except urllib.error.HTTPError as exc:
-        raw = exc.read().decode("utf-8", errors="replace")
-        try:
-            parsed = json.loads(raw)
-        except json.JSONDecodeError:
-            parsed = {"error": raw[:500]}
-        return exc.code, parsed
-
-
-def plan_rollout_slugs(cp_url: str, token: str, body: dict, redeploy=None) -> list[str]:
-    if redeploy is None:
-        redeploy = redeploy_scoped
-    dry_run_body = dict(body)
-    dry_run_body["dry_run"] = True
-    status, resp = redeploy(cp_url, token, dry_run_body)
-    if status != 200:
-        raise RuntimeError(f"dry-run redeploy-fleet returned HTTP {status}: {resp.get('error', '')}")
-    if resp.get("ok") is not True:
-        raise RuntimeError(f"dry-run redeploy-fleet reported ok={resp.get('ok')}: {resp.get('error', '')}")
-    slugs = slugs_from_redeploy_response(resp)
-    if not slugs:
-        raise RuntimeError("dry-run redeploy-fleet returned no rollout candidates")
-    return slugs
-
-
-def redeploy_scoped(cp_url: str, token: str, body: dict) -> tuple[int, dict]:
-    return cp_api_json("POST", f"{cp_url}{REDEPLOY_PATH}", token, body)
-
-
-def _raise_for_redeploy_result(status: int, body: dict, slugs: list[str]) -> None:
-    if status != 200 or body.get("ok") is not True:
-        raise RuntimeError(
-            "redeploy scoped call failed for "
-            f"{','.join(slugs)}: HTTP {status}, ok={body.get('ok')}"
-        )
-
-
-def execute_scoped_rollout(
-    plan: dict,
-    token: str,
-    list_slugs=plan_rollout_slugs,
-    redeploy=redeploy_scoped,
-    sleep=time.sleep,
-) -> dict:
-    cp_url = plan["cp_url"]
-    base_body = plan["body"]
-    all_slugs = list_slugs(cp_url, token, base_body)
-    batch_size = int(base_body.get("batch_size") or 1)
-    canary_slug = str(base_body.get("canary_slug") or "").strip()
-    dry_run = bool(base_body.get("dry_run"))
-    aggregate = {"ok": True, "results": []}
-
-    if canary_slug:
-        if canary_slug not in all_slugs:
-            raise RuntimeError(f"configured canary slug {canary_slug!r} is not a running tenant")
-        body = scoped_redeploy_body(base_body, [canary_slug])
-        print(f"POST {cp_url}{REDEPLOY_PATH} only_slugs={','.join(body['only_slugs'])}")
-        status, resp = redeploy(cp_url, token, body)
-        aggregate["results"].extend(resp.get("results") or [])
-        try:
-            _raise_for_redeploy_result(status, resp, [canary_slug])
-        except RuntimeError as exc:
-            aggregate["ok"] = False
-            aggregate["error"] = str(exc)
-            raise RolloutFailed(str(exc), aggregate) from exc
-        soak_seconds = int(base_body.get("soak_seconds") or 0)
-        if soak_seconds > 0 and not dry_run:
-            print(f"Canary passed; soaking locally for {soak_seconds}s")
-            sleep(soak_seconds)
-
-    remaining = [slug for slug in all_slugs if slug != canary_slug]
-    for group in chunks(remaining, batch_size):
-        body = scoped_redeploy_body(base_body, group)
-        print(f"POST {cp_url}{REDEPLOY_PATH} only_slugs={','.join(group)}")
-        status, resp = redeploy(cp_url, token, body)
-        aggregate["results"].extend(resp.get("results") or [])
-        try:
-            _raise_for_redeploy_result(status, resp, group)
-        except RuntimeError as exc:
-            aggregate["ok"] = False
-            aggregate["error"] = str(exc)
-            raise RolloutFailed(str(exc), aggregate) from exc
-
-    return aggregate
-
-
-def rollout_from_plan_file(plan_path: str, response_path: str, env: dict[str, str]) -> None:
-    token = env.get("CP_ADMIN_API_TOKEN", "").strip()
-    if not token:
-        raise ValueError("CP_ADMIN_API_TOKEN is required for production auto-deploy")
-    with open(plan_path, "r", encoding="utf-8") as fh:
-        plan = json.load(fh)
-    if not plan.get("enabled"):
-        raise RuntimeError("production auto-deploy plan is disabled")
-    try:
-        response = execute_scoped_rollout(plan, token)
-    except RolloutFailed as exc:
-        response = exc.response
-        with open(response_path, "w", encoding="utf-8") as fh:
-            json.dump(response, fh, sort_keys=True)
-            fh.write("\n")
-        raise
-    with open(response_path, "w", encoding="utf-8") as fh:
-        json.dump(response, fh, sort_keys=True)
-        fh.write("\n")
-
-
 def _api_json(url: str, token: str) -> dict:
     req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
     try:
@@ -379,9 +229,6 @@ def main() -> int:
     sub.add_parser("plan", help="print production deploy plan as JSON")
     sub.add_parser("assert-enabled", help="fail if production deploy is currently disabled")
     sub.add_parser("wait-ci", help="block until required CI context is green")
-    rollout_parser = sub.add_parser("rollout", help="execute canary-first scoped production rollout")
-    rollout_parser.add_argument("--plan", required=True, help="path to prod-auto-deploy plan JSON")
-    rollout_parser.add_argument("--response", required=True, help="path to write aggregate response JSON")
     args = parser.parse_args()
 
     try:
@@ -394,9 +241,6 @@ def main() -> int:
         if args.command == "wait-ci":
             wait_for_ci_context(dict(os.environ))
             return 0
-        if args.command == "rollout":
-            rollout_from_plan_file(args.plan, args.response, dict(os.environ))
-            return 0
     except Exception as exc:  # noqa: BLE001 - CLI should render operator-friendly errors.
         print(f"::error::{exc}", file=sys.stderr)
         return 1

.gitea/scripts/review-check.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC2016,SC2329
 # review-check — evaluate whether a PR satisfies a single team-review gate.
 #
 # RFC#324 Step 1 of 5 — qa-review + security-review check workflows.
@@ -12,7 +11,6 @@
 #   ≥ 1 review on the PR where:
 #     • state == APPROVED
 #     • review.dismissed == false
-#     • review.official != false (excludes draft/mis-filed APPROVED reviews)
 #     • review.user.login != PR.user.login (non-author)
 #     • review.user.login ∈ team-members
 #
@@ -102,12 +100,11 @@ printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$CURL_AUTH_FILE"
 # (bash trap 'function' EXIT expands variables at trap-fire time, not def time).
 PR_JSON=$(mktemp)
 REVIEWS_JSON=$(mktemp)
-COMMENTS_JSON=$(mktemp)
 TEAM_PROBE_TMP=$(mktemp)
 NA_STATUSES_TMP=""  # declared here so cleanup() always has the var
 
 cleanup() {
-  rm -f "$CURL_AUTH_FILE" "$PR_JSON" "$REVIEWS_JSON" "$COMMENTS_JSON" "$TEAM_PROBE_TMP" "${NA_STATUSES_TMP-}"
+  rm -f "$CURL_AUTH_FILE" "$PR_JSON" "$REVIEWS_JSON" "$TEAM_PROBE_TMP" "${NA_STATUSES_TMP-}"
 }
 trap cleanup EXIT
 
@@ -130,7 +127,6 @@ fi
 PR_AUTHOR=$(jq -r '.user.login // ""' "$PR_JSON")
 PR_HEAD_SHA=$(jq -r '.head.sha // ""' "$PR_JSON")
 PR_BASE_REF=$(jq -r '.base.ref // ""' "$PR_JSON")
-PR_BASE_SHA=$(jq -r '.base.sha // ""' "$PR_JSON")
 PR_STATE=$(jq -r '.state // ""' "$PR_JSON")
 DEFAULT_BRANCH="${DEFAULT_BRANCH:-main}"
 debug "pr_author=${PR_AUTHOR} pr_head=${PR_HEAD_SHA:0:7} pr_base=${PR_BASE_REF} pr_state=${PR_STATE}"
@@ -139,10 +135,6 @@ if [ "$PR_STATE" != "open" ]; then
   echo "::notice::PR ${PR_NUMBER} is ${PR_STATE} — exiting 0 (closed PRs do not gate)"
   exit 0
 fi
-if [ "$PR_HEAD_SHA" = "$PR_BASE_SHA" ]; then
-  echo "::notice::PR ${PR_NUMBER} has no diff (head == base) — exiting 0 (empty PRs do not gate)"
-  exit 0
-fi
 if [ "$PR_BASE_REF" != "$DEFAULT_BRANCH" ]; then
   echo "::notice::PR ${PR_NUMBER} targets ${PR_BASE_REF:-<unknown>} not ${DEFAULT_BRANCH} — ${TEAM}-review gate not applicable"
   exit 0
@@ -202,7 +194,6 @@ fi
 JQ_FILTER='.[]
   | select(.state == "APPROVED")
   | select(.dismissed != true)
-  | select(.official != false)
   | select(.user.login != $author)'
 if [ "${REVIEW_CHECK_STRICT:-}" = "1" ]; then
   JQ_FILTER="${JQ_FILTER}
@@ -211,82 +202,11 @@ fi
 JQ_FILTER="${JQ_FILTER}
   | .user.login"
 
-REVIEW_CANDIDATES=$(jq -r --arg author "$PR_AUTHOR" --arg head "$PR_HEAD_SHA" "$JQ_FILTER" "$REVIEWS_JSON" | sort -u)
-debug "candidate non-author approvers: $(echo "$REVIEW_CANDIDATES" | tr '\n' ' ')"
+CANDIDATES=$(jq -r --arg author "$PR_AUTHOR" --arg head "$PR_HEAD_SHA" "$JQ_FILTER" "$REVIEWS_JSON" | sort -u)
+debug "candidate non-author approvers: $(echo "$CANDIDATES" | tr '\n' ' ')"
 
-if [ -z "$REVIEW_CANDIDATES" ]; then
-  # --- Guardrail (internal#503): explain the most common false
-  # "no candidates" red. Gitea's review event enum is EXACTLY
-  # APPROVED/REQUEST_CHANGES/COMMENT/PENDING. A wrong value ("APPROVE",
-  # lowercase, ...) is silently accepted (HTTP 200) and stored as
-  # state=PENDING. A correctly-started draft review has an EMPTY body;
-  # a NON-empty body + state==PENDING by a non-author == an intended
-  # verdict mis-filed by a wrong event string. Surface it actionably.
-  # This does NOT change the gate result (still fail-closed below) — it
-  # only converts a mystery red into a named, self-fixing error.
-  MISFILED_FILTER='.[]
-    | select(.state == "PENDING")
-    | select(.dismissed != true)
-    | select(.user.login != $author)
-    | select(((.body // "") | gsub("^\\s+|\\s+$";"") | length) > 0)
-    | "\(.id)\t\(.user.login)"'
-  MISFILED=$(jq -r --arg author "$PR_AUTHOR" "$MISFILED_FILTER" "$REVIEWS_JSON" 2>/dev/null || true)
-  if [ -n "$MISFILED" ]; then
-    echo "::error::${TEAM}-review: non-author review(s) were SUBMITTED but stored as PENDING — almost certainly the wrong Gitea review event string (internal#503)."
-    echo "::error::Gitea accepts ONLY the exact enum APPROVED / REQUEST_CHANGES / COMMENT. 'APPROVE' or lowercase is silently (HTTP 200) filed as PENDING and is invisible to this gate."
-    printf '%s\n' "$MISFILED" | while IFS="$(printf '\t')" read -r _rid _rl; do
-      [ -n "${_rid:-}" ] && echo "::error::  review id=${_rid} by '${_rl}': RE-SUBMIT via POST ${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews with {\"event\":\"APPROVED\"} (correct enum) — do NOT edit the DB."
-    done
-  fi
-
-fi
-
-# --- Fallback/extension (internal#348): check issue comments for agent-approval ---
-# core-qa-agent and core-security-agent can approve via issue comments. Always
-# include comment candidates, even if the reviews API returned approvals for a
-# different team; team membership below is the authoritative filter.
-COMMENT_CANDIDATES=""
-AGENT_PATTERN=""
-case "$TEAM" in
-  qa)       AGENT_PATTERN="\\[core-qa-agent\\]" ;;
-  security) AGENT_PATTERN="\\[core-security-agent\\]" ;;
-esac
-HTTP_CODE=$(curl -sS -o "$COMMENTS_JSON" -w '%{http_code}' \
-  -K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/comments")
-debug "GET /issues/${PR_NUMBER}/comments → HTTP ${HTTP_CODE}"
-if [ "$HTTP_CODE" = "200" ]; then
-  # JQ expression: select non-author comments that match either the
-  # agent-prefix pattern (case-insensitive) OR a generic approval keyword.
-  JQ_APPROVALS='
-    .[] |
-    select(.user.login != $author) |
-    . as $cmt |
-    if ($agent_pattern | length) > 0 and ($cmt.body // "" | test($agent_pattern; "i")) then
-      $cmt.user.login
-    elif ($cmt.body // "" | test("\\b(APPROVED|LGTM|ACCEPTED)\\b"; "i")) then
-      $cmt.user.login
-    else
-      empty
-    end
-  '
-  COMMENT_CANDIDATES=$(jq -r \
-    --arg author "$PR_AUTHOR" \
-    --arg agent_pattern "$AGENT_PATTERN" \
-    "$JQ_APPROVALS" \
-    "$COMMENTS_JSON" 2>/dev/null | sort -u)
-  debug "comment-based approval candidates: $(echo "$COMMENT_CANDIDATES" | tr '\n' ' ')"
-
-  if [ -n "$COMMENT_CANDIDATES" ]; then
-    echo "::notice::${TEAM}-review: found $(echo "$COMMENT_CANDIDATES" | wc -w | xargs) comment-based approval candidate(s) — verifying team membership..."
-  fi
-else
-  debug "could not fetch issue comments (HTTP ${HTTP_CODE})"
-fi
-
-CANDIDATES=$(printf '%s\n%s\n' "$REVIEW_CANDIDATES" "$COMMENT_CANDIDATES" | sed '/^$/d' | sort -u)
-
-if [ -z "${CANDIDATES:-}" ]; then
-  echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (no candidates from reviews API or issue comments)"
+if [ -z "$CANDIDATES" ]; then
+  echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (no candidates yet)"
   exit 1
 fi
 
@@ -306,15 +226,12 @@ for U in $CANDIDATES; do
       exit 0
       ;;
     403)
-      # Token owner is not in the team being probed; Gitea 1.22.6 refuses
-      # to confirm membership in this case. Do NOT hard-fail the gate on a
-      # 403 — doing so would fail the entire gate if ANY candidate triggers
-      # a 403, even when other valid team-members exist. Instead skip this
-      # candidate and continue checking others. If all candidates produce
-      # 403 (token owner can't query any of them) the final exit fires.
-      echo "::warning::team-probe for ${U} in ${TEAM} returned 403 (token owner not in ${TEAM} team — skipping; cannot confirm membership)"
+      # Token owner is not in the team being probed; the API refuses to
+      # confirm membership. This is the RFC#324 follow-up token-scope gap.
+      # Fail closed — never grant approval on a 403; surface clearly.
+      echo "::error::team-probe for ${U} in ${TEAM} returned 403 (token owner not in ${TEAM} team — RFC#324 token-scope follow-up). Cannot confirm membership; failing closed."
       cat "$TEAM_PROBE_TMP" >&2
-      continue
+      exit 1
       ;;
     404)
       debug "${U} not a member of ${TEAM}"

.gitea/scripts/sop-checklist.py

@@ -64,41 +64,11 @@ import argparse
 import json
 import os
 import re
-import resource
 import sys
 import urllib.error
 import urllib.parse
 import urllib.request
-from typing import Any, Callable, Iterator
-
-# ---------------------------------------------------------------------------
-# Address-space guardrail (RFC#369 / task #369 follow-up to mc#1242-class OOM).
-#
-# `get_issue_comments` paginates the full comment history of a PR. On
-# bot-relay-heavy PRs (e.g. mc#291, mc#1242) this can balloon past the
-# runner's cgroup memory limit and 137 the job. Cap virtual-address-space
-# at 2 GiB so the script OOMs as a `MemoryError` (catchable / surfaceable)
-# rather than a SIGKILL we can't post a status for.
-#
-# 2 GiB is generous — a 5000-comment PR with 1 KiB minimal-dicts (see
-# get_issue_comments below) fits in ~10 MiB, leaving plenty of headroom
-# for the Python runtime + urllib + json buffers.
-#
-# Skipped under pytest / dry-run where RLIMIT_AS would interfere with
-# test runner memory needs (set SOP_CHECKLIST_NO_RLIMIT=1 to opt out).
-if not os.environ.get("SOP_CHECKLIST_NO_RLIMIT"):
-    try:
-        resource.setrlimit(resource.RLIMIT_AS, (2 * 1024**3, 2 * 1024**3))
-    except (ValueError, OSError):
-        # macOS sometimes refuses RLIMIT_AS; not fatal — the Linux runner
-        # is the only place this matters for the OOM-prevention goal.
-        pass
-
-# Per-comment body cap (task #369). The directive parser walks the body
-# line-by-line looking for ^/sop-ack ^/sop-revoke ^/sop-n/a markers — only
-# the first few KiB matter for that. Cap each comment body so a single
-# pasted-log comment can't push us past the cgroup limit.
-_MAX_BODY_BYTES = int(os.environ.get("SOP_CHECKLIST_MAX_BODY_BYTES") or 8 * 1024)
+from typing import Any, Callable
 
 
 # ---------------------------------------------------------------------------
@@ -298,7 +268,6 @@ def compute_ack_state(
     items_by_slug: dict[str, dict[str, Any]],
     numeric_aliases: dict[int, str],
     team_membership_probe: "callable[[str, list[str]], list[str]]",
-    high_risk: bool = False,
 ) -> dict[str, dict[str, Any]]:
     """Compute per-item ack state.
 
@@ -338,6 +307,7 @@ def compute_ack_state(
     # Filter out self-acks and unknown slugs.
     ackers_per_slug: dict[str, list[str]] = {s: [] for s in items_by_slug}
     rejected_self: dict[str, list[str]] = {s: [] for s in items_by_slug}
+    rejected_unknown: dict[str, list[str]] = {s: [] for s in items_by_slug}
     pending_team_check: dict[str, list[str]] = {s: [] for s in items_by_slug}
 
     for (user, slug), kind in latest_directive.items():
@@ -360,16 +330,11 @@ def compute_ack_state(
     for slug, candidates in pending_team_check.items():
         if not candidates:
             continue
-        # Risk-class-aware required-teams resolution (RFC#450 Option C):
-        # high-risk PRs use `required_teams_high_risk` (when set on the
-        # item); default class uses `required_teams`. The probe closure
-        # is built with the same high_risk flag so the two reads are
-        # always consistent (both sites share `resolve_required_teams`).
-        required = resolve_required_teams(items_by_slug[slug], high_risk)
+        required = items_by_slug[slug]["required_teams"]
         approved = team_membership_probe(slug, candidates)  # returns subset
         rejected_not_in_team[slug] = [u for u in candidates if u not in approved]
         ackers_per_slug[slug] = approved
-        # Stash resolved teams for description rendering.
+        # Stash required teams for description rendering.
         items_by_slug[slug]["_required_resolved"] = required
 
     return {
@@ -489,35 +454,16 @@ class GiteaClient:
             raise RuntimeError(f"GET pulls/{pr} → HTTP {code}: {data!r}")
         return data
 
-    def iter_issue_comments(
-        self, owner: str, repo: str, issue: int, page_size: int = 50
-    ) -> Iterator[dict[str, Any]]:
-        """Stream comments page-by-page, yielding ONE minimal-dict per comment.
-
-        Each yielded comment carries ONLY the fields the gate actually reads
-        — `{"user": {"login": str}, "body": str}` — and DROPS the much
-        larger Gitea-API extras (html_url, pull_request_url, issue_url,
-        assets, created_at, updated_at, id, original_author_*).
-
-        Memory motivation (task #369 / mc#1242-class OOM): full Gitea
-        comment dicts are ~2 KiB median + ~3 KiB p95. On PRs with several
-        thousand bot-relay comments the eager `list[full_dict]` shape used
-        previously pushed runner anon-rss past the cgroup limit. The
-        minimal-dict shape is ~10-20x smaller (typically ~50-100B Python
-        overhead + the body string).
-
-        The two downstream consumers (`compute_ack_state`,
-        `compute_na_state`) each iterate the comment list exactly once and
-        read only `body` + `user.login`, so dropping every other field is
-        safe. They still receive `list[dict[str, Any]]`-shaped objects so
-        the test fixtures (which already used the minimal shape) keep
-        working with no fixture changes.
-        """
+    def get_issue_comments(
+        self, owner: str, repo: str, issue: int
+    ) -> list[dict[str, Any]]:
+        # Paginate. Gitea default page size 50.
+        out: list[dict[str, Any]] = []
         page = 1
         while True:
             code, data = self._req(
                 "GET",
-                f"/repos/{owner}/{repo}/issues/{issue}/comments?limit={page_size}&page={page}",
+                f"/repos/{owner}/{repo}/issues/{issue}/comments?limit=50&page={page}",
             )
             if code != 200:
                 raise RuntimeError(
@@ -525,41 +471,10 @@ class GiteaClient:
                 )
             if not data:
                 break
-            for c in data:
-                # Minimal projection — drop ALL fields the gate doesn't read.
-                user_login = ((c.get("user") or {}).get("login") or "") if isinstance(c, dict) else ""
-                body = (c.get("body") if isinstance(c, dict) else "") or ""
-                # Body-size guardrail: huge comments (e.g. pasted CI logs) can
-                # individually be MiBs. The directive parser only needs the
-                # first ~8 KiB to find /sop-ack /sop-revoke /sop-n/a markers
-                # — anything past that is filler. Truncate at 8 KiB so a
-                # single oversized comment can't OOM the runner.
-                if len(body) > _MAX_BODY_BYTES:
-                    body = body[:_MAX_BODY_BYTES]
-                yield {"user": {"login": user_login}, "body": body}
-            if len(data) < page_size:
+            out.extend(data)
+            if len(data) < 50:
                 break
             page += 1
-
-    def get_issue_comments(
-        self,
-        owner: str,
-        repo: str,
-        issue: int,
-        max_comments: int | None = None,
-    ) -> list[dict[str, Any]]:
-        """Paginate + collect minimal comment dicts. See `iter_issue_comments`
-        for the per-comment shape and the OOM-prevention rationale.
-
-        `max_comments` (optional, default unbounded): hard cap. When the cap
-        is hit we stop fetching further pages and the caller surfaces a
-        soft 'skipping due to volume' status (see main()).
-        """
-        out: list[dict[str, Any]] = []
-        for c in self.iter_issue_comments(owner, repo, issue):
-            out.append(c)
-            if max_comments is not None and len(out) >= max_comments:
-                break
         return out
 
     def resolve_team_id(self, org: str, team_name: str) -> int | None:
@@ -636,11 +551,6 @@ def load_config(path: str) -> dict[str, Any]:
     dep by keeping the config shape constrained.
     """
     try:
-        # yaml is an optional dep; the canonical loader is used when available,
-        # but the SOP runs on runners that may not have PyYAML installed. The
-        # fallback _load_config_minimal covers the same config shape without
-        # requiring the dep, so the ignore is safe: if yaml loads, we use it;
-        # otherwise we fall back silently.
         import yaml  # type: ignore[import-not-found]
         with open(path) as f:
             return yaml.safe_load(f)
@@ -661,14 +571,8 @@ def _load_config_minimal(path: str) -> dict[str, Any]:
     return _parse_minimal_yaml(lines)
 
 
-def _parse_minimal_yaml(lines: list[str]) -> dict[str, Any]:
-    """Hand-rolled subset parser. See _load_config_minimal docstring.
-
-    C901: function is necessarily long — it implements a finite-state YAML
-    subset (scalars, maps, lists of maps at fixed depth). No utility refactors
-    meaningfully reduce length without degrading readability. All branches
-    are exhaustively tested in test_parse_minimal_yaml.py.
-    """
+def _parse_minimal_yaml(lines: list[str]) -> dict[str, Any]:  # noqa: C901
+    """Hand-rolled subset parser. See _load_config_minimal docstring."""
     # Strip comments + blank lines but preserve indentation.
     cleaned: list[tuple[int, str]] = []
     for raw in lines:
@@ -852,7 +756,7 @@ def render_status(
 def get_tier_mode(pr: dict[str, Any], cfg: dict[str, Any]) -> str:
     """Read tier label, return 'hard' or 'soft' per cfg.tier_failure_mode."""
     labels = pr.get("labels") or []
-    tier_labels = [label.get("name", "") for label in labels if (label.get("name", "") or "").startswith("tier:")]
+    tier_labels = [l.get("name", "") for l in labels if (l.get("name", "") or "").startswith("tier:")]
     mode_map = cfg.get("tier_failure_mode") or {}
     default_mode = cfg.get("default_mode", "hard")
     for tl in tier_labels:
@@ -861,42 +765,6 @@ def get_tier_mode(pr: dict[str, Any], cfg: dict[str, Any]) -> str:
     return default_mode
 
 
-def is_high_risk(pr: dict[str, Any], cfg: dict[str, Any]) -> bool:
-    """Return True when the PR is high-risk per RFC#450 Option C.
-
-    A PR is high-risk when ANY of:
-      - it carries the `tier:high` label (mechanically strictest tier), or
-      - it carries any label listed in cfg.high_risk_labels.
-
-    High-risk PRs use `required_teams_high_risk` (when set on an item)
-    instead of the default `required_teams`. Items without
-    `required_teams_high_risk` are unaffected (the default applies).
-
-    Governance fix for internal#442 — closes the inconsistency between
-    sop-tier-check (tier-aware) and sop-checklist (was tier-blind).
-    """
-    label_set = {(label.get("name") or "") for label in (pr.get("labels") or [])}
-    if "tier:high" in label_set:
-        return True
-    high_risk_labels = set(cfg.get("high_risk_labels") or [])
-    return bool(label_set & high_risk_labels)
-
-
-def resolve_required_teams(item: dict[str, Any], high_risk: bool) -> list[str]:
-    """Pick the active required_teams list for an item.
-
-    When high_risk is True AND the item declares a non-empty
-    `required_teams_high_risk`, return that. Else fall back to
-    `required_teams`. Keeping this in one helper means the gate's
-    decision shape stays single-sited even as items grow.
-    """
-    if high_risk:
-        elevated = item.get("required_teams_high_risk") or []
-        if elevated:
-            return list(elevated)
-    return list(item.get("required_teams") or [])
-
-
 def main(argv: list[str] | None = None) -> int:
     p = argparse.ArgumentParser()
     p.add_argument("--owner", required=True)
@@ -922,17 +790,6 @@ def main(argv: list[str] | None = None) -> int:
             "thing BP sees is the POSTed status. Useful for local debugging."
         ),
     )
-    p.add_argument(
-        "--max-comments",
-        type=int,
-        default=int(os.environ.get("SOP_CHECKLIST_MAX_COMMENTS") or 5000),
-        help=(
-            "Hard cap on comments fetched from the PR. Above this we post "
-            "a SOFT-pending status with a 'skipping due to volume' note "
-            "instead of OOM'ing the runner (task #369). Override with the "
-            "SOP_CHECKLIST_MAX_COMMENTS env var. Set 0 to disable the cap."
-        ),
-    )
     args = p.parse_args(argv)
 
     token = os.environ.get("GITEA_TOKEN", "")
@@ -966,24 +823,7 @@ def main(argv: list[str] | None = None) -> int:
         print("::error::PR payload missing user.login or head.sha", file=sys.stderr)
         return 1
 
-    max_comments_cap = args.max_comments if args.max_comments and args.max_comments > 0 else None
-    comments = client.get_issue_comments(
-        args.owner, args.repo, args.pr, max_comments=max_comments_cap
-    )
-
-    # Volume short-circuit: PRs with thousands of bot-relay comments
-    # (the mc#1242-class OOM source) get a soft 'volume-skipped' status
-    # so the gate doesn't churn the runner; reviewers can re-trigger by
-    # editing the PR or filing a fresh PR with the housekeeping comments
-    # split off. Cap-hit means we couldn't see the WHOLE history, so we
-    # can't fairly post failure — pending is the safe default.
-    volume_skipped = bool(max_comments_cap and len(comments) >= max_comments_cap)
-
-    # High-risk classification (RFC#450 Option C, governance fix for
-    # internal#442). Computed ONCE per PR — used by both the probe
-    # closure and compute_ack_state so the elevation decision is
-    # single-sited.
-    high_risk = is_high_risk(pr, cfg)
+    comments = client.get_issue_comments(args.owner, args.repo, args.pr)
 
     # Build team-membership probe closure that caches results per
     # (user, team-id) so a user acking multiple items only triggers
@@ -991,34 +831,8 @@ def main(argv: list[str] | None = None) -> int:
     team_member_cache: dict[tuple[str, int], bool | None] = {}
 
     def probe(slug: str, users: list[str]) -> list[str]:
-        # `slug` may be either an items-key (compute_ack_state caller) OR
-        # an n/a-gate key (compute_na_state caller). Previously this hard
-        # KeyError'd on the n/a-gate path when slug was e.g. "security-review"
-        # — that's a config gate, not an item — so the gate would crash
-        # instead of falling back to the gate's own required_teams. Fix
-        # task #369 follow-up to issue #355.
-        if slug in items_by_slug:
-            item = items_by_slug[slug]
-            team_names: list[str] = resolve_required_teams(item, high_risk)
-        elif slug in na_gates:
-            # n/a-gate configs carry `required_teams` directly (see
-            # sop-checklist-config.yaml: n/a_gates.<gate>.required_teams).
-            gate_cfg = na_gates[slug] or {}
-            team_names = list(gate_cfg.get("required_teams") or [])
-            if not team_names:
-                print(
-                    f"::warning::n/a-gate '{slug}' has no required_teams; "
-                    "fail-closed (no users will be approved)",
-                    file=sys.stderr,
-                )
-        else:
-            # Unknown slug — fail closed, log so we can find config drift.
-            print(
-                f"::warning::probe() called with slug '{slug}' which is "
-                f"neither an items entry nor an n/a-gate; fail-closed",
-                file=sys.stderr,
-            )
-            return []
+        item = items_by_slug[slug]
+        team_names: list[str] = item["required_teams"]
         # Resolve names → ids. NOTE: orgs/{org}/teams/search may not be
         # available — fall back to the list endpoint.
         team_ids: list[int] = []
@@ -1026,14 +840,14 @@ def main(argv: list[str] | None = None) -> int:
             tid = client.resolve_team_id(args.owner, tn)
             if tid is None:
                 # Try the list endpoint as a fallback.
-                code, data = client._req(  # noqa: SLF001  # internal helper; called from loop in caller context
+                code, data = client._req(  # noqa: SLF001
                     "GET", f"/orgs/{args.owner}/teams"
                 )
                 if code == 200 and isinstance(data, list):
                     for t in data:
                         if t.get("name") == tn:
                             tid = t.get("id")
-                            client._team_id_cache[(args.owner, tn)] = tid  # noqa: SLF001  # internal write-through cache
+                            client._team_id_cache[(args.owner, tn)] = tid  # noqa: SLF001
                             break
             if tid is not None:
                 team_ids.append(tid)
@@ -1063,9 +877,7 @@ def main(argv: list[str] | None = None) -> int:
                     # may still find membership in another team.
         return approved
 
-    ack_state = compute_ack_state(
-        comments, author, items_by_slug, numeric_aliases, probe, high_risk=high_risk
-    )
+    ack_state = compute_ack_state(comments, author, items_by_slug, numeric_aliases, probe)
     body_state = {it["slug"]: section_marker_present(body, it["pr_section_marker"]) for it in items}
 
     state, description = render_status(items, ack_state, body_state)
@@ -1076,21 +888,9 @@ def main(argv: list[str] | None = None) -> int:
         # were not required (vs a tier:medium+ PR that truly passed all acks).
         state = "success"
         description = f"[info tier:low] {description}"
-    if volume_skipped:
-        # Above the comment-cap — we may have a partial view. Soft-pend
-        # so neither BP nor the author gets stuck; surface the cap so
-        # reviewers know what's up. No-block at the gate level.
-        state = "pending"
-        description = (
-            f"[volume-skipped] comment-cap={max_comments_cap} hit; please file "
-            f"a fresh PR with bot-relay history split off (#369). {description}"
-        )
 
     # Diagnostics to job log.
-    print(
-        f"::notice::PR #{args.pr} author={author} head={head_sha[:7]} "
-        f"mode={mode} risk_class={'high' if high_risk else 'default'}"
-    )
+    print(f"::notice::PR #{args.pr} author={author} head={head_sha[:7]} mode={mode}")
     for it in items:
         slug = it["slug"]
         ackers = ack_state[slug]["ackers"]

.gitea/scripts/sop-tier-refire.sh

@@ -104,13 +104,10 @@ if [ "${SOP_REFIRE_DISABLE_RATE_LIMIT:-}" != "1" ]; then
   fi
 fi
 
-# 3. Invoke sop-tier-check.sh with the env it expects.
-# The canonical workflow intentionally fail-opens the job conclusion
-# (`bash .gitea/scripts/sop-tier-check.sh || true`) while Gitea branch
-# protection enforces reviewer approvals separately. Keep the refire path
-# aligned with that workflow status behavior; otherwise /refire-tier-check can
-# post a hard failure that the canonical pull_request_target workflow would
-# not publish.
+# 3. Invoke sop-tier-check.sh with the env it expects. Capture exit code.
+# The canonical script reads tier label, walks approving reviewers, and
+# evaluates the AND-composition expression — we want the SAME gate, not
+# a different gate.
 #
 # SOP_REFIRE_TIER_CHECK_SCRIPT env var lets tests substitute a mock —
 # sop-tier-check.sh uses bash 4+ associative arrays which trigger a known
@@ -126,6 +123,7 @@ fi
 
 # Re-invoke. Pipe stdout/stderr through so the runner log shows the
 # tier-check decision inline.
+set +e
 GITEA_TOKEN="$GITEA_TOKEN" \
   GITEA_HOST="$GITEA_HOST" \
   REPO="$REPO" \
@@ -133,8 +131,9 @@ GITEA_TOKEN="$GITEA_TOKEN" \
   PR_AUTHOR="$PR_AUTHOR" \
   SOP_DEBUG="${SOP_DEBUG:-0}" \
   SOP_LEGACY_CHECK="${SOP_LEGACY_CHECK:-0}" \
-  bash "$SCRIPT" || true
-TIER_EXIT=0
+  bash "$SCRIPT"
+TIER_EXIT=$?
+set -e
 debug "sop-tier-check.sh exit=$TIER_EXIT"
 
 # 4. POST the resulting status.

.gitea/scripts/status-reaper.py

@@ -47,9 +47,7 @@ What this script does, per `.gitea/workflows/status-reaper.yml` invocation:
          Parse context as `<workflow_name> / <job_name> (push)`.
          Look up workflow_name in the trigger map:
            - missing → log ::notice:: and skip (conservative).
-           - has_push_trigger=True and description == "Has been cancelled"
-             → compensate cancelled/superseded push noise.
-           - has_push_trigger=True otherwise → preserve (real defect signal).
+           - has_push_trigger=True → preserve (real defect signal).
            - has_push_trigger=False → POST a compensating
              `state=success` status to /statuses/{sha} with the same
              context (Gitea de-dups by context) and a description
@@ -143,11 +141,6 @@ PR_SHADOW_COMPENSATION_DESCRIPTION = (
     "shadowed by successful push status on same SHA; see "
     ".gitea/scripts/status-reaper.py)"
 )
-CANCELLED_PUSH_COMPENSATION_DESCRIPTION = (
-    "Compensated by status-reaper (push run was cancelled/superseded; "
-    "Gitea 1.22.6 reports cancelled runs as failure statuses)"
-)
-CANCELLED_DESCRIPTION = "Has been cancelled"
 
 # Context suffix the reaper acts on. Gitea hardcodes this for ALL
 # default-branch workflow runs.
@@ -483,7 +476,7 @@ def reap(
       {compensated, preserved_real_push, preserved_unknown,
        preserved_non_failure, preserved_non_push_suffix,
        preserved_unparseable, compensated_pr_shadowed_by_push_success,
-       preserved_pr_without_push_success, compensated_cancelled_push,
+       preserved_pr_without_push_success,
        compensated_contexts: [<context>, ...]}
 
     `compensated_contexts` is rev2-added so `reap_branch` can build
@@ -497,7 +490,6 @@ def reap(
         "preserved_non_push_suffix": 0,
         "preserved_unparseable": 0,
         "compensated_pr_shadowed_by_push_success": 0,
-        "compensated_cancelled_push": 0,
         "preserved_pr_without_push_success": 0,
         "compensated_contexts": [],
     }
@@ -575,27 +567,8 @@ def reap(
             counters["preserved_unknown"] += 1
             continue
 
-        if (s.get("description") or "").strip() == CANCELLED_DESCRIPTION:
-            # Gitea 1.22.6 maps cancelled action runs to failure commit
-            # statuses. During merge bursts, older push runs can be
-            # superseded and cancelled even though a newer run for the
-            # same branch is the real signal. Compensate only the exact
-            # Gitea cancellation description; real push failures remain red.
-            post_compensating_status(
-                sha,
-                context,
-                s.get("target_url"),
-                description=CANCELLED_PUSH_COMPENSATION_DESCRIPTION,
-                dry_run=dry_run,
-            )
-            counters["compensated"] += 1
-            counters["compensated_cancelled_push"] += 1
-            counters["compensated_contexts"].append(context)
-            continue
-
         if workflow_trigger_map[workflow_name]:
-            # Real push trigger with a non-cancelled failure description
-            # remains a defect signal. Preserve.
+            # Real push trigger → real defect signal. Preserve.
             counters["preserved_real_push"] += 1
             continue
 
@@ -701,7 +674,6 @@ def reap_branch(
             "preserved_non_push_suffix": 0,
             "preserved_unparseable": 0,
             "compensated_pr_shadowed_by_push_success": 0,
-            "compensated_cancelled_push": 0,
             "preserved_pr_without_push_success": 0,
             "compensated_per_sha": {},
             "skipped": True,
@@ -717,7 +689,6 @@ def reap_branch(
         "preserved_non_push_suffix": 0,
         "preserved_unparseable": 0,
         "compensated_pr_shadowed_by_push_success": 0,
-        "compensated_cancelled_push": 0,
         "preserved_pr_without_push_success": 0,
         "compensated_per_sha": {},
     }
@@ -757,7 +728,6 @@ def reap_branch(
             "preserved_non_push_suffix",
             "preserved_unparseable",
             "compensated_pr_shadowed_by_push_success",
-            "compensated_cancelled_push",
             "preserved_pr_without_push_success",
         ):
             aggregate[key] += per_sha[key]

.gitea/scripts/tests/_review_check_fixture.py

@@ -17,10 +17,6 @@ Scenarios:
   T8_team_not_member          — team membership → 404 (not a member) → exit 1
   T9_team_403                — team membership → 403 (token not in team) → exit 1
   T14_non_default_base        — open PR targeting staging → script exits 0 (no-op)
-  T15_comments_agent_approval — reviews empty; comments have "[core-qa-agent] APPROVED" → exit 0
-  T16_comments_generic_approval — reviews empty; comments have "APPROVED" by team member → exit 0
-  T17_comments_no_approval   — reviews empty; comments have no approval keywords → exit 1
-  T18_review_wrong_team_comment_right_team — review candidate 404s, comment candidate passes
 
 Usage:
   FIXTURE_STATE_DIR=/tmp/x python3 _review_check_fixture.py 8080
@@ -33,6 +29,7 @@ import re
 import sys
 import urllib.parse
 
+
 STATE_DIR = os.environ.get("FIXTURE_STATE_DIR", "/tmp")
 
 
@@ -80,7 +77,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
         # GET /repos/{owner}/{name}/pulls/{pr_number}
         m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/pulls/(\d+)$", path)
         if m:
-            pr_num = m.group(3)
+            owner, name, pr_num = m.group(1), m.group(2), m.group(3)
             if sc == "T2_pr_closed":
                 return self._json(200, {
                     "number": int(pr_num),
@@ -100,9 +97,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
         # GET /repos/{owner}/{name}/pulls/{pr_number}/reviews
         m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/pulls/(\d+)/reviews$", path)
         if m:
-            if sc in ("T4_reviews_empty", "T5_reviews_only_author",
-                      "T15_comments_agent_approval", "T16_comments_generic_approval",
-                      "T17_comments_no_approval"):
+            if sc in ("T4_reviews_empty", "T5_reviews_only_author"):
                 return self._json(200, [])
             if sc == "T6_reviews_dismissed":
                 return self._json(200, [{
@@ -121,51 +116,17 @@ class Handler(http.server.BaseHTTPRequestHandler):
                 {"state": "APPROVED", "dismissed": False, "user": {"login": "core-devops"}, "commit_id": "abc1234"},
             ])
 
-        # GET /repos/{owner}/{name}/issues/{pr_number}/comments
-        m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/issues/(\d+)/comments$", path)
-        if m:
-            if sc == "T15_comments_agent_approval":
-                return self._json(200, [
-                    {"user": {"login": "core-qa-agent"}, "body": "[core-qa-agent] APPROVED this PR. Good changes.", "id": 1},
-                    {"user": {"login": "alice"}, "body": "I authored this PR", "id": 2},
-                    {"user": {"login": "random-user"}, "body": "Looks okay to me", "id": 3},
-                ])
-            if sc == "T16_comments_generic_approval":
-                return self._json(200, [
-                    {"user": {"login": "core-qa-agent"}, "body": "APPROVED — all acceptance criteria met", "id": 1},
-                    {"user": {"login": "alice"}, "body": "-authored", "id": 2},
-                ])
-            if sc == "T17_comments_no_approval":
-                return self._json(200, [
-                    {"user": {"login": "alice"}, "body": "I authored this PR", "id": 1},
-                    {"user": {"login": "random-user"}, "body": "Looks okay to me", "id": 2},
-                ])
-            if sc == "T18_review_wrong_team_comment_right_team":
-                return self._json(200, [
-                    {"user": {"login": "core-qa-agent"}, "body": "[core-qa-agent] APPROVED after focused review", "id": 1},
-                ])
-            # Default scenarios (T1–T9, T14): no comments
-            return self._json(200, [])
-
         # GET /teams/{team_id}/members/{username}
         m = re.match(r"^/api/v1/teams/(\d+)/members/([^/]+)$", path)
         if m:
-            login = m.group(2)
+            team_id, login = m.group(1), m.group(2)
             if sc == "T8_team_not_member":
                 return self._empty(404)
             if sc == "T9_team_403":
                 return self._empty(403)
-            if sc == "T18_review_wrong_team_comment_right_team" and login == "core-devops":
-                return self._empty(404)
             # T7_team_member: member
             return self._empty(204)
 
-        # GET /repos/{owner}/{name}/statuses/{sha} — for N/A declaration check
-        m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/statuses/([a-f0-9]+)$", path)
-        if m:
-            # All comment-based scenarios have no N/A declarations
-            return self._json(200, [])
-
         return self._json(404, {"path": path, "msg": "fixture: no route"})
 
     def do_POST(self):

.gitea/scripts/tests/test_ci_required_drift.py

@@ -1,176 +0,0 @@
-import importlib.util
-import sys
-from pathlib import Path
-from unittest.mock import patch
-
-SCRIPT = Path(__file__).resolve().parents[1] / "ci-required-drift.py"
-spec = importlib.util.spec_from_file_location("ci_required_drift", SCRIPT)
-drift = importlib.util.module_from_spec(spec)
-sys.modules[spec.name] = drift
-spec.loader.exec_module(drift)
-
-# Module-level constants are loaded from env at import time; set them
-# explicitly so unit tests can import without the full env contract.
-drift.SENTINEL_JOB = "all-required"
-drift.CI_WORKFLOW_PATH = ".gitea/workflows/ci.yml"
-drift.AUDIT_WORKFLOW_PATH = ".gitea/workflows/audit-force-merge.yml"
-
-
-# ---------------------------------------------------------------------------
-# Helper fixtures
-# ---------------------------------------------------------------------------
-
-def _make_ci_doc(jobs: dict) -> dict:
-    return {"jobs": jobs}
-
-
-def _make_audit_doc(required_checks: list[str]) -> dict:
-    return {
-        "jobs": {
-            "audit": {
-                "steps": [
-                    {"env": {"REQUIRED_CHECKS": "\n".join(required_checks)}}
-                ]
-            }
-        }
-    }
-
-
-# ---------------------------------------------------------------------------
-# sentinel_needs
-# ---------------------------------------------------------------------------
-
-def test_sentinel_needs_returns_empty_when_absent():
-    doc = _make_ci_doc({"all-required": {"runs-on": "ubuntu-latest"}})
-    assert drift.sentinel_needs(doc) == set()
-
-
-def test_sentinel_needs_parses_list():
-    doc = _make_ci_doc(
-        {"all-required": {"needs": ["platform-build", "canvas-build"]}}
-    )
-    assert drift.sentinel_needs(doc) == {"platform-build", "canvas-build"}
-
-
-def test_sentinel_needs_parses_string():
-    doc = _make_ci_doc({"all-required": {"needs": "platform-build"}})
-    assert drift.sentinel_needs(doc) == {"platform-build"}
-
-
-# ---------------------------------------------------------------------------
-# ci_job_names / ci_jobs_all
-# ---------------------------------------------------------------------------
-
-def test_ci_job_names_excludes_sentinel_and_event_gated():
-    doc = _make_ci_doc(
-        {
-            "platform-build": {},
-            "canvas-build": {"if": "github.event_name == 'pull_request'"},
-            "main-push": {"if": "github.ref == 'refs/heads/main'"},
-            "all-required": {},
-        }
-    )
-    assert drift.ci_job_names(doc) == {"platform-build"}
-
-
-def test_ci_jobs_all_includes_event_gated():
-    doc = _make_ci_doc(
-        {
-            "platform-build": {},
-            "canvas-build": {"if": "github.event_name == 'pull_request'"},
-            "all-required": {},
-        }
-    )
-    assert drift.ci_jobs_all(doc) == {"platform-build", "canvas-build"}
-
-
-# ---------------------------------------------------------------------------
-# detect_drift — F1 / F1b with mocked I/O
-# ---------------------------------------------------------------------------
-
-SAMPLE_PROTECTION = {
-    "status_check_contexts": [
-        "CI / all-required (pull_request)",
-        "Secret scan / Scan diff for credential-shaped strings (pull_request)",
-    ]
-}
-
-
-def test_detect_drift_no_needs_sentinel_skips_f1():
-    """Post-#1766 contract: all-required has no needs: → F1 is a false positive."""
-    ci = _make_ci_doc(
-        {
-            "platform-build": {},
-            "canvas-build": {},
-            "all-required": {},
-        }
-    )
-    audit = _make_audit_doc(
-        [
-            "CI / all-required (pull_request)",
-            "Secret scan / Scan diff for credential-shaped strings (pull_request)",
-        ]
-    )
-
-    with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
-        with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
-            findings, debug = drift.detect_drift("main")
-
-    assert findings == []
-    assert debug["sentinel_needs"] == []
-
-
-def test_detect_drift_typo_in_needs_triggers_f1b():
-    """F1b still catches typos when needs exists."""
-    ci = _make_ci_doc(
-        {
-            "platform-build": {},
-            "all-required": {"needs": ["platfom-build"]},  # typo
-        }
-    )
-    audit = _make_audit_doc(["CI / all-required (pull_request)"])
-
-    with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
-        with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
-            findings, _ = drift.detect_drift("main")
-
-    assert any("F1b" in f for f in findings)
-    assert any("platfom-build" in f for f in findings)
-
-
-def test_detect_drift_missing_job_in_needs_triggers_f1():
-    """F1 still fires when needs is non-empty and jobs are missing."""
-    ci = _make_ci_doc(
-        {
-            "platform-build": {},
-            "canvas-build": {},
-            "all-required": {"needs": ["platform-build"]},
-        }
-    )
-    audit = _make_audit_doc(["CI / all-required (pull_request)"])
-
-    with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
-        with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
-            findings, _ = drift.detect_drift("main")
-
-    assert any("F1 —" in f for f in findings)
-    assert any("canvas-build" in f for f in findings)
-    assert not any("F1b" in f for f in findings)
-
-
-def test_detect_drift_no_f1_when_needs_empty_even_with_jobs():
-    """Explicit regression guard: empty needs + existing jobs = no F1."""
-    ci = _make_ci_doc(
-        {
-            "platform-build": {},
-            "canvas-build": {},
-            "all-required": {"needs": []},
-        }
-    )
-    audit = _make_audit_doc(["CI / all-required (pull_request)"])
-
-    with patch.object(drift, "load_yaml", side_effect=[ci, audit]):
-        with patch.object(drift, "api", return_value=(200, SAMPLE_PROTECTION)):
-            findings, _ = drift.detect_drift("main")
-
-    assert not any("F1 —" in f for f in findings)

.gitea/scripts/tests/test_ci_workflow_bookkeeping.py

@@ -1,31 +0,0 @@
-from pathlib import Path
-
-import yaml
-
-
-ROOT = Path(__file__).resolve().parents[2]
-
-
-def load_workflow(name: str) -> dict:
-    with (ROOT / "workflows" / name).open() as f:
-        return yaml.safe_load(f)
-
-
-def test_all_required_uses_dedicated_meta_runner_lane():
-    workflow = load_workflow("ci.yml")
-    all_required = workflow["jobs"]["all-required"]
-
-    assert all_required["runs-on"] == "ci-meta"
-    assert "needs" not in all_required
-
-
-def test_all_required_reuses_path_filter_before_polling():
-    workflow = load_workflow("ci.yml")
-    all_required = workflow["jobs"]["all-required"]
-    rendered = str(all_required)
-
-    assert "--profile ci" in rendered
-    assert ".gitea/scripts/detect-changes.py" in rendered
-    assert "REQUIRE_PLATFORM" in rendered
-    assert "REQUIRE_CANVAS" in rendered
-    assert "REQUIRE_SCRIPTS" in rendered

.gitea/scripts/tests/test_gitea_merge_queue.py

@@ -2,6 +2,7 @@ import importlib.util
 import sys
 from pathlib import Path
 
+
 SCRIPT = Path(__file__).resolve().parents[1] / "gitea-merge-queue.py"
 spec = importlib.util.spec_from_file_location("gitea_merge_queue", SCRIPT)
 mq = importlib.util.module_from_spec(spec)

.gitea/scripts/tests/test_lint_pre_flip_continue_on_error.py

@@ -15,6 +15,7 @@ Mirrors the pattern in scripts/ops/test_check_migration_collisions.py
 from __future__ import annotations
 
 import importlib.util
+import os
 import sys
 import unittest
 from pathlib import Path

.gitea/scripts/tests/test_main_red_watchdog.py

@@ -1,282 +0,0 @@
-import importlib.util
-import sys
-from pathlib import Path
-from unittest.mock import patch, MagicMock
-
-SCRIPT = Path(__file__).resolve().parents[1] / "main-red-watchdog.py"
-spec = importlib.util.spec_from_file_location("main_red_watchdog", SCRIPT)
-wd = importlib.util.module_from_spec(spec)
-sys.modules[spec.name] = wd
-spec.loader.exec_module(wd)
-
-# Module-level constants are loaded from env at import time; set them
-# explicitly so unit tests can import without the full env contract.
-wd.GITEA_TOKEN = "fake-token"
-wd.GITEA_HOST = "git.example.com"
-wd.REPO = "molecule-ai/molecule-core"
-wd.OWNER = "molecule-ai"
-wd.NAME = "molecule-core"
-wd.WATCH_BRANCH = "main"
-wd.RED_LABEL = "tier:high"
-wd.API = "https://git.example.com/api/v1"
-
-
-# ---------------------------------------------------------------------------
-# _is_scheduled_context
-# ---------------------------------------------------------------------------
-
-def test_is_scheduled_context_matches_staging_saas_smoke():
-    assert wd._is_scheduled_context("Staging SaaS smoke") is True
-
-
-def test_is_scheduled_context_matches_case_insensitive():
-    assert wd._is_scheduled_context("continuous synthetic e2e") is True
-
-
-def test_is_scheduled_context_no_match_for_required_ci():
-    assert wd._is_scheduled_context("CI / all-required") is False
-
-
-# ---------------------------------------------------------------------------
-# _entry_state
-# ---------------------------------------------------------------------------
-
-def test_entry_state_prefers_status_over_state():
-    """Gitea 1.22.6 per-entry key is `status`; `state` is fallback."""
-    assert wd._entry_state({"status": "failure", "state": "success"}) == "failure"
-
-
-def test_entry_state_falls_back_to_state():
-    assert wd._entry_state({"state": "pending"}) == "pending"
-
-
-def test_entry_state_empty_when_neither_key_present():
-    assert wd._entry_state({"context": "foo"}) == ""
-
-
-# ---------------------------------------------------------------------------
-# is_red
-# ---------------------------------------------------------------------------
-
-def test_is_red_combined_failure_no_statuses():
-    """Combined failure with empty statuses[] still trips red."""
-    red, failed = wd.is_red({"state": "failure", "statuses": []})
-    assert red is True
-    assert failed == []
-
-
-def test_is_red_cancel_cascade_filtered():
-    """status=3 (cancelled) mapped to failure string must be filtered."""
-    status = {
-        "state": "failure",
-        "statuses": [
-            {"context": "CI / build", "status": "failure", "description": "Has been cancelled"},
-        ],
-    }
-    red, failed = wd.is_red(status)
-    assert red is False
-    assert failed == []
-
-
-def test_is_red_real_failure_not_filtered():
-    """Real failures with different descriptions are kept."""
-    status = {
-        "state": "failure",
-        "statuses": [
-            {"context": "CI / build", "status": "failure", "description": "Failing after 12s"},
-        ],
-    }
-    red, failed = wd.is_red(status)
-    assert red is True
-    assert len(failed) == 1
-    assert failed[0]["context"] == "CI / build"
-
-
-def test_is_red_uses_entry_state_not_top_level_state():
-    """Regression: per-entry key is `status`, not `state`."""
-    status = {
-        "state": "failure",
-        "statuses": [
-            # Only `status` present; pre-rev4 code read `state` and got None
-            {"context": "CI / test", "status": "failure"},
-        ],
-    }
-    red, failed = wd.is_red(status)
-    assert red is True
-    assert len(failed) == 1
-
-
-# ---------------------------------------------------------------------------
-# list_open_red_issues — pagination (mc#1789)
-# ---------------------------------------------------------------------------
-
-def test_list_open_red_issues_exhausts_pagination():
-    """Backlog can exceed 50 issues; all pages must be fetched."""
-    calls = []
-
-    def fake_api(method, path, **kwargs):
-        calls.append((method, path, kwargs))
-        query = (kwargs.get("query") or {})
-        page = int(query.get("page", "1"))
-        limit = int(query.get("limit", "50"))
-        # Page 1 returns full limit; page 2 returns partial → break
-        if page == 1:
-            return 200, [
-                {"title": f"[main-red] molecule-ai/molecule-core: sha{i:04d}"}
-                for i in range(limit)
-            ]
-        if page == 2:
-            return 200, [
-                {"title": "[main-red] molecule-ai/molecule-core: extra1"},
-                {"title": "[main-red] molecule-ai/molecule-core: extra2"},
-                {"title": " unrelated issue "},  # filtered out
-            ]
-        return 200, []
-
-    with patch.object(wd, "api", side_effect=fake_api):
-        issues = wd.list_open_red_issues()
-
-    assert len(issues) == 52  # 50 + 2 matched
-    titles = {i["title"] for i in issues}
-    assert "[main-red] molecule-ai/molecule-core: extra1" in titles
-    assert "[main-red] molecule-ai/molecule-core: extra2" in titles
-
-
-def test_list_open_red_issues_single_page():
-    """When results < limit, loop breaks after first page."""
-    def fake_api(method, path, **kwargs):
-        return 200, [
-            {"title": "[main-red] molecule-ai/molecule-core: abc123"},
-        ]
-
-    with patch.object(wd, "api", side_effect=fake_api):
-        issues = wd.list_open_red_issues()
-
-    assert len(issues) == 1
-
-
-# ---------------------------------------------------------------------------
-# run_once — close logic (mc#1789)
-# ---------------------------------------------------------------------------
-
-def test_run_once_green_closes_stale_issues(monkeypatch):
-    """Combined success → close stale issues."""
-    monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
-    monkeypatch.setattr(wd, "get_combined_status", lambda s: {"state": "success", "statuses": []})
-    monkeypatch.setattr(wd, "is_red", lambda s: (False, []))
-
-    closed = []
-
-    def capture_close(current_sha, *, dry_run=False, close_same_sha=False):
-        closed.append(current_sha)
-        return 1
-
-    monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", capture_close)
-    monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
-
-    assert wd.run_once(dry_run=True) == 0
-    assert closed == ["abc123"]
-
-
-def test_run_once_pending_scheduled_only_closes_stale_issues(monkeypatch):
-    """Combined pending, but only scheduled contexts pending → close stale."""
-    monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
-    monkeypatch.setattr(
-        wd, "get_combined_status",
-        lambda s: {
-            "state": "pending",
-            "statuses": [
-                {"context": "CI / all-required", "status": "success"},
-                {"context": "Staging SaaS smoke", "status": "pending"},
-            ],
-        }
-    )
-    monkeypatch.setattr(wd, "is_red", lambda s: (False, []))
-
-    closed = []
-
-    def capture_close(current_sha, *, dry_run=False, close_same_sha=False):
-        closed.append(current_sha)
-        return 1
-
-    monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", capture_close)
-    monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
-
-    assert wd.run_once(dry_run=True) == 0
-    assert closed == ["abc123"]
-
-
-def test_run_once_pending_required_does_not_close(monkeypatch):
-    """Combined pending with a real required context still pending → no close."""
-    monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
-    monkeypatch.setattr(
-        wd, "get_combined_status",
-        lambda s: {
-            "state": "pending",
-            "statuses": [
-                {"context": "CI / all-required", "status": "pending"},
-                {"context": "Staging SaaS smoke", "status": "success"},
-            ],
-        }
-    )
-    monkeypatch.setattr(wd, "is_red", lambda s: (False, []))
-
-    closed = []
-
-    def capture_close(current_sha, *, dry_run=False, close_same_sha=False):
-        closed.append(current_sha)
-        return 0
-
-    monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", capture_close)
-    monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
-
-    assert wd.run_once(dry_run=True) == 0
-    assert closed == []
-
-
-def test_run_once_failure_does_not_close(monkeypatch):
-    """Real failure in non-scheduled context → no close."""
-    monkeypatch.setattr(wd, "get_head_sha", lambda b: "abc123")
-    monkeypatch.setattr(
-        wd, "get_combined_status",
-        lambda s: {
-            "state": "failure",
-            "statuses": [
-                {"context": "CI / all-required", "status": "failure"},
-            ],
-        }
-    )
-    # is_red will return True, so we enter the red path, not the green close path
-    monkeypatch.setattr(wd, "is_red", lambda s: (True, s.get("statuses", [])))
-    monkeypatch.setattr(wd, "time", MagicMock(sleep=lambda x: None))
-    monkeypatch.setattr(wd, "emit_loki_event", lambda *a, **k: None)
-
-    filed = []
-
-    def capture_file(sha, failed, debug, *, dry_run=False):
-        filed.append(sha)
-
-    monkeypatch.setattr(wd, "file_or_update_red", capture_file)
-    monkeypatch.setattr(wd, "close_open_red_issues_for_other_shas", lambda *a, **k: 0)
-
-    assert wd.run_once(dry_run=True) == 0
-    assert filed == ["abc123"]
-
-
-# ---------------------------------------------------------------------------
-# title_for / find_open_issue_for_sha
-# ---------------------------------------------------------------------------
-
-def test_title_for_uses_short_sha():
-    assert wd.title_for("abcdef123456") == "[main-red] molecule-ai/molecule-core: abcdef1234"
-
-
-def test_find_open_issue_for_sha_matches_exact_title(monkeypatch):
-    fake_issue = {"title": "[main-red] molecule-ai/molecule-core: abc1234567", "number": 42}
-    monkeypatch.setattr(wd, "list_open_red_issues", lambda: [fake_issue])
-    assert wd.find_open_issue_for_sha("abc1234567") == fake_issue
-
-
-def test_find_open_issue_for_sha_returns_none_when_no_match(monkeypatch):
-    monkeypatch.setattr(wd, "list_open_red_issues", lambda: [])
-    assert wd.find_open_issue_for_sha("abc123") is None

.gitea/scripts/tests/test_prod_auto_deploy.py

@@ -36,37 +36,9 @@ def test_build_plan_defaults_to_staging_sha_target_and_prod_cp():
         "soak_seconds": 60,
         "batch_size": 3,
         "dry_run": False,
-        # cp#228 / task #308: fleet-wide intent must carry confirm:true.
-        "confirm": True,
     }
 
 
-def test_build_plan_always_sets_confirm_true_for_fleet_intent():
-    """Regression guard: every plan body MUST carry confirm:true.
-
-    CP /cp/admin/tenants/redeploy-fleet (cp#228) returns 400 on empty
-    body / {confirm:false} / {only_slugs:[]} to prevent accidental
-    fleet-wide mutation. This caller is fleet-wide intent (canary +
-    fan-out, no slug scoping), so the plan MUST carry confirm:true.
-    Pairs with cp#228's TestRedeployFleet_EmptyBodyReturns400 +
-    TestRedeployFleet_ConfirmTrueProceeds.
-    """
-    plan = prod.build_plan({"GITHUB_SHA": "abcdef1234567890"})
-    assert plan["body"]["confirm"] is True
-
-    # Operator-overridable knobs do NOT drop the ack.
-    plan = prod.build_plan(
-        {
-            "GITHUB_SHA": "abcdef1234567890",
-            "PROD_AUTO_DEPLOY_SOAK_SECONDS": "0",
-            "PROD_AUTO_DEPLOY_BATCH_SIZE": "10",
-            "PROD_AUTO_DEPLOY_DRY_RUN": "true",
-            "PROD_AUTO_DEPLOY_CANARY_SLUG": "",
-        }
-    )
-    assert plan["body"]["confirm"] is True
-
-
 def test_build_plan_rejects_non_prod_cp_without_explicit_override():
     try:
         prod.build_plan(
@@ -146,212 +118,3 @@ def test_context_is_terminal_failure_rejects_cancelled_and_skipped():
         assert prod.context_is_terminal_failure(state) is True
     for state in ("pending", "missing", "success"):
         assert prod.context_is_terminal_failure(state) is False
-
-
-def test_default_required_contexts_delegate_path_gating_to_all_required():
-    assert prod.required_contexts({}) == [
-        "CI / all-required (push)",
-        "Secret scan / Scan diff for credential-shaped strings (push)",
-    ]
-
-
-def test_slugs_from_redeploy_response_uses_controlplane_plan_rows():
-    body = {
-        "results": [
-            {"slug": "hongming", "phase": "canary", "ssm_status": "DryRun"},
-            {"slug": "tenant-a", "phase": "batch-1", "ssm_status": "DryRun"},
-            {"slug": "", "phase": "batch-1", "ssm_status": "DryRun"},
-            {"phase": "batch-1", "ssm_status": "DryRun"},
-        ]
-    }
-
-    assert prod.slugs_from_redeploy_response(body) == ["hongming", "tenant-a"]
-
-
-def test_plan_rollout_slugs_asks_controlplane_for_dry_run_plan():
-    calls = []
-
-    def fake_redeploy(_cp_url, _token, body):
-        calls.append(body)
-        return 200, {
-            "ok": True,
-            "results": [
-                {"slug": "hongming", "phase": "canary", "ssm_status": "DryRun"},
-                {"slug": "tenant-a", "phase": "batch-1", "ssm_status": "DryRun"},
-            ],
-        }
-
-    slugs = prod.plan_rollout_slugs(
-        "https://api.moleculesai.app",
-        "secret",
-        {
-            "target_tag": "staging-abcdef1",
-            "canary_slug": "hongming",
-            "soak_seconds": 60,
-            "batch_size": 3,
-            "dry_run": False,
-            "confirm": True,
-        },
-        redeploy=fake_redeploy,
-    )
-
-    assert slugs == ["hongming", "tenant-a"]
-    assert calls == [
-        {
-            "target_tag": "staging-abcdef1",
-            "canary_slug": "hongming",
-            "soak_seconds": 60,
-            "batch_size": 3,
-            "dry_run": True,
-            "confirm": True,
-        }
-    ]
-
-
-def test_scoped_redeploy_body_removes_canary_and_local_soak():
-    base = {
-        "target_tag": "staging-abcdef1",
-        "canary_slug": "hongming",
-        "soak_seconds": 60,
-        "batch_size": 3,
-        "dry_run": False,
-        "confirm": True,
-    }
-
-    scoped = prod.scoped_redeploy_body(base, ["tenant-a", "tenant-b"])
-
-    assert scoped == {
-        "target_tag": "staging-abcdef1",
-        "soak_seconds": 0,
-        "batch_size": 2,
-        "dry_run": False,
-        "confirm": True,
-        "only_slugs": ["tenant-a", "tenant-b"],
-    }
-
-
-def test_plan_scoped_rollout_preserves_canary_then_batches():
-    calls, sleeps = [], []
-
-    def fake_list(_cp_url, _token, _body):
-        return ["tenant-a", "hongming", "tenant-b", "tenant-c"]
-
-    def fake_redeploy(_cp_url, _token, body):
-        calls.append(body)
-        return 200, {
-            "ok": True,
-            "results": [{"slug": slug, "healthz_ok": True} for slug in body["only_slugs"]],
-        }
-
-    aggregate = prod.execute_scoped_rollout(
-        {
-            "cp_url": "https://api.moleculesai.app",
-            "body": {
-                "target_tag": "staging-abcdef1",
-                "canary_slug": "hongming",
-                "soak_seconds": 60,
-                "batch_size": 2,
-                "dry_run": False,
-                "confirm": True,
-            },
-        },
-        token="secret",
-        list_slugs=fake_list,
-        redeploy=fake_redeploy,
-        sleep=sleeps.append,
-    )
-
-    assert [call["only_slugs"] for call in calls] == [
-        ["hongming"],
-        ["tenant-a", "tenant-b"],
-        ["tenant-c"],
-    ]
-    assert sleeps == [60]
-    assert aggregate["ok"] is True
-    assert [result["slug"] for result in aggregate["results"]] == [
-        "hongming",
-        "tenant-a",
-        "tenant-b",
-        "tenant-c",
-    ]
-
-
-def test_scoped_rollout_halts_after_failed_canary():
-    calls = []
-
-    def fake_redeploy(_cp_url, _token, body):
-        calls.append(body)
-        return 200, {"ok": False, "results": [{"slug": body["only_slugs"][0], "error": "bad"}]}
-
-    try:
-        prod.execute_scoped_rollout(
-            {
-                "cp_url": "https://api.moleculesai.app",
-                "body": {
-                    "target_tag": "staging-abcdef1",
-                    "canary_slug": "hongming",
-                    "soak_seconds": 60,
-                    "batch_size": 2,
-                    "dry_run": False,
-                    "confirm": True,
-                },
-            },
-            token="secret",
-            list_slugs=lambda _cp_url, _token, _body: ["hongming", "tenant-a"],
-            redeploy=fake_redeploy,
-            sleep=lambda _seconds: None,
-        )
-    except prod.RolloutFailed as exc:
-        assert "redeploy scoped call failed" in str(exc)
-        assert exc.response["ok"] is False
-        assert exc.response["results"] == [{"slug": "hongming", "error": "bad"}]
-    else:
-        raise AssertionError("expected failed canary to halt rollout")
-
-    assert [call["only_slugs"] for call in calls] == [["hongming"]]
-
-
-def test_rollout_from_plan_file_writes_partial_response_on_failure(tmp_path):
-    plan_path = tmp_path / "plan.json"
-    response_path = tmp_path / "response.json"
-    plan_path.write_text(
-        """
-        {
-          "enabled": true,
-          "cp_url": "https://api.moleculesai.app",
-          "body": {"target_tag": "staging-abcdef1", "confirm": true}
-        }
-        """,
-        encoding="utf-8",
-    )
-
-    original = prod.execute_scoped_rollout
-
-    def fake_execute(_plan, _token):
-        raise prod.RolloutFailed(
-            "redeploy scoped call failed for hongming: HTTP 500, ok=false",
-            {
-                "ok": False,
-                "error": "redeploy scoped call failed for hongming: HTTP 500, ok=false",
-                "results": [{"slug": "hongming", "error": "bad"}],
-            },
-        )
-
-    prod.execute_scoped_rollout = fake_execute
-    try:
-        try:
-            prod.rollout_from_plan_file(
-                str(plan_path),
-                str(response_path),
-                {"CP_ADMIN_API_TOKEN": "secret"},
-            )
-        except prod.RolloutFailed:
-            pass
-        else:
-            raise AssertionError("expected rollout failure")
-    finally:
-        prod.execute_scoped_rollout = original
-
-    assert response_path.read_text(encoding="utf-8").strip()
-    assert '"ok": false' in response_path.read_text(encoding="utf-8")
-    assert '"slug": "hongming"' in response_path.read_text(encoding="utf-8")

.gitea/scripts/tests/test_review_check.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# shellcheck disable=SC2034
 # Regression tests for .gitea/scripts/review-check.sh (RFC#324 Step 1).
 #
 # Covers:
@@ -17,7 +16,6 @@
 #   T12 — jq filter: non-author APPROVED → in candidate list; dismissed → excluded
 #   T13 — missing required env GITEA_TOKEN → exits 1 with error
 #   T14 — non-default-base PR exits 0 without requiring review
-#   T18 — wrong-team review candidate does not block right-team comment approval
 #
 # Hostile-self-review (per feedback_assert_exact_not_substring):
 # this test MUST FAIL if the script is absent. Verified by running
@@ -140,7 +138,7 @@ fi
 echo
 echo "== T13 missing GITEA_TOKEN =="
 set +e
-T13_OUT=$(PATH="/tmp:$PATH" GITEA_TOKEN='' GITEA_HOST=git.example.com REPO=x/y PR_NUMBER=1 TEAM=qa TEAM_ID=1 bash "$SCRIPT" 2>&1 || true)
+T13_OUT=$(PATH="/tmp:$PATH" GITEA_TOKEN= GITEA_HOST=git.example.com REPO=x/y PR_NUMBER=1 TEAM=qa TEAM_ID=1 bash "$SCRIPT" 2>&1 || true)
 set -e
 assert_contains "T13 exits non-zero when GITEA_TOKEN missing" "GITEA_TOKEN required" "$T13_OUT"
 
@@ -308,12 +306,12 @@ echo
 echo "== T10 CURL_AUTH_FILE =="
 # Verify the token-file logic directly: create a temp file with the
 # same mktemp pattern, write the header with printf, chmod 600, then assert.
-T10_TOKEN="secret-fixture-token-abc123"
+T10_TOKEN="secret-test-token-abc123"
 T10_AUTHFILE=$(mktemp "${TMPDIR:-/tmp}/curl-auth.test.XXXXXX")
 chmod 600 "$T10_AUTHFILE"
 printf 'header = "Authorization: token %s"\n' "$T10_TOKEN" > "$T10_AUTHFILE"
 assert_file_mode "T10a mktemp authfile mode 600 (CURL_AUTH_FILE pattern)" "$T10_AUTHFILE" "600"
-assert_file_contains "T10b printf header format (CURL_AUTH_FILE content)" "$T10_AUTHFILE" "Authorization: token secret-fixture-token-abc123"
+assert_file_contains "T10b printf header format (CURL_AUTH_FILE content)" "$T10_AUTHFILE" "Authorization: token secret-test-token-abc123"
 assert_file_contains "T10c 'header =' curl-config syntax" "$T10_AUTHFILE" 'header = "Authorization: token '
 rm -f "$T10_AUTHFILE"
 
@@ -336,42 +334,6 @@ assert_contains "T12 jq: core-devops (non-author APPROVED) in candidates" "core-
 assert_eq "T12 jq: alice (author) NOT in candidates" "" "$(echo "$T12_CANDIDATES" | grep '^alice$' || true)"
 assert_eq "T12 jq: carol (dismissed) NOT in candidates" "" "$(echo "$T12_CANDIDATES" | grep '^carol$' || true)"
 
-# T15 — comment-based approval via agent prefix pattern → exit 0
-echo
-echo "== T15 comment agent-prefix approval =="
-T15_OUT=$(run_review_check "T15_comments_agent_approval")
-T15_RC=$(cat "$FIX_STATE_DIR/last_rc")
-assert_eq "T15 exit code 0 (agent-comment approval + team member)" "0" "$T15_RC"
-assert_contains "T15 comment fallback notice" "comment-based approval" "$T15_OUT"
-assert_contains "T15 core-qa-agent APPROVED" "APPROVED by core-qa-agent" "$T15_OUT"
-
-# T16 — comment-based approval via generic APPROVED keyword → exit 0
-echo
-echo "== T16 comment generic keyword approval =="
-T16_OUT=$(run_review_check "T16_comments_generic_approval")
-T16_RC=$(cat "$FIX_STATE_DIR/last_rc")
-assert_eq "T16 exit code 0 (generic-approval comment + team member)" "0" "$T16_RC"
-assert_contains "T16 comment fallback notice" "comment-based approval" "$T16_OUT"
-
-# T17 — no approval keywords in comments → exit 1
-echo
-echo "== T17 comments with no approval keywords =="
-T17_OUT=$(run_review_check "T17_comments_no_approval")
-T17_RC=$(cat "$FIX_STATE_DIR/last_rc")
-assert_eq "T17 exit code 1 (no candidates from comments)" "1" "$T17_RC"
-assert_contains "T17 no candidates error" "no candidates from reviews API or issue comments" "$T17_OUT"
-
-# T18 — a wrong-team PR review candidate must not suppress a right-team
-# comment approval. This matches PR #1790, where QA had an APPROVED review
-# and security approved via the agent comment convention.
-echo
-echo "== T18 review candidate wrong team, comment candidate right team =="
-T18_OUT=$(run_review_check "T18_review_wrong_team_comment_right_team")
-T18_RC=$(cat "$FIX_STATE_DIR/last_rc")
-assert_eq "T18 exit code 0 (comment approval still considered)" "0" "$T18_RC"
-assert_contains "T18 comment candidate notice" "comment-based approval" "$T18_OUT"
-assert_contains "T18 comment approver accepted" "APPROVED by core-qa-agent" "$T18_OUT"
-
 echo
 echo "------"
 echo "PASS=$PASS FAIL=$FAIL"

.gitea/scripts/tests/test_sop_checklist.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 
 import os
 import sys
+import tempfile
 import unittest
 
 # Resolve sibling script regardless of where pytest is invoked from.
@@ -601,405 +602,4 @@ class TestComputeNaState(unittest.TestCase):
         self.assertEqual(len(na_directives), 1)
         self.assertEqual(na_directives[0][0], "sop-n/a")
         self.assertEqual(na_directives[0][1], "qa-review")
-
-
-# ---------------------------------------------------------------------------
-# RFC#450 Option C — risk-classed two-eyes (governance fix for internal#442)
-# ---------------------------------------------------------------------------
-
-
-class TestIsHighRisk(unittest.TestCase):
-    """The high-risk predicate decides which required_teams list applies.
-
-    Predicate: tier:high label OR any label in cfg.high_risk_labels.
-    """
-
-    def setUp(self):
-        self.cfg = sop.load_config(CONFIG_PATH)
-
-    def test_no_labels_is_default_class(self):
-        pr = {"labels": []}
-        self.assertFalse(sop.is_high_risk(pr, self.cfg))
-
-    def test_tier_high_is_high_risk(self):
-        pr = {"labels": [{"name": "tier:high"}]}
-        self.assertTrue(sop.is_high_risk(pr, self.cfg))
-
-    def test_tier_low_is_default_class(self):
-        pr = {"labels": [{"name": "tier:low"}]}
-        self.assertFalse(sop.is_high_risk(pr, self.cfg))
-
-    def test_tier_medium_is_default_class(self):
-        # tier:medium alone is NOT high-risk (Option C — medium routes
-        # to the wider engineers OR-set).
-        pr = {"labels": [{"name": "tier:medium"}]}
-        self.assertFalse(sop.is_high_risk(pr, self.cfg))
-
-    def test_area_security_label_is_high_risk(self):
-        pr = {"labels": [{"name": "tier:medium"}, {"name": "area:security"}]}
-        self.assertTrue(sop.is_high_risk(pr, self.cfg))
-
-    def test_area_schema_label_is_high_risk(self):
-        pr = {"labels": [{"name": "area:schema"}]}
-        self.assertTrue(sop.is_high_risk(pr, self.cfg))
-
-    def test_area_identity_label_is_high_risk(self):
-        pr = {"labels": [{"name": "area:identity"}]}
-        self.assertTrue(sop.is_high_risk(pr, self.cfg))
-
-    def test_area_fleet_image_label_is_high_risk(self):
-        pr = {"labels": [{"name": "area:fleet-image"}]}
-        self.assertTrue(sop.is_high_risk(pr, self.cfg))
-
-    def test_area_gate_meta_label_is_high_risk(self):
-        # Gate-meta = changes to sop-checklist/sop-tier-check itself.
-        pr = {"labels": [{"name": "area:gate-meta"}]}
-        self.assertTrue(sop.is_high_risk(pr, self.cfg))
-
-    def test_unknown_area_label_is_default_class(self):
-        pr = {"labels": [{"name": "area:docs"}]}
-        self.assertFalse(sop.is_high_risk(pr, self.cfg))
-
-
-class TestResolveRequiredTeams(unittest.TestCase):
-    """The team resolver picks the elevated list only for high-risk PRs
-    AND only when the item declares one — items without an elevated
-    list always use the default required_teams."""
-
-    def test_default_class_uses_default_teams(self):
-        item = {"required_teams": ["engineers", "managers", "ceo"], "required_teams_high_risk": ["ceo"]}
-        self.assertEqual(
-            sop.resolve_required_teams(item, high_risk=False),
-            ["engineers", "managers", "ceo"],
-        )
-
-    def test_high_risk_uses_elevated_teams(self):
-        item = {"required_teams": ["engineers", "managers", "ceo"], "required_teams_high_risk": ["ceo"]}
-        self.assertEqual(
-            sop.resolve_required_teams(item, high_risk=True),
-            ["ceo"],
-        )
-
-    def test_high_risk_without_elevated_falls_back_to_default(self):
-        # Items that don't declare required_teams_high_risk (e.g.
-        # comprehensive-testing, staging-smoke) are unaffected by risk-class.
-        item = {"required_teams": ["engineers"]}
-        self.assertEqual(
-            sop.resolve_required_teams(item, high_risk=True),
-            ["engineers"],
-        )
-
-    def test_empty_elevated_list_falls_back_to_default(self):
-        # A defensive case: required_teams_high_risk: [] should not
-        # silently lock out all approvers — fall back to the default
-        # so the gate stays satisfiable. (Tightening should remove the
-        # key, not set it to empty.)
-        item = {"required_teams": ["engineers"], "required_teams_high_risk": []}
-        self.assertEqual(
-            sop.resolve_required_teams(item, high_risk=True),
-            ["engineers"],
-        )
-
-
-class TestRootCauseAckEligibilityWidened(unittest.TestCase):
-    """Closes internal#442: a non-author engineers-team ack now satisfies
-    root-cause / no-backwards-compat for the default class.
-
-    The dead-managers/ceo-persona-token gridlock is the symptom; the
-    root cause is that sop-checklist ignored tier-class. These tests
-    pin the new wider-default behavior so it can't regress silently.
-    """
-
-    def setUp(self):
-        self.items = _items_by_slug()
-        self.aliases = _numeric_aliases()
-
-    @staticmethod
-    def _approve_only(allowed):
-        return lambda slug, users: [u for u in users if u in allowed]
-
-    def test_engineers_ack_satisfies_root_cause_default_class(self):
-        # Bob is in engineers only (not managers, not ceo). Default class.
-        comments = [_comment("bob", "/sop-ack root-cause")]
-        # Probe: bob is approved because root-cause now lists engineers.
-        probe = self._approve_only({"bob"})
-        state = sop.compute_ack_state(
-            comments, "alice", self.items, self.aliases, probe, high_risk=False
-        )
-        self.assertEqual(state["root-cause"]["ackers"], ["bob"])
-
-    def test_engineers_ack_satisfies_no_backwards_compat_default_class(self):
-        comments = [_comment("bob", "/sop-ack no-backwards-compat")]
-        probe = self._approve_only({"bob"})
-        state = sop.compute_ack_state(
-            comments, "alice", self.items, self.aliases, probe, high_risk=False
-        )
-        self.assertEqual(state["no-backwards-compat"]["ackers"], ["bob"])
-
-    def test_engineers_ack_alone_fails_root_cause_when_high_risk(self):
-        # High-risk PR: only ceo can ack. Engineers-only ack must fail.
-        comments = [_comment("bob", "/sop-ack root-cause")]
-        # Probe: bob is in engineers, not ceo. Under high_risk,
-        # required_teams_high_risk=[ceo] → bob is NOT approved.
-        # Probe receives the items + flag indirectly via main(); for
-        # the unit-test path we inject a probe that rejects bob.
-        probe = self._approve_only(set())  # nobody is in ceo
-        state = sop.compute_ack_state(
-            comments, "alice", self.items, self.aliases, probe, high_risk=True
-        )
-        self.assertEqual(state["root-cause"]["ackers"], [])
-        self.assertIn("bob", state["root-cause"]["rejected"]["not_in_team"])
-
-    def test_ceo_ack_satisfies_root_cause_when_high_risk(self):
-        # High-risk PR + ceo-team approver → passes (the senior path).
-        comments = [_comment("hongming", "/sop-ack root-cause")]
-        probe = self._approve_only({"hongming"})
-        state = sop.compute_ack_state(
-            comments, "alice", self.items, self.aliases, probe, high_risk=True
-        )
-        self.assertEqual(state["root-cause"]["ackers"], ["hongming"])
-
-    def test_self_ack_still_forbidden_even_with_widened_eligibility(self):
-        # Author cannot self-ack — widening teams must NOT weaken
-        # the non-author rule.
-        comments = [_comment("alice", "/sop-ack root-cause")]
-        probe = self._approve_only({"alice"})
-        state = sop.compute_ack_state(
-            comments, "alice", self.items, self.aliases, probe, high_risk=False
-        )
-        self.assertEqual(state["root-cause"]["ackers"], [])
-        self.assertIn("alice", state["root-cause"]["rejected"]["self_ack"])
-
-
-class TestHighRiskClassUsesElevatedListInConfig(unittest.TestCase):
-    """End-to-end: the shipped config + RFC#450 predicate must keep
-    root-cause / no-backwards-compat gated on ceo for high-risk PRs."""
-
-    def test_root_cause_high_risk_elevated_to_ceo_only(self):
-        items = _items_by_slug()
-        # tier:high alone makes the PR high-risk → root-cause needs ceo.
-        self.assertEqual(
-            sop.resolve_required_teams(items["root-cause"], high_risk=True),
-            ["ceo"],
-        )
-        # Default class accepts engineers/managers/ceo.
-        self.assertEqual(
-            sorted(sop.resolve_required_teams(items["root-cause"], high_risk=False)),
-            sorted(["engineers", "managers", "ceo"]),
-        )
-
-    def test_no_backwards_compat_high_risk_elevated_to_ceo_only(self):
-        items = _items_by_slug()
-        self.assertEqual(
-            sop.resolve_required_teams(items["no-backwards-compat"], high_risk=True),
-            ["ceo"],
-        )
-        self.assertEqual(
-            sorted(sop.resolve_required_teams(items["no-backwards-compat"], high_risk=False)),
-            sorted(["engineers", "managers", "ceo"]),
-        )
-
-    def test_other_items_unchanged_by_risk_class(self):
-        # Items without required_teams_high_risk are unaffected.
-        items = _items_by_slug()
-        for slug in (
-            "comprehensive-testing",
-            "local-postgres-e2e",
-            "staging-smoke",
-            "five-axis-review",
-            "memory-consulted",
-        ):
-            self.assertEqual(
-                sop.resolve_required_teams(items[slug], high_risk=False),
-                sop.resolve_required_teams(items[slug], high_risk=True),
-                f"item {slug} should not be affected by risk-class",
-            )
-
-
-# ---------------------------------------------------------------------------
-# get_issue_comments — streaming + minimal-dict shape (task #369 / OOM fix)
-# ---------------------------------------------------------------------------
-
-
-class _FakeReq:
-    """Stand-in for GiteaClient._req that serves canned pages."""
-
-    def __init__(self, pages):
-        # pages: list[list[dict]]; one page per call, exhausted in order.
-        self._pages = list(pages)
-        self.calls = []
-
-    def __call__(self, method, path, body=None, ok_codes=(200, 201, 204)):
-        self.calls.append((method, path))
-        if not self._pages:
-            return 200, []
-        return 200, self._pages.pop(0)
-
-
-class TestGetIssueCommentsStreaming(unittest.TestCase):
-    """Verify the OOM-fix invariants — minimal-dict shape + page break."""
-
-    def _client_with_pages(self, pages):
-        client = sop.GiteaClient("git.example.com", "tok")
-        client._req = _FakeReq(pages)  # type: ignore[method-assign]
-        return client
-
-    def test_minimal_dict_shape_drops_large_fields(self):
-        """get_issue_comments must DROP html_url/assets/timestamps/etc. and
-        keep ONLY {user.login, body} — that's the whole OOM-prevention."""
-        full_page = [
-            {
-                "id": 1234,
-                "html_url": "https://example.com/some-huge-url",
-                "pull_request_url": "https://example.com/some-other-huge-url",
-                "issue_url": "https://example.com/yet-another-url",
-                "user": {"login": "bob", "avatar_url": "x" * 4000, "id": 99},
-                "original_author": "",
-                "original_author_id": 0,
-                "body": "/sop-ack comprehensive-testing\n\nlooks good",
-                "assets": ["x" * 1000, "y" * 1000],
-                "created_at": "2026-05-19T01:02:03Z",
-                "updated_at": "2026-05-19T01:02:03Z",
-            }
-        ]
-        client = self._client_with_pages([full_page])
-        out = client.get_issue_comments("o", "r", 1)
-        self.assertEqual(len(out), 1)
-        # Only the two whitelisted keys + nested user.login
-        self.assertEqual(set(out[0].keys()), {"user", "body"})
-        self.assertEqual(set(out[0]["user"].keys()), {"login"})
-        self.assertEqual(out[0]["user"]["login"], "bob")
-        self.assertEqual(out[0]["body"], "/sop-ack comprehensive-testing\n\nlooks good")
-        # Critical: avatar/assets/timestamps/etc. must be gone (~4KB+ each).
-        self.assertNotIn("html_url", out[0])
-        self.assertNotIn("assets", out[0])
-        self.assertNotIn("created_at", out[0])
-
-    def test_pagination_break_on_short_page(self):
-        # Page-size 50; a page of <50 means no more pages.
-        page1 = [{"user": {"login": "u"}, "body": "x"}] * 7
-        client = self._client_with_pages([page1])
-        out = client.get_issue_comments("o", "r", 2)
-        self.assertEqual(len(out), 7)
-        # Should have made exactly 1 _req call (no page-2 probe).
-        self.assertEqual(len(client._req.calls), 1)
-
-    def test_pagination_continues_until_empty(self):
-        # Two full pages + one short page.
-        page1 = [{"user": {"login": "u"}, "body": "x"}] * 50
-        page2 = [{"user": {"login": "u"}, "body": "y"}] * 50
-        page3 = [{"user": {"login": "u"}, "body": "z"}] * 3
-        client = self._client_with_pages([page1, page2, page3])
-        out = client.get_issue_comments("o", "r", 3)
-        self.assertEqual(len(out), 103)
-        self.assertEqual(len(client._req.calls), 3)
-
-    def test_max_comments_caps_collection(self):
-        page1 = [{"user": {"login": "u"}, "body": "x"}] * 50
-        page2 = [{"user": {"login": "u"}, "body": "y"}] * 50
-        page3 = [{"user": {"login": "u"}, "body": "z"}] * 50
-        client = self._client_with_pages([page1, page2, page3])
-        out = client.get_issue_comments("o", "r", 4, max_comments=75)
-        self.assertEqual(len(out), 75)
-        # Stops short: shouldn't have requested page-3.
-        self.assertLessEqual(len(client._req.calls), 2)
-
-    def test_oversized_body_truncated(self):
-        # An individual comment with a multi-MiB body (e.g. pasted CI log)
-        # must NOT pull the whole thing into memory. The directive parser
-        # only needs the first ~8 KiB to find /sop-* markers.
-        huge_body = "/sop-ack comprehensive-testing\n" + ("X" * (4 * 1024 * 1024))
-        page = [{"user": {"login": "bob"}, "body": huge_body}]
-        client = self._client_with_pages([page])
-        out = client.get_issue_comments("o", "r", 99)
-        self.assertEqual(len(out), 1)
-        # Cap is 8 KiB; comment body must be <= 8 KiB after streaming.
-        self.assertLessEqual(len(out[0]["body"]), 8 * 1024)
-        # Marker still discoverable at the start.
-        self.assertTrue(out[0]["body"].startswith("/sop-ack comprehensive-testing"))
-
-    def test_iter_handles_missing_user_or_body(self):
-        # Defensive: Gitea has been seen to return user=null on deleted users.
-        page = [
-            {"user": None, "body": "abandoned-author"},
-            {"user": {"login": "alice"}, "body": None},
-            {"body": "no-user-key"},
-            {"user": {"login": "bob"}, "body": "ok"},
-        ]
-        client = self._client_with_pages([page])
-        out = client.get_issue_comments("o", "r", 5)
-        self.assertEqual(len(out), 4)
-        self.assertEqual(out[0]["user"]["login"], "")
-        self.assertEqual(out[0]["body"], "abandoned-author")
-        self.assertEqual(out[1]["user"]["login"], "alice")
-        self.assertEqual(out[1]["body"], "")
-        self.assertEqual(out[2]["user"]["login"], "")
-        self.assertEqual(out[3]["user"]["login"], "bob")
-
-    def test_minimal_dicts_work_with_compute_ack_state(self):
-        """Round-trip: minimal dicts feed back through compute_ack_state."""
-        page = [{"user": {"login": "bob"}, "body": "/sop-ack comprehensive-testing"}]
-        client = self._client_with_pages([page])
-        comments = client.get_issue_comments("o", "r", 6)
-        items = _items_by_slug()
-        aliases = _numeric_aliases()
-        state = sop.compute_ack_state(
-            comments, "alice", items, aliases, lambda slug, users: list(users)
-        )
-        self.assertEqual(state["comprehensive-testing"]["ackers"], ["bob"])
-
-
-# ---------------------------------------------------------------------------
-# probe() na-gate fallback — fix for #355-class KeyError 'security-review'
-# ---------------------------------------------------------------------------
-
-
-class TestComputeNaStateAcceptsGateNotInItems(unittest.TestCase):
-    """compute_na_state passes the gate NAME to probe(); when the gate is
-    NOT also an items entry (the common case for `security-review`,
-    `qa-review`), probe must fall back to the gate's own required_teams
-    instead of KeyError'ing on items_by_slug[slug].
-
-    This test exercises the public surface (compute_na_state) rather than
-    the inline `probe` closure, because the closure is built inside main().
-    We simulate the fallback by passing a probe that mirrors the production
-    contract — slug may be either an item OR an n/a-gate key, both are valid.
-    """
-
-    def test_na_gate_with_required_teams_resolves_without_keyerror(self):
-        na_gates = {
-            "security-review": {
-                "required_teams": ["security", "managers", "ceo"],
-                "description": "security N/A",
-            },
-        }
-        comments = [
-            {"user": {"login": "carol"}, "body": "/sop-n/a security-review docs-only"},
-        ]
-        # Probe approves any user in the security team; importantly it does
-        # NOT try items_by_slug[slug] for the gate name.
-        called_with = []
-
-        def probe(slug, users):
-            called_with.append(slug)
-            # production probe accepts gate-name OR item-slug; for this test
-            # we just approve everyone.
-            return list(users)
-
-        na_state = sop.compute_na_state(comments, "alice", na_gates, probe)
-        self.assertTrue(na_state["security-review"]["declared"])
-        self.assertEqual(na_state["security-review"]["decl_ackers"], ["carol"])
-        # probe must have been called with the GATE name, not an item slug.
-        self.assertEqual(called_with, ["security-review"])
-
-    def test_na_gate_self_declaration_rejected(self):
-        # Author cannot self-declare N/A — pre-existing invariant; pin it
-        # so the new probe-fallback doesn't regress this.
-        na_gates = {"security-review": {"required_teams": ["security"]}}
-        comments = [
-            {"user": {"login": "alice"}, "body": "/sop-n/a security-review"},
-        ]
-        na_state = sop.compute_na_state(
-            comments, "alice", na_gates, lambda *_: ["alice"]
-        )
-        self.assertFalse(na_state["security-review"]["declared"])
+        self.assertIn("no surface", na_directives[0][2])

.gitea/scripts/tests/test_sop_tier_refire.sh

@@ -6,10 +6,9 @@
 #   T1: PR open + APPROVED via tier:low → script invokes sop-tier-check
 #       and POSTs status=success.
 #   T2: PR open + missing tier label → sop-tier-check exits non-zero;
-#       refire still POSTs status=success, matching the canonical
-#       pull_request_target workflow's fail-open job conclusion.
+#       refire POSTs status=failure (description mentions failure).
 #   T3: PR open + tier:low but NO approving reviews → sop-tier-check
-#       exits non-zero; refire still POSTs status=success for the same reason.
+#       exits non-zero; refire POSTs status=failure.
 #   T4: PR CLOSED → refire exits 0 with no status POST (no-op on closed).
 #   T5: Rate-limit — recent status update within 30s → refire skips,
 #       no new POST.
@@ -33,7 +32,7 @@ THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
 SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
 WORKFLOW_DIR="$(cd "$THIS_DIR/../../workflows" && pwd)"
 WORKFLOW="$WORKFLOW_DIR/sop-tier-refire.yml"
-DISPATCH_WORKFLOW="$WORKFLOW_DIR/sop-checklist.yml"
+DISPATCH_WORKFLOW="$WORKFLOW_DIR/review-refire-comments.yml"
 SCRIPT="$SCRIPT_DIR/sop-tier-refire.sh"
 
 PASS=0
@@ -89,7 +88,7 @@ assert_file_exists() {
 echo
 echo "== existence =="
 assert_file_exists "workflow file exists"  "$WORKFLOW"
-assert_file_exists "SSOT dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
+assert_file_exists "dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
 assert_file_exists "script file exists"    "$SCRIPT"
 if [ "$FAIL" -gt 0 ]; then
   echo
@@ -134,15 +133,15 @@ else
 fi
 
 DISPATCH_PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$DISPATCH_WORKFLOW" 2>&1 || true)
-assert_eq "T6e SSOT dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
+assert_eq "T6e dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
 DISPATCH_CONTENT=$(cat "$DISPATCH_WORKFLOW")
-assert_contains "T6f SSOT dispatcher listens on issue_comment" \
+assert_contains "T6f dispatcher listens on issue_comment" \
   "issue_comment" "$DISPATCH_CONTENT"
-assert_contains "T6g SSOT dispatcher handles /qa-recheck" \
+assert_contains "T6g dispatcher handles /qa-recheck" \
   "/qa-recheck" "$DISPATCH_CONTENT"
-assert_contains "T6h SSOT dispatcher handles /security-recheck" \
+assert_contains "T6h dispatcher handles /security-recheck" \
   "/security-recheck" "$DISPATCH_CONTENT"
-assert_contains "T6i SSOT dispatcher handles /refire-tier-check" \
+assert_contains "T6i dispatcher handles /refire-tier-check" \
   "/refire-tier-check" "$DISPATCH_CONTENT"
 
 # T1-T5 — script behavior against a local Gitea-fixture
@@ -246,21 +245,34 @@ assert_contains "T1 POST context is sop-tier-check / tier-check" \
   '"context": "sop-tier-check / tier-check (pull_request)"' "$POSTED"
 assert_contains "T1 description names commenter" "test-runner" "$POSTED"
 
-# T2: missing tier label → tier-check fails internally, but refire status
-# matches the canonical workflow's fail-open job conclusion.
+# T2: missing tier label → tier-check fails → failure status POSTed
 run_scenario "T2_no_tier_label" "fail_no_label"
 RC=$(cat "$FIX_STATE_DIR/last_rc")
 POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
-assert_eq "T2 exit code 0 (canonical fail-open)" "0" "$RC"
-assert_contains "T2 POSTed state=success" '"state": "success"' "$POSTED"
+# tier-check.sh exits 1; refire script forwards that exit, so RC != 0
+if [ "$RC" -ne 0 ]; then
+  echo "  PASS  T2 exit code non-zero (got $RC)"
+  PASS=$((PASS + 1))
+else
+  echo "  FAIL  T2 exit code should be non-zero, got 0"
+  FAIL=$((FAIL + 1))
+  FAILED_TESTS="${FAILED_TESTS} T2_rc"
+fi
+assert_contains "T2 POSTed state=failure" '"state": "failure"' "$POSTED"
 
-# T3: tier:low present but ZERO approving reviews → internal tier check fails,
-# refire status remains aligned with the canonical workflow.
+# T3: tier:low present but ZERO approving reviews → failure
 run_scenario "T3_no_approvals" "fail_no_approvals"
 RC=$(cat "$FIX_STATE_DIR/last_rc")
 POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
-assert_eq "T3 exit code 0 (canonical fail-open)" "0" "$RC"
-assert_contains "T3 POSTed state=success" '"state": "success"' "$POSTED"
+if [ "$RC" -ne 0 ]; then
+  echo "  PASS  T3 exit code non-zero (got $RC)"
+  PASS=$((PASS + 1))
+else
+  echo "  FAIL  T3 exit code should be non-zero, got 0"
+  FAIL=$((FAIL + 1))
+  FAILED_TESTS="${FAILED_TESTS} T3_rc"
+fi
+assert_contains "T3 POSTed state=failure" '"state": "failure"' "$POSTED"
 
 # T4: closed PR — refire is a no-op (no POST, exit 0)
 run_scenario "T4_closed" "pass"

.gitea/scripts/tests/test_status_reaper_api.py

@@ -14,7 +14,7 @@ def load_reaper():
     assert spec.loader is not None
     spec.loader.exec_module(mod)
     mod.API = "https://git.example.test/api/v1"
-    mod.GITEA_TOKEN = "fixture-token"
+    mod.GITEA_TOKEN = "test-token"
     mod.API_TIMEOUT_SEC = 1
     mod.API_RETRIES = 3
     mod.API_RETRY_SLEEP_SEC = 0

.gitea/sop-checklist-config.yaml

@@ -50,34 +50,6 @@ tier_failure_mode:
   "tier:low": soft
 default_mode: hard  # used when no tier:* label is present
 
-# High-risk class (RFC#450 Option C, governance-fix for internal#442).
-#
-# A PR is "high-risk" when ANY of the listed labels are applied OR when
-# the PR has `tier:high` (mechanically the strictest existing tier).
-# High-risk items use `required_teams_high_risk` (when present on the
-# item); non-high-risk items use the default `required_teams`.
-#
-# This closes the inconsistency that the SOP charter already mandates
-# `tier:high → ceo only` for the sibling `sop-tier-check` gate; the
-# sop-checklist's `root-cause` and `no-backwards-compat` items now
-# follow the same risk-classed two-eyes shape:
-#   - Default class (tier:low/medium, not high-risk): a non-author
-#     engineers/managers/ceo ack satisfies the item — 25+ live
-#     identities, no dependency on a dead/inactive senior persona
-#     token.
-#   - High-risk class (tier:high OR any high_risk_label): still
-#     requires a non-author ceo ack (durable human team).
-#
-# Tightening: add labels to high_risk_labels.
-# Loosening: remove labels.
-high_risk_labels:
-  - "risk:high"
-  - "area:security"
-  - "area:schema"
-  - "area:fleet-image"
-  - "area:identity"
-  - "area:gate-meta"
-
 items:
   - slug: comprehensive-testing
     numeric_alias: 1
@@ -106,15 +78,11 @@ items:
   - slug: root-cause
     numeric_alias: 4
     pr_section_marker: "Root-cause not symptom"
-    required_teams: [engineers, managers, ceo]
-    required_teams_high_risk: [ceo]
+    required_teams: [managers, ceo]
     description: >-
-      One-sentence root-cause statement. Default class: non-author
-      engineers/managers/ceo ack suffices (engineers can attest
-      root-cause-vs-symptom for routine fixes). High-risk class
-      (see `high_risk_labels`): non-author ceo ack required —
-      senior judgment for irreversible/security/identity/gate
-      changes. Closes internal#442 + tracks RFC#450.
+      One-sentence root-cause statement. Ack from managers tier
+      (team-leads) or ceo. Senior judgment required to attest
+      root-cause-versus-symptom.
 
   - slug: five-axis-review
     numeric_alias: 5
@@ -127,14 +95,10 @@ items:
   - slug: no-backwards-compat
     numeric_alias: 6
     pr_section_marker: "No backwards-compat shim / dead code added"
-    required_teams: [engineers, managers, ceo]
-    required_teams_high_risk: [ceo]
+    required_teams: [managers, ceo]
     description: >-
-      Yes/no + justification if no. Default class: non-author
-      engineers/managers/ceo ack suffices. High-risk class
-      (see `high_risk_labels`): non-author ceo ack required —
-      senior judgment for shim-versus-real-fix on irreversible
-      surfaces. Closes internal#442 + tracks RFC#450.
+      Yes/no + justification if no. Senior ack required because
+      backward-compat shims are how dead-code accretes.
 
   - slug: memory-consulted
     numeric_alias: 7

.gitea/workflows/audit-force-merge.yml

@@ -54,6 +54,5 @@ jobs:
           # read-only by design (least-privilege).
           REQUIRED_CHECKS: |
             CI / all-required (pull_request)
-            E2E API Smoke Test / E2E API Smoke Test (pull_request)
-            Handlers Postgres Integration / Handlers Postgres Integration (pull_request)
+            sop-checklist / all-items-acked (pull_request)
         run: bash .gitea/scripts/audit-force-merge.sh

.gitea/workflows/cascade-list-drift-gate.yml

@@ -0,0 +1,60 @@
+name: cascade-list-drift-gate
+
+# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - on.paths reference .gitea/workflows/publish-runtime.yml (the active
+#     Gitea workflow file) instead of .github/workflows/publish-runtime.yml
+#     (which Category A of this sweep deletes).
+#   - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
+#     .gitea/ workflow (the script's default is still .github/... which
+#     will not exist post-Cat-A).
+#   - Workflow-level env.GITHUB_SERVER_URL set per
+#     feedback_act_runner_github_server_url.
+#   - `continue-on-error: true` on the job (RFC §1 contract — surface
+#     defects without blocking; follow-up PR flips after triage).
+#
+# Structural gate: TEMPLATES list in publish-runtime.yml must match
+# manifest.json's workspace_templates exactly. Closes the recurrence
+# path of PR #2556 (the data fix) and is the first concrete deliverable
+# of RFC #388 PR-3.
+#
+# Triggers narrowly to keep CI quiet: only on PRs that actually change
+# one of the two files. The path-filtered split + always-emit-result
+# pattern (memory: "Required check names need a job that always runs")
+# is unnecessary here because the workflow IS the check name and PR
+# branch protection should require it directly. Future-proof: if this
+# becomes a required check, add a no-op aggregator with always() so the
+# name still emits when paths don't match.
+
+on:
+  pull_request:
+    branches: [staging, main]
+    paths:
+      - manifest.json
+      - .gitea/workflows/publish-runtime.yml
+      - scripts/check-cascade-list-vs-manifest.sh
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+permissions:
+  contents: read
+
+jobs:
+  # bp-exempt: drift visibility gate; CI / all-required remains the required aggregate.
+  check:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Check cascade list matches manifest
+        # Pass the .gitea/ workflow path explicitly — the script's
+        # default still points at .github/... which Category A of this
+        # sweep removes.
+        run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml

.gitea/workflows/ci-arm64-advisory.yml

@@ -1,187 +0,0 @@
-# ci-arm64-advisory — Mac arm64 self-hosted ADVISORY fast-check lane.
-#
-# === WHY ===
-#
-# The amd64 Gitea runner pool (molecule-runner-1..20) is queue-contended
-# (internal#418). This lane offloads the *genuinely container-independent*
-# fast checks (Go build/vet/lint, shellcheck, Python lint) onto the Mac
-# arm64 self-hosted runner so developers get a fast arm64 signal WITHOUT
-# adding load to the starved amd64 pool — capability-honestly, as an
-# additive pilot. Pilot ② of the Mac-CI strategy (CTO-delegated 2026-05-17).
-#
-# === NON-NEGOTIABLE SAFETY CONTRACT (the prime directive) ===
-#
-# This lane is **ADVISORY ONLY**. It is provably incapable of hanging a
-# merge. Concretely:
-#
-#   1. It is a SEPARATE workflow file. `ci.yml` is byte-for-byte
-#      untouched by this PR. The `CI / all-required` aggregator sentinel
-#      and the five contexts it polls
-#      (`CI / Detect changes|Platform (Go)|Canvas (Next.js)|
-#      Shellcheck (E2E scripts)|Python Lint & Test (pull_request)`)
-#      are unchanged. The canonical required gate stays 100% on the
-#      existing amd64 pool.
-#
-#   2. The context this workflow emits is
-#      `ci-arm64-advisory / fast-checks (pull_request)`. That string is
-#      DELIBERATELY NOT present in, and this PR does NOT add it to:
-#        - branch_protections/{main,staging}.status_check_contexts
-#          (DB-verified pb 86/75 = exactly
-#           ["CI / all-required (pull_request)",
-#            "sop-checklist / all-items-acked (pull_request)"])
-#        - audit-force-merge.yml REQUIRED_CHECKS env
-#        - ci.yml `all-required` sentinel's hardcoded `required[]` list
-#      Branch protection therefore never waits on this context. If the
-#      Mac runner is absent / offline / removed, this workflow's status
-#      simply never appears — and because nothing requires it, every
-#      merge proceeds exactly as it does today. There is no path by
-#      which a missing/red arm64 status blocks a merge.
-#
-#   3. `continue-on-error: true` on the job — even a genuine arm64-only
-#      failure (toolchain drift, arch-specific test flake) is surfaced
-#      as information, never as a merge blocker, for the duration of
-#      the pilot.
-#
-#   4. The job carries a `github.event_name` `if:` gate. Beyond its
-#      functional purpose this also keeps the job OUT of
-#      `ci-required-drift.py:ci_job_names()` (which excludes
-#      `github.event_name`/`github.ref`-gated jobs), so the hourly
-#      ci-required-drift sentinel's F1 ("job not under sentinel needs")
-#      cannot ever flag this advisory job. F2/F3 are untouched because
-#      this context is absent from BP and from REQUIRED_CHECKS.
-#      `lint-bp-context-emit-match` only fails on BP→emitter gaps; an
-#      emitter without a BP context is explicitly informational there.
-#
-# === RUNNER TARGETING ===
-#
-# The Mac runner is `hongming-pc-runner-1`. The bare `self-hosted`
-# label is POLLUTED in this Gitea instance: molecule-runner-1..20
-# (the contended amd64 pool) also advertise `self-hosted`. Targeting
-# bare `self-hosted` would route back onto the very pool we are trying
-# to relieve — and onto amd64 hardware. We therefore require an
-# AND-set of labels that ONLY the Mac satisfies. `macos-self-hosted`
-# is Mac-exclusive (the amd64 pool does not carry it). Until the
-# label-install burst (a10862b2) lands `self-hosted`+`macos-self-hosted`
-# on the Mac, the runner's current unique label `hongming-pc-laptop`
-# is also listed; AND-semantics over the labels a runner advertises
-# means a job requiring [self-hosted, macos-self-hosted] can ONLY be
-# claimed once the Mac advertises both. If neither label set is yet
-# present on the Mac, the workflow stays queued harmlessly and is
-# garbage-collected by the normal stale-run reaper — it blocks nothing
-# (see safety contract point 2).
-#
-# === ROLLBACK ===
-#
-# Delete this single file (`git rm .gitea/workflows/ci-arm64-advisory.yml`)
-# and merge. No branch-protection edit, no ci.yml edit, no
-# REQUIRED_CHECKS edit is required to roll back, because none were made
-# to roll forward. Zero blast radius either direction.
-
-name: ci-arm64-advisory
-
-on:
-  push:
-    branches: [main, staging]
-  pull_request:
-    branches: [main, staging]
-
-# Per-ref cancel: a newer commit on the same ref supersedes the older
-# advisory run. Distinct from ci.yml's `ci-${ref}` group so this lane
-# never cancels (or is cancelled by) the canonical required CI.
-concurrency:
-  group: ci-arm64-advisory-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  GITHUB_SERVER_URL: https://git.moleculesai.app
-
-jobs:
-  fast-checks:
-    name: fast-checks
-    # AND-set: only the Mac arm64 runner advertises macos-self-hosted.
-    # See "RUNNER TARGETING" header note for why bare self-hosted is unsafe.
-    runs-on: [self-hosted, macos-self-hosted]
-    # ADVISORY: never blocks. See safety contract point 3. mc#774
-    # internal#418 — tracked: arm64 advisory pilot, non-gating by design.
-    continue-on-error: true
-    # event_name gate: functional (only meaningful on push/PR) AND keeps
-    # this job out of ci-required-drift.py:ci_job_names() so F1 can never
-    # flag it. See safety contract point 4.
-    if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
-    timeout-minutes: 20
-    steps:
-      - name: Provenance — advisory lane, non-gating
-        run: |
-          echo "This is the arm64 ADVISORY fast-check lane."
-          echo "It does NOT gate merges. Canonical required CI is ci.yml"
-          echo "on the amd64 pool. Arch: $(uname -m) on $(uname -s)."
-
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      # ---- Go: build + vet + lint (container-independent: needs only the
-      # Go toolchain; no amd64 ECR image, no docker-in-job). Race-detector
-      # unit-test + coverage gates are deliberately NOT duplicated here —
-      # those stay authoritative on amd64 ci.yml `Platform (Go)`. This lane
-      # is fast-feedback for the compile/vet/lint surface only. ----
-      - name: Setup Go
-        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
-        with:
-          go-version: 'stable'
-      - name: Go build + vet (workspace-server)
-        working-directory: workspace-server
-        run: |
-          go mod download
-          go build ./cmd/server
-          go vet ./...
-      - name: golangci-lint (workspace-server)
-        working-directory: workspace-server
-        run: |
-          go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
-          "$(go env GOPATH)/bin/golangci-lint" run --timeout 3m ./...
-
-      # ---- Shellcheck (container-independent: shellcheck binary only).
-      # Mirrors ci.yml `Shellcheck (E2E scripts)` bulk pass scope. ----
-      - name: Install shellcheck (arm64)
-        run: |
-          if ! command -v shellcheck >/dev/null 2>&1; then
-            echo "shellcheck not preinstalled on this self-hosted runner."
-            echo "Attempting Homebrew install (Mac arm64)."
-            brew install shellcheck || {
-              echo "::warning::shellcheck unavailable on runner; advisory shellcheck skipped."
-              exit 0
-            }
-          fi
-          shellcheck --version
-      - name: Shellcheck tests/e2e + infra/scripts
-        run: |
-          command -v shellcheck >/dev/null 2>&1 || { echo "skip"; exit 0; }
-          find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
-            | xargs -0 shellcheck --severity=warning
-
-      # ---- Python lint/compile (container-independent: CPython only).
-      # Lint + import-compile surface; the authoritative pytest + coverage
-      # floors stay on amd64 ci.yml `Python Lint & Test`. ----
-      - name: Setup Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
-        with:
-          python-version: '3.11'
-      - name: Python byte-compile (workspace)
-        working-directory: workspace
-        run: |
-          python -m pip install --quiet ruff || true
-          python -m compileall -q .
-          if command -v ruff >/dev/null 2>&1; then
-            ruff check . || echo "::warning::ruff findings (advisory only)"
-          fi
-
-      - name: Advisory summary
-        if: always()
-        run: |
-          {
-            echo "## arm64 advisory fast-checks complete"
-            echo ""
-            echo "This lane is **advisory** — it does not gate merges."
-            echo "Authoritative required CI remains \`CI / all-required\`"
-            echo "on the amd64 pool (\`ci.yml\`, unchanged by this PR)."
-          } >> "$GITHUB_STEP_SUMMARY"

.gitea/workflows/ci-mcp-stdio-transport.yml

@@ -0,0 +1,165 @@
+name: MCP Stdio Transport Regression
+
+# Regression test for molecule-ai-workspace-runtime#61:
+# asyncio.connect_read_pipe / connect_write_pipe fail with
+# ValueError: "Pipe transport is only for pipes, sockets and character devices"
+# when stdout is a regular file (openclaw capture, CI tee, debugging).
+#
+# This workflow reproduces the exact failure mode and verifies the
+# fallback to direct buffer I/O works. It runs on every PR that
+# touches the MCP server or this workflow, plus nightly cron.
+#
+# Why a separate workflow (not folded into ci.yml python-lint):
+#   - The test needs to spawn the MCP server with stdout redirected
+#     to a regular file (not a TTY/pipe), which conflicts with
+#     pytest's own capture mechanism.
+#   - It exercises the actual process spawn path (python a2a_mcp_server.py)
+#     not just unit-test mocks — closer to the real openclaw integration.
+#   - A dedicated workflow surfaces stdio-specific regressions without
+#     coupling to the broader Python test suite's coverage gate.
+
+on:
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace/a2a_mcp_server.py'
+      - 'workspace/mcp_cli.py'
+      - 'workspace/tests/test_a2a_mcp_server.py'
+      - '.gitea/workflows/ci-mcp-stdio-transport.yml'
+  push:
+    branches: [main, staging]
+    paths:
+      - 'workspace/a2a_mcp_server.py'
+      - 'workspace/mcp_cli.py'
+      - 'workspace/tests/test_a2a_mcp_server.py'
+      - '.gitea/workflows/ci-mcp-stdio-transport.yml'
+  schedule:
+    # Nightly at 04:00 UTC — catches drift from dependency updates
+    # (e.g. asyncio behavior changes in new Python patch releases).
+    - cron: '0 4 * * *'
+
+concurrency:
+  group: mcp-stdio-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  # bp-exempt: regression canary for runtime#61; not a merge gate — informational only until promoted to required.
+  # mc#774: continue-on-error mask — new workflow, flip to false once it's green on ≥3 consecutive main runs.
+  mcp-stdio-regular-file:
+    name: MCP stdio with regular-file stdout
+    runs-on: ubuntu-latest
+    continue-on-error: true  # mc#774
+    timeout-minutes: 5
+    env:
+      WORKSPACE_ID: "00000000-0000-0000-0000-000000000001"
+    defaults:
+      run:
+        working-directory: workspace
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov
+
+      - name: Reproduce runtime#61 — stdout as regular file
+        run: |
+          set -euo pipefail
+          echo "=== Reproducing molecule-ai-workspace-runtime#61 ==="
+          echo ""
+          echo "Before the fix, this command would fail with:"
+          echo '  ValueError: Pipe transport is only for pipes, sockets and character devices'
+          echo ""
+
+          # Spawn the MCP server with stdout redirected to a regular file.
+          # This is exactly what openclaw does when capturing MCP output.
+          OUTPUT=$(mktemp)
+          trap 'rm -f "$OUTPUT"' EXIT
+
+          # Send initialize request, then tools/list, then exit
+          {
+            echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}'
+            echo '{"jsonrpc":"2.0","id":2,"method":"tools/list"}'
+          } | python a2a_mcp_server.py > "$OUTPUT" 2>&1 || {
+            RC=$?
+            echo "FAIL: MCP server exited with code $RC"
+            echo "--- stdout+stderr ---"
+            cat "$OUTPUT"
+            exit 1
+          }
+
+          echo "PASS: MCP server handled regular-file stdout without crashing"
+          echo ""
+          echo "--- Output (first 20 lines) ---"
+          head -20 "$OUTPUT"
+          echo ""
+
+          # Verify we got valid JSON-RPC responses
+          if grep -q '"result"' "$OUTPUT"; then
+            echo "PASS: JSON-RPC responses found in output"
+          else
+            echo "FAIL: No JSON-RPC responses in output"
+            cat "$OUTPUT"
+            exit 1
+          fi
+
+      - name: Reproduce runtime#61 — stdin from regular file
+        run: |
+          set -euo pipefail
+          echo "=== stdin as regular file (CI tee / capture pattern) ==="
+
+          INPUT=$(mktemp)
+          OUTPUT=$(mktemp)
+          trap 'rm -f "$INPUT" "$OUTPUT"' EXIT
+
+          cat > "$INPUT" <<'EOF'
+          {"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}
+          {"jsonrpc":"2.0","id":2,"method":"tools/list"}
+          EOF
+
+          python a2a_mcp_server.py < "$INPUT" > "$OUTPUT" 2>&1 || {
+            RC=$?
+            echo "FAIL: MCP server exited with code $RC"
+            cat "$OUTPUT"
+            exit 1
+          }
+
+          echo "PASS: MCP server handled regular-file stdin without crashing"
+
+          if grep -q '"result"' "$OUTPUT"; then
+            echo "PASS: JSON-RPC responses found in output"
+          else
+            echo "FAIL: No JSON-RPC responses in output"
+            cat "$OUTPUT"
+            exit 1
+          fi
+
+      - name: Verify warning is emitted for non-pipe stdio
+        run: |
+          set -euo pipefail
+          echo "=== Verify diagnostic warning ==="
+
+          OUTPUT=$(mktemp)
+          trap 'rm -f "$OUTPUT"' EXIT
+
+          {
+            echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}'
+          } | python a2a_mcp_server.py > "$OUTPUT" 2>&1
+
+          # The warning should mention "not a pipe" for operator visibility
+          if grep -qi "not a pipe" "$OUTPUT"; then
+            echo "PASS: Diagnostic warning emitted for non-pipe stdio"
+          else
+            echo "NOTE: No warning in output (may be suppressed by log level)"
+          fi
+
+      - name: Run unit tests for stdio transport
+        run: |
+          set -euo pipefail
+          echo "=== Running stdio transport unit tests ==="
+          python -m pytest tests/test_a2a_mcp_server.py::TestStdioPipeAssertion -v --no-cov

.gitea/workflows/ci.yml

@@ -86,25 +86,53 @@ jobs:
         with:
           fetch-depth: 0
       - id: check
-        env:
-          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
-          PUSH_BEFORE: ${{ github.event.before }}
         run: |
-          python3 .gitea/scripts/detect-changes.py \
-            --profile ci \
-            --event-name "${{ github.event_name }}" \
-            --pr-base-sha "$PR_BASE_SHA" \
-            --base-ref "$PR_BASE_REF" \
-            --push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}"
+          # For PR events: diff against the base branch (not HEAD~1 of the branch,
+          # which may be unrelated after force-pushes). When a push updates a PR,
+          # both pull_request and push events fire — prefer the PR base so that
+          # the diff is always computed against the actual merge base, not the
+          # previous SHA on the branch which may be on a different history line.
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          # GITHUB_BASE_REF is set for PR events (the base branch name).
+          # For pull_request events we use the stored base.sha; for push events
+          # (or when base.sha is unavailable) fall back to github.event.before.
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          # Fallback: if BASE is empty or all zeros (new branch), run everything
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "platform=true" >> "$GITHUB_OUTPUT"
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            echo "python=true" >> "$GITHUB_OUTPUT"
+            echo "scripts=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # Workflow-only edits are covered by the workflow lint family
+          # and by this workflow's always-present required jobs. Do not fan
+          # those edits out into Go/Canvas/Python/shellcheck work; the
+          # downstream jobs still emit their required contexts via no-op
+          # steps when their surface flag is false.
+          #
+          # If the diff itself cannot be trusted, fail open by running every
+          # surface instead of silently under-testing the PR.
+          if ! DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null); then
+            echo "platform=true" >> "$GITHUB_OUTPUT"
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            echo "python=true" >> "$GITHUB_OUTPUT"
+            echo "scripts=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "python=$(echo "$DIFF" | grep -qE '^workspace/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
 
-  # Platform (Go) — Go build/vet/test/lint + coverage gates. The job always
-  # emits the required context, but expensive steps are path-scoped on every
-  # event so docs/E2E/Canvas-only main pushes do not block deploy on unrelated
-  # Go bootstrap work.
+  # Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
+  # + per-step gating shape preserves the GitHub-side required-check name
+  # contract (so when this Gitea port becomes a required check in Phase 4,
+  # the name match works on PRs that don't touch workspace-server/).
   platform-build:
     name: Platform (Go)
-    needs: changes
     runs-on: ubuntu-latest
     # mc#774 (closed 2026-05-14): Phase 4 flip of the platform-build job.
     # Phase 4 (#656) originally flipped this to continue-on-error: false based on
@@ -117,43 +145,45 @@ jobs:
     # the diagnostic step with its own continue-on-error: true (line 203).
     # Flip confirmed by CI / Platform (Go) status = success on main HEAD 363905d3.
     continue-on-error: false
-    # Job-level ceiling. The go test step below runs with a per-step 10m timeout;
-    # this cap catches any step that leaks past that. Set well above 10m so
+    # Job-level ceiling. The go test step below runs with a per-step 30m timeout;
+    # this cap catches any step that leaks past that. Set well above 30m so
     # the per-step timeout is the active constraint.
-    timeout-minutes: 15
+    timeout-minutes: 35
     defaults:
       run:
         working-directory: workspace-server
     steps:
-      - if: ${{ needs.changes.outputs.platform != 'true' }}
+      - if: false
         working-directory: .
-        run: echo "No workspace-server/** changes — Platform (Go) gate satisfied without running Go build/test/lint."
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+        run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: always()
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
         with:
           go-version: 'stable'
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         run: go mod download
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         run: go build ./cmd/server
       # CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         run: go vet ./...
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         name: Install golangci-lint
         run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         name: Run golangci-lint
         run: $(go env GOPATH)/bin/golangci-lint run --timeout 3m ./...
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
-        name: Diagnostic — per-package verbose 60s
+      - if: always()
+        name: Diagnostic — per-package verbose (300s timeout)
         run: |
           set +e
-          go test -race -v -timeout 60s ./internal/handlers/... 2>&1 | tee /tmp/test-handlers.log
+          # 300s allows handlers + pendinguploads packages to complete on cold
+          # runners with -race instrumentation (~60-120s each vs ~14s non-race).
+          go test -race -v -timeout 300s ./internal/handlers/... 2>&1 | tee /tmp/test-handlers.log
           handlers_exit=$?
-          go test -race -v -timeout 60s ./internal/pendinguploads/... 2>&1 | tee /tmp/test-pu.log
+          go test -race -v -timeout 300s ./internal/pendinguploads/... 2>&1 | tee /tmp/test-pu.log
           pu_exit=$?
           echo "::group::handlers exit=$handlers_exit (last 100 lines)"
           tail -100 /tmp/test-handlers.log
@@ -163,15 +193,15 @@ jobs:
           echo "::endgroup::"
         # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
         continue-on-error: true
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         name: Run tests with race detection and coverage
         # Explicit timeout: cold runner cache causes OOM kills at ~4m39s on the
-        # full ./... suite with race detection + coverage. A 10m per-step timeout
-        # lets the suite complete on cold cache (~5-7m) while failing cleanly
-        # instead of OOM-killing. The job-level timeout (15m) is a backstop.
-        run: go test -race -timeout 10m -coverprofile=coverage.out ./...
+        # full ./... suite with race detection + coverage. A 30m per-step timeout
+        # lets the suite complete on cold cache (~13-25m) while failing cleanly
+        # instead of OOM-killing. The job-level timeout (35m) is a backstop.
+        run: go test -race -timeout 30m -coverprofile=coverage.out ./...
 
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         name: Per-file coverage report
         # Advisory — lists every source file with its coverage so reviewers
         # can see at-a-glance where gaps are. Sorted ascending so the worst
@@ -185,7 +215,7 @@ jobs:
                    END {for (f in s) printf "%6.1f%%  %s\n", s[f]/c[f], f}' \
             | sort -n
 
-      - if: ${{ needs.changes.outputs.platform == 'true' }}
+      - if: always()
         name: Check coverage thresholds
         # Enforces two gates from #1823 Layer 1:
         #   1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
@@ -239,7 +269,7 @@ jobs:
               # Strip the package-import prefix so we can match .coverage-allowlist.txt
               # entries written as paths relative to workspace-server/.
               # Handle both module paths: platform/workspace-server/... and platform/...
-              rel=$(echo "$file" | sed 's|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/workspace-server/||; s|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/||')
+              rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
 
               if echo "$ALLOWLIST" | grep -qxF "$rel"; then
                 echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
@@ -273,7 +303,6 @@ jobs:
   # siblings — verified empirically on PR #2314).
   canvas-build:
     name: Canvas (Next.js)
-    needs: changes
     runs-on: ubuntu-latest
     timeout-minutes: 20
     # Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
@@ -282,20 +311,20 @@ jobs:
       run:
         working-directory: canvas
     steps:
-      - if: ${{ needs.changes.outputs.canvas != 'true' }}
+      - if: false
         working-directory: .
-        run: echo "No canvas/** changes — Canvas (Next.js) gate satisfied without running npm build/test."
-      - if: ${{ needs.changes.outputs.canvas == 'true' }}
+        run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: always()
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - if: ${{ needs.changes.outputs.canvas == 'true' }}
+      - if: always()
         uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
           node-version: '22'
-      - if: ${{ needs.changes.outputs.canvas == 'true' }}
-        run: npm ci --include=optional --prefer-offline
-      - if: ${{ needs.changes.outputs.canvas == 'true' }}
+      - if: always()
+        run: rm -f package-lock.json && npm install
+      - if: always()
         run: npm run build
-      - if: ${{ needs.changes.outputs.canvas == 'true' }}
+      - if: always()
         name: Run tests with coverage
         # Coverage instrumentation is configured in canvas/vitest.config.ts
         # (provider: v8, reporters: text + html + json-summary). Step 2 of
@@ -304,7 +333,7 @@ jobs:
         # tracked in #1815) after the team sees what current coverage is.
         run: npx vitest run --coverage
       - name: Upload coverage summary as artifact
-        if: ${{ needs.changes.outputs.canvas == 'true' }}
+        if: always()
         # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
         # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
         # implement, surfacing as `GHESNotSupportedError: @actions/artifact
@@ -318,19 +347,18 @@ jobs:
           retention-days: 7
           if-no-files-found: warn
 
-  # Shellcheck (E2E scripts) — required context, path-scoped heavy steps.
+  # Shellcheck (E2E scripts) — required check, always runs.
   shellcheck:
     name: Shellcheck (E2E scripts)
-    needs: changes
     runs-on: ubuntu-latest
     # Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
     continue-on-error: false
     steps:
-      - if: ${{ needs.changes.outputs.scripts != 'true' }}
-        run: echo "No tests/e2e, scripts, or infra/scripts changes — Shellcheck gate satisfied without running script checks."
-      - if: ${{ needs.changes.outputs.scripts == 'true' }}
+      - if: false
+        run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
+      - if: always()
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - if: ${{ needs.changes.outputs.scripts == 'true' }}
+      - if: always()
         name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
         # shellcheck is pre-installed on ubuntu-latest runners (via apt).
         # infra/scripts/ is included because setup.sh + nuke.sh gate the
@@ -341,16 +369,16 @@ jobs:
           find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
             | xargs -0 shellcheck --severity=warning
 
-      - if: ${{ needs.changes.outputs.scripts == 'true' }}
+      - if: always()
         name: Lint cleanup-trap hygiene (RFC #2873)
         run: bash tests/e2e/lint_cleanup_traps.sh
 
-      - if: ${{ needs.changes.outputs.scripts == 'true' }}
+      - if: always()
         name: Run E2E bash unit tests (no live infra)
         run: |
           bash tests/e2e/test_model_slug.sh
 
-      - if: ${{ needs.changes.outputs.scripts == 'true' }}
+      - if: always()
         name: Test ECR promote-tenant-image script (mock-driven, no live infra)
         # Covers scripts/promote-tenant-image.sh — the codified
         # :staging-latest → :latest ECR promote + tenant fleet redeploy
@@ -360,7 +388,7 @@ jobs:
         run: |
           bash scripts/test-promote-tenant-image.sh
 
-      - if: ${{ needs.changes.outputs.scripts == 'true' }}
+      - if: always()
         name: Shellcheck promote-tenant-image script
         # scripts/ is excluded from the bulk shellcheck pass above (legacy
         # SC3040/SC3043 cleanup pending). Run shellcheck explicitly on
@@ -375,7 +403,7 @@ jobs:
 
   canvas-deploy-reminder:
     name: Canvas Deploy Reminder
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # mc#774 root-fix: added job-level `if:` so ci-required-drift.py's
     # ci_job_names() detects this as github.ref-gated and skips it from F1.
     # The step-level exit 0 handles the "not main push" case; the job-level
@@ -430,40 +458,93 @@ jobs:
           cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
 
   # Python Lint & Test — required check, always runs.
-  # Runtime Python moved to molecule-ai-workspace-runtime. Keep this context as
-  # a guard so branch protection still catches attempts to reintroduce an
-  # editable runtime copy under molecule-core/workspace/.
   python-lint:
     name: Python Lint & Test
     runs-on: ubuntu-latest
+    # Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
     continue-on-error: false
+    env:
+      WORKSPACE_ID: test
+    defaults:
+      run:
+        working-directory: workspace
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - name: Runtime SSOT guard
+      - if: false
+        working-directory: .
+        run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
+      - if: always()
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: always()
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - if: always()
+        run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
+      # Coverage flags + fail-under floor moved into workspace/pytest.ini
+      # (issue #1817) so local `pytest` and CI use identical config.
+      - if: always()
+        run: python -m pytest --tb=short
+
+      - if: always()
+        name: Per-file critical-path coverage (MCP / inbox / auth)
+        # MCP-critical Python files have a per-file floor on top of the
+        # 86% total floor in pytest.ini. See issue #2790 for full rationale.
         run: |
-          set -eu
-          if [ -d workspace ]; then
-            echo "::error file=workspace::Runtime source must live in molecule-ai-workspace-runtime, not molecule-core/workspace."
-            exit 1
-          fi
-          for f in scripts/build_runtime_package.py scripts/test_build_runtime_package.py; do
-            if [ -e "$f" ]; then
-              echo "::error file=$f::Legacy build-from-workspace packaging script must not be restored."
-              exit 1
+          set -e
+          PER_FILE_FLOOR=75
+          CRITICAL_FILES=(
+            "a2a_mcp_server.py"
+            "mcp_cli.py"
+            "a2a_tools.py"
+            "a2a_tools_inbox.py"
+            "inbox.py"
+            "platform_auth.py"
+          )
+
+          # pytest already wrote .coverage; emit a JSON view scoped to
+          # the critical files so jq/python can read the per-file pct
+          # without parsing tabular text.
+          INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
+          INCLUDES="${INCLUDES%,}"
+          python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
+
+          FAILED=0
+          for f in "${CRITICAL_FILES[@]}"; do
+            pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
+            if [ "$pct" = "MISSING" ]; then
+              echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
+              FAILED=$((FAILED+1))
+              continue
+            fi
+            echo "$f: ${pct}%"
+            if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
+              echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
+              FAILED=$((FAILED+1))
             fi
           done
-          echo "Runtime SSOT guard passed; core consumes the standalone runtime package."
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
+            echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
+            echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
+            echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
+            echo "  (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
+            echo "  (b) if this is unavoidable historical debt, file an issue and propose"
+            echo "      adjusting the floor with rationale in COVERAGE_FLOOR.md."
+            exit 1
+          fi
 
   all-required:
     # Aggregator sentinel — RFC internal#219 §2 (Phase 4 — closes internal#286).
     #
-    # Emits `CI / all-required (<event>)` where <event> is the workflow trigger
-    # (e.g. `CI / all-required (pull_request)`, `CI / all-required (push)`).
-    # Branch protection MUST be updated to require the event-suffixed name —
-    # requiring `CI / all-required` (bare, no suffix) silently blocks all merges
-    # because Gitea treats absent status contexts as pending (not skipped), and
-    # no workflow emits the bare name. Fixed: BP now requires
-    # `CI / all-required (pull_request)` per issue #1473.
+    # Single stable required-status name that branch protection points at;
+    # CI churns underneath in `needs:` without any protection edits. Mirrors
+    # the molecule-controlplane Phase 2a impl shipped in CP PR#112 and
+    # referenced by `internal#286` ("Phase 4 is a single small PR... mirrors
+    # CP's existing one").
     #
     # Closes the failure mode where status_check_contexts on molecule-core/main
     # only listed `Secret scan` + `sop-tier-check` (the 2 meta-gates), so real
@@ -476,11 +557,7 @@ jobs:
     # jobs settle, leaving branch protection with a permanent pending
     # `CI / all-required` context. Instead, this independent sentinel polls the
     # required commit-status contexts for this SHA and fails if any fail, skip,
-    # or never emit. It runs the same path detector as `changes` and only waits
-    # for path-relevant jobs; Gitea can otherwise leave needs/output-skipped
-    # jobs permanently pending with "Blocked by required conditions". It runs on
-    # the dedicated `ci-meta` lane so the poller does not occupy the same
-    # general runner pool as the jobs it is waiting for.
+    # or never emit.
     #
     # canvas-deploy-reminder is intentionally NOT included in all-required.needs.
     # It is an informational main-push reminder, not a PR quality gate. Keeping
@@ -488,24 +565,9 @@ jobs:
     # sentinel before the `always()` guard can emit a branch-protection status.
     #
     continue-on-error: false
-    runs-on: ci-meta
+    runs-on: ubuntu-latest
     timeout-minutes: 45
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          fetch-depth: 0
-      - id: check
-        env:
-          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
-          PUSH_BEFORE: ${{ github.event.before }}
-        run: |
-          python3 .gitea/scripts/detect-changes.py \
-            --profile ci \
-            --event-name "${{ github.event_name }}" \
-            --pr-base-sha "$PR_BASE_SHA" \
-            --base-ref "$PR_BASE_REF" \
-            --push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}"
       - name: Wait for required CI contexts
         env:
           GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -513,9 +575,6 @@ jobs:
           REPOSITORY: ${{ github.repository }}
           COMMIT_SHA: ${{ github.sha }}
           EVENT_NAME: ${{ github.event_name }}
-          REQUIRE_PLATFORM: ${{ steps.check.outputs.platform }}
-          REQUIRE_CANVAS: ${{ steps.check.outputs.canvas }}
-          REQUIRE_SCRIPTS: ${{ steps.check.outputs.scripts }}
         run: |
           set -euo pipefail
           python3 - <<'PY'
@@ -533,14 +592,11 @@ jobs:
           event = os.environ["EVENT_NAME"]
           required = [
               f"CI / Detect changes ({event})",
+              f"CI / Platform (Go) ({event})",
+              f"CI / Canvas (Next.js) ({event})",
+              f"CI / Shellcheck (E2E scripts) ({event})",
               f"CI / Python Lint & Test ({event})",
           ]
-          if os.environ.get("REQUIRE_PLATFORM") == "true":
-              required.append(f"CI / Platform (Go) ({event})")
-          if os.environ.get("REQUIRE_CANVAS") == "true":
-              required.append(f"CI / Canvas (Next.js) ({event})")
-          if os.environ.get("REQUIRE_SCRIPTS") == "true":
-              required.append(f"CI / Shellcheck (E2E scripts) ({event})")
           terminal_bad = {"failure", "error"}
           deadline = time.time() + 40 * 60
           last_summary = None

.gitea/workflows/continuous-synth-e2e.yml

@@ -43,18 +43,6 @@ name: Continuous synthetic E2E (staging)
 
 on:
   schedule:
-    # Every 30 minutes, on :02 and :32. This keeps a recurring SaaS
-    # behavior probe while cutting runner occupancy from this workflow by
-    # roughly two thirds; fast liveness belongs in the lighter smoke/heartbeat
-    # probes, not in a full tenant/workspace synth every 10 minutes.
-    #
-    # Previous cadence was every 10 minutes (:02 :12 :22 :32 :42 :52).
-    # The current operator-host runner pool is the bottleneck, so full
-    # synth E2E is deliberately lower-cadence until it moves to a dedicated
-    # runner host or warm-runtime pool.
-    #
-    # Historical notes from the 10-minute shape:
-    #
     # Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
     #   1. Stay off the top-of-hour. GitHub Actions scheduler drops
     #      :00 firings under high load (own docs:
@@ -78,7 +66,7 @@ on:
     # fires = ~30 min cadence; closer to the 20-min target than the
     # current shape and provides a real degradation alarm if drops
     # get worse.
-    - cron: '2,32 * * * *'
+    - cron: '2,12,22,32,42,52 * * * *'
 permissions:
   contents: read
   # No issue-write here — failures surface as red runs in the workflow
@@ -118,7 +106,7 @@ jobs:
     timeout-minutes: 20
     env:
       # claude-code default: cold-start ~5 min (comparable to langgraph),
-      # but uses MiniMax-M2 via the template's third-party-
+      # but uses MiniMax-M2.7-highspeed via the template's third-party-
       # Anthropic-compat path (workspace-configs-templates/claude-code-
       # default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
       # gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
@@ -131,9 +119,9 @@ jobs:
       # on the per-runtime default ("sonnet" → routes to direct
       # Anthropic, defeats the cost saving). Operators can override
       # via workflow_dispatch by setting a different E2E_MODEL_SLUG
-      # input if they need to exercise a specific model. MiniMax-M2 is the
-      # stable staging MiniMax path used by the full-SaaS smoke.
-      E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2' }}
+      # input if they need to exercise a specific model. M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
       # Bound to 10 min so a stuck provision fails the run instead of
       # holding up the next cron firing. 15-min default in the script
       # is for the on-PR full lifecycle where we have more headroom.
@@ -145,11 +133,6 @@ jobs:
       E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
       MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
       MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: us-east-2
-      E2E_AWS_LEAK_CHECK: required
-      E2E_AWS_TERMINATE_LEAKS: '1'
       # MiniMax key is the canary's PRIMARY auth path. claude-code
       # template's `minimax` provider routes ANTHROPIC_BASE_URL to
       # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
@@ -190,12 +173,6 @@ jobs:
             echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
             exit 1
           fi
-          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
-            if [ -z "${!var:-}" ]; then
-              echo "::error::$var secret missing — EC2 leak verification cannot run"
-              exit 1
-            fi
-          done
 
           # LLM-key requirement is per-runtime: claude-code accepts
           # EITHER MiniMax OR direct-Anthropic (whichever is set first),

.gitea/workflows/e2e-api.yml

@@ -108,20 +108,7 @@ env:
 
 jobs:
   detect-changes:
-    # mc#1529 follow-on: pin to `docker-host` so the e2e-api lane lands
-    # on Linux operator-host runners (molecule-runner-*) that carry the
-    # `molecule-core-net` bridge network + a working `aws ecr get-login-
-    # password | docker login` path. The bare `ubuntu-latest` label is
-    # also accepted by hongming-pc-runner-* (Windows act_runner v1.0.3),
-    # where the docker.sock-bound steps below fail non-deterministically
-    # (e.g. `docker run -d --name pg-e2e-api-...` with port-bind +
-    # `docker exec ... pg_isready` cannot work against a Windows daemon).
-    # detect-changes itself doesn't bind docker.sock, but pinning here too
-    # keeps both jobs on the same lane so we don't re-roll the dice on
-    # workspace-volume cross-host surprises and the routing rule is
-    # discoverable in one place. Mirror of mc#1543 (handlers-postgres-
-    # integration). See internal#512 for the class defect.
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -132,13 +119,31 @@ jobs:
         with:
           fetch-depth: 0
       - id: decide
+        # Inline replacement for dorny/paths-filter — same pattern PR#372's
+        # ci.yml port used. Diffs against the PR base or push BEFORE SHA,
+        # then matches against the api-relevant path set.
         run: |
-          python3 .gitea/scripts/detect-changes.py \
-            --profile e2e-api \
-            --event-name "${{ github.event_name }}" \
-            --pr-base-sha "${{ github.event.pull_request.base.sha }}" \
-            --base-ref "${{ github.event.pull_request.base.ref }}" \
-            --push-before "${GITHUB_EVENT_BEFORE:-${{ github.event.before }}}"
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! git cat-file -e "$BASE" 2>/dev/null; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "api=false" >> "$GITHUB_OUTPUT"
+          fi
 
   # ONE job (no job-level `if:`) that always runs and reports under the
   # required-check name `E2E API Smoke Test`. Real work is gated per-step
@@ -155,10 +160,7 @@ jobs:
   e2e-api:
     needs: detect-changes
     name: E2E API Smoke Test
-    # mc#1529 follow-on: must run on operator-host Linux runners (where
-    # docker.sock + `molecule-core-net` + `aws ecr ...` work). See
-    # detect-changes for the full rationale.
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -348,9 +350,6 @@ jobs:
             exit 1
           fi
           echo "Migrations OK"
-      - name: Run today's-PR-coverage E2E (mc#1525/1535/1536/1539/1542 fix-specific assertions)
-        if: needs.detect-changes.outputs.api == 'true'
-        run: bash tests/e2e/test_today_pr_coverage_e2e.sh
       - name: Run E2E API tests
         if: needs.detect-changes.outputs.api == 'true'
         run: bash tests/e2e/test_api.sh
@@ -360,12 +359,6 @@ jobs:
       - name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
         if: needs.detect-changes.outputs.api == 'true'
         run: bash tests/e2e/test_priority_runtimes_e2e.sh
-      - name: Install standalone runtime parser from Gitea registry
-        if: needs.detect-changes.outputs.api == 'true'
-        run: |
-          python3 -m pip install --no-deps \
-            --index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ \
-            molecule-ai-workspace-runtime
       - name: Run poll-mode + since_id cursor E2E (#2339)
         if: needs.detect-changes.outputs.api == 'true'
         run: bash tests/e2e/test_poll_mode_e2e.sh
@@ -389,3 +382,4 @@ jobs:
         run: |
           docker rm -f "$PG_CONTAINER" 2>/dev/null || true
           docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
+

.gitea/workflows/e2e-chat.yml

@@ -1,10 +1,8 @@
 name: E2E Chat
 
 # Comprehensive Playwright E2E for the unified chat stack (desktop
-# ChatTab + mobile MobileChat). Heavy browser execution is intentionally
-# outside the normal required PR path: PRs run it only after entering the
-# `merge-queue`, while push/main, nightly, and manual dispatch preserve
-# coverage without making every PR pay the full runtime/browser cost.
+# ChatTab + mobile MobileChat). Runs on every PR that touches canvas,
+# workspace-server, or this workflow file.
 #
 # Architecture:
 #   1. Ephemeral Postgres + Redis (docker, unique container names)
@@ -24,11 +22,6 @@ on:
     branches: [main, staging]
   pull_request:
     branches: [main, staging]
-  schedule:
-    # Nightly at 09:00 UTC. Keeps coverage for the currently non-required
-    # heavy browser lane without spending runner time on every PR.
-    - cron: '0 9 * * *'
-  workflow_dispatch:
 
 concurrency:
   group: e2e-chat-${{ github.event.pull_request.head.sha || github.sha }}
@@ -40,13 +33,7 @@ env:
 jobs:
   # bp-exempt: helper job; real gate is E2E Chat / E2E Chat (pull_request)
   detect-changes:
-    # mc#1529 follow-on: pin to `docker-host` (Linux operator-host
-    # runners). The bare `ubuntu-latest` label is also advertised by
-    # hongming-pc-runner-* (Windows act_runner v1.0.3) where the
-    # docker.sock-bound steps below fail. Mirror of mc#1543
-    # (handlers-postgres-integration). See internal#512 for the class
-    # defect.
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -57,14 +44,7 @@ jobs:
         with:
           fetch-depth: 0
       - id: decide
-        env:
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          QUEUE_LABEL: merge-queue
         run: |
-          if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            echo "chat=true" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
           BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
           if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
             BASE="${{ github.event.pull_request.base.sha }}"
@@ -81,26 +61,9 @@ jobs:
             exit 0
           fi
           CHANGED=$(git diff --name-only "$BASE" HEAD)
-          if ! echo "$CHANGED" | grep -qE '^(canvas/|workspace-server/|\.gitea/workflows/e2e-chat\.yml$)'; then
-            echo "chat=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          if [ "${{ github.event_name }}" != "pull_request" ]; then
-            echo "chat=true" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          authfile=$(mktemp)
-          chmod 600 "$authfile"
-          printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
-          labels=$(curl -fsS -K "$authfile" \
-            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
-            | python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
-          rm -f "$authfile"
-          if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
+          if echo "$CHANGED" | grep -qE '^(canvas/|workspace-server/|\.gitea/workflows/e2e-chat\.yml$)'; then
             echo "chat=true" >> "$GITHUB_OUTPUT"
           else
-            echo "PR is not in merge-queue; skipping heavy E2E Chat for normal PR path."
             echo "chat=false" >> "$GITHUB_OUTPUT"
           fi
 
@@ -108,9 +71,7 @@ jobs:
   e2e-chat:
     needs: detect-changes
     name: E2E Chat
-    # mc#1529 follow-on: docker run/exec for postgres + redis containers.
-    # Must land on operator-host Linux (docker-host).
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -261,14 +222,7 @@ jobs:
       - name: Install Playwright browsers
         if: needs.detect-changes.outputs.chat == 'true'
         working-directory: canvas
-        run: |
-          PREBAKED_PLAYWRIGHT=/ms-playwright
-          if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
-            echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
-            echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
-            exit 0
-          fi
-          npx playwright install --with-deps chromium
+        run: npx playwright install --with-deps chromium
 
       - name: Start canvas dev server (background)
         if: needs.detect-changes.outputs.chat == 'true'

.gitea/workflows/e2e-legacy-advisory.yml

@@ -1,242 +0,0 @@
-name: E2E Legacy Advisory
-
-# Advisory lane for older/manual E2E scripts that are too broad or
-# environment-dependent for required PR CI. This intentionally does not run on
-# pull_request or push so it cannot block merges/deploys; scheduled/manual reds
-# still surface drift in scripts that would otherwise only be shellchecked.
-#
-# Gitea 1.22.6 rejects workflow_dispatch.inputs, so keep dispatch input-free.
-
-on:
-  schedule:
-    # Stagger after the staging smoke/canvas morning lanes.
-    - cron: '15 9 * * *'
-  workflow_dispatch:
-
-concurrency:
-  group: e2e-legacy-advisory
-  cancel-in-progress: false
-
-permissions:
-  contents: read
-
-env:
-  GITHUB_SERVER_URL: https://git.moleculesai.app
-
-jobs:
-  legacy-local-platform:
-    name: Legacy local-platform E2E
-    runs-on: docker-host
-    timeout-minutes: 45
-    env:
-      PG_CONTAINER: pg-e2e-legacy-${{ github.run_id }}-${{ github.run_attempt }}
-      REDIS_CONTAINER: redis-e2e-legacy-${{ github.run_id }}-${{ github.run_attempt }}
-      MOLECULE_ENV: development
-      BIND_ADDR: 127.0.0.1
-      MOLECULE_IN_DOCKER: "false"
-      A2A_TIMEOUT: "30"
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
-        with:
-          go-version: 'stable'
-          cache: true
-          cache-dependency-path: workspace-server/go.sum
-
-      - name: Prepare local platform dependencies
-        run: |
-          set -euo pipefail
-          docker pull postgres:16 >/dev/null
-          docker pull redis:7 >/dev/null
-          docker pull alpine:latest >/dev/null
-          docker network create molecule-core-net >/dev/null 2>&1 || true
-
-      - name: Start Postgres
-        run: |
-          set -euo pipefail
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
-          docker run -d --name "$PG_CONTAINER" \
-            -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
-            -p 0:5432 postgres:16 >/dev/null
-          PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
-          if [ -z "$PG_PORT" ]; then
-            PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
-          fi
-          if [ -z "$PG_PORT" ]; then
-            echo "::error::Could not resolve host port for $PG_CONTAINER"
-            docker port "$PG_CONTAINER" 5432/tcp || true
-            docker logs "$PG_CONTAINER" || true
-            exit 1
-          fi
-          echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
-          for i in $(seq 1 30); do
-            docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1 && exit 0
-            sleep 1
-          done
-          docker logs "$PG_CONTAINER" || true
-          exit 1
-
-      - name: Start Redis
-        run: |
-          set -euo pipefail
-          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
-          docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
-          REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
-          if [ -z "$REDIS_PORT" ]; then
-            REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
-          fi
-          if [ -z "$REDIS_PORT" ]; then
-            echo "::error::Could not resolve host port for $REDIS_CONTAINER"
-            docker port "$REDIS_CONTAINER" 6379/tcp || true
-            docker logs "$REDIS_CONTAINER" || true
-            exit 1
-          fi
-          echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
-          for i in $(seq 1 15); do
-            docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG && exit 0
-            sleep 1
-          done
-          docker logs "$REDIS_CONTAINER" || true
-          exit 1
-
-      - name: Pick platform port
-        run: |
-          set -euo pipefail
-          PLATFORM_PORT=$(python3 - <<'PY'
-          import socket
-          with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-              s.bind(("127.0.0.1", 0))
-              print(s.getsockname()[1])
-          PY
-          )
-          echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
-          echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
-
-      - name: Build platform
-        working-directory: workspace-server
-        run: go build -o platform-server ./cmd/server
-
-      - name: Populate template manifests for dev-mode E2E
-        run: |
-          set -euo pipefail
-          if command -v jq >/dev/null 2>&1; then
-            bash scripts/clone-manifest.sh manifest.json workspace-configs-templates org-templates plugins
-          else
-            echo "::warning::jq unavailable; dev-mode template assertion may fail if templates are absent"
-          fi
-
-      - name: Start platform
-        run: |
-          set -euo pipefail
-          ./workspace-server/platform-server > workspace-server/platform.log 2>&1 &
-          echo $! > workspace-server/platform.pid
-          for i in $(seq 1 30); do
-            curl -sf "$BASE/health" >/dev/null && exit 0
-            sleep 1
-          done
-          cat workspace-server/platform.log || true
-          exit 1
-
-      - name: Run comprehensive E2E
-        run: bash tests/e2e/test_comprehensive_e2e.sh
-
-      - name: Run workspace abilities E2E
-        run: bash tests/e2e/test_workspace_abilities_e2e.sh
-
-      - name: Run dev-mode E2E
-        run: bash tests/e2e/test_dev_mode.sh
-
-      - name: Start stub A2A agents
-        run: |
-          set -euo pipefail
-          cat > /tmp/molecule-stub-a2a.py <<'PY'
-          import json
-          from http.server import BaseHTTPRequestHandler, HTTPServer
-
-          class Handler(BaseHTTPRequestHandler):
-              def do_POST(self):
-                  length = int(self.headers.get("content-length", "0"))
-                  raw = self.rfile.read(length) if length else b"{}"
-                  try:
-                      req = json.loads(raw)
-                  except Exception:
-                      req = {}
-                  method = req.get("method")
-                  if method not in ("message/send", None):
-                      body = {"jsonrpc": "2.0", "id": req.get("id"), "error": {"code": -32601, "message": "method not found"}}
-                  else:
-                      body = {
-                          "jsonrpc": "2.0",
-                          "id": req.get("id", "stub"),
-                          "result": {
-                              "role": "agent",
-                              "parts": [{"kind": "text", "type": "text", "text": "stub agent response"}],
-                          },
-                      }
-                  data = json.dumps(body, separators=(",", ":")).encode()
-                  self.send_response(200)
-                  self.send_header("content-type", "application/json")
-                  self.send_header("content-length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-              def log_message(self, *_):
-                  return
-
-          HTTPServer(("127.0.0.1", 18080), Handler).serve_forever()
-          PY
-          python3 /tmp/molecule-stub-a2a.py > /tmp/molecule-stub-a2a.log 2>&1 &
-          echo $! > /tmp/molecule-stub-a2a.pid
-
-      - name: Seed external agents for legacy A2A/activity scripts
-        run: |
-          set -euo pipefail
-          create_agent() {
-            local name="$1" role="$2"
-            curl -sS -X POST "$BASE/workspaces" \
-              -H "Content-Type: application/json" \
-              -d "{\"name\":\"${name}\",\"role\":\"${role}\",\"tier\":1,\"runtime\":\"external\",\"external\":true,\"url\":\"http://127.0.0.1:18080\"}" \
-              | python3 -c "import json,sys; print(json.load(sys.stdin)['id'])"
-          }
-          ECHO_ID=$(create_agent "Echo Agent" "Echo")
-          SEO_ID=$(create_agent "SEO Agent" "SEO")
-          curl -sS -X POST "$BASE/registry/register" -H "Content-Type: application/json" \
-            -d "{\"id\":\"$ECHO_ID\",\"url\":\"http://127.0.0.1:18080\",\"agent_card\":{\"name\":\"Echo Agent\",\"skills\":[{\"id\":\"echo\",\"name\":\"Echo\"}]}}" >/dev/null
-          curl -sS -X POST "$BASE/registry/register" -H "Content-Type: application/json" \
-            -d "{\"id\":\"$SEO_ID\",\"url\":\"http://127.0.0.1:18080\",\"agent_card\":{\"name\":\"SEO Agent\",\"skills\":[{\"id\":\"seo\",\"name\":\"SEO\"}]}}" >/dev/null
-
-      - name: Run activity E2E
-        run: bash tests/e2e/test_activity_e2e.sh
-
-      - name: Run A2A E2E
-        run: bash tests/e2e/test_a2a_e2e.sh
-
-      - name: Runtime-dependent legacy E2E preflight
-        run: |
-          set -euo pipefail
-          if [ -f workspace-configs-templates/claude-code-default/.auth-token ] && docker image inspect workspace:latest >/dev/null 2>&1; then
-            bash tests/e2e/test_claude_code_e2e.sh
-            bash tests/e2e/test_chat_upload_e2e.sh
-          else
-            echo "::notice::Skipping test_claude_code_e2e.sh and test_chat_upload_e2e.sh: require workspace:latest plus workspace-configs-templates/claude-code-default/.auth-token"
-          fi
-
-      - name: Dump platform log on failure
-        if: failure()
-        run: cat workspace-server/platform.log || true
-
-      - name: Stop platform and stub agents
-        if: always()
-        run: |
-          if [ -f workspace-server/platform.pid ]; then
-            kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
-          fi
-          if [ -f /tmp/molecule-stub-a2a.pid ]; then
-            kill "$(cat /tmp/molecule-stub-a2a.pid)" 2>/dev/null || true
-          fi
-
-      - name: Stop service containers
-        if: always()
-        run: |
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
-          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true

.gitea/workflows/e2e-peer-visibility.yml

@@ -44,8 +44,6 @@ name: E2E Peer Visibility (literal MCP list_peers)
 #   - No cross-repo `uses:` (feedback_gitea_cross_repo_uses_blocked). The
 #     actions/checkout SHA is the one e2e-staging-canvas.yml already uses
 #     successfully (a mirrored SHA — see #1277/PR#1292 root-cause).
-#   - 2026-05-21 retrigger: verify fresh platform-tenant image after the
-#     publish Buildx DOCKER_CONFIG fix restored staging-latest image updates.
 #   - Per-SHA concurrency, not global (feedback_concurrency_group_per_sha).
 #   - Workflow-level GITHUB_SERVER_URL pinned
 #     (feedback_act_runner_github_server_url).
@@ -54,27 +52,6 @@ name: E2E Peer Visibility (literal MCP list_peers)
 #     flip-to-required-ready (mirrors e2e-staging-saas.yml's proven shape;
 #     real EC2-provisioning E2E is push/dispatch/cron only — it is 30+ min
 #     and cannot run per-PR-update).
-#
-# LOCAL BACKEND (added 2026-05-15 — feedback_local_must_mimic_production,
-# feedback_mandatory_local_e2e_before_ship, feedback_local_test_before_
-# staging_e2e)
-# --------------------------------------------------------------------
-# The standing rule is that the local prod-mimic stack runs a MANDATORY
-# local-Postgres E2E BEFORE staging E2E. A staging-only peer-visibility
-# gate caught regressions late + expensively (cold EC2). The
-# `peer-visibility-local` job below runs the SAME byte-identical
-# assertion (tests/e2e/lib/peer_visibility_assert.sh) against the local
-# docker-compose stack — built + booted exactly like e2e-api.yml's
-# proven E2E API Smoke Test job (ephemeral pg/redis ports, go build,
-# background platform-server). It runs on PR + push (local boot is
-# minutes, not the 30+ min cold-EC2 path), so peer-visibility is part of
-# the local gate that fires before the staging E2E.
-#
-# It is its OWN non-required status context `E2E Peer Visibility (local)`.
-# The local backend uses external-mode workspaces by default so it tests
-# the literal platform MCP list_peers path without depending on local
-# template container boot/heartbeat. Container-mode runtime boot remains
-# available via PV_LOCAL_PROVISION_MODE=container for targeted debugging.
 
 on:
   push:
@@ -85,10 +62,9 @@ on:
       - 'workspace-server/internal/middleware/**'
       - 'workspace-server/internal/handlers/registry.go'
       - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace/a2a_mcp_server.py'
+      - 'workspace/platform_tools/registry.py'
       - 'tests/e2e/test_peer_visibility_mcp_staging.sh'
-      - 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
-      - 'tests/e2e/test_peer_visibility_mcp_local.sh'
-      - 'tests/e2e/lib/peer_visibility_assert.sh'
       - '.gitea/workflows/e2e-peer-visibility.yml'
   pull_request:
     branches: [main]
@@ -98,10 +74,9 @@ on:
       - 'workspace-server/internal/middleware/**'
       - 'workspace-server/internal/handlers/registry.go'
       - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace/a2a_mcp_server.py'
+      - 'workspace/platform_tools/registry.py'
       - 'tests/e2e/test_peer_visibility_mcp_staging.sh'
-      - 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
-      - 'tests/e2e/test_peer_visibility_mcp_local.sh'
-      - 'tests/e2e/lib/peer_visibility_assert.sh'
       - '.gitea/workflows/e2e-peer-visibility.yml'
   workflow_dispatch:
   schedule:
@@ -133,169 +108,16 @@ jobs:
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - name: Validate driving scripts + shared assertion lib
+      - name: Validate driving script
         run: |
-          bash -n tests/e2e/lib/peer_visibility_assert.sh
-          echo "lib/peer_visibility_assert.sh — bash syntax OK"
           bash -n tests/e2e/test_peer_visibility_mcp_staging.sh
           echo "test_peer_visibility_mcp_staging.sh — bash syntax OK"
-          bash -n tests/e2e/test_peer_visibility_token_mint_staging.sh
-          echo "test_peer_visibility_token_mint_staging.sh — bash syntax OK"
-          bash -n tests/e2e/test_peer_visibility_mcp_local.sh
-          echo "test_peer_visibility_mcp_local.sh — bash syntax OK"
-          legacy_token_suffix="test""-token"
-          if rg -n "$legacy_token_suffix" tests/e2e/test_*staging*.sh; then
-            echo "::error::staging E2E must use production-safe admin token minting"
-            exit 1
-          fi
-          echo "Staging fresh-provision MCP list_peers E2E runs on push to"
+          echo "Real fresh-provision MCP list_peers E2E runs on push to"
           echo "main / workflow_dispatch / daily cron (30+ min EC2 boot)."
-          echo "The LOCAL backend runs in the peer-visibility-local job"
-          echo "below on this same PR (local docker-compose stack)."
 
-  # LOCAL gate: same byte-identical assertion against the local prod-mimic
-  # docker-compose stack — the MANDATORY local-E2E that must run BEFORE
-  # the staging E2E (feedback_mandatory_local_e2e_before_ship,
-  # feedback_local_test_before_staging_e2e). Bootstrap mirrors
-  # e2e-api.yml's proven E2E API Smoke Test job (per-run container names +
-  # ephemeral host ports so concurrent host-network act_runner runs don't
-  # collide; go build; background platform-server). Its OWN non-required
-  # status context `E2E Peer Visibility (local)` — non-required-by-design
-  # exactly like the staging job (flip-to-required tracked at
-  # molecule-core#1296). HONEST gate, NO continue-on-error mask
-  # (feedback_fix_root_not_symptom). Runs on PR +
-  # push (local boot is minutes, not the 30+ min cold-EC2 path).
-  # bp-required: pending #1296
-  peer-visibility-local:
-    name: E2E Peer Visibility (local)
-    runs-on: docker-host
-    timeout-minutes: 30
-    env:
-      # Per-run names + ephemeral ports — same collision-avoidance as
-      # e2e-api.yml (host-network act_runner; feedback_act_runner_*).
-      PG_CONTAINER: pg-e2e-pv-${{ github.run_id }}-${{ github.run_attempt }}
-      REDIS_CONTAINER: redis-e2e-pv-${{ github.run_id }}-${{ github.run_attempt }}
-      # LLM keys so hermes/openclaw can actually boot. The local script
-      # SKIPs (not fails) any runtime whose key is absent, so a partially
-      # keyed CI env still exercises whatever it can.
-      CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.E2E_CLAUDE_CODE_OAUTH_TOKEN }}
-      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
-      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
-      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
-      PV_RUNTIMES: "hermes openclaw claude-code"
-      PV_LOCAL_PROVISION_MODE: external
-      ADMIN_TOKEN: local-e2e-admin-token
-      MOLECULE_ADMIN_TOKEN: local-e2e-admin-token
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
-        with:
-          go-version: 'stable'
-          cache: true
-          cache-dependency-path: workspace-server/go.sum
-      - name: Pre-pull alpine + ensure provisioner network
-        run: |
-          docker pull alpine:latest >/dev/null
-          docker network create molecule-core-net >/dev/null 2>&1 || true
-          echo "alpine:latest pre-pulled; molecule-core-net ensured."
-      - name: Start Postgres (docker, ephemeral port)
-        run: |
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
-          docker run -d --name "$PG_CONTAINER" \
-            -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
-            -p 0:5432 postgres:16 >/dev/null
-          PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
-          [ -n "$PG_PORT" ] || PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
-          if [ -z "$PG_PORT" ]; then
-            echo "::error::Could not resolve host port for $PG_CONTAINER"
-            docker logs "$PG_CONTAINER" || true; exit 1
-          fi
-          echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
-          for i in $(seq 1 30); do
-            docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1 && { echo "Postgres ready after ${i}s"; exit 0; }
-            sleep 1
-          done
-          echo "::error::Postgres did not become ready in 30s"; docker logs "$PG_CONTAINER" || true; exit 1
-      - name: Start Redis (docker, ephemeral port)
-        run: |
-          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
-          docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
-          REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
-          [ -n "$REDIS_PORT" ] || REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
-          if [ -z "$REDIS_PORT" ]; then
-            echo "::error::Could not resolve host port for $REDIS_CONTAINER"
-            docker logs "$REDIS_CONTAINER" || true; exit 1
-          fi
-          echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
-          for i in $(seq 1 15); do
-            docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG && { echo "Redis ready after ${i}s"; exit 0; }
-            sleep 1
-          done
-          echo "::error::Redis did not become ready in 15s"; docker logs "$REDIS_CONTAINER" || true; exit 1
-      - name: Build platform
-        working-directory: workspace-server
-        run: go build -o platform-server ./cmd/server
-      - name: Pick platform port
-        run: |
-          PLATFORM_PORT=$(python3 - <<'PY'
-          import socket
-          with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-              s.bind(("127.0.0.1", 0))
-              print(s.getsockname()[1])
-          PY
-          )
-          echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
-          echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
-          echo "Platform host port: ${PLATFORM_PORT}"
-      - name: Kill stale platform-server before start
-        run: |
-          killed=0
-          for pid in $(grep -l "platform-serve" /proc/[0-9]*/comm 2>/dev/null); do
-            kpid="${pid%/comm}"; kpid="${kpid##*/}"
-            cmdline=$(cat "/proc/${kpid}/cmdline" 2>/dev/null | tr '\0' ' ')
-            if echo "$cmdline" | grep -q "platform-server"; then
-              echo "Killing stale platform-server pid ${kpid}"
-              kill "$kpid" 2>/dev/null || true; killed=$((killed + 1))
-            fi
-          done
-          [ "$killed" -gt 0 ] && sleep 2 || true
-          echo "stale-kill done ($killed killed)"
-      - name: Start platform (background)
-        working-directory: workspace-server
-        run: |
-          ./platform-server > platform.log 2>&1 &
-          echo $! > platform.pid
-      - name: Wait for /health
-        run: |
-          for i in $(seq 1 30); do
-            curl -sf "$BASE/health" > /dev/null && { echo "Platform up after ${i}s"; exit 0; }
-            sleep 1
-          done
-          echo "::error::Platform did not become healthy in 30s"
-          cat workspace-server/platform.log || true; exit 1
-      - name: Run LOCAL fresh-provision peer-visibility E2E (literal MCP list_peers)
-        # HONEST gate — NO continue-on-error. The local backend uses
-        # external-mode workspaces so this context tests the literal MCP
-        # peer-visibility path without coupling to template container boot.
-        run: bash tests/e2e/test_peer_visibility_mcp_local.sh
-      - name: Dump platform log on failure
-        if: failure()
-        run: cat workspace-server/platform.log || true
-      - name: Stop platform
-        if: always()
-        run: |
-          if [ -f workspace-server/platform.pid ]; then
-            kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
-          fi
-      - name: Stop service containers
-        if: always()
-        run: |
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
-          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
-
-  # Real STAGING gate: provisions a throwaway org + sibling-per-runtime,
-  # drives the LITERAL list_peers MCP call per runtime, asserts 200 +
-  # expected peer set, then scoped teardown. push(main)/dispatch/cron only.
+  # Real gate: provisions a throwaway org + sibling-per-runtime, drives
+  # the LITERAL list_peers MCP call per runtime, asserts 200 + expected
+  # peer set, then scoped teardown. push(main)/dispatch/cron only.
   peer-visibility:
     name: E2E Peer Visibility
     runs-on: ubuntu-latest

.gitea/workflows/e2e-staging-canvas.yml

@@ -16,9 +16,9 @@ name: E2E Staging Canvas (Playwright)
 # e2e-staging-saas.yml (which tests the API shape) by exercising the
 # actual browser + canvas bundle against live staging.
 #
-# Triggers: push to main, PR touching canvas sources + this workflow only
-# after the PR enters `merge-queue`, manual dispatch, and scheduled cron to
-# catch browser/runtime drift even when canvas is quiet.
+# Triggers: push to main/staging or PR touching canvas sources + this workflow,
+# manual dispatch, and weekly cron to catch browser/runtime drift even
+# when canvas is quiet.
 # Added staging to push/pull_request branches so the auto-promote gate
 # check (--event push --branch staging) can see a completed run for this
 # workflow — mirrors what PR #1891 does for e2e-api.yml.
@@ -37,10 +37,9 @@ on:
   pull_request:
     branches: [main]
   schedule:
-    # Nightly at 08:00 UTC — catches Chrome / Playwright / Next.js
+    # Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
     # release-note-shaped regressions that don't ride in with a PR.
-    - cron: '0 8 * * *'
-  workflow_dispatch:
+    - cron: '0 8 * * 0'
 
 concurrency:
   # Per-SHA grouping (changed 2026-04-28 from a single global group). The
@@ -80,13 +79,10 @@ jobs:
         with:
           fetch-depth: 0
       - id: decide
-        env:
-          GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          QUEUE_LABEL: merge-queue
         # Inline replacement for dorny/paths-filter — see e2e-api.yml.
-        # Cron and manual triggers always run real work (no diff context).
+        # Cron triggers always run real work (no diff context).
         run: |
-          if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+          if [ "${{ github.event_name }}" = "schedule" ]; then
             echo "canvas=true" >> "$GITHUB_OUTPUT"
             exit 0
           fi
@@ -106,26 +102,9 @@ jobs:
             exit 0
           fi
           CHANGED=$(git diff --name-only "$BASE" HEAD)
-          if ! echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
-            echo "canvas=false" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          if [ "${{ github.event_name }}" != "pull_request" ]; then
-            echo "canvas=true" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          authfile=$(mktemp)
-          chmod 600 "$authfile"
-          printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
-          labels=$(curl -fsS -K "$authfile" \
-            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
-            | python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
-          rm -f "$authfile"
-          if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
+          if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
             echo "canvas=true" >> "$GITHUB_OUTPUT"
           else
-            echo "PR is not in merge-queue; skipping heavy E2E Staging Canvas for normal PR path."
             echo "canvas=false" >> "$GITHUB_OUTPUT"
           fi
 
@@ -190,14 +169,7 @@ jobs:
       - name: Install Playwright browsers
         if: needs.detect-changes.outputs.canvas == 'true'
         timeout-minutes: 10
-        run: |
-          PREBAKED_PLAYWRIGHT=/ms-playwright
-          if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
-            echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
-            echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
-            exit 0
-          fi
-          npx playwright install --with-deps chromium
+        run: npx playwright install --with-deps chromium
 
       - name: Run staging canvas E2E
         if: needs.detect-changes.outputs.canvas == 'true'

.gitea/workflows/e2e-staging-saas.yml

@@ -49,8 +49,6 @@ on:
       - 'workspace-server/internal/middleware/**'
       - 'workspace-server/internal/provisioner/**'
       - 'tests/e2e/test_staging_full_saas.sh'
-      - 'tests/e2e/lib/aws_leak_check.sh'
-      - 'tests/e2e/test_aws_leak_check.sh'
       - '.gitea/workflows/e2e-staging-saas.yml'
   pull_request:
     branches: [main]
@@ -61,8 +59,6 @@ on:
       - 'workspace-server/internal/middleware/**'
       - 'workspace-server/internal/provisioner/**'
       - 'tests/e2e/test_staging_full_saas.sh'
-      - 'tests/e2e/lib/aws_leak_check.sh'
-      - 'tests/e2e/test_aws_leak_check.sh'
       - '.gitea/workflows/e2e-staging-saas.yml'
   workflow_dispatch:
   schedule:
@@ -108,13 +104,13 @@ jobs:
         # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
         continue-on-error: true
 
-  # Actual E2E: runs on trunk pushes and PRs that touch provisioning-critical
-  # paths. pr-validate remains as the lightweight workflow-shape check for PRs,
-  # but it is not a substitute for live staging proof when this workflow or the
-  # staging harness changes.
+  # Actual E2E: runs on trunk pushes (main + staging). NOT the PR-fire-only
+  # path — pr-validate above posts success for workflow-only PRs.
   e2e-staging-saas:
     name: E2E Staging SaaS
     runs-on: ubuntu-latest
+    # Only runs on trunk pushes. PR paths get pr-validate instead.
+    if: github.event.pull_request.base.ref == ''
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -131,11 +127,6 @@ jobs:
       # (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
       # internal#322 — see this PR for the cross-workflow sweep.
       MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: us-east-2
-      E2E_AWS_LEAK_CHECK: required
-      E2E_AWS_TERMINATE_LEAKS: '1'
       # MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
       # from hermes+OpenAI default after #2578 (the staging OpenAI key
       # account went over quota and stayed dead for 36+ hours, taking
@@ -152,7 +143,7 @@ jobs:
       # block). See #2578 PR comment for the rationale.
       E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
       # OpenAI fallback — kept wired so an operator-dispatched run with
-      # E2E_RUNTIME=hermes or =codex via workflow_dispatch can still
+      # E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
       # exercise the OpenAI path.
       E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
       E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
@@ -161,7 +152,7 @@ jobs:
       # and defeats the cost saving. Operators can override via the
       # workflow_dispatch flow (no input wired here yet — runtime
       # override is enough for ad-hoc).
-      E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'codex' && 'openai/gpt-4o' || 'MiniMax-M2' }}
+      E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
       E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
       E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
 
@@ -174,18 +165,12 @@ jobs:
             echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
             exit 2
           fi
-          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
-            if [ -z "${!var:-}" ]; then
-              echo "::error::$var not set — EC2 leak verification cannot run"
-              exit 2
-            fi
-          done
           echo "Admin token present ✓"
 
       - name: Verify LLM key present
         run: |
           # Per-runtime key check — claude-code uses MiniMax; hermes /
-          # codex (operator-dispatched only) use OpenAI. Hard-fail
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
           # rather than soft-skip per #2578's lesson — empty key
           # silently falls through to the wrong SECRETS_JSON branch and
           # produces a confusing auth error 5 min later instead of the
@@ -206,7 +191,7 @@ jobs:
                 required_secret_value=""
               fi
               ;;
-            codex|hermes)
+            langgraph|hermes)
               required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
               required_secret_value="${E2E_OPENAI_API_KEY:-}"
               ;;

.gitea/workflows/e2e-staging-sanity.yml

@@ -47,11 +47,6 @@ jobs:
       # (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
       # internal#322 — see this PR for the cross-workflow sweep.
       MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: us-east-2
-      E2E_AWS_LEAK_CHECK: required
-      E2E_AWS_TERMINATE_LEAKS: '1'
       E2E_MODE: smoke
       E2E_RUNTIME: hermes
       E2E_RUN_ID: "sanity-${{ github.run_id }}"
@@ -66,12 +61,6 @@ jobs:
             echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
             exit 2
           fi
-          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
-            if [ -z "${!var:-}" ]; then
-              echo "::error::$var not set — EC2 leak verification cannot run"
-              exit 2
-            fi
-          done
 
       # Inverted assertion: the run MUST fail. If it passes, the
       # E2E_INTENTIONAL_FAILURE path is broken.

.gitea/workflows/gate-check-v3.yml

@@ -7,11 +7,10 @@
 #   PR_NUMBER  — set via ${{ github.event.pull_request.number }} from the trigger
 #   POST_COMMENT — "true" to post/update comment on PR
 #
-# Gating logic (MVP signals 1,2,3,4,6):
+# Gating logic (MVP signals 1,2,3,6):
 #   1. Author-aware agent-tag comment scan
 #   2. REQUEST_CHANGES reviews state machine
 #   3. Staleness detection (SOP-12: review.commit_id != PR.head_sha + >1 working day)
-#   4. Branch divergence / scope-creep guard (base-sha vs target HEAD; mc#365)
 #   6. CI required-checks awareness
 #
 # Exit code: 0=CLEAR, 1=BLOCKED, 2=ERROR
@@ -33,24 +32,6 @@ on:
   # iterating all open PRs when PR_NUMBER is empty.
   workflow_dispatch:
 
-# Serialize per PR (or per repo for schedule/manual ticks) to prevent
-# the fan-out OOM class documented in
-# `reference_operator_host_python3_oom_storm_2026_05_18`. `edited`
-# events fan out on every PR-body edit; combined with the hourly cron
-# and synchronize bursts this workflow can stack runs of the same
-# workflow_id on the same PR (each ~4GB anon-RSS) and trip the
-# `--memory=4g --memory-swap=8g` per-container cap.
-#
-# NO `cancel-in-progress` (defaults to false). Per
-# `feedback_janitor_supersede_must_group_by_workflow_id`, cancelling
-# in-flight runs of any required-check-shaped workflow risks the
-# dismiss_stale_approvals + empty-commit-rerun dance (Gitea 1.22.6 has
-# no REST rerun). The gate-check is `continue-on-error: true` +
-# idempotent (POST/PATCH gate-check comment by context) so sequential
-# ticks are strictly safe.
-concurrency:
-  group: gate-check-v3-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
-
 permissions:
   # read: contents — for checkout (base ref, not PR head for security)
   # read: pull-requests — for reading PR info via API

.gitea/workflows/gitea-merge-queue.yml

@@ -13,12 +13,8 @@ name: gitea-merge-queue
 #   - add `merge-queue-hold` to pause a queued PR without removing it
 
 on:
-  # Schedule moved to operator-config:
-  #   /etc/cron.d/molecule-core-merge-queue ->
-  #   /usr/local/bin/molecule-core-cron-bot.sh merge-queue
-  #
-  # The queue bot still processes one PR per tick, but no longer occupies
-  # one of the shared Actions runners just to poll.
+  schedule:
+    - cron: '*/5 * * * *'
   workflow_dispatch:
 
 permissions:
@@ -56,9 +52,5 @@ jobs:
           # explicitly instead of the combined state avoids false-pause when
           # non-blocking jobs (continue-on-error: true) have failed — those
           # failures pollute combined state but do not gate merges.
-          # NOTE: the event-suffixed context name is intentional — branch protection
-          # MUST require `CI / all-required (pull_request)` (with suffix), NOT the
-          # bare `CI / all-required`. Gitea treats absent contexts as pending, not
-          # skipped; requiring the bare name silently blocks all merges (issue #1473).
           PUSH_REQUIRED_CONTEXTS: CI / all-required (push)
         run: python3 .gitea/scripts/gitea-merge-queue.py

.gitea/workflows/handlers-postgres-integration.yml

@@ -77,16 +77,7 @@ env:
 jobs:
   detect-changes:
     name: detect-changes
-    # mc#1529 §1: pin to `docker-host` so the integration job runs on the
-    # operator-host runners (molecule-runner-*), which carry the
-    # `molecule-core-net` bridge network this workflow depends on. PC2
-    # runners (hongming-pc-runner-*) also advertise ubuntu-latest but
-    # don't have that network — the previous `runs-on: ubuntu-latest`
-    # rolled the dice and hard-failed the bridge-inspect step ~30% of
-    # the time. detect-changes itself doesn't need the bridge, but keeping
-    # both jobs on the same label avoids workspace-volume cross-host
-    # surprises and keeps the routing rule discoverable in one place.
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # mc#774 Phase 3 (RFC §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -101,13 +92,36 @@ jobs:
           # not present in the shallow checkout.
           fetch-depth: 2
       - id: filter
+        # Inline replacement for dorny/paths-filter — see e2e-api.yml.
         run: |
-          python3 .gitea/scripts/detect-changes.py \
-            --profile handlers-postgres \
-            --event-name "${{ github.event_name }}" \
-            --pr-base-sha "${{ github.event.pull_request.base.sha }}" \
-            --base-ref "${{ github.event.pull_request.base.ref }}" \
-            --push-before "${GITHUB_EVENT_BEFORE:-}"
+          # Gitea Actions evaluates github.event.before to empty string in shell
+          # scripts. Use GITHUB_EVENT_BEFORE shell env var instead (Gitea
+          # correctly populates it for push events). PR case uses template var.
+          BASE=""
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          elif [ -n "$GITHUB_EVENT_BEFORE" ]; then
+            BASE="$GITHUB_EVENT_BEFORE"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "handlers=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # timeout 30 guards against the case where BASE points to a ref that
+          # git can resolve but cat-file hangs (rare on corrupted objects).
+          if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
+            echo "handlers=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
+            echo "handlers=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "handlers=false" >> "$GITHUB_OUTPUT"
+          fi
 
   # Single-job-with-per-step-if pattern: always runs to satisfy the
   # required-check name on branch protection; real work gates on the
@@ -115,9 +129,7 @@ jobs:
   integration:
     name: Handlers Postgres Integration
     needs: detect-changes
-    # mc#1529 §1: must run on operator-host (where `molecule-core-net`
-    # exists). See detect-changes for the full routing rationale.
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # mc#774 Phase 3 (RFC §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true

.gitea/workflows/harness-replays.yml

@@ -62,13 +62,7 @@ env:
 jobs:
   # bp-exempt: change detector only; downstream Harness Replays is the meaningful gate.
   detect-changes:
-    # mc#1529 follow-on: pin to `docker-host` so this lane lands on
-    # Linux operator-host runners (the only ones with a working
-    # docker.sock + `molecule-core-net`). The bare `ubuntu-latest`
-    # label is also matched by hongming-pc-runner-* (Windows act_runner
-    # v1.0.3), where the `docker compose ...` exec below fails. Mirror
-    # of mc#1543; see internal#512 for class defect.
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true
@@ -168,9 +162,7 @@ jobs:
   harness-replays:
     needs: detect-changes
     name: Harness Replays
-    # mc#1529 follow-on: `docker compose ... ps/logs` against tenant-alpha/
-    # beta containers. Must run on operator-host Linux (docker-host).
-    runs-on: docker-host
+    runs-on: ubuntu-latest
     # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
     # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
     continue-on-error: true

.gitea/workflows/lint-forbidden-env-keys.yml

@@ -1,168 +0,0 @@
-name: Lint forbidden tenant-env keys
-
-# RFC#523 Layer 3 (task #146): scan workspace_secrets-writer Go code
-# under workspace-server/ for new code that hardcodes a forbidden
-# operator-scope env var NAME (GITEA_TOKEN, CP_ADMIN_API_TOKEN,
-# RAILWAY_TOKEN, INFISICAL_OPERATOR_TOKEN, MOLECULE_OPERATOR_*, …).
-#
-# Catches the class "a new writer accidentally widens the propagation
-# set" — e.g. a future env-mutator plugin that sets envVars["GITEA_TOKEN"]
-# directly. Today the L1 runtime guard would abort the provision, but
-# this lint surfaces the offending code at PR review time instead of
-# at first provision attempt.
-#
-# Companion layers:
-#   - L1: workspace-server/internal/handlers/workspace_provision_forbidden_env.go
-#         (fail-closed abort at provision time)
-#   - L2: workspace/entrypoint.sh top-of-file env-grep + exit 1
-#
-# Open-source-template-friendly: the deny pattern is generic. A fork
-# can copy this workflow and replace OPERATOR_KEY_PATTERN with its
-# own operator-scope key names.
-#
-# Path-filter discipline:
-#   This workflow runs on every PR (no paths: filter — see
-#   feedback_path_filtered_workflow_cant_be_required). The scan itself
-#   targets workspace_secrets-writer paths via grep -r; it's fast
-#   (sub-second) so unconditional run is fine.
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-  push:
-    branches: [main, staging]
-
-env:
-  GITHUB_SERVER_URL: https://git.moleculesai.app
-
-jobs:
-  scan:
-    name: Scan workspace_secrets writers for forbidden env keys
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          fetch-depth: 1
-
-      - name: Scan for forbidden operator-scope env key NAMES in writer paths
-        run: |
-          set -euo pipefail
-
-          # Forbidden EXACT-MATCH env var names. Kept in lockstep with
-          # workspace-server/internal/handlers/workspace_provision_forbidden_env.go
-          # forbiddenTenantEnvKeys. The Go-side test
-          # TestIsForbiddenTenantEnvKey_ExactMatches is the source of
-          # truth — if Go-side adds a key, also add it here (and
-          # vice-versa). Drift between the two is the failure mode this
-          # entire 3-layer guardrail is designed to catch.
-          FORBIDDEN_KEYS=(
-            "GITEA_TOKEN" "GITEA_PAT"
-            "GITHUB_TOKEN" "GITHUB_PAT" "GH_TOKEN"
-            "GITLAB_TOKEN" "GL_TOKEN"
-            "BITBUCKET_TOKEN"
-            "CP_ADMIN_API_TOKEN" "CP_ADMIN_TOKEN"
-            "INFISICAL_OPERATOR_TOKEN" "INFISICAL_BOOTSTRAP_TOKEN"
-            "RAILWAY_TOKEN" "RAILWAY_PERSONAL_API_TOKEN"
-            "HETZNER_TOKEN" "HETZNER_API_TOKEN"
-          )
-
-          # Forbidden PREFIX patterns — operator-scope families.
-          FORBIDDEN_PREFIXES=(
-            "MOLECULE_OPERATOR_"
-          )
-
-          # Writer paths: Go source under workspace-server/ that
-          # writes to the env-vars map or to workspace_secrets DB rows.
-          # Tests, the forbidden-env source itself, and the silent-
-          # strip denylist are exempt (they LIST the keys by design).
-          SCAN_ROOT="workspace-server/internal"
-          # Exempt paths fall in two classes:
-          #   1. The deny-set definitions + the silent-strip denylist:
-          #      they LIST the forbidden names by design.
-          #   2. Pre-RFC#523 persona-merge / config-read paths that
-          #      already handle these names correctly (the silent-
-          #      strip downstream + the new L1 fail-closed cover the
-          #      runtime risk; these reads are unchanged).
-          # New code MUST NOT be added to this list without reviewer
-          # signoff and a one-line justification in this diff.
-          EXEMPT_PATHS=(
-            # Class 1 — deny-set definitions
-            "workspace-server/internal/handlers/workspace_provision_forbidden_env.go"
-            "workspace-server/internal/handlers/workspace_provision_forbidden_env_test.go"
-            "workspace-server/internal/provisioner/provisioner.go"
-            "workspace-server/internal/provisioner/provisioner_test.go"
-            # Class 2 — pre-existing persona-fallback / org-helper paths
-            # that set the GITEA_TOKEN fallback lane (stripped downstream
-            # by provisioner.buildContainerEnv per forensic #145). The
-            # new L1 fail-closed runs BEFORE these writers, so any
-            # operator-scope leak via global/workspace_secrets is
-            # already caught. See applyAgentGitHTTPCreds doc-comment.
-            "workspace-server/internal/handlers/agent_git_identity.go"
-            "workspace-server/internal/handlers/org_helpers.go"
-            "workspace-server/internal/handlers/org.go"
-            # Class 2 — CP→platform admin auth (NOT a tenant env write;
-            # this is the control-plane HTTP auth header source).
-            "workspace-server/internal/provisioner/cp_provisioner.go"
-          )
-
-          # Build a single grep -F pattern: every forbidden key wrapped
-          # in quotes (Go string-literal form, which is how env-map
-          # writes appear). e.g. envVars["GITEA_TOKEN"] = ... or
-          # `"GITEA_TOKEN":` in a literal-map declaration.
-          #
-          # We deliberately match the quoted form so a comment that
-          # happens to spell the name without quotes (e.g. "see
-          # GITEA_TOKEN below") doesn't trip the lint.
-          PATTERN=""
-          for k in "${FORBIDDEN_KEYS[@]}"; do
-            PATTERN="${PATTERN}\"${k}\"\n"
-          done
-          for p in "${FORBIDDEN_PREFIXES[@]}"; do
-            # Prefix match needs a regex; switch to grep -E below for
-            # this slice. Kept conceptually here so the deny set lives
-            # in one place; scan is run twice (literal + prefix).
-            true
-          done
-
-          # Build exempt-paths grep filter — `grep -v -f` style.
-          EXEMPT_FILTER=$(mktemp)
-          trap 'rm -f "$EXEMPT_FILTER"' EXIT
-          for p in "${EXEMPT_PATHS[@]}"; do
-            echo "$p" >> "$EXEMPT_FILTER"
-          done
-
-          # --- Exact-match scan ---
-          HITS=""
-          for k in "${FORBIDDEN_KEYS[@]}"; do
-            # Only .go files; skip _test.go for the writer-path scan
-            # since tests legitimately reference the names. The
-            # writer-path lint targets PRODUCTION code only.
-            found=$(grep -rn --include='*.go' --exclude='*_test.go' "\"${k}\"" "$SCAN_ROOT" 2>/dev/null \
-                    | grep -v -F -f "$EXEMPT_FILTER" || true)
-            if [ -n "$found" ]; then
-              HITS="${HITS}${found}\n"
-            fi
-          done
-
-          # --- Prefix scan ---
-          for prefix in "${FORBIDDEN_PREFIXES[@]}"; do
-            found=$(grep -rnE --include='*.go' --exclude='*_test.go' "\"${prefix}[A-Z0-9_]+\"" "$SCAN_ROOT" 2>/dev/null \
-                    | grep -v -F -f "$EXEMPT_FILTER" || true)
-            if [ -n "$found" ]; then
-              HITS="${HITS}${found}\n"
-            fi
-          done
-
-          if [ -n "$HITS" ]; then
-            echo "::error::RFC#523 Layer 3: forbidden operator-scope env var name(s) hardcoded in tenant-workspace writer paths:"
-            printf "$HITS"
-            echo ""
-            echo "These env-var NAMES are on the operator-scope deny list (see"
-            echo "workspace-server/internal/handlers/workspace_provision_forbidden_env.go)."
-            echo "If your code legitimately needs to inject one of these for a"
-            echo "non-tenant code path, add the file to EXEMPT_PATHS in this"
-            echo "workflow with a one-line justification — reviewer signoff required."
-            exit 1
-          fi
-
-          echo "OK No forbidden operator-scope env key names hardcoded in writer paths."

.gitea/workflows/lint-no-tenant-gitea-token.yml

@@ -1,182 +0,0 @@
-name: Lint no tenant GITEA or GITHUB token write
-
-# Task #146 — CI guardrail companion to RFC#523's `lint-forbidden-env-keys.yml`.
-#
-# `lint-forbidden-env-keys.yml` (Layer 3) catches code that hardcodes a
-# forbidden env-var key NAME as a quoted literal in workspace_secrets
-# writer paths under workspace-server/internal/.
-#
-# This workflow catches a BROADER class: any code path that reads a
-# repo-host token (GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN) and then writes
-# it into a TENANT WORKSPACE's env, secret store, user-data, or
-# provision payload. This is the actual RFC#523 threat-model statement —
-# the goal is "no tenant workspace ever receives an operator-scope repo
-# token," not just "no _quoted_ literal `GITEA_TOKEN`." A future writer
-# could route the value via a variable, a struct field, or a config key
-# and slip past the existing literal scan; this lint catches those
-# routing patterns at PR review time.
-#
-# Scope
-#   Scans the WHOLE repo's Go sources (not just workspace-server/) for
-#   co-occurrences of:
-#     - a repo-host token NAME (GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN /
-#       GITEA_PAT / GITHUB_PAT) used as os.Getenv argument or string
-#       literal
-#     - within a file that ALSO references a tenant-writer surface
-#       (`tenant`, `workspace_secrets`, `global_secrets`, `seedAllowList`,
-#       `/settings/secrets`, `userData`, `provisionPayload`,
-#       `envVars[`, `containerEnv`).
-#
-#   Co-occurrence (not single-line) is the false-positive control: a
-#   file that just LOGS the variable name (e.g. "missing GITEA_TOKEN")
-#   without touching any tenant surface won't fire.
-#
-# Drift contract with lint-forbidden-env-keys.yml
-#   Both lints share the same FORBIDDEN_KEYS list (a subset — only the
-#   repo-host tokens, since this lint's threat model is "tenant gets
-#   write access to operator's git host"). If RFC#523's deny set grows,
-#   update BOTH this file AND lint-forbidden-env-keys.yml AND the Go
-#   source-of-truth in
-#   workspace-server/internal/handlers/workspace_provision_forbidden_env.go.
-#
-# Open-source-template-friendly
-#   The patterns scanned are generic (no MOLECULE_-prefix literals).
-#   A fork can copy this workflow as-is and adjust FORBIDDEN_KEYS.
-#
-# Path-filter discipline
-#   No `paths:` filter — required-status workflows must run on every PR
-#   per `feedback_path_filtered_workflow_cant_be_required`. Scan is
-#   sub-second.
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-  push:
-    branches: [main, staging]
-
-env:
-  GITHUB_SERVER_URL: https://git.moleculesai.app
-
-jobs:
-  # bp-exempt: advisory RFC#523 lint; PR review gate is review-driven, not BP-driven.
-  # (Carried with the workflow-name rename in PR mc#1593 so the renamed
-  # context emission satisfies lint_required_context_exists_in_bp Tier 2g.)
-  scan:
-    name: Scan for repo-host token write into tenant workspace surface
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          fetch-depth: 1
-
-      - name: Find Go files referencing a tenant-writer surface AND a repo-host token
-        run: |
-          set -euo pipefail
-
-          # Repo-host token NAMES — the threat-model subset. Operator-fleet
-          # tokens (CP_ADMIN_API_TOKEN, RAILWAY_TOKEN, INFISICAL_*) are
-          # caught by lint-forbidden-env-keys.yml's broader deny set; this
-          # lint focuses on the git-host class so a single co-occurrence
-          # match has a low false-positive rate.
-          FORBIDDEN_KEYS=(
-            "GITEA_TOKEN"
-            "GITEA_PAT"
-            "GITHUB_TOKEN"
-            "GITHUB_PAT"
-            "GH_TOKEN"
-          )
-
-          # Tenant-writer surface markers. A file matches the surface set
-          # if it references ANY of these strings. This is the "is this
-          # code path writing into a tenant workspace?" heuristic.
-          # Curated to catch the actual code shapes used in this repo
-          # (verified by grep against current main 2026-05-19):
-          #   - "workspace_secrets" / "global_secrets"  → DB table writes
-          #   - "seedAllowList"                          → CP-side seed table
-          #   - "/settings/secrets"                      → tenant HTTP API write
-          #   - "envVars["                               → in-memory env map write
-          #   - "containerEnv"                           → docker-run env-set
-          #   - "userData"                               → EC2 user-data script
-          #   - "provisionPayload" / "provisionContext"  → provision-request shape
-          SURFACE_PATTERN='workspace_secrets|global_secrets|seedAllowList|/settings/secrets|envVars\[|containerEnv|userData|provisionPayload|provisionContext'
-
-          # Files that legitimately reference these names AND a surface
-          # marker, but do so for guard / strip / test / doc-comment
-          # reasons. New entries require reviewer signoff and a one-line
-          # justification in the diff.
-          EXEMPT_FILES=(
-            # RFC#523 L1 deny-set source-of-truth + tests
-            "workspace-server/internal/handlers/workspace_provision_forbidden_env.go"
-            "workspace-server/internal/handlers/workspace_provision_forbidden_env_test.go"
-            # Forensic-#145 silent-strip denylist (defense-in-depth, by design lists the names)
-            "workspace-server/internal/provisioner/provisioner.go"
-            "workspace-server/internal/provisioner/provisioner_test.go"
-            # Pre-RFC#523 persona-fallback / org-helper paths. The L1
-            # fail-closed runs BEFORE these writers; downstream silent-strip
-            # also covers them. See applyAgentGitHTTPCreds doc-comment.
-            "workspace-server/internal/handlers/agent_git_identity.go"
-            "workspace-server/internal/handlers/org_helpers.go"
-            "workspace-server/internal/handlers/org.go"
-            # CP→platform admin auth (NOT a tenant env write).
-            "workspace-server/internal/provisioner/cp_provisioner.go"
-          )
-
-          # Build an extended-regex alternation of forbidden keys.
-          KEY_ALT="$(IFS='|'; echo "${FORBIDDEN_KEYS[*]}")"
-
-          # Find candidate files: Go non-test sources that contain a
-          # tenant-writer surface marker.
-          mapfile -t CANDIDATES < <(
-            grep -rlE --include='*.go' --exclude='*_test.go' \
-              "${SURFACE_PATTERN}" . 2>/dev/null \
-            | sed 's|^\./||' \
-            | sort -u
-          )
-
-          if [ "${#CANDIDATES[@]}" -eq 0 ]; then
-            echo "OK No tenant-writer-surface files found in tree (unexpected, but not a lint failure)."
-            exit 0
-          fi
-
-          HITS=""
-          for f in "${CANDIDATES[@]}"; do
-            # Skip exempt files.
-            skip=0
-            for ex in "${EXEMPT_FILES[@]}"; do
-              if [ "$f" = "$ex" ]; then skip=1; break; fi
-            done
-            [ "$skip" = "1" ] && continue
-
-            # File contains a surface marker; now grep for a forbidden
-            # key NAME. We require a QUOTED-literal match to avoid
-            # firing on a comment like "// also handle GITEA_TOKEN".
-            #
-            # The literal form catches:
-            #   - os.Getenv("GITEA_TOKEN")
-            #   - envVars["GITEA_TOKEN"] = ...
-            #   - {envKey: "GITEA_TOKEN", tenantKey: "GITEA_TOKEN"}
-            # but not:
-            #   - // see GITEA_TOKEN below   (no quotes)
-            found=$(grep -nE "\"(${KEY_ALT})\"" "$f" 2>/dev/null || true)
-            if [ -n "$found" ]; then
-              HITS="${HITS}--- ${f} ---\n${found}\n"
-            fi
-          done
-
-          if [ -n "$HITS" ]; then
-            echo "::error::Task #146 lint: repo-host token name(s) quoted in a tenant-writer-surface file:"
-            printf "$HITS"
-            echo ""
-            echo "These files reference a tenant-writer surface (workspace_secrets,"
-            echo "seedAllowList, /settings/secrets, containerEnv, userData, etc.)"
-            echo "AND quote a repo-host token name (GITEA_TOKEN/GITHUB_TOKEN/…)."
-            echo "Per RFC#523 threat model, tenant workspaces MUST NOT receive"
-            echo "operator-scope repo-host tokens. If your code legitimately needs"
-            echo "to reference one of these names in a tenant-writer file (e.g."
-            echo "a deny-set definition or silent-strip list), add the file to"
-            echo "EXEMPT_FILES with a one-line justification — reviewer signoff"
-            echo "required."
-            exit 1
-          fi
-
-          echo "OK No tenant-writer-surface file co-mentions a repo-host token literal."

.gitea/workflows/lint-required-workflows-docker-host-pinned.yml

@@ -1,164 +0,0 @@
-name: lint-required-workflows-docker-host-pinned
-
-# Fail-closed lint that catches workflows touching docker.sock without
-# pinning `runs-on:` to a Linux-only label.
-#
-# Class defect (internal#512 + mc#1529 + today's oc#81/82/83 + autogen#8):
-# the `ubuntu-latest` label is advertised by BOTH the Linux operator-host
-# runners (molecule-runner-*) AND the Windows act_runner v1.0.3 on
-# hongming-pc-runner-*. Job placement is non-deterministic. When a docker-
-# bound job lands on a Windows runner, `docker run`/`docker login`/
-# `docker compose` fail with platform-specific errors ("protocol not
-# available", "cannot exec", etc.) — placement-dependent, not transient.
-#
-# This lint enforces the convention: any workflow whose YAML body
-# contains a docker exec (`docker run|build|buildx|compose|pull|push|
-# exec|tag|login|cp|inspect|ps` OR `docker/build-push-action|docker/
-# login-action|docker/setup-buildx`) MUST pin every job's `runs-on:` to
-# one of:
-#   - docker-host  (general docker.sock work — molecule-runner-*)
-#   - publish      (image build/push — molecule-runner-publish-*)
-#
-# Comments and heredoc/markdown bodies that merely MENTION docker are
-# excluded by the detection rule (see scan.py below).
-#
-# Per `feedback_never_skip_ci`: this is fail-closed (exit 1 on miss).
-
-on:
-  pull_request:
-    paths:
-      - '.gitea/workflows/**'
-  push:
-    branches: [main, staging]
-    paths:
-      - '.gitea/workflows/**'
-
-permissions:
-  contents: read
-
-env:
-  GITHUB_SERVER_URL: https://git.moleculesai.app
-
-jobs:
-  lint-docker-host-pin:
-    name: Lint docker-host pin on docker-touching workflows
-    runs-on: docker-host
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Scan workflows for docker-bound jobs missing docker-host/publish pin
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import os
-          import re
-          import sys
-
-          # Docker-step detection: real exec, not just word-mention in comments.
-          # We strip comment-only lines, then look for the docker subcommand
-          # tokens at word-boundary, OR uses: docker/* actions.
-          DOCKER_EXEC = re.compile(
-              r'(?<!\w)docker\s+(run|build|buildx|compose|pull|push|exec|tag|login|cp|inspect|ps)\b'
-          )
-          DOCKER_ACTION = re.compile(
-              r'uses:\s*docker/(build-push-action|login-action|setup-buildx-action|setup-qemu-action)'
-          )
-          # Detect a job header line like `  myjob:` (2-space indent) AND its runs-on.
-          JOB_HEADER = re.compile(r'^( {2})([a-zA-Z0-9_-]+):\s*$')
-          RUNS_ON    = re.compile(r'^( {4})runs-on:\s*(.+?)\s*$')
-
-          ALLOWED_LABELS = {'docker-host', 'publish'}
-
-          fails = []
-          warnings = []
-
-          # Gitea is SSOT for molecule-core CI per task #347 / memory
-          # reference_molecule_core_actions_gitea_only. The legacy
-          # .github/workflows/ tree was deleted in SSOT-Instance-4 (#331).
-          roots = []
-          for root in ('.gitea/workflows',):
-              if os.path.isdir(root):
-                  roots.append(root)
-
-          for root in roots:
-              for fn in sorted(os.listdir(root)):
-                  if not (fn.endswith('.yml') or fn.endswith('.yaml')):
-                      continue
-                  path = os.path.join(root, fn)
-                  with open(path) as f:
-                      raw_lines = f.readlines()
-
-                  # Parse job headers + their runs-on. Simple line scan; relies on
-                  # 2-space job indent + 4-space runs-on indent under `jobs:`.
-                  jobs = []
-                  current = None
-                  in_jobs = False
-                  for i, line in enumerate(raw_lines, 1):
-                      if re.match(r'^jobs:\s*$', line):
-                          in_jobs = True
-                          continue
-                      if not in_jobs:
-                          continue
-                      mh = JOB_HEADER.match(line)
-                      if mh:
-                          if current:
-                              current['end'] = i - 1
-                              jobs.append(current)
-                          current = {'name': mh.group(2), 'line': i, 'end': len(raw_lines), 'runs_on': None}
-                          continue
-                      mr = RUNS_ON.match(line)
-                      if mr and current and current['runs_on'] is None:
-                          current['runs_on'] = mr.group(2).strip()
-                  if current:
-                      jobs.append(current)
-
-                  for j in jobs:
-                      # Strip pure-comment lines for docker-exec detection so
-                      # documentation comments don't trigger the lint. Scan the
-                      # current job body only: a workflow may contain one
-                      # docker-bound job and several harmless metadata jobs.
-                      job_lines = raw_lines[j['line'] - 1:j['end']]
-                      scan_text = ''.join(
-                          l for l in job_lines
-                          if not re.match(r'^\s*#', l)
-                      )
-                      has_docker = bool(DOCKER_EXEC.search(scan_text)) or bool(DOCKER_ACTION.search(scan_text))
-                      if not has_docker:
-                          continue
-                      ro = j['runs_on']
-                      if ro is None:
-                          # Reusable workflow caller (`uses:` instead of `runs-on:`) —
-                          # skip; rule enforced in the called workflow.
-                          continue
-                      # Strip surrounding [ ] and quotes.
-                      ro_norm = ro.strip('[]').strip().strip('"\'')
-                      # Multi-label "[a, b]" — split.
-                      labels = [t.strip().strip('"\'') for t in ro_norm.split(',') if t.strip()]
-                      if any(lbl in ALLOWED_LABELS for lbl in labels):
-                          continue
-                      # Allow caller-supplied label expressions; spell the
-                      # marker indirectly so Gitea's expression parser does
-                      # not try to parse this Python heredoc.
-                      expression_marker = '$' + '{{'
-                      if any(expression_marker in lbl for lbl in labels):
-                          continue
-                      fails.append(
-                          f"{path}:{j['line']}: job `{j['name']}` uses docker but runs-on={ro!r} "
-                          f"(must be one of {sorted(ALLOWED_LABELS)})"
-                      )
-
-          if fails:
-              print("FAIL: docker-bound jobs missing docker-host/publish pin:")
-              for f in fails:
-                  print(f"  - {f}")
-              print()
-              print("Why this rule exists (internal#512 + mc#1529):")
-              print("  Bare `ubuntu-latest` is advertised by BOTH Linux operator-host")
-              print("  runners AND Windows hongming-pc-runner-* (act_runner v1.0.3).")
-              print("  Docker-bound jobs that land on Windows fail non-deterministically.")
-              print("  Pin to `docker-host` (general) or `publish` (image build/push).")
-              sys.exit(1)
-
-          print("OK: all docker-bound jobs are pinned to docker-host or publish.")
-          PY

.gitea/workflows/lint-shellcheck-arm64-pilot.yml

@@ -1,117 +0,0 @@
-name: Lint shellcheck (arm64 pilot)
-
-# Mac-CI dual-track pilot (#233). ADDITIVE / NOT REQUIRED.
-#
-# Validates the arm64 self-hosted lane (no docker.sock, no privileged
-# ops) before any required gate moves onto it.
-#
-# Runner label mapping (2026-05-22 fix): the actual Mac mini runner
-# registered in this Gitea ships labels
-#   ["self-hosted","macos-self-hosted-arm64","arm64-darwin"]
-# — no plain `arm64`. The earlier `runs-on: [self-hosted, arm64]`
-# could not match any registered runner so every fire of this workflow
-# was assigned task_id=0 / runner_id=NULL → Gitea cancelled it. The
-# rows showed up as Cancelled in the action status feed (not Failed)
-# but the lane never actually ran. Workflow now selects on
-# `arm64-darwin` which is the canonical Mac-arm64 label per the
-# Mac mini's registration (per internal#494 capability-honest labels).
-#
-# If we later want to add a Linux-arm64 runner to the same lane, add
-# both labels to that runner's registration AND broaden the selector
-# here — don't rename `arm64-darwin` (it's Mac-specific by design and
-# `feedback_pc2_runner_labels_must_stay_narrow` rule applies).
-#
-# Pairs with internal#543 (RFC: Mac arm64 multi-arch runner-base) and
-# internal#494 (multi-arch runner-base capability-honest labels).
-# No paths: filter on purpose (feedback_path_filtered_workflow_cant_be_required).
-
-on:
-  pull_request:
-    branches:
-      - main
-      - staging
-  push:
-    branches:
-      - main
-
-permissions:
-  contents: read
-
-jobs:
-  shellcheck-arm64:
-    name: shellcheck-arm64 (pilot)
-    runs-on: [self-hosted, arm64-darwin]
-    # NOT a required check; safe to sit pending until Mac runner is up.
-    # If the Mac runner has trouble pulling actions/checkout we fall
-    # back to a plain git clone (see step 'fallback clone').
-    timeout-minutes: 10
-    env:
-      GITHUB_SERVER_URL: https://git.moleculesai.app
-    steps:
-      - name: Identify runner
-        run: |
-          set -eu
-          echo "arch=$(uname -m)"
-          echo "kernel=$(uname -sr)"
-          echo "shell=$BASH_VERSION"
-          # Sanity: must actually be arm64. If amd64 sneaks in here,
-          # fail fast — that means the label routing is wrong.
-          case "$(uname -m)" in
-            aarch64|arm64) echo "arm64 confirmed" ;;
-            *) echo "ERROR: expected arm64, got $(uname -m)"; exit 1 ;;
-          esac
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Install shellcheck (arm64)
-        continue-on-error: true
-        run: |
-          set -eu
-          if command -v shellcheck >/dev/null 2>&1; then
-            echo "shellcheck already present: $(shellcheck --version | head -1)"
-          else
-            # Prefer apt if the runner base ships it; else download arm64 binary.
-            if command -v apt-get >/dev/null 2>&1; then
-              sudo apt-get update -qq
-              sudo apt-get install -y --no-install-recommends shellcheck
-            else
-              SC_VER=v0.10.0
-              curl -fsSL "https://github.com/koalaman/shellcheck/releases/download/${SC_VER}/shellcheck-${SC_VER}.linux.aarch64.tar.xz" \
-                | tar -xJf - --strip-components=1
-              sudo mv shellcheck /usr/local/bin/
-            fi
-          fi
-          shellcheck --version | head -2
-
-      - name: Run shellcheck on .gitea/scripts/*.sh
-        continue-on-error: true
-        run: |
-          set -eu
-          # Only the scripts we control under .gitea/scripts. Pilot
-          # scope is intentionally narrow — broaden in a follow-up
-          # once the lane is proven.
-          if ! command -v shellcheck >/dev/null 2>&1; then
-            echo "WARN: shellcheck binary not found — skipping (pilot mode)"
-            exit 0
-          fi
-          # NOTE: macOS ships Bash 3.2 (Apple license), no `mapfile`
-          # (Bash 4+ builtin). Mac mini runner empirically failed at
-          # `mapfile: command not found` (run 79275 / task 145654).
-          # Use the portable `while read` pattern instead — works on
-          # both Bash 3.2 (macOS) and Bash 4+ (Linux).
-          TARGETS=()
-          while IFS= read -r f; do
-            TARGETS+=("$f")
-          done < <(find .gitea/scripts -maxdepth 2 -type f -name '*.sh' | sort)
-          if [ "${#TARGETS[@]}" -eq 0 ]; then
-            echo "No .sh files found under .gitea/scripts — nothing to check"
-            exit 0
-          fi
-          echo "Checking ${#TARGETS[@]} file(s):"
-          printf '  %s\n' "${TARGETS[@]}"
-          # SC1091 = couldn't follow non-constant source; expected for
-          # CI-time analysis without the full runtime layout.
-          shellcheck --severity=error --exclude=SC1091 "${TARGETS[@]}"

.gitea/workflows/publish-canvas-image.yml

@@ -42,13 +42,7 @@ permissions:
   packages: write
 
 env:
-  # SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
-  # sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
-  # bootstrap fallback. When the org var is set, the fallback becomes dead code and
-  # switching accounts/regions is a one-line change at the org level (instead of
-  # touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
-  # use below in this repo's staging-verify.yml.
-  IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/canvas
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas
   GITHUB_SERVER_URL: https://git.moleculesai.app
 
 jobs:
@@ -73,17 +67,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
-      # Keep Docker auth/buildx state inside the job temp dir. Publish
-      # runners can inherit a HOME/DOCKER_CONFIG path that is host-owned
-      # and not writable from the job container; docker login otherwise
-      # fails before the image build starts.
-      - name: Prepare writable Docker config
-        run: |
-          set -euo pipefail
-          export DOCKER_CONFIG="$RUNNER_TEMP/docker-config"
-          mkdir -p "$DOCKER_CONFIG/buildx/certs"
-          echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
-
       - name: Log in to ECR
         env:
           IMAGE_NAME: ${{ env.IMAGE_NAME }}

.gitea/workflows/publish-runtime-autobump.yml

@@ -0,0 +1,152 @@
+name: publish-runtime-autobump
+
+# Auto-bump-on-workspace-edit half of the publish pipeline.
+#
+# Why this file exists (issue #351):
+#   Gitea Actions does not correctly disambiguate `paths:` from `tags:`
+#   when both are bundled under a single `on.push` key. The result is
+#   that tag pushes get filtered out and `publish-runtime.yml` never
+#   fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
+#   because PYPI_TOKEN was absent (publishes would have failed anyway).
+#
+#   Split design:
+#     - publish-runtime.yml         : on.push.tags only        (the publisher)
+#     - publish-runtime-autobump.yml: on.push.branches+paths   (this file — the version-bumper)
+#
+#   This file computes the next version from PyPI's latest, pushes a
+#   `runtime-v$VERSION` tag, and exits. The tag push then triggers
+#   publish-runtime.yml via its tags-only trigger.
+#
+# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
+# so concurrent workspace pushes serialize at the bump step. Without
+# this, two pushes minutes apart could both read PyPI latest=0.1.129
+# and try to tag 0.1.130 simultaneously, only one of which would land.
+
+on:
+  # Run on PR pushes to post a success status so Gitea can merge the PR.
+  # All steps use continue-on-error: true so operational failures
+  # (PyPI unreachable, DISPATCH_TOKEN missing) do not block merge.
+  pull_request:
+    paths:
+      - "workspace/**"
+  # Bump-and-tag on main/staging push (the actual operational trigger).
+  push:
+    branches:
+      - main
+      - staging
+    paths:
+      - "workspace/**"
+  # Manual dispatch — useful when Gitea Actions API (/actions/*) is
+  # unreachable (e.g. act_runner 404 on Gitea 1.22.6) and we cannot
+  # re-trigger via curl.
+  workflow_dispatch:
+
+permissions:
+  contents: write  # required to push tags back
+
+concurrency:
+  group: publish-runtime
+  cancel-in-progress: false
+
+jobs:
+  # PR-validation path: always succeeds so Gitea can merge workflow-only PRs.
+  # Operational failures (PyPI unreachable, missing DISPATCH_TOKEN) are
+  # surfaced via continue-on-error: true rather than blocking the merge.
+  # The actual bump work happens on the main/staging push after merge.
+  # bp-exempt: advisory validation for runtime publication; not a branch-protection gate.
+  pr-validate:
+    runs-on: ubuntu-latest
+    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
+    continue-on-error: true  # do not block PR merge on operational failures
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Validate PyPI connectivity (best-effort)
+        run: |
+          set -eu
+          echo "=== Checking PyPI accessibility ==="
+          LATEST=$(curl -fsS --retry 3 --max-time 10 \
+            https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
+            | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])" \
+            || echo "PyPI unreachable (non-blocking for PR validation)")
+          echo "Latest: ${LATEST:-unknown}"
+
+  # Actual bump-and-tag: runs on main/staging pushes, posts real success/failure.
+  # No continue-on-error — operational failures here trip the main-red
+  # watchdog, which is the desired signal for infrastructure degradation.
+  # bp-exempt: post-merge tag publication side effect; CI / all-required gates source changes.
+  bump-and-tag:
+    runs-on: ubuntu-latest
+    # Only fire on push events (main/staging after PR merge). Pull_request
+    # events are handled by pr-validate above; we do NOT bump on every
+    # push-synchronize because that would race with the PR head.
+    #
+    # NOTE: the prior condition `github.event.pull_request.base.ref == ''`
+    # was broken — on a PR-merge push in Gitea Actions, the pull_request
+    # context is still attached (base.ref='main'), so the condition always
+    # evaluated to false and bump-and-tag was permanently skipped.
+    if: github.event_name == 'push'
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 1
+
+      - name: Fetch tags for collision check
+        run: git fetch origin --tags --depth=1
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Compute next version from PyPI latest
+        id: bump
+        run: |
+          set -eu
+          LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
+            | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
+          MAJOR=$(echo "$LATEST" | cut -d. -f1)
+          MINOR=$(echo "$LATEST" | cut -d. -f2)
+          PATCH=$(echo "$LATEST" | cut -d. -f3)
+          VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
+          echo "PyPI latest=$LATEST -> next=$VERSION"
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
+            exit 1
+          fi
+          if git tag --list | grep -qx "runtime-v$VERSION"; then
+            echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
+            exit 1
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Push runtime-v$VERSION tag
+        env:
+          DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+          VERSION: ${{ steps.bump.outputs.version }}
+          GITEA_URL: https://git.moleculesai.app
+        run: |
+          set -eu
+          if [ -z "$DISPATCH_TOKEN" ]; then
+            echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
+            exit 1
+          fi
+          git config user.name  "publish-runtime autobump"
+          git config user.email "publish-runtime@moleculesai.app"
+          git tag -a "runtime-v$VERSION" \
+            -m "Auto-bump on workspace/** edit on $GITHUB_REF" \
+            -m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
+            -m "publish-runtime.yml will pick up this tag and upload to PyPI"
+          # Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
+          # ensures the resulting tag-push event is dispatched to
+          # publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
+          # trigger downstream workflows.
+          git remote set-url origin "${GITEA_URL#https://}"
+          git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
+          git push origin "runtime-v$VERSION"
+          echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"

.gitea/workflows/publish-runtime.yml

@@ -0,0 +1,345 @@
+name: publish-runtime
+
+# Gitea Actions port of .github/workflows/publish-runtime.yml.
+#
+# Ported 2026-05-10 (issue #206). Key differences from the GitHub version:
+#   - Gitea Actions reads .gitea/workflows/, not .github/workflows/
+#   - Dropped `environment: pypi-publish` — Gitea Actions does not support
+#     named environments or OIDC trusted publishers
+#   - Replaced `pypa/gh-action-pypi-publish@release/v1` (OIDC) with
+#     `twine upload` using PYPI_TOKEN secret — same mechanism as a local
+#     `python -m twine upload` with a PyPI token
+#   - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
+#     — Gitea Actions exposes github.ref (the full ref) but not ref_name
+#   - Dropped `merge_group` trigger (Gitea has no merge queue)
+#
+# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
+# `workspace/**` path-filter trigger in PR #349.
+#
+# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
+# file. Bundling `paths` with `tags` under a single `on.push` key caused
+# Gitea Actions to never dispatch the workflow for tag-push events (0
+# runs in `action_run` for workflow_id='publish-runtime.yml' since the
+# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
+# 0.1.129 despite a v1.0.0 Gitea tag existing).
+#
+# The auto-bump-on-workspace-edit trigger now lives in
+# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
+# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
+# which THIS file then picks up via the tags-only trigger below.
+#
+# This decoupling means Gitea's path-vs-tag evaluator never has to
+# disambiguate — each file has a single unambiguous trigger shape.
+#
+# PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
+# Set via: repo Settings → Actions → Variables and Secrets → New Secret.
+# The token should be a PyPI API token scoped to molecule-ai-workspace-runtime.
+#
+# The DISPATCH_TOKEN cascade (git push to template repos) is unchanged —
+# it uses the Gitea API directly and was already Gitea-compatible.
+
+on:
+  push:
+    tags:
+      - "runtime-v*"
+  workflow_dispatch:
+  # 2026-05-11 (root cause of #351 / 0 runs ever):
+  # Gitea 1.22.6's workflow parser rejects `workflow_dispatch.inputs.version`
+  # with "unknown on type" — it mis-treats the inputs sub-keys as top-level
+  # `on:` event types. Log line:
+  #   actions/workflows.go:DetectWorkflows() [W] ignore invalid workflow
+  #   "publish-runtime.yml": unknown on type: map["version": {...}]
+  # That `[W] ignore invalid workflow` is silent UX — the workflow never
+  # registers, so it never fires for ANY event (push.tags included).
+  # Removing the inputs block restores parsing. Manual dispatch from the
+  # Gitea UI now triggers the PyPI auto-bump fallback in `Derive version`
+  # below (no `inputs.version` to read).
+
+permissions:
+  contents: read
+
+# Serialize publishes so two concurrent tag pushes don't both compute
+# "latest+1" and race on PyPI upload. The second one waits.
+concurrency:
+  group: publish-runtime
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    # Dedicated publish/release lane (internal#462 / #394 / #399). Ship
+    # path (on: push tag runtime-v*) — reserved capacity, never FIFO
+    # behind PR-CI. `publish` resolves only to molecule-runner-publish-*.
+    runs-on: publish
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: Derive version (tag or PyPI auto-bump)
+        id: version
+        run: |
+          if echo "$GITHUB_REF" | grep -q "^refs/tags/runtime-v"; then
+            # Tag is `runtime-vX.Y.Z` — strip the prefix.
+            VERSION="${GITHUB_REF#refs/tags/runtime-v}"
+          else
+            # workflow_dispatch path (no inputs supported on Gitea 1.22.6) or
+            # any other non-tag trigger: derive from PyPI latest + patch bump.
+            LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
+              | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
+            MAJOR=$(echo "$LATEST" | cut -d. -f1)
+            MINOR=$(echo "$LATEST" | cut -d. -f2)
+            PATCH=$(echo "$LATEST" | cut -d. -f3)
+            VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
+            echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
+          fi
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
+            echo "::error::version $VERSION does not match PEP 440"
+            exit 1
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Publishing molecule-ai-workspace-runtime $VERSION"
+
+      - name: Install build tooling
+        run: pip install build twine
+
+      - name: Build package from workspace/
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "${{ steps.version.outputs.version }}" \
+            --out "${{ runner.temp }}/runtime-build"
+
+      - name: Build wheel + sdist
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: python -m build
+
+      - name: Capture wheel SHA256 for cascade content-verification
+        id: wheel_hash
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: |
+          set -eu
+          WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
+          if [ -z "$WHEEL" ]; then
+            echo "::error::No .whl in dist/ — \`python -m build\` must have failed silently"
+            exit 1
+          fi
+          HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
+          echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
+          echo "Local wheel SHA256 (pre-upload): ${HASH}"
+          echo "Wheel filename: $(basename "$WHEEL")"
+
+      - name: Verify package contents (sanity)
+        working-directory: ${{ runner.temp }}/runtime-build
+        run: |
+          python -m twine check dist/*
+          python -m venv /tmp/smoke
+          /tmp/smoke/bin/pip install --quiet dist/*.whl
+          /tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
+
+      - name: Publish to PyPI
+        # working-directory matches the preceding Build/Verify steps. Without
+        # this, twine runs from the default workspace checkout dir where
+        # `dist/` doesn't exist and fails with:
+        #   ERROR InvalidDistribution: Cannot find file (or expand pattern): 'dist/*'
+        # Caught on the first-ever successful dispatch of this workflow
+        # (run 5097, 2026-05-11 02:08Z) — every other step in the publish
+        # job already had this working-directory; Publish was missing it.
+        working-directory: ${{ runner.temp }}/runtime-build
+        env:
+          # PYPI_TOKEN: repository secret scoped to molecule-ai-workspace-runtime.
+          # Set via: Settings → Actions → Variables and Secrets → New Secret.
+          # Format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          if [ -z "$PYPI_TOKEN" ]; then
+            echo "::error::PYPI_TOKEN secret is not set — set it at Settings → Actions → Variables and Secrets → New Secret."
+            echo "::error::Required format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+            exit 1
+          fi
+          python -m twine upload \
+            --verbose \
+            --repository pypi \
+            --username __token__ \
+            --password "$PYPI_TOKEN" \
+            dist/*
+
+  cascade:
+    needs: publish
+    # Publish/release lane (internal#462) — downstream of the runtime
+    # publish ship job; keep it on the reserved lane too.
+    runs-on: publish
+    steps:
+      - name: Wait for PyPI to propagate the new version
+        env:
+          RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
+          EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
+        run: |
+          set -eu
+          if [ -z "$EXPECTED_SHA256" ]; then
+            echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
+            exit 1
+          fi
+          python -m venv /tmp/propagation-probe
+          PROBE=/tmp/propagation-probe/bin
+          $PROBE/pip install --upgrade --quiet pip
+          for i in $(seq 1 30); do
+            if $PROBE/pip install \
+                  --quiet \
+                  --no-cache-dir \
+                  --force-reinstall \
+                  --no-deps \
+                  "molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
+                  >/dev/null 2>&1; then
+              INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
+                          | awk -F': ' '/^Version:/{print $2}')
+              if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
+                echo "✓ PyPI resolved $RUNTIME_VERSION (install check)"
+                break
+              fi
+            fi
+            if [ $i -eq 30 ]; then
+              echo "::error::pip install --no-cache-dir molecule-ai-workspace-runtime==${RUNTIME_VERSION} never resolved within ~5 min."
+              echo "::error::Refusing to fan out cascade against a potentially stale PyPI index."
+              exit 1
+            fi
+            echo "  [$i/30] waiting for PyPI to propagate ${RUNTIME_VERSION}..."
+            sleep 4
+          done
+
+          # Stage (b): download wheel + SHA256 compare against what we built.
+          # Catches Fastly stale-content serving old bytes under a new version URL.
+          #
+          # Caught run 5196 (first-ever successful publish, 2026-05-11): the
+          # previous one-liner `HASH=$(pip download ... && sha256sum ...)`
+          # captured pip's stdout (`Collecting molecule-ai-workspace-runtime
+          # ==X.Y.Z`) into HASH, then the SHA comparison failed against the
+          # leaked `Collecting...` string. `2>/dev/null` silences stderr but
+          # NOT stdout; pip writes its progress to stdout by default.
+          # Fix: split into two steps, silence pip's stdout explicitly, capture
+          # only sha256sum's output into HASH.
+          python -m pip download \
+            --no-deps \
+            --no-cache-dir \
+            --dest /tmp/wheel-probe \
+            --quiet \
+            "molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
+            >/dev/null 2>&1
+          HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
+          if [ "$HASH" != "$EXPECTED_SHA256" ]; then
+            echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
+            echo "::error::Expected: $EXPECTED_SHA256"
+            echo "::error::Got:      $HASH"
+            echo "::error::Fastly may be serving stale content. Refusing to fan out cascade."
+            exit 1
+          fi
+          echo "✓ PyPI CDN verified (SHA256 match)"
+
+      - name: Fan out via push to .runtime-version
+        env:
+          # Gitea PAT with write:repository scope on the 8 cascade-active
+          # template repos. Used for git push to each template repo's main
+          # branch, which trips their `on: push: branches: [main]` trigger
+          # on publish-image.yml.
+          DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+          RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
+        run: |
+          set +e   # don't abort on a single repo failure — collect them all
+
+          if [ -z "$DISPATCH_TOKEN" ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
+              echo "::warning::set it at Settings → Actions → Variables and Secrets → New Secret."
+              exit 0
+            fi
+            echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
+            echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version."
+            exit 1
+          fi
+          VERSION="$RUNTIME_VERSION"
+          if [ -z "$VERSION" ]; then
+            echo "::error::publish job did not expose a version output"
+            exit 1
+          fi
+
+          GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
+          TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
+          FAILED=""
+          SKIPPED=""
+
+          git config --global user.name  "publish-runtime cascade"
+          git config --global user.email "publish-runtime@moleculesai.app"
+
+          WORKDIR="$(mktemp -d)"
+          for tpl in $TEMPLATES; do
+            REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
+            CLONE="$WORKDIR/$tpl"
+
+            HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
+              -H "Authorization: token $DISPATCH_TOKEN" \
+              "$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
+            if [ "$HTTP" = "404" ]; then
+              echo "↷ $tpl has no publish-image.yml — soft-skip"
+              SKIPPED="$SKIPPED $tpl"
+              continue
+            fi
+
+            attempt=0
+            success=false
+            while [ $attempt -lt 3 ]; do
+              attempt=$((attempt + 1))
+              rm -rf "$CLONE"
+              if ! git clone --depth=1 \
+                  "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
+                  "$CLONE" >/tmp/clone.log 2>&1; then
+                echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
+                sleep 2
+                continue
+              fi
+
+              cd "$CLONE"
+              echo "$VERSION" > .runtime-version
+
+              if git diff --quiet -- .runtime-version; then
+                echo "✓ $tpl already at $VERSION — no commit needed"
+                success=true
+                cd - >/dev/null
+                break
+              fi
+
+              git add .runtime-version
+              git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
+                -m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
+                >/dev/null
+
+              if git push origin HEAD:main >/tmp/push.log 2>&1; then
+                echo "✓ $tpl pushed $VERSION on attempt $attempt"
+                success=true
+                cd - >/dev/null
+                break
+              fi
+
+              echo "::warning::push $tpl attempt $attempt failed, pull-rebasing"
+              git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
+              cd - >/dev/null
+            done
+
+            if [ "$success" != "true" ]; then
+              FAILED="$FAILED $tpl"
+            fi
+          done
+          rm -rf "$WORKDIR"
+
+          if [ -n "$FAILED" ]; then
+            echo "::error::Cascade incomplete after 3 retries each. Failed:$FAILED"
+            exit 1
+          fi
+          if [ -n "$SKIPPED" ]; then
+            echo "Cascade complete: pinned $VERSION. Soft-skipped (no publish-image.yml):$SKIPPED"
+          else
+            echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
+          fi

.gitea/workflows/publish-workspace-server-image.yml

@@ -25,12 +25,8 @@ name: publish-workspace-server-image
 #   staging-<sha>. Set repo variable or secret PROD_AUTO_DEPLOY_DISABLED=true
 #   to stop production rollout while keeping image publishing enabled.
 #
-# Primary ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
-# Optional staging tenant mirror target:
-#   004947743811.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
+# ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
 # Required secrets: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AUTO_SYNC_TOKEN
-# Staging ECR grants the primary SSOT-managed publisher principal repository
-# policy access, so no persistent staging AWS access keys are required.
 #
 # mc#711: Docker daemon not accessible on ubuntu-latest runner (molecule-canonical-1
 # shows client-only in `docker info` — daemon not running). DinD mount is present but
@@ -47,29 +43,14 @@ on:
 # `cancel-in-progress: false`; that is not acceptable for a workflow with a
 # production deploy job. Per-SHA image tags are immutable, and staging-latest is
 # best-effort last-writer-wins metadata.
-#
-# 2026-05-20 retrigger: run #86994 on mc#1589 merge sha 0f0f1ba2 failed at
-# setup-buildx-action with EACCES on PC2 WSL publish runner — the runner's
-# DOCKER_CONFIG=/home/hongming/.docker-ecr/ dir didn't have a buildx/certs
-# subdir writable by the container's UID 1001. Hot-patched the dir perms;
-# this chore push retriggers the workflow. Proper fix (per-runner
-# DOCKER_CONFIG owned by 1001, internal#597 --env HOME=/home/runner pattern)
-# is tracked as a CI-hygiene follow-up — not in scope here.
 
 permissions:
   contents: read
   packages: write
 
 env:
-  # SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
-  # sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
-  # bootstrap fallback. When the org var is set, the fallback becomes dead code and
-  # switching accounts/regions is a one-line change at the org level (instead of
-  # touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
-  # use below in this repo's staging-verify.yml.
-  IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
-  TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
-  STAGING_TENANT_IMAGE_NAME: ${{ vars.STAGING_ECR_REGISTRY || '004947743811.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
+  TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
 
 jobs:
   build-and-push:
@@ -140,18 +121,6 @@ jobs:
         run: |
           echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
 
-      # Keep Buildx state inside the job temp dir. The publish runner's
-      # inherited DOCKER_CONFIG can point at a host-owned ECR config path
-      # (/home/hongming/.docker-ecr), which caused setup-buildx-action to
-      # fail before image build with EACCES creating buildx/certs.
-      - name: Prepare writable Docker config
-        run: |
-          set -euo pipefail
-          export DOCKER_CONFIG="$RUNNER_TEMP/docker-config"
-          mkdir -p "$DOCKER_CONFIG/buildx/certs"
-          echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
-          docker buildx version
-
       # Build + push platform image (inline ECR auth — mirrors the operator-host
       # approach; credentials come from GITHUB_SECRET_AWS_ACCESS_KEY_ID /
       # GITHUB_SECRET_AWS_SECRET_ACCESS_KEY in Gitea Actions).
@@ -187,14 +156,9 @@ jobs:
             --push .
 
       # Build + push tenant image (Go platform + Next.js canvas in one image).
-      # Push the same build to the staging account too so fresh staging/E2E
-      # tenants can pull without cross-account ECR reads. The staging ECR repo
-      # policy trusts the primary SSOT-managed publisher principal; do not add
-      # separate persistent staging AWS access keys here.
       - name: Build & push tenant image to ECR (staging-<sha> + staging-latest)
         env:
           TENANT_IMAGE_NAME: ${{ env.TENANT_IMAGE_NAME }}
-          STAGING_TENANT_IMAGE_NAME: ${{ env.STAGING_TENANT_IMAGE_NAME }}
           TAG_SHA: staging-${{ steps.tags.outputs.sha }}
           TAG_LATEST: staging-latest
           GIT_SHA: ${{ github.sha }}
@@ -205,19 +169,8 @@ jobs:
         run: |
           set -euo pipefail
           ECR_REGISTRY="${TENANT_IMAGE_NAME%%/*}"
-          STAGING_ECR_REGISTRY="${STAGING_TENANT_IMAGE_NAME%%/*}"
           aws ecr get-login-password --region us-east-2 | \
             docker login --username AWS --password-stdin "${ECR_REGISTRY}"
-          aws ecr get-login-password --region us-east-2 | \
-            docker login --username AWS --password-stdin "${STAGING_ECR_REGISTRY}"
-
-          build_tags=(
-            --tag "${TENANT_IMAGE_NAME}:${TAG_SHA}"
-            --tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}"
-            --tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_SHA}"
-            --tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_LATEST}"
-          )
-
           docker buildx build \
             --file ./workspace-server/Dockerfile.tenant \
             --build-arg NEXT_PUBLIC_PLATFORM_URL= \
@@ -226,7 +179,8 @@ jobs:
             --label "org.opencontainers.image.revision=${GIT_SHA}" \
             --label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
             --label "molecule.workflow.run_id=${GITHUB_RUN_ID}" \
-            "${build_tags[@]}" \
+            --tag "${TENANT_IMAGE_NAME}:${TAG_SHA}" \
+            --tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}" \
             --push .
 
   # bp-exempt: production deploy side-effect; merge is gated by CI / all-required and this job waits for push CI before acting.
@@ -234,18 +188,15 @@ jobs:
     name: Production auto-deploy
     needs: build-and-push
     if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-    # Side-effect deploy only; image publish success is the durable artifact. mc#774
-    continue-on-error: true
     # Publish/release lane (internal#462) — production deploy of a merged
     # fix; reserved capacity, never queued behind PR-CI.
     runs-on: publish
-    timeout-minutes: 90
+    timeout-minutes: 75
     env:
       CP_URL: ${{ vars.PROD_CP_URL || 'https://api.moleculesai.app' }}
       CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
       GITEA_HOST: git.moleculesai.app
       GITEA_TOKEN: ${{ secrets.PROD_AUTO_DEPLOY_CONTROL_TOKEN || secrets.AUTO_SYNC_TOKEN }}
-      CI_STATUS_TIMEOUT_SECONDS: "3600"
       PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
       PROD_AUTO_DEPLOY_CANARY_SLUG: ${{ vars.PROD_AUTO_DEPLOY_CANARY_SLUG || 'hongming' }}
       PROD_AUTO_DEPLOY_SOAK_SECONDS: ${{ vars.PROD_AUTO_DEPLOY_SOAK_SECONDS || '60' }}
@@ -304,19 +255,26 @@ jobs:
           python3 .gitea/scripts/prod-auto-deploy.py assert-enabled
           PLAN="$RUNNER_TEMP/prod-auto-deploy-plan.json"
           TARGET_TAG="$(jq -r '.target_tag' "$PLAN")"
+          BODY="$(jq -c '.body' "$PLAN")"
+
+          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  target_tag: $TARGET_TAG"
+          echo "  body: $BODY"
 
           HTTP_RESPONSE="$RUNNER_TEMP/prod-redeploy-response.json"
+          HTTP_CODE_FILE="$RUNNER_TEMP/prod-redeploy-http-code.txt"
           set +e
-          python3 .gitea/scripts/prod-auto-deploy.py rollout \
-            --plan "$PLAN" \
-            --response "$HTTP_RESPONSE"
-          ROLLOUT_EXIT=$?
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" > "$HTTP_CODE_FILE"
           set -e
 
-          if [ ! -s "$HTTP_RESPONSE" ]; then
-            jq -nc --arg error "rollout command exited $ROLLOUT_EXIT before writing a response" \
-              '{ok:false, results:[], error:$error}' > "$HTTP_RESPONSE"
-          fi
+          HTTP_CODE="$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")"
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+          echo "HTTP $HTTP_CODE"
           jq '{ok, result_count: (.results // [] | length)}' "$HTTP_RESPONSE" || true
 
           {
@@ -324,6 +282,7 @@ jobs:
             echo ""
             echo "**Commit:** \`${GITHUB_SHA:0:7}\`"
             echo "**Target tag:** \`$TARGET_TAG\`"
+            echo "**HTTP:** $HTTP_CODE"
             echo ""
             echo "### Per-tenant result"
             echo ""
@@ -332,15 +291,15 @@ jobs:
             jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \((.error // "") != "") |"' "$HTTP_RESPONSE" || true
           } >> "$GITHUB_STEP_SUMMARY"
 
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
+            exit 1
+          fi
           OK="$(jq -r '.ok' "$HTTP_RESPONSE")"
           if [ "$OK" != "true" ]; then
             echo "::error::redeploy-fleet reported ok=false; production rollout halted."
             exit 1
           fi
-          if [ "$ROLLOUT_EXIT" -ne 0 ]; then
-            echo "::error::redeploy-fleet rollout failed with exit code $ROLLOUT_EXIT."
-            exit "$ROLLOUT_EXIT"
-          fi
 
       - name: Verify reachable tenants report this SHA
         if: ${{ steps.plan.outputs.enabled == 'true' }}

.gitea/workflows/qa-review.yml

@@ -89,7 +89,6 @@ on:
 permissions:
   contents: read
   pull-requests: read
-  secrets: read
 
 jobs:
   # bp-exempt: PR review bot signal; required merge state is enforced by CI / all-required.

.gitea/workflows/redeploy-tenants-on-main.yml

@@ -151,11 +151,6 @@ jobs:
             exit 1
           fi
 
-          # confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
-          # contract (cp#228 / task #308) for fleet-wide intent. Empty body
-          # / {confirm:false} / {only_slugs:[]} → 400. This caller redeploys
-          # the entire prod fleet (canary + fan-out), no slug scoping, so
-          # confirm:true is correct.
           BODY=$(jq -nc \
             --arg tag "$TARGET_TAG" \
             --arg canary "$CANARY_SLUG" \
@@ -167,8 +162,7 @@ jobs:
               canary_slug: $canary,
               soak_seconds: $soak,
               batch_size: $batch,
-              dry_run: $dry,
-              confirm: true
+              dry_run: $dry
             }')
 
           echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"

.gitea/workflows/redeploy-tenants-on-staging.yml

@@ -123,11 +123,6 @@ jobs:
             exit 1
           fi
 
-          # confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
-          # contract (cp#228 / task #308) for fleet-wide intent. Empty body
-          # / {confirm:false} / {only_slugs:[]} → 400. Staging IS the
-          # canary, no slug scoping; this rolls the entire staging fleet,
-          # so confirm:true is correct.
           BODY=$(jq -nc \
             --arg tag "$TARGET_TAG" \
             --arg canary "$CANARY_SLUG" \
@@ -139,8 +134,7 @@ jobs:
               canary_slug: $canary,
               soak_seconds: $soak,
               batch_size: $batch,
-              dry_run: $dry,
-              confirm: true
+              dry_run: $dry
             }')
 
           echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"

.gitea/workflows/review-refire-comments.yml

@@ -1,16 +1,11 @@
-# DEPRECATED — superseded by `.gitea/workflows/sop-checklist.yml`.
+# Consolidated comment dispatcher for manual review/tier refires.
 #
-# The review-refire logic (qa/security/tier slash-command dispatch) has been
-# merged into sop-checklist.yml as the `review-refire` job. This workflow
-# is kept as a no-op stub to avoid a gap during the transition window where
-# this file may be deleted while sop-checklist.yml has not yet been merged.
-#
-# After sop-checklist.yml lands, this file will be deleted (issue #1280).
-#
-# Historical behavior (superseded):
 # Gitea 1.22 queues one run per workflow subscribed to `issue_comment` before
-# evaluating job-level `if:`. Previously this workflow was the single
-# non-SOP comment subscriber for qa/security/tier refire slash commands.
+# evaluating job-level `if:`. SOP-heavy PRs therefore created queue storms when
+# qa-review, security-review, sop-checklist, and sop-tier-refire all
+# listened to comments. This workflow is the single non-SOP comment subscriber:
+# ordinary comments no-op quickly; slash commands post the required status
+# contexts to the PR head SHA.
 
 name: review-refire-comments
 
@@ -28,12 +23,91 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # No-op stub — all refire logic moved to sop-checklist.yml review-refire job.
-  # Kept to avoid transition gap; will be deleted after sop-checklist.yml merges.
   dispatch:
     runs-on: ubuntu-latest
     steps:
-      - name: Deprecated — refire logic moved to sop-checklist.yml
+      - name: Classify comment
+        id: classify
+        env:
+          COMMENT_BODY: ${{ github.event.comment.body }}
+          IS_PR: ${{ github.event.issue.pull_request != null }}
         run: |
-          echo "::warning::review-refire-comments.yml is deprecated. Refire logic is now in sop-checklist.yml review-refire job. This workflow is a no-op stub pending deletion (issue #1280)."
-          exit 0
+          set -euo pipefail
+          {
+            echo "run_qa=false"
+            echo "run_security=false"
+            echo "run_tier=false"
+          } >> "$GITHUB_OUTPUT"
+          if [ "$IS_PR" != "true" ]; then
+            echo "::notice::not a PR comment; no-op"
+            exit 0
+          fi
+          first_line=$(printf '%s\n' "$COMMENT_BODY" | sed -n '1p')
+          case "$first_line" in
+            /qa-recheck*)
+              echo "run_qa=true" >> "$GITHUB_OUTPUT"
+              ;;
+            /security-recheck*)
+              echo "run_security=true" >> "$GITHUB_OUTPUT"
+              ;;
+            /refire-tier-check*)
+              echo "run_tier=true" >> "$GITHUB_OUTPUT"
+              ;;
+            *)
+              echo "::notice::no supported review refire slash command; no-op"
+              ;;
+          esac
+
+      - name: Check out BASE ref for trusted scripts
+        if: |
+          steps.classify.outputs.run_qa == 'true' ||
+          steps.classify.outputs.run_security == 'true' ||
+          steps.classify.outputs.run_tier == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Refire qa-review status
+        if: steps.classify.outputs.run_qa == 'true'
+        env:
+          GITEA_TOKEN: ${{ secrets.RFC_324_TEAM_READ_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.issue.number }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          TEAM: qa
+          TEAM_ID: '20'
+          REVIEW_CHECK_DEBUG: '0'
+          REVIEW_CHECK_STRICT: '0'
+          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+        run: |
+          set -euo pipefail
+          .gitea/scripts/review-refire-status.sh
+
+      - name: Refire security-review status
+        if: steps.classify.outputs.run_security == 'true'
+        env:
+          GITEA_TOKEN: ${{ secrets.RFC_324_TEAM_READ_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.issue.number }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          TEAM: security
+          TEAM_ID: '21'
+          REVIEW_CHECK_DEBUG: '0'
+          REVIEW_CHECK_STRICT: '0'
+          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+        run: |
+          set -euo pipefail
+          .gitea/scripts/review-refire-status.sh
+
+      - name: Refire sop-tier-check status
+        if: steps.classify.outputs.run_tier == 'true'
+        env:
+          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
+          GITEA_HOST: git.moleculesai.app
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.issue.number }}
+          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+          SOP_DEBUG: '0'
+        run: bash .gitea/scripts/sop-tier-refire.sh

.gitea/workflows/runtime-pin-compat.yml

@@ -0,0 +1,101 @@
+name: Runtime Pin Compatibility
+
+# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
+# RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `merge_group:` (no Gitea merge queue) and
+#     `workflow_dispatch:` (no inputs, but the trigger itself is
+#     parser-rejected when inputs are absent in some Gitea 1.22.x
+#     builds; safest to drop entirely — manual runs go via cron-trigger
+#     bump or push-with-paths-filter).
+#   - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
+#     file) instead of the .github/ one.
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on the job (RFC §1 contract).
+#
+# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
+# recurring (controlplane#253). The original failure mode:
+#   1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
+#      requires_dist metadata (incorrect — it actually imports
+#      a2a.server.routes which only exists in a2a-sdk 1.0+)
+#   2. `pip install molecule-ai-workspace-runtime` resolved cleanly
+#   3. `from molecule_runtime.main import main_sync` raised ImportError
+#   4. Every tenant workspace crashed; the canary tenant caught it but
+#      only after 5 hours of degraded staging
+#
+# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
+# top of `workspace/requirements.txt` and smoke-imports. Catches:
+#   - Upstream PyPI yanks
+#   - Bad re-releases of molecule-ai-workspace-runtime
+#   - Already-shipped wheels that stop importing because a transitive
+#     dep moved underneath
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      # Narrow filter: pypi-latest is sensitive only to changes that
+      # affect what we're INSTALLING (requirements.txt) or WHAT THE
+      # CHECK ITSELF DOES (this workflow file). Edits to workspace/
+      # source code don't change what's on PyPI right now, so they
+      # don't change this gate's verdict.
+      - 'workspace/requirements.txt'
+      - '.gitea/workflows/runtime-pin-compat.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace/requirements.txt'
+      - '.gitea/workflows/runtime-pin-compat.yml'
+  # Daily catch for upstream PyPI publishes that break the pin combo
+  # without any change in our repo (e.g. someone re-yanks an a2a-sdk
+  # release or molecule-ai-workspace-runtime publishes a bad bump).
+  schedule:
+    - cron: '0 13 * * *'  # 06:00 PT
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pypi-latest-install:
+    name: PyPI-latest install + import smoke
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking
+    # the PR. Follow-up PR flips this off after surfaced defects are
+    # triaged.
+    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install runtime + workspace requirements
+        # Install order is load-bearing: install the runtime FIRST so pip
+        # honors whatever a2a-sdk constraint the runtime metadata declares
+        # (this is the surface that broke in 2026-04-24 — runtime declared
+        # `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
+        # of workspace/requirements.txt then upgrades a2a-sdk to the
+        # constraint our runtime image actually pins. The import smoke
+        # below verifies the upgraded combination is consistent.
+        run: |
+          python -m venv /tmp/venv
+          /tmp/venv/bin/pip install --upgrade pip
+          /tmp/venv/bin/pip install molecule-ai-workspace-runtime
+          /tmp/venv/bin/pip install -r workspace/requirements.txt
+          /tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import — fail if metadata declares deps that don't satisfy real imports
+        # WORKSPACE_ID is validated at import time by platform_auth.py — EC2
+        # user-data sets it from the cloud-init template; set a placeholder
+        # here so the import smoke doesn't trip on the env-var guard.
+        env:
+          WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
+        run: |
+          /tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"

.gitea/workflows/runtime-prbuild-compat.yml

@@ -0,0 +1,150 @@
+name: Runtime PR-Built Compatibility
+
+# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
+# per RFC internal#219 §1 sweep.
+#
+# Differences from the GitHub version:
+#   - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
+#     (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
+#     finicky without them).
+#   - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
+#     pattern for ci.yml port).
+#   - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
+#   - Workflow-level env.GITHUB_SERVER_URL set.
+#   - `continue-on-error: true` on every job (RFC §1 contract).
+#
+# Companion to `runtime-pin-compat.yml`. That workflow tests what's
+# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
+# PUBLISHED if THIS PR merges.
+#
+# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
+# wheel" job to the original runtime-pin-compat.yml, but both jobs
+# shared a `paths:` filter that was the union of their needs
+# (`workspace/**`). That meant the PyPI-latest job ran on every doc
+# edit even though the upstream PyPI artifact can't change with our
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter that matches the inputs it actually depends on.
+#
+# Catches the failure mode where a PR adds an import requiring a newer
+# SDK than `workspace/requirements.txt` pins:
+#   1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
+#      passes (it imports the OLD main.py from the wheel, not the PR's
+#      new main.py).
+#   2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
+#   3. Tenant images redeploy -> all crash on first boot with ImportError.
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+concurrency:
+  # event_name + sha keeps PR sync and the subsequent staging push on the
+  # same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
+    continue-on-error: true
+    outputs:
+      wheel: ${{ steps.decide.outputs.wheel }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: decide
+        run: |
+          # Inline replacement for dorny/paths-filter — same pattern
+          # PR#372's ci.yml port used. Diffs against the PR base or the
+          # previous push SHA, then matches against the wheel-relevant
+          # path set.
+          #
+          # NOTE: Gitea Actions does not expose github.event.before as a
+          # shell environment variable. The ${{ github.event.before }} template
+          # expression works inside YAML run: blocks but is evaluated to an
+          # empty string for push events, making the ${VAR:-fallback} always
+          # use the fallback. Use GITHUB_EVENT_BEFORE instead — it IS set in
+          # the runner's shell environment for push events.
+          BASE=""
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          elif [ -n "$GITHUB_EVENT_BEFORE" ]; then
+            BASE="$GITHUB_EVENT_BEFORE"
+          fi
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            # New branch or no previous SHA: treat as wheel-relevant.
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
+            git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+          fi
+          if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          CHANGED=$(git diff --name-only "$BASE" HEAD)
+          if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "wheel=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `PR-built wheel + import smoke`. Real work is
+  # gated per-step on `needs.detect-changes.outputs.wheel`.
+  local-build-install:
+    needs: detect-changes
+    name: PR-built wheel + import smoke
+    runs-on: ubuntu-latest
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
+    continue-on-error: true
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.wheel != 'true'
+        run: |
+          echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
+          echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install build tooling
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: pip install build
+      - name: Build wheel from PR source (mirrors publish-runtime.yml)
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Use a fixed test version so the wheel filename is predictable.
+        # Doesn't reach PyPI — this build is local-only for the smoke.
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "0.0.0.dev0+pin-compat" \
+            --out /tmp/runtime-build
+          cd /tmp/runtime-build && python -m build
+      - name: Install built wheel + workspace requirements
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: |
+          python -m venv /tmp/venv-built
+          /tmp/venv-built/bin/pip install --upgrade pip
+          /tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
+          /tmp/venv-built/bin/pip install -r workspace/requirements.txt
+          /tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import the PR-built wheel
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Same script publish-runtime.yml runs against the to-be-PyPI wheel.
+        run: |
+          /tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"

.gitea/workflows/secret-scan.yml

@@ -30,11 +30,6 @@ jobs:
   scan:
     name: Scan diff for credential-shaped strings
     runs-on: ubuntu-latest
-    # Hard CI gate — must complete or the PR is unmergable. 10-minute ceiling
-    # is generous for a diff-scan against a single SHA. If this times out, the
-    # runner is frozen and holding a slot — the step timeout triggers clean
-    # failure, releasing the runner for the next job.
-    timeout-minutes: 10
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -138,14 +133,6 @@ jobs:
             [ -z "$f" ] && continue
             [ "$f" = "$SELF_GITHUB" ] && continue
             [ "$f" = "$SELF_GITEA" ] && continue
-            # Test-fixture exclude (internal#425): the secrets-detector's OWN
-            # unit-test corpus deliberately embeds credential-SHAPED example
-            # strings to exercise the detector. Verified 2026-05-18 synthetic
-            # (fabricated ghp_* fixtures, not real). Without this the scanner
-            # self-trips on its own fixtures and fail-closes every deploy.
-            # Same rationale as the SELF_* excludes above; gate NOT weakened
-            # (all other paths still fully scanned).
-            [ "$f" = "workspace-server/internal/secrets/patterns_test.go" ] && continue
             if [ -n "$DIFF_RANGE" ]; then
               ADDED=$(git diff --no-color --unified=0 "$BASE" "$HEAD" -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
             else

.gitea/workflows/security-review.yml

@@ -16,7 +16,6 @@ on:
 permissions:
   contents: read
   pull-requests: read
-  secrets: read
 
 jobs:
   # bp-exempt: PR security review bot signal; required merge state is enforced by CI / all-required.

.gitea/workflows/sop-checklist.yml

@@ -2,20 +2,24 @@
 #
 # RFC#351 Step 2 of 6 (implementation MVP).
 #
-# === CONSOLIDATION (issue #1280) ===
+# === DESIGN ===
 #
-# This workflow is the SINGLE `issue_comment` subscriber — the logic from
-# `review-refire-comments.yml` has been merged in. Before this change:
-#   - sop-checklist.yml (pre-2026-05-16) → issue_comment:[created,edited,deleted] → runner slot used, job no-oped
-#   - review-refire-comments.yml           → issue_comment:[created]            → runner slot used, job no-oped
-#   → every non-refire comment occupied 2 runner slots for ~800 s each
-#      (~650 no-op runs/day, ~1,300 runner-slot-occupancy-hours/day).
+# Goal: each PR must answer 7 SOP-checklist questions in its body,
+# and each item must have at least one /sop-ack <slug> comment from
+# a non-author peer in the required team. BP requires the
+# `sop-checklist / all-items-acked (pull_request)` status to merge.
 #
-# Fix (PR #1345 / issue #1280):
-#   - ONE workflow, ONE issue_comment:[created] subscription (no edited/deleted)
-#   - all-items-acked job: pull_request_target OR sop slash-command comments
-#   - review-refire job: qa/security/tier refire slash commands
-#   → ~50% reduction in comment-triggered runner occupancy vs pre-fix.
+# Triggers:
+#   - `pull_request_target`: opened, edited, synchronize, reopened
+#       → fires when PR opens, body is edited (refire — RFC#351 §4),
+#         or new code is pushed (head.sha changes → stale status would
+#         be auto-discarded by BP via dismiss_stale_reviews, but the
+#         status itself is per-SHA so we re-post on the new head).
+#   - `issue_comment`: created, edited, deleted
+#       → fires on any new comment so /sop-ack / /sop-revoke take
+#         effect immediately (Gitea 1.22.6 doesn't refire on
+#         pull_request_review per feedback_pull_request_review_no_refire,
+#         so issue_comment is the canonical refire channel).
 #
 # Trust boundary (mirrors RFC#324 §A4 + sop-tier-check security note):
 #   `pull_request_target` (not `pull_request`) — workflow def is loaded
@@ -47,7 +51,7 @@
 #   /sop-ack <slug-or-numeric-alias> [optional note]
 #       — register a peer-ack for one checklist item.
 #       — slug accepts kebab-case, snake_case, or natural-spaces
-#         (all normalized to canonical kebab-case).
+#         (all normalize to canonical kebab-case).
 #       — numeric 1..7 maps via config.items[*].numeric_alias.
 #       — most-recent (user, slug) directive wins.
 #
@@ -57,13 +61,6 @@
 #       — most-recent (user, slug) directive wins, so a later /sop-ack
 #         re-restores the ack.
 #
-#   /sop-n/a <gate> [reason]
-#       — declare a gate (qa-review, security-review) N/A.
-#       — see sop-checklist-config.yaml n/a_gates section.
-#
-#   /qa-recheck /security-recheck /refire-tier-check
-#       — refire the corresponding status check on the PR head.
-#
 # The eval is read-only + idempotent (read PR + comments + team
 # membership, compute, post status). Re-running on any event is safe —
 # the new status overwrites the previous one for the same context.
@@ -82,22 +79,22 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize, reopened, labeled, unlabeled]
   issue_comment:
-    types: [created]   # NOT [created, edited, deleted] — Gitea 1.22.6 holds a runner slot
-    # at job-parsing time, before job-level if: guards run. edited/deleted events
-    # occupied ~1,300 runner-slot-hours/day on this workflow alone during the
-    # 2026-05-16 freeze. Per PR #1345 fix.
+    types: [created, edited, deleted]
 
 permissions:
   contents: read
   pull-requests: read
+  # NOTE: `statuses: write` is the GitHub-Actions name for POST /statuses.
+  # Gitea 1.22.6 may not gate on this permission key (it just checks the
+  # token), but listing it explicitly documents intent for the next
+  # platform-version upgrade.
   statuses: write
-  secrets: read
 
 jobs:
-  # sop-checklist gate: runs on PR lifecycle events OR sop slash commands.
-  # All other comment types (no-op text comments) no longer assign a runner
-  # because this job's if: guard short-circuits before runner assignment.
   all-items-acked:
+    # Run on pull_request_target events always. On issue_comment events,
+    # only when the comment is on a PR (issue_comment fires for issues
+    # too) and the body contains one of the slash-commands.
     if: |
       github.event_name == 'pull_request_target' ||
       (github.event_name == 'issue_comment' &&
@@ -131,95 +128,3 @@ jobs:
             --pr "$PR_NUMBER" \
             --config .gitea/sop-checklist-config.yaml \
             --gitea-host git.moleculesai.app
-
-  # bp-exempt: informational refire handler, not a merge gate. Emits
-  # qa-review/security-review status updates on /qa-recheck et al slash commands.
-  review-refire:
-    if: |
-      github.event_name == 'issue_comment' &&
-      github.event.issue.pull_request != null
-    runs-on: ubuntu-latest
-    steps:
-      - name: Classify comment
-        id: classify
-        env:
-          COMMENT_BODY: ${{ github.event.comment.body }}
-        run: |
-          set -euo pipefail
-          {
-            echo "run_qa=false"
-            echo "run_security=false"
-            echo "run_tier=false"
-          } >> "$GITHUB_OUTPUT"
-          first_line=$(printf '%s\n' "$COMMENT_BODY" | sed -n '1p')
-          case "$first_line" in
-            /qa-recheck*)
-              echo "run_qa=true" >> "$GITHUB_OUTPUT"
-              ;;
-            /security-recheck*)
-              echo "run_security=true" >> "$GITHUB_OUTPUT"
-              ;;
-            /refire-tier-check*)
-              echo "run_tier=true" >> "$GITHUB_OUTPUT"
-              ;;
-            *)
-              echo "::notice::no supported review refire slash command; no-op"
-              ;;
-          esac
-
-      - name: Check out BASE ref for trusted scripts
-        if: |
-          steps.classify.outputs.run_qa == 'true' ||
-          steps.classify.outputs.run_security == 'true' ||
-          steps.classify.outputs.run_tier == 'true'
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          ref: ${{ github.event.repository.default_branch }}
-
-      - name: Refire qa-review status
-        if: steps.classify.outputs.run_qa == 'true'
-        env:
-          # RFC_324_TEAM_READ_TOKEN is read-only (team membership read scope only).
-          # review-refire-status.sh POSTs to /statuses — requires write scope.
-          # SOP_TIER_CHECK_TOKEN carries write:repository + write:issue + read:organization.
-          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
-          GITEA_HOST: git.moleculesai.app
-          REPO: ${{ github.repository }}
-          PR_NUMBER: ${{ github.event.issue.number }}
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          TEAM: qa
-          TEAM_ID: '20'
-          REVIEW_CHECK_DEBUG: '0'
-          REVIEW_CHECK_STRICT: '0'
-        run: |
-          set -euo pipefail
-          .gitea/scripts/review-refire-status.sh
-
-      - name: Refire security-review status
-        if: steps.classify.outputs.run_security == 'true'
-        env:
-          # RFC_324_TEAM_READ_TOKEN is read-only (team membership read scope only).
-          # review-refire-status.sh POSTs to /statuses — requires write scope.
-          # SOP_TIER_CHECK_TOKEN carries write:repository + write:issue + read:organization.
-          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
-          GITEA_HOST: git.moleculesai.app
-          REPO: ${{ github.repository }}
-          PR_NUMBER: ${{ github.event.issue.number }}
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          TEAM: security
-          TEAM_ID: '21'
-          REVIEW_CHECK_DEBUG: '0'
-          REVIEW_CHECK_STRICT: '0'
-        run: |
-          set -euo pipefail
-          .gitea/scripts/review-refire-status.sh
-
-      - name: Refire sop-tier-check status
-        if: steps.classify.outputs.run_tier == 'true'
-        env:
-          GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
-          GITEA_HOST: git.moleculesai.app
-          REPO: ${{ github.repository }}
-          PR_NUMBER: ${{ github.event.issue.number }}
-          SOP_DEBUG: '0'
-        run: bash .gitea/scripts/sop-tier-refire.sh

.gitea/workflows/sop-tier-check.yml

@@ -71,7 +71,6 @@ jobs:
     permissions:
       contents: read
       pull-requests: read
-      secrets: read
     steps:
       - name: Check out base branch (for the script)
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

.gitea/workflows/staging-smoke.yml

@@ -81,11 +81,6 @@ jobs:
       # (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
       # internal#322 — see this PR for the cross-workflow sweep.
       MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_DEFAULT_REGION: us-east-2
-      E2E_AWS_LEAK_CHECK: required
-      E2E_AWS_TERMINATE_LEAKS: '1'
       # MiniMax is the smoke's PRIMARY LLM auth path post-2026-05-04.
       # Switched from hermes+OpenAI after #2578 (the staging OpenAI key
       # account went over quota and stayed dead for 36+ hours, taking
@@ -112,9 +107,9 @@ jobs:
       E2E_RUNTIME: claude-code
       # Pin the smoke to a specific MiniMax model rather than relying
       # on the per-runtime default (which could resolve to "sonnet" →
-      # direct Anthropic and defeat the cost saving). MiniMax-M2 is the
-      # stable staging MiniMax path used by the full-SaaS smoke.
-      E2E_MODEL_SLUG: MiniMax-M2
+      # direct Anthropic and defeat the cost saving). M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
       E2E_RUN_ID: "smoke-${{ github.run_id }}"
       # Debug-only: when an operator dispatches with keep_on_failure=true,
       # the smoke script's E2E_KEEP_ORG=1 path skips teardown so the
@@ -134,12 +129,6 @@ jobs:
             echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
             exit 2
           fi
-          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
-            if [ -z "${!var:-}" ]; then
-              echo "::error::$var not set — EC2 leak verification cannot run"
-              exit 2
-            fi
-          done
 
       - name: Verify LLM key present
         run: |

.gitea/workflows/staging-verify.yml

@@ -75,12 +75,8 @@ permissions:
 env:
   # ECR registry (post-2026-05-06 SSOT for tenant images).
   # publish-workspace-server-image.yml pushes here.
-  # SSOT-Instance-10 (#333): triplet sourced from org/repo var `ECR_REGISTRY` with
-  # the current prod-account literal as bootstrap fallback. When the org var is set,
-  # the fallback becomes dead code and switching accounts/regions is a one-line
-  # change at the org level. Pattern mirrors `vars.CP_URL || 'literal'` below.
-  IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
-  TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
+  TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
   # CP endpoint for redeploy-fleet (used in promote step below).
   CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
   GITHUB_SERVER_URL: https://git.moleculesai.app
@@ -239,11 +235,6 @@ jobs:
           set -euo pipefail
 
           TARGET_TAG="staging-${SHA}"
-          # confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
-          # contract (cp#228 / task #308) for fleet-wide intent. Empty body
-          # / {confirm:false} / {only_slugs:[]} → 400. This caller promotes
-          # the verified staging image across the entire prod fleet (canary
-          # + fan-out), no slug scoping, so confirm:true is correct.
           BODY=$(jq -nc \
             --arg tag "$TARGET_TAG" \
             --argjson soak "${SOAK_SECONDS:-120}" \
@@ -253,8 +244,7 @@ jobs:
               target_tag: $tag,
               soak_seconds: $soak,
               batch_size: $batch,
-              dry_run: $dry,
-              confirm: true
+              dry_run: $dry
             }')
 
           if [ -n "${CANARY_SLUG:-}" ]; then

.gitea/workflows/status-reaper.yml

@@ -53,12 +53,19 @@ name: status-reaper
 # `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
 # "unknown on type" when `workflow_dispatch.inputs.X` is present.
 on:
-  # Schedule moved to operator-config:
-  #   /etc/cron.d/molecule-core-status-reaper ->
-  #   /usr/local/bin/molecule-core-cron-bot.sh status-reaper
-  #
-  # This keeps the 5-minute compensation cadence but stops a maintenance
-  # bot from consuming Gitea Actions runner slots during PR merge waves.
+  # SCHEDULE RE-ENABLED 2026-05-12 rev3 — interim disable (mc#645) reverted now that
+  # rev3 widens DEFAULT_SWEEP_LIMIT 10 → 30 (covers retroactive-failure timing window).
+  # Sibling watchdog re-enabled in the same PR with timeout-minutes raised 5 → 15.
+  schedule:
+    # Every 5 minutes. Off-zero alignment with sibling cron workflows:
+    # ci-required-drift (`:17`), main-red-watchdog (`:05`),
+    # railway-pin-audit (`:23`). 5-min cadence gives a tight enough
+    # close on schedule-triggered false-reds that main-red-watchdog
+    # (hourly :05) almost never files an issue on the false case.
+    # rev3 keeps `*/5` unchanged per hongming-pc2 03:25Z review:
+    # "trades window-width-cheap for cadence-loady" — N=30 widens
+    # the lookback cheaply without doubling runner load via `*/2`.
+    - cron: '*/5 * * * *'
   workflow_dispatch:
 
 # Compensating-status POST needs write on repo statuses; no other

.gitea/workflows/sweep-aws-secrets.yml

@@ -40,12 +40,14 @@ name: Sweep stale AWS Secrets Manager secrets
 # the mostly-orphan tunnels) refuses to nuke past the threshold.
 
 on:
-  schedule:
-    # Hourly at :30, offset from sweep-cf-orphans (:15) and
-    # sweep-cf-tunnels (:45). This janitor is intentionally schedule-only
-    # for deletes; manual dispatch is forced to dry-run below because Gitea
-    # 1.22.6 rejects workflow_dispatch.inputs.
-    - cron: '30 * * * *'
+  # Disabled as an hourly schedule until the dedicated
+  # AWS_SECRETS_JANITOR_* key exists in the key-management SSOT and is
+  # mirrored into Gitea. Falling back to the molecule-cp app principal is
+  # intentionally not allowed: it lacks account-wide ListSecrets, and
+  # granting that to an application credential would weaken least privilege.
+  #
+  # Keep the manual trigger so operators can validate the workflow immediately
+  # after provisioning the janitor key, then restore the hourly :30 schedule.
   workflow_dispatch:
 # Don't let two sweeps race the same AWS account.
 concurrency:
@@ -62,24 +64,22 @@ jobs:
   sweep:
     name: Sweep AWS Secrets Manager
     runs-on: ubuntu-latest
-    # This is a cost/leak janitor. A scheduled failure must be red so
-    # operators know tenant bootstrap secrets may be leaking.
+    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
+    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
+    continue-on-error: true
     # 30 min cap, mirroring the other janitors. AWS DeleteSecret is
     # fast (~0.3s/call) so even a 100+ backlog drains in seconds
     # under the 8-way xargs parallelism, but the cap is set generously
     # to leave headroom for any actual API hang.
     timeout-minutes: 30
     env:
-      # Keep this literal. Gitea/act_runner 1.22.6 can mis-render
-      # secret-backed expressions with `||`, which produced an invalid
-      # Secrets Manager endpoint in the scheduled janitor.
-      AWS_REGION: us-east-2
+      AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_SECRETS_JANITOR_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETS_JANITOR_SECRET_ACCESS_KEY }}
       CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
       CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
-      MAX_DELETE_PCT: 50
-      GRACE_HOURS: 24
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
+      GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
 
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -114,25 +114,17 @@ jobs:
 
       - name: Run sweep
         if: steps.verify.outputs.skip != 'true'
-        # Schedule-vs-dispatch dry-run asymmetry:
-        #   - schedule: execute (the whole point of an hourly janitor).
-        #   - workflow_dispatch: dry-run. Gitea 1.22.6 rejects
-        #     workflow_dispatch.inputs, so there is no safe manual
-        #     "flip it to execute" toggle in this workflow.
-        # The script's MAX_DELETE_PCT gate (default 50%) remains the
-        # second line of defense regardless of trigger.
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
         run: |
           set -euo pipefail
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
             echo "Running in dry-run mode — no deletions"
             bash scripts/ops/sweep-aws-secrets.sh
           else
             echo "Running with --execute — will delete identified orphans"
             bash scripts/ops/sweep-aws-secrets.sh --execute
           fi
-
-      - name: Notify on sweep failure
-        if: failure()
-        run: |
-          echo "::error::sweep-aws-secrets FAILED — AWS tenant bootstrap secrets may be leaking. Check missing Gitea secrets, staging/prod CP admin tokens, AWS janitor IAM permissions, or the script safety gate."
-          exit 1

.gitea/workflows/test-ops-scripts.yml

@@ -58,20 +58,14 @@ jobs:
           python-version: '3.11'
       - name: Install .gitea script test dependencies
         run: python -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2'
-      - name: Run scripts/ unittests, if any
-        # Top-level scripts/ tests live alongside their target file. The
-        # runtime packaging tests moved to molecule-ai-workspace-runtime, so
-        # this pass may legitimately find no tests.
+      - name: Run scripts/ unittests (build_runtime_package, ...)
+        # Top-level scripts/ tests live alongside their target file
+        # (e.g. scripts/test_build_runtime_package.py exercises
+        # scripts/build_runtime_package.py). discover from scripts/
+        # picks up only top-level test_*.py because scripts/ops/ has
+        # no __init__.py — that's intentional, so we run two passes.
         working-directory: scripts
-        run: |
-          set +e
-          python -m unittest discover -t . -p 'test_*.py' -v
-          rc=$?
-          if [ "$rc" -eq 5 ]; then
-            echo "No top-level scripts/ unittest files found; skipping."
-            exit 0
-          fi
-          exit "$rc"
+        run: python -m unittest discover -t . -p 'test_*.py' -v
       - name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
         working-directory: scripts/ops
         run: python -m unittest discover -p 'test_*.py' -v

.gitea/workflows/weekly-platform-go.yml

@@ -106,7 +106,7 @@ jobs:
               [[ "$file" == *_test.go ]] && continue
               [[ "$file" == *"$path"* ]] || continue
               awk "BEGIN{exit !(\$pct < 10)}" || continue
-              rel=$(echo "$file" | sed 's|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/workspace-server/||; s|^git.moleculesai.app/molecule-ai/molecule-core/workspace-server/||')
+              rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
               if echo "$ALLOWLIST" | grep -qxF "$rel"; then
                 continue
               fi

.github/workflows/block-internal-paths.yml

@@ -0,0 +1,154 @@
+name: Block internal-flavored paths
+
+# Hard CI gate. Internal content (positioning, competitive briefs, sales
+# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
+# this public monorepo must never re-acquire those paths. CEO directive
+# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
+#
+# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
+# briefs into the easiest path their cwd resolves to (root /research,
+# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
+# or a stale gitignore line. This workflow is the mechanical backstop.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches: [main, staging]
+  # Required for GitHub merge queue: the queue's pre-merge CI run on
+  # `gh-readonly-queue/...` refs needs this check to fire so the queue
+  # gets a real result instead of stalling forever AWAITING_CHECKS.
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  check:
+    name: Block forbidden paths
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 2  # need previous commit to diff against on push events
+
+      # For pull_request events the diff base is github.event.pull_request.base.sha,
+      # which may be many commits behind HEAD and therefore absent from the
+      # shallow clone above.  Fetch it explicitly (depth=1 keeps it fast).
+      - name: Fetch PR base SHA (pull_request events only)
+        if: github.event_name == 'pull_request'
+        run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
+
+      # For merge_group events the queue's pre-merge ref is a commit on
+      # `gh-readonly-queue/...` whose parent is the queue's base_sha.
+      # That parent isn't part of the queue branch's shallow clone, so
+      # we fetch it explicitly. Mirrors the equivalent step in
+      # secret-scan.yml (#2120) — same shallow-clone bug class.
+      - name: Fetch merge_group base SHA (merge_group events only)
+        if: github.event_name == 'merge_group'
+        run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
+
+      - name: Refuse if forbidden paths appear
+        env:
+          # Plumb event-specific SHAs through env so the script doesn't
+          # need conditional `${{ ... }}` interpolation per event type.
+          # github.event.before/after only exist on push events;
+          # merge_group has its own base_sha/head_sha; pull_request has
+          # pull_request.base.sha / pull_request.head.sha.
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
+          MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
+          PUSH_BEFORE: ${{ github.event.before }}
+          PUSH_AFTER: ${{ github.event.after }}
+        run: |
+          # Paths that must NEVER live in the public monorepo. Add to this
+          # list narrowly — broader patterns belong in .gitignore so day-to-day
+          # docs work isn't accidentally blocked.
+          FORBIDDEN_PATTERNS=(
+            "^research/"
+            "^marketing/"
+            "^docs/marketing/"
+            "^comment-[0-9]+\.json$"
+            "^test-pmm.*\.(txt|md)$"
+            "^tick-reflections.*\.(txt|md)$"
+            ".*-temp\.(md|txt)$"
+          )
+
+          # Determine the diff base. Each event type stores its SHAs in
+          # a different place — see the env block above.
+          case "${{ github.event_name }}" in
+            pull_request)
+              BASE="$PR_BASE_SHA"
+              HEAD="$PR_HEAD_SHA"
+              ;;
+            merge_group)
+              BASE="$MG_BASE_SHA"
+              HEAD="$MG_HEAD_SHA"
+              ;;
+            *)
+              BASE="$PUSH_BEFORE"
+              HEAD="$PUSH_AFTER"
+              ;;
+          esac
+
+          # On push events with shallow clones, BASE may be present in
+          # the event payload but absent from the local object DB
+          # (fetch-depth=2 doesn't always reach the previous commit
+          # across true merges). Try fetching it on demand. If the
+          # fetch fails — e.g. the SHA was force-overwritten — we fall
+          # through to the empty-BASE branch below, which scans the
+          # entire tree as if every file were new. Correct, just slow.
+          # Same recovery shape as secret-scan.yml (#2120 — incident
+          # 2026-04-27 06:50Z block-internal-paths exit 128 with
+          # "fatal: bad object <sha>" on staging push).
+          if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
+            if ! git cat-file -e "$BASE" 2>/dev/null; then
+              git fetch --depth=1 origin "$BASE" 2>/dev/null || true
+            fi
+          fi
+
+          # Files added or modified in this change.
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
+            # New branch / no previous SHA / BASE unreachable — check
+            # the entire tree as if every file were new. Slower but
+            # correct on first push or post-fetch-failure recovery.
+            CHANGED=$(git ls-tree -r --name-only HEAD)
+          else
+            CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
+          fi
+
+          if [ -z "$CHANGED" ]; then
+            echo "No changed files to inspect."
+            exit 0
+          fi
+
+          OFFENDING=""
+          for path in $CHANGED; do
+            for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
+              if echo "$path" | grep -qE "$pattern"; then
+                OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
+                break
+              fi
+            done
+          done
+
+          if [ -n "$OFFENDING" ]; then
+            echo "::error::Forbidden internal-flavored paths detected:"
+            printf "$OFFENDING"
+            echo ""
+            echo "These paths belong in molecule-ai/internal, not this public repo."
+            echo "See docs/internal-content-policy.md for canonical locations."
+            echo ""
+            echo "If your file is genuinely public-facing (e.g. a blog post"
+            echo "ready to ship), use one of these alternatives instead:"
+            echo "  • Public-bound blog posts:  docs/blog/<slug>.md"
+            echo "  • Public-bound tutorials:   docs/tutorials/<slug>.md"
+            echo "  • Public devrel content:    docs/devrel/<slug>.md"
+            echo ""
+            echo "If you legitimately need to add a new top-level path that"
+            echo "happens to match a forbidden pattern, edit"
+            echo ".github/workflows/block-internal-paths.yml and update the"
+            echo "FORBIDDEN_PATTERNS list with reviewer signoff."
+            exit 1
+          fi
+
+          echo "✓ No forbidden paths in this change."

.github/workflows/canary-staging.yml

@@ -0,0 +1,320 @@
+name: Canary — staging SaaS smoke (every 30 min)
+
+# Minimum viable health check: provisions one Hermes workspace on a fresh
+# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
+# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
+# issue on the next green run.
+#
+# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
+# but runs only on provisioning-critical pushes + nightly — this one
+# catches drift in the 30-min window between those runs (AMI health, CF
+# cert rotation, WorkOS session stability, etc.).
+#
+# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
+# peers/activity checks. One parent workspace + one A2A turn is enough
+# to signal "SaaS stack end-to-end is alive."
+
+on:
+  schedule:
+    # Every 30 min. Cron on GitHub-hosted runners has a known drift of
+    # a few minutes under load — that's fine for a canary.
+    - cron: '*/30 * * * *'
+  workflow_dispatch:
+    inputs:
+      keep_on_failure:
+        description: >-
+          Skip teardown when the canary fails (debugging only). The
+          tenant org + EC2 + CF tunnel + DNS stay alive so an operator
+          can SSM into the workspace EC2 and capture docker logs of the
+          failing claude-code container. REMEMBER to manually delete
+          via DELETE /cp/admin/tenants/<slug> when done so the org
+          doesn't accumulate cost. Only honored on workflow_dispatch;
+          cron runs always tear down (we don't want unattended cron
+          to leak resources).
+        type: boolean
+        default: false
+
+# Serialise with the full-SaaS workflow so they don't contend for the
+# same org-create quota on staging. Different group key from
+# e2e-staging-saas since we don't mind queueing canaries behind one
+# full run, but two canaries SHOULD queue against each other.
+concurrency:
+  group: canary-staging
+  cancel-in-progress: false
+
+permissions:
+  # Needed to open / close the alerting issue.
+  issues: write
+  contents: read
+
+jobs:
+  canary:
+    name: Canary smoke
+    runs-on: ubuntu-latest
+    # 25 min headroom over the 15-min TLS-readiness deadline in
+    # tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
+    # the job is killed at the wall-clock 15:00 mark BEFORE the bash
+    # `fail` + diagnostic burst can fire, leaving every cancellation
+    # silent. Sibling staging E2E jobs run at 20-45 min — keeping
+    # canary tighter than them so a true wedge still surfaces here
+    # first.
+    timeout-minutes: 25
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      # MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
+      # Switched from hermes+OpenAI after #2578 (the staging OpenAI key
+      # account went over quota and stayed dead for 36+ hours, taking
+      # the canary red the entire time). claude-code template's
+      # `minimax` provider routes ANTHROPIC_BASE_URL to
+      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
+      # ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
+      # billing account, so OpenAI quota collapse no longer wedges the
+      # canary. Mirrors the migration continuous-synth-e2e.yml made on
+      # 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
+      # full_saas.sh branches SECRETS_JSON on which key is present —
+      # MiniMax wins when set.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so an operator-dispatched run with
+      # E2E_RUNTIME=hermes overridden via workflow_dispatch can still
+      # exercise the OpenAI path without re-editing the workflow.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
+      E2E_MODE: canary
+      E2E_RUNTIME: claude-code
+      # Pin the canary to a specific MiniMax model rather than relying
+      # on the per-runtime default (which could resolve to "sonnet" →
+      # direct Anthropic and defeat the cost saving). M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
+      E2E_RUN_ID: "canary-${{ github.run_id }}"
+      # Debug-only: when an operator dispatches with keep_on_failure=true,
+      # the canary script's E2E_KEEP_ORG=1 path skips teardown so the
+      # tenant org + EC2 stay alive for SSM-based log capture. Cron runs
+      # never set this (the input only exists on workflow_dispatch) so
+      # unattended cron always tears down. See molecule-core#129
+      # failure mode #1 — capturing the actual exception requires
+      # docker logs from the live container.
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
+            exit 2
+          fi
+
+      - name: Verify LLM key present
+        run: |
+          # Per-runtime key check — claude-code uses MiniMax; hermes /
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
+          # rather than soft-skip per the lesson from synth E2E #2578:
+          # an empty key silently falls through to the wrong
+          # SECRETS_JSON branch and the canary fails 5 min later with
+          # a confusing auth error instead of the clean "secret
+          # missing" message at the top.
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              # Either MiniMax OR direct-Anthropic works — first
+              # non-empty wins in the test script's secrets-injection
+              # priority chain. Operators only need to set ONE of these
+              # secrets; we don't force a choice between them.
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
+            exit 2
+          fi
+          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
+
+      - name: Canary run
+        id: canary
+        run: bash tests/e2e/test_staging_full_saas.sh
+
+      # Alerting: open a sticky issue on the FIRST failure; comment on
+      # subsequent failures; auto-close on next green. Comment-on-existing
+      # de-duplicates so a single open issue accumulates the streak —
+      # ops sees one issue with N comments rather than N issues.
+      #
+      # Why no consecutive-failures threshold (e.g., wait 3 runs before
+      # filing): the prior threshold check used
+      # `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
+      # not expose (returns 404). On Gitea Actions the threshold call
+      # ALWAYS failed, breaking the entire alerting step and going days
+      # silent on real regressions (38h+ chronic red on 2026-05-07/08
+      # before this fix; tracked in molecule-core#129). Filing on first
+      # failure is also better UX — we want to know about the first red,
+      # not wait 90 min for it to "count." Real flakes get one issue +
+      # a quick close-on-green; persistent reds accumulate comments.
+      - name: Open issue on failure
+        if: failure()
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const title = '🔴 Canary failing: staging SaaS smoke';
+            const runURL = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+            // Find an existing open canary issue (stable title match).
+            // If one exists, this isn't a "first failure" — comment and exit.
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'canary-staging',
+              per_page: 10,
+            });
+            const match = existing.find(i => i.title === title);
+            if (match) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: match.number,
+                body: `Canary still failing. ${runURL}`,
+              });
+              core.info(`Commented on existing issue #${match.number}`);
+              return;
+            }
+
+            // No open issue yet — file one on this first failure. The
+            // comment-on-existing branch above means subsequent failures
+            // accumulate as comments on this same issue, so we don't
+            // spam new issues per run.
+            const body =
+              `Canary run failed at ${new Date().toISOString()}.\n\n` +
+              `Run: ${runURL}\n\n` +
+              `This issue auto-closes on the next green canary run. ` +
+              `Consecutive failures add a comment here rather than a new issue.`;
+            await github.rest.issues.create({
+              owner: context.repo.owner, repo: context.repo.repo,
+              title, body,
+              labels: ['canary-staging', 'bug'],
+            });
+            core.info('Opened canary failure issue (first red)');
+
+      - name: Auto-close canary issue on success
+        if: success()
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const title = '🔴 Canary failing: staging SaaS smoke';
+            const { data: open } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'canary-staging',
+              per_page: 10,
+            });
+            const match = open.find(i => i.title === title);
+            if (match) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: match.number,
+                body: `Canary recovered at ${new Date().toISOString()}. Closing.`,
+              });
+              await github.rest.issues.update({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: match.number,
+                state: 'closed',
+              });
+              core.info(`Closed recovered canary issue #${match.number}`);
+            }
+
+      - name: Teardown safety net
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          # Slug prefix matches what test_staging_full_saas.sh emits
+          # in canary mode:
+          #   SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
+          # Earlier this was `e2e-{today}-canary-` — that was the
+          # full-mode pattern (date FIRST, mode SECOND); canary slugs
+          # have mode FIRST, date SECOND. The mismatch silently
+          # never matched, leaving every cancelled-canary EC2 alive
+          # until the once-an-hour sweep eventually caught it
+          # (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
+          # cleanup; same gap on three earlier cancellations today).
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # Scope to slugs from THIS canary run when GITHUB_RUN_ID is
+          # available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
+          # so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
+          # full-mode safety net's per-run scoping (e2e-staging-saas.yml)
+          # added after the 2026-04-21 cross-run cleanup incident.
+          # Sweep both today AND yesterday's UTC dates so a run that
+          # crosses midnight still cleans up its own slug — see the
+          # 2026-04-26→27 canvas-safety-net incident.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if run_id:
+              prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
+          else:
+              prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
+          candidates = [o['slug'] for o in d.get('orgs', [])
+                        if any(o.get('slug','').startswith(p) for p in prefixes)
+                        and o.get('status') not in ('purged',)]
+          print('\n'.join(candidates))
+          " 2>/dev/null)
+          # Per-slug DELETE with HTTP-code verification. The previous
+          # `... >/dev/null || true` swallowed every failure, so a 5xx
+          # or timeout from CP looked identical to "successfully cleaned
+          # up" and the tenant kept eating ~2 vCPU until the hourly
+          # stale sweep caught it (up to 2h later). Now we capture the
+          # response code and surface non-2xx as a workflow warning, so
+          # the run page shows which slug leaked. We still don't `exit 1`
+          # on cleanup failure — a single-canary cleanup miss shouldn't
+          # fail-flag the canary itself when the actual smoke check
+          # passed. The sweep-stale-e2e-orgs cron (now every 15 min,
+          # 30-min threshold) is the safety net for whatever slips past.
+          # See molecule-controlplane#420.
+          leaks=()
+          for slug in $orgs; do
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
+            set -e
+            code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
+          done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
+          exit 0

.github/workflows/canary-verify.yml

@@ -0,0 +1,255 @@
+name: canary-verify
+
+# Runs the canary smoke suite against the staging canary tenant fleet
+# after a new :staging-<sha> image lands in ECR. On green, calls the
+# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
+# the prod tenant fleet's 5-minute auto-updater picks up the verified
+# digest. On red, :latest stays on the prior known-good digest and
+# prod is untouched.
+#
+# Registry note (2026-05-10): This workflow previously used GHCR
+# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
+# during the 2026-05-06 Gitea suspension migration when publish-
+# workspace-server-image.yml switched to the operator's ECR org
+# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
+# platform-tenant). The GHCR → ECR migration was never applied to
+# this file, so canary-verify was silently smoke-testing the stale
+# GHCR image while the actual staging/prod tenants ran the ECR image.
+# Result: smoke tests could not catch a broken ECR build. Fix:
+#   - Wait step: reads SHA from running canary /health (tenant-
+#     agnostic, works regardless of registry).
+#   - Promote step: calls CP redeploy-fleet endpoint with target_tag=
+#     staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
+#     No longer attempts GHCR crane ops.
+#
+# Dependencies:
+#   - publish-workspace-server-image.yml publishes :staging-<sha>
+#     to ECR on staging and main merges.
+#   - Canary tenants are configured to pull :staging-<sha> from ECR
+#     (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
+#   - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
+#     CANARY_CP_SHARED_SECRET are populated.
+
+on:
+  workflow_run:
+    workflows: ["publish-workspace-server-image"]
+    types: [completed]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  packages: write
+  actions: read
+
+env:
+  # ECR registry (post-2026-05-06 SSOT for tenant images).
+  # publish-workspace-server-image.yml pushes here.
+  IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
+  TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
+  # CP endpoint for redeploy-fleet (used in promote step below).
+  CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
+
+jobs:
+  canary-smoke:
+    # Skip when the upstream workflow failed — no image to test against.
+    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    outputs:
+      sha: ${{ steps.compute.outputs.sha }}
+      smoke_ran: ${{ steps.smoke.outputs.ran }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Compute sha
+        id: compute
+        run: echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
+      - name: Wait for canary tenants to pick up :staging-<sha>
+        # Poll canary health endpoints every 30s for up to 7 min instead
+        # of a fixed 6-min sleep. Exits as soon as ALL canaries report
+        # the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
+        # proceeding after 7 min even if not all canaries responded —
+        # the smoke suite will catch any that didn't update.
+        #
+        # NOTE: The SHA is read from the running tenant's /health response,
+        # NOT from a registry lookup. This is registry-agnostic and works
+        # regardless of whether the tenant pulls from ECR, GHCR, or any
+        # other registry — the canary is telling us what it's actually
+        # running, which is the ground truth for smoke testing.
+        env:
+          CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
+          EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
+        run: |
+          if [ -z "$CANARY_TENANT_URLS" ]; then
+            echo "No canary URLs configured — falling back to 60s wait"
+            sleep 60
+            exit 0
+          fi
+          IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
+          MAX_WAIT=420  # 7 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $MAX_WAIT ]; do
+            ALL_READY=true
+            for url in "${URLS[@]}"; do
+              HEALTH=$(curl -s --max-time 5 "${url}/health" 2>/dev/null || echo "{}")
+              SHA=$(echo "$HEALTH" | grep -o "\"sha\":\"[^\"]*\"" | head -1 | cut -d'"' -f4)
+              if [ "$SHA" != "$EXPECTED_SHA" ]; then
+                ALL_READY=false
+                break
+              fi
+            done
+            if $ALL_READY; then
+              echo "All canaries running staging-${EXPECTED_SHA} after ${ELAPSED}s"
+              exit 0
+            fi
+            echo "Waiting for canaries... (${ELAPSED}s / ${MAX_WAIT}s)"
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
+
+      - name: Run canary smoke suite
+        id: smoke
+        # Graceful-skip when no canary fleet is configured (Phase 2 not yet
+        # stood up — see molecule-controlplane/docs/canary-tenants.md).
+        # Sets `ran=false` on skip so promote-to-latest stays off (we don't
+        # want every main merge auto-promoting without gating). Manual
+        # promote-latest.yml is the release gate while canary is absent.
+        # Once the fleet is real: delete the early-exit branch.
+        env:
+          CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
+          CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
+          CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
+          CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
+        run: |
+          set -euo pipefail
+          if [ -z "${CANARY_TENANT_URLS:-}" ] \
+            || [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
+            || [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
+            {
+              echo "## ⚠️ canary-verify skipped"
+              echo
+              echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
+              echo "Phase 2 canary fleet has not been stood up yet —"
+              echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
+              echo
+              echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "ran=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::canary-verify: skipped — no canary fleet configured"
+            exit 0
+          fi
+          bash scripts/canary-smoke.sh
+          echo "ran=true" >> "$GITHUB_OUTPUT"
+
+      - name: Summary on failure
+        if: ${{ failure() }}
+        run: |
+          {
+            echo "## Canary smoke FAILED"
+            echo
+            echo "Canary tenants rejected image \`staging-${{ steps.compute.outputs.sha }}\`."
+            echo ":latest stays pinned to the prior good digest — prod is untouched."
+            echo
+            echo "Fix forward and merge again, or investigate the specific failed"
+            echo "assertions in the canary-smoke step log above."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  promote-to-latest:
+    # On green, calls the CP redeploy-fleet endpoint with target_tag=
+    # staging-<sha> to promote the verified ECR image. This is the same
+    # mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
+    #
+    # Pre-fix history: the old GHCR promote step used `crane tag` against
+    # ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
+    # image.yml had already migrated to ECR on 2026-05-07 (commit
+    # 10e510f5). The GHCR tags were never updated, so this step was
+    # silently promoting a stale GHCR image while actual prod tenants
+    # pulled from ECR. Canary smoke tests were GHCR-targeted and could
+    # not catch a broken ECR build.
+    needs: canary-smoke
+    if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
+    runs-on: ubuntu-latest
+    env:
+      SHA: ${{ needs.canary-smoke.outputs.sha }}
+      CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
+      # CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
+      # Stored at the repo level so all workflows pick it up automatically.
+      CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+      # canary_slug pin: deploy the verified :staging-<sha> to the canary
+      # first (soak 120s), then fan out to the rest of the fleet.
+      CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
+      SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
+      BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
+    steps:
+      - name: Check CP credentials
+        run: |
+          if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
+            echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
+            echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
+            exit 1
+          fi
+
+      - name: Promote verified ECR image to :latest
+        run: |
+          set -euo pipefail
+
+          TARGET_TAG="staging-${SHA}"
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --argjson soak "${SOAK_SECONDS:-120}" \
+            --argjson batch "${BATCH_SIZE:-3}" \
+            --argjson dry false \
+            '{
+              target_tag: $tag,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          if [ -n "${CANARY_SLUG:-}" ]; then
+            BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
+          fi
+
+          echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  target_tag: $TARGET_TAG"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          CURL_EXIT=$?
+          set -e
+
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
+            exit 1
+          fi
+
+      - name: Summary
+        run: |
+          {
+            echo "## Canary verified — :latest promoted via CP redeploy-fleet"
+            echo ""
+            echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
+            echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
+            echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
+            echo "- **Batch size:** ${BATCH_SIZE:-3}"
+            echo ""
+            echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
+            echo "The fleet's 5-minute health-check loop will pick up the update automatically."
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/cascade-list-drift-gate.yml

@@ -0,0 +1,39 @@
+name: cascade-list-drift-gate
+
+# Structural gate: TEMPLATES list in publish-runtime.yml must match
+# manifest.json's workspace_templates exactly. Closes the recurrence
+# path of PR #2556 (the data fix) and is the first concrete deliverable
+# of RFC #388 PR-3.
+#
+# Why a gate, not just discipline: PR #2536 pruned the manifest, but the
+# cascade list wasn't updated for ~weeks before someone (PR #2556)
+# noticed during an unrelated audit. During that window, codex never
+# rebuilt on a runtime publish. A structural gate catches the drift
+# the same day either file changes.
+#
+# Triggers narrowly to keep CI quiet: only on PRs that actually change
+# one of the two files. The path-filtered split + always-emit-result
+# pattern (memory: "Required check names need a job that always runs")
+# is unnecessary here because the workflow IS the check name and PR
+# branch protection should require it directly. Future-proof: if this
+# becomes a required check, add a no-op aggregator with always() so the
+# name still emits when paths don't match.
+
+on:
+  pull_request:
+    branches: [staging, main]
+    paths:
+      - manifest.json
+      - .github/workflows/publish-runtime.yml
+      - scripts/check-cascade-list-vs-manifest.sh
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Check cascade list matches manifest
+        run: bash scripts/check-cascade-list-vs-manifest.sh

.github/workflows/check-migration-collisions.yml

@@ -0,0 +1,58 @@
+name: Check migration collisions
+
+# Hard gate (#2341): fails a PR that adds a migration prefix already
+# claimed by the base branch or another open PR. Caught manually 2026-04-30
+# during PR #2276 rebase: 044_runtime_image_pins collided with
+# 044_platform_inbound_secret from RFC #2312. This workflow makes that
+# check automatic.
+#
+# Trigger model: pull_request only — there's no value running this on
+# pushes to staging or main (those are post-merge; the gate must fire
+# pre-merge to be useful). Path filter scopes to PRs that actually touch
+# migrations.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'workspace-server/migrations/**'
+      - 'scripts/ops/check_migration_collisions.py'
+      - '.github/workflows/check-migration-collisions.yml'
+
+permissions:
+  contents: read
+  # gh pr list/diff need read access to other PRs
+  pull-requests: read
+
+jobs:
+  check:
+    name: Migration version collision check
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Need history to diff against base ref
+          fetch-depth: 0
+
+      - name: Detect collisions
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          BASE_REF: origin/${{ github.event.pull_request.base.ref }}
+          HEAD_REF: ${{ github.event.pull_request.head.sha }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          # gh CLI uses GH_TOKEN from env
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Ensure the named base ref exists locally. checkout@v4 with
+          # fetch-depth=0 pulls full history, but the explicit fetch is
+          # cheap insurance against form-of-ref differences across runs.
+          #
+          # IMPORTANT: do NOT pass --depth=1 here. The script below uses
+          # `git diff origin/<base>...<head>` (three-dot, merge-base form),
+          # which fails with "fatal: no merge base" if the base ref is
+          # shallow. The auto-promote staging→main PR (#2361) was blocked
+          # by exactly this for ~5h on 2026-04-30 — the depth=1 fetch
+          # overwrote checkout@v4's full-history clone with a shallow tip.
+          git fetch origin "${{ github.event.pull_request.base.ref }}" || true
+          python3 scripts/ops/check_migration_collisions.py

.github/workflows/ci.yml

@@ -0,0 +1,443 @@
+name: CI
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  # GitHub merge queue fires `merge_group` for the queue's pre-merge CI run.
+  # Required so the queue gets a real check result instead of a false-green
+  # from the absence of a triggered workflow. Safe to add unconditionally —
+  # the event simply doesn't fire until the queue is enabled on the branch.
+  merge_group:
+    types: [checks_requested]
+
+# Cancel in-progress CI runs when a new commit arrives on the same ref.
+# This prevents stale runs from queuing behind each other. The merge_group
+# refs (refs/heads/gh-readonly-queue/...) get their own concurrency group
+# automatically because github.ref differs from the PR ref.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Detect which paths changed so downstream jobs can skip when only
+  # docs/markdown files were modified.
+  changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      platform: ${{ steps.check.outputs.platform }}
+      canvas: ${{ steps.check.outputs.canvas }}
+      python: ${{ steps.check.outputs.python }}
+      scripts: ${{ steps.check.outputs.scripts }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+      - id: check
+        run: |
+          # For PR events: diff against the base branch (not HEAD~1 of the branch,
+          # which may be unrelated after force-pushes). When a push updates a PR,
+          # both pull_request and push events fire — prefer the PR base so that
+          # the diff is always computed against the actual merge base, not the
+          # previous SHA on the branch which may be on a different history line.
+          BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
+          # GITHUB_BASE_REF is set by GitHub for PR events (the base branch name).
+          # For pull_request events we use the stored base.sha; for push events
+          # (or when base.sha is unavailable) fall back to github.event.before.
+          if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          fi
+          # Fallback: if BASE is empty or all zeros (new branch), run everything
+          if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
+            echo "platform=true" >> "$GITHUB_OUTPUT"
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+            echo "python=true" >> "$GITHUB_OUTPUT"
+            echo "scripts=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".github/workflows/ci.yml")
+          echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
+
+  # Platform (Go) is a required check on staging. Always-run + per-step
+  # gating (see Canvas (Next.js) for the rationale and the failure mode
+  # this avoids).
+  platform-build:
+    name: Platform (Go)
+    needs: changes
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: workspace-server
+    steps:
+      - if: needs.changes.outputs.platform != 'true'
+        working-directory: .
+        run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.platform == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.platform == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+      - if: needs.changes.outputs.platform == 'true'
+        run: go mod download
+      - if: needs.changes.outputs.platform == 'true'
+        run: go build ./cmd/server
+      # CLI (molecli) moved to standalone repo: github.com/molecule-ai/molecule-cli
+      - if: needs.changes.outputs.platform == 'true'
+        run: go vet ./... || true
+      - if: needs.changes.outputs.platform == 'true'
+        name: Run golangci-lint
+        run: golangci-lint run --timeout 3m ./... || true
+      - if: needs.changes.outputs.platform == 'true'
+        name: Run tests with race detection and coverage
+        run: go test -race -coverprofile=coverage.out ./...
+
+      - if: needs.changes.outputs.platform == 'true'
+        name: Per-file coverage report
+        # Advisory — lists every source file with its coverage so reviewers
+        # can see at-a-glance where gaps are. Sorted ascending so the worst
+        # offenders float to the top. Does NOT fail the build; the hard
+        # gate is the threshold check below. (#1823)
+        run: |
+          echo "=== Per-file coverage (worst first) ==="
+          go tool cover -func=coverage.out \
+            | grep -v '^total:' \
+            | awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
+                   END {for (f in s) printf "%6.1f%%  %s\n", s[f]/c[f], f}' \
+            | sort -n
+
+      - if: needs.changes.outputs.platform == 'true'
+        name: Check coverage thresholds
+        # Enforces two gates from #1823 Layer 1:
+        #   1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
+        #   2. Per-file floor — non-test .go files in security-critical
+        #      paths with coverage <10% fail the build, UNLESS the file
+        #      path is listed in .coverage-allowlist.txt (acknowledged
+        #      historical debt with a tracking issue + expiry).
+        run: |
+          set -e
+          TOTAL_FLOOR=25
+          # Security-critical paths where a 0%-coverage file is a real risk.
+          CRITICAL_PATHS=(
+            "internal/handlers/tokens"
+            "internal/handlers/workspace_provision"
+            "internal/handlers/a2a_proxy"
+            "internal/handlers/registry"
+            "internal/handlers/secrets"
+            "internal/middleware/wsauth"
+            "internal/crypto"
+          )
+
+          TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
+          echo "Total coverage: ${TOTAL}%"
+          if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
+            echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
+            exit 1
+          fi
+
+          # Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
+          go tool cover -func=coverage.out \
+            | grep -v '^total:' \
+            | awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
+                   END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
+            > /tmp/perfile.txt
+
+          # Build allowlist — paths relative to workspace-server, one per line.
+          # Lines starting with # are comments.
+          ALLOWLIST=""
+          if [ -f ../.coverage-allowlist.txt ]; then
+            ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
+          fi
+
+          FAILED=0
+          WARNED=0
+          for path in "${CRITICAL_PATHS[@]}"; do
+            while read -r file pct; do
+              [[ "$file" == *_test.go ]] && continue
+              [[ "$file" == *"$path"* ]] || continue
+              awk "BEGIN{exit !($pct < 10)}" || continue
+
+              # Strip the package-import prefix so we can match .coverage-allowlist.txt
+              # entries written as paths relative to workspace-server/.
+              # Handle both module paths: platform/workspace-server/... and platform/...
+              rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
+
+              if echo "$ALLOWLIST" | grep -qxF "$rel"; then
+                echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
+                WARNED=$((WARNED+1))
+              else
+                echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
+                FAILED=$((FAILED+1))
+              fi
+            done < /tmp/perfile.txt
+          done
+
+          echo ""
+          echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "$FAILED security-critical file(s) have <10% test coverage and are"
+            echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
+            echo "workspace provisioning — a 0% file here is the exact gap that let"
+            echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
+            echo "  (a) add tests to raise coverage above 10%, or"
+            echo "  (b) add the path to .coverage-allowlist.txt with an expiry date"
+            echo "      and a tracking issue reference."
+            exit 1
+          fi
+
+  # Canvas (Next.js) — required check, always runs. See platform-build
+  # comment above for the rationale.
+  #
+  # Supersedes the canvas-build-noop pattern attempted in PR #2321: two
+  # jobs sharing `name:` doesn't actually satisfy branch protection
+  # because the SKIPPED check run sibling is treated as not-passed
+  # regardless of how many SUCCESS siblings it has. Verified empirically
+  # on PR #2314 — mergeStateStatus stayed BLOCKED until I collapsed to
+  # a single-job-with-conditional-steps shape.
+  canvas-build:
+    name: Canvas (Next.js)
+    needs: changes
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: canvas
+    steps:
+      - if: needs.changes.outputs.canvas != 'true'
+        working-directory: .
+        run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.canvas == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.canvas == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '22'
+      - if: needs.changes.outputs.canvas == 'true'
+        run: rm -f package-lock.json && npm install
+      - if: needs.changes.outputs.canvas == 'true'
+        run: npm run build
+      - if: needs.changes.outputs.canvas == 'true'
+        name: Run tests with coverage
+        # Coverage instrumentation is configured in canvas/vitest.config.ts
+        # (provider: v8, reporters: text + html + json-summary). Step 2 of
+        # #1815 — wires coverage into CI so we get a baseline visible on
+        # every PR. No threshold gate yet; thresholds dial in (Step 3, also
+        # tracked in #1815) after the team sees what current coverage is.
+        # Per the inline comment in vitest.config.ts: "first land
+        # observability so we can see the baseline, then dial in
+        # thresholds + a hard gate" — this PR ships the observability half.
+        run: npx vitest run --coverage
+      - name: Upload coverage summary as artifact
+        if: needs.changes.outputs.canvas == 'true' && always()
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
+        # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
+        # implement, surfacing as `GHESNotSupportedError: @actions/artifact
+        # v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
+        # currently supported on GHES`. Drop this pin when Gitea ships
+        # the v4 protocol (tracked: post-Gitea-1.23 followup).
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: canvas-coverage-${{ github.run_id }}
+          path: canvas/coverage/
+          retention-days: 7
+          if-no-files-found: warn
+
+  # MCP Server + SDK removed from CI — now in standalone repos:
+  # - github.com/molecule-ai/molecule-mcp-server (npm CI)
+  # - github.com/molecule-ai/molecule-sdk-python (PyPI CI)
+
+  # e2e-api job moved to .github/workflows/e2e-api.yml (issue #458).
+  # It now has workflow-level concurrency (cancel-in-progress: false) so
+  # new pushes queue the E2E run rather than cancelling it at the run level.
+
+  # Shellcheck (E2E scripts) — required check, always runs. See
+  # platform-build for the rationale.
+  shellcheck:
+    name: Shellcheck (E2E scripts)
+    needs: changes
+    runs-on: ubuntu-latest
+    steps:
+      - if: needs.changes.outputs.scripts != 'true'
+        run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.scripts == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
+        # shellcheck is pre-installed on ubuntu-latest runners (via apt).
+        # infra/scripts/ is included because setup.sh + nuke.sh gate the
+        # README quickstart — a shellcheck regression there silently breaks
+        # new-user onboarding. scripts/ is intentionally excluded until its
+        # pre-existing SC3040/SC3043 warnings are cleaned up.
+        run: |
+          find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
+            | xargs -0 shellcheck --severity=warning
+
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Lint cleanup-trap hygiene (RFC #2873)
+        # Asserts every shell E2E test that calls `mktemp` also installs
+        # an EXIT trap. Catches the /tmp-leak class — a missing trap
+        # silently leaks scratch into CI runners (~10-100KB per run).
+        # See tests/e2e/lint_cleanup_traps.sh for the rule + fix pattern.
+        run: bash tests/e2e/lint_cleanup_traps.sh
+
+      - if: needs.changes.outputs.scripts == 'true'
+        name: Run E2E bash unit tests (no live infra)
+        # Pure-bash unit tests for E2E helper libs (lib/*.sh). These pin
+        # behavior of dispatch logic that — when broken — silently masks as
+        # "Could not resolve authentication method" only after a successful
+        # tenant + workspace provision (PR #2571 incident, 2026-05-03). Add
+        # new self-contained unit tests here as the lib/ directory grows;
+        # tests requiring live CP/tenant credentials belong in the dedicated
+        # e2e-staging-* workflows, not this job.
+        run: |
+          bash tests/e2e/test_model_slug.sh
+
+  canvas-deploy-reminder:
+    name: Canvas Deploy Reminder
+    runs-on: ubuntu-latest
+    needs: [changes, canvas-build]
+    # Only fires on direct pushes to main (i.e. after staging→main promotion).
+    if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - name: Write deploy reminder to step summary
+        env:
+          COMMIT_SHA: ${{ github.sha }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          # Write body to a temp file — avoids backtick escaping in shell.
+          cat > /tmp/deploy-reminder.md << 'BODY'
+          ## Canvas build passed ✅ — deploy required
+
+          The `publish-canvas-image` workflow is now building a fresh Docker image
+          (`ghcr.io/molecule-ai/canvas:latest`) in the background.
+
+          Once it completes (~3–5 min), apply on the host machine with:
+          ```bash
+          cd <runner-workspace>
+          git pull origin main
+          docker compose pull canvas && docker compose up -d canvas
+          ```
+
+          If you need to rebuild from local source instead (e.g. testing unreleased
+          changes or a new `NEXT_PUBLIC_*` URL), use:
+          ```bash
+          docker compose build canvas && docker compose up -d canvas
+          ```
+          BODY
+          printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
+            "$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
+
+          # Gitea has no commit-comments API (no equivalent of
+          # POST /repos/{owner}/{repo}/commits/{commit_sha}/comments).
+          # Write to GITHUB_STEP_SUMMARY instead — both GitHub Actions and
+          # Gitea Actions render this as the workflow run's summary page,
+          # which is where operators look for post-deploy action items.
+          # (#75 / PR-D)
+          cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
+
+  # Python Lint & Test — required check, always runs. See platform-build
+  # for the rationale.
+  python-lint:
+    name: Python Lint & Test
+    needs: changes
+    runs-on: ubuntu-latest
+    env:
+      WORKSPACE_ID: test
+    defaults:
+      run:
+        working-directory: workspace
+    steps:
+      - if: needs.changes.outputs.python != 'true'
+        working-directory: .
+        run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
+      - if: needs.changes.outputs.python == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.changes.outputs.python == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - if: needs.changes.outputs.python == 'true'
+        run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
+      # Coverage flags + fail-under floor moved into workspace/pytest.ini
+      # (issue #1817) so local `pytest` and CI use identical config.
+      - if: needs.changes.outputs.python == 'true'
+        run: python -m pytest --tb=short
+
+      - if: needs.changes.outputs.python == 'true'
+        name: Per-file critical-path coverage (MCP / inbox / auth)
+        # MCP-critical Python files have a per-file floor on top of the
+        # 86% total floor in pytest.ini. Rationale (issue #2790, after
+        # the PR #2766 → PR #2771 cycle): the total floor averages ~6000
+        # lines, so a single MCP file could regress to ~50% with no
+        # complaint as long as other modules compensate. These five
+        # files handle multi-tenant routing + auth + inbox dispatch —
+        # a coverage drop here is the same risk shape as a Go-side
+        # workspace-server token/secrets file dropping below 10%.
+        #
+        # Floor 75% sits below current actuals (80-96%) so this gate is
+        # strictly additive — no existing PR fails. Ratchet plan in
+        # COVERAGE_FLOOR.md.
+        run: |
+          set -e
+          PER_FILE_FLOOR=75
+          CRITICAL_FILES=(
+            "a2a_mcp_server.py"
+            "mcp_cli.py"
+            "a2a_tools.py"
+            "a2a_tools_inbox.py"
+            "inbox.py"
+            "platform_auth.py"
+          )
+
+          # pytest already wrote .coverage; emit a JSON view scoped to
+          # the critical files so jq/python can read the per-file pct
+          # without parsing tabular text. --include uses fnmatch, and
+          # the leading "*" allows the file to live anywhere under the
+          # workspace root (today they sit at workspace/<name>.py).
+          INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
+          INCLUDES="${INCLUDES%,}"
+          python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
+
+          FAILED=0
+          for f in "${CRITICAL_FILES[@]}"; do
+            # Match by top-level path key (e.g. "a2a_tools.py", not
+            # "builtin_tools/a2a_tools.py" — different file at 100%).
+            # The keys in coverage.json are paths relative to the run
+            # cwd (workspace/), so the critical-path entry sits at the
+            # bare basename.
+            pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
+            if [ "$pct" = "MISSING" ]; then
+              echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
+              FAILED=$((FAILED+1))
+              continue
+            fi
+            echo "$f: ${pct}%"
+            if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
+              echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
+              FAILED=$((FAILED+1))
+            fi
+          done
+
+          if [ "$FAILED" -gt 0 ]; then
+            echo ""
+            echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
+            echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
+            echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
+            echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
+            echo "  (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
+            echo "  (b) if this is unavoidable historical debt, file an issue and propose"
+            echo "      adjusting the floor with rationale in COVERAGE_FLOOR.md."
+            exit 1
+          fi
+
+      # SDK + plugin validation moved to standalone repo:
+      # github.com/molecule-ai/molecule-sdk-python
+

.github/workflows/continuous-synth-e2e.yml

@@ -0,0 +1,257 @@
+name: Continuous synthetic E2E (staging)
+
+# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
+# regressions visible only at runtime — schema drift, deployment-pipeline
+# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
+#
+# Why this gate exists:
+#   PR-time CI catches code-level regressions but not deployment-time or
+#   integration-time ones. Today's empirical data:
+#     • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
+#       JSON-RPC parse layer between sender and receiver. Visible only
+#       to a sender exercising the full path.
+#     • RFC #2312 chat upload — landed on staging-branch but never
+#       reached staging tenants because publish-workspace-server-image
+#       was main-only. Caught by manual dogfooding hours after deploy.
+#   Both would have surfaced within 15-20 min of regression if a
+#   continuous synth-E2E was running.
+#
+# Cadence: every 20 min (3x/hour). The script is conservatively
+# bounded at 10 min wall-clock; even on degraded staging it should
+# finish before the next firing. cron-overlap is guarded by the
+# concurrency group below.
+#
+# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
+# Plus a fresh tenant provisioned + torn down each run (Railway +
+# AWS pennies). Negligible.
+#
+# Failure handling: when the run fails, the workflow exits non-zero
+# and GitHub's standard email/notification path fires. Operators
+# can subscribe to this workflow's failure channel for paging-grade
+# alerting.
+
+on:
+  schedule:
+    # Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
+    #   1. Stay off the top-of-hour. GitHub Actions scheduler drops
+    #      :00 firings under high load (own docs:
+    #      https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
+    #      Prior history: cron was '0,20,40' (2026-05-02) — only :00
+    #      ever survived. Bumped to '10,30,50' (2026-05-03) on the
+    #      theory that further-from-:00 wins. Empirically 2026-05-04
+    #      that ALSO dropped to ~60 min effective cadence (only ~1
+    #      schedule fire per hour — see molecule-core#2726). Detection
+    #      latency was claimed 20 min, actual 60 min.
+    #   2. Avoid colliding with the existing :15 sweep-cf-orphans
+    #      and :45 sweep-cf-tunnels — both hit the CF API and we
+    #      don't want to fight for rate-limit tokens.
+    #   3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
+    #      secrets, sweep-stale-e2e-orgs every :15) — multiple
+    #      overlapping cron registrations on the same minute is part
+    #      of what GH drops under load.
+    # Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
+    # lanes (1-3 min away from any other cron). Even with empirically-
+    # observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
+    # fires = ~30 min cadence; closer to the 20-min target than the
+    # current shape and provides a real degradation alarm if drops
+    # get worse.
+    - cron: '2,12,22,32,42,52 * * * *'
+  workflow_dispatch:
+    inputs:
+      runtime:
+        description: "Runtime to provision (claude-code = default + cheapest via MiniMax; langgraph = OpenAI-only; hermes = SDK-native path, slower)"
+        required: false
+        default: "claude-code"
+        type: string
+      model_slug:
+        description: "Model id to provision the workspace with (default MiniMax-M2.7-highspeed; e.g. 'sonnet' to test direct Anthropic, 'openai/gpt-4o' for hermes)"
+        required: false
+        default: "MiniMax-M2.7-highspeed"
+        type: string
+      keep_org:
+        description: "Skip teardown for post-mortem debugging (only manual dispatch — never set this for cron runs)"
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  # No issue-write here — failures surface as red runs in the workflow
+  # history. If you want auto-issue-on-fail, add a follow-up step that
+  # uses gh issue create gated on `if: failure()`. Keeping the surface
+  # minimal until that's actually wanted.
+
+# Serialize so two firings can never overlap. Cron firing every 20 min
+# but scripts conservatively bounded at 10 min — overlap shouldn't
+# happen in steady state, but if a run hangs we don't want N more
+# stacking up.
+concurrency:
+  group: continuous-synth-e2e
+  cancel-in-progress: false
+
+jobs:
+  synth:
+    name: Synthetic E2E against staging
+    runs-on: ubuntu-latest
+    # Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
+    # (apt-get update + install docker.io/jq/awscli/caddy + snap install
+    # ssm-agent) runs from raw Ubuntu on every boot — none of it is
+    # pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
+    # across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
+    # latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
+    # A 12min budget leaves only ~2min for the workspace (which needs
+    # ~3.5min for claude-code cold boot) on slow-apt days, blowing the
+    # budget. 20min absorbs the worst tenant tail so the workspace probe
+    # gets the full ~7min it needs even on a slow apt day. Real fix:
+    # pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
+    timeout-minutes: 20
+    env:
+      # claude-code default: cold-start ~5 min (comparable to langgraph),
+      # but uses MiniMax-M2.7-highspeed via the template's third-party-
+      # Anthropic-compat path (workspace-configs-templates/claude-code-
+      # default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
+      # gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
+      # exhaustion class that took the canary down 2026-05-03 (#265).
+      # Operators can pick langgraph / hermes via workflow_dispatch
+      # when they specifically need to exercise the OpenAI or SDK-
+      # native paths.
+      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
+      # Pin the canary to a specific MiniMax model rather than relying
+      # on the per-runtime default ("sonnet" → routes to direct
+      # Anthropic, defeats the cost saving). Operators can override
+      # via workflow_dispatch by setting a different E2E_MODEL_SLUG
+      # input if they need to exercise a specific model. M2.7-highspeed
+      # is "Token Plan only" but cheap-per-token and fast.
+      E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
+      # Bound to 10 min so a stuck provision fails the run instead of
+      # holding up the next cron firing. 15-min default in the script
+      # is for the on-PR full lifecycle where we have more headroom.
+      E2E_PROVISION_TIMEOUT_SECS: '600'
+      # Slug suffix — namespaced "synth-" so these runs are
+      # distinguishable from PR-driven runs in CP admin.
+      E2E_RUN_ID: synth-${{ github.run_id }}
+      # Forced false for cron; respected for manual dispatch
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
+      MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      # MiniMax key is the canary's PRIMARY auth path. claude-code
+      # template's `minimax` provider routes ANTHROPIC_BASE_URL to
+      # api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
+      # tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
+      # which key is present — MiniMax wins when set.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so operators can dispatch with
+      # E2E_RUNTIME=langgraph or =hermes and still have a working
+      # canary path. The script picks the right blob shape based on
+      # which key is non-empty.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        run: |
+          # Hard-fail on missing secret REGARDLESS of trigger. Previously
+          # this step soft-skipped on workflow_dispatch via `exit 0`, but
+          # `exit 0` only ends the STEP — subsequent steps still ran with
+          # the empty secret, the synth script fell through to the wrong
+          # SECRETS_JSON branch, and the canary failed 5 min later with a
+          # confusing "Agent error (Exception)" instead of the clean
+          # "secret missing" message at the top. Caught 2026-05-04 by
+          # dispatched run 25296530706: claude-code + missing MINIMAX
+          # silently used OpenAI keys but kept model=MiniMax-M2.7, then
+          # the workspace 401'd against MiniMax once it tried to call.
+          # Fix: exit 1 in both cron and dispatch paths. Operators who
+          # want to verify a YAML change without setting up the secret
+          # can read the verify-secrets step's stderr — the failure is
+          # itself the verification signal.
+          if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
+            echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
+            echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+            exit 1
+          fi
+
+          # LLM-key requirement is per-runtime: claude-code accepts
+          # EITHER MiniMax OR direct-Anthropic (whichever is set first),
+          # langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_KEY).
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
+            echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
+            exit 1
+          fi
+
+      - name: Install required tools
+        run: |
+          # The script depends on jq + curl (already on ubuntu-latest)
+          # and python3 (likewise). Verify they're all present so we
+          # fail fast on a runner image regression rather than mid-script.
+          for cmd in jq curl python3; do
+            command -v "$cmd" >/dev/null 2>&1 || {
+              echo "::error::required tool '$cmd' not on PATH — runner image regression?"
+              exit 1
+            }
+          done
+
+      - name: Run synthetic E2E
+        # The script handles its own teardown via EXIT trap; even on
+        # failure (timeout, assertion), the org is deprovisioned and
+        # leaks are reported. Exit code propagates from the script.
+        run: |
+          bash tests/e2e/test_staging_full_saas.sh
+
+      - name: Failure summary
+        # Runs only on failure. Adds a job summary so the workflow run
+        # page shows a quick "what happened" instead of forcing readers
+        # to scroll through script output.
+        if: failure()
+        run: |
+          {
+            echo "## Continuous synth E2E failed"
+            echo ""
+            echo "**Run ID:** ${{ github.run_id }}"
+            echo "**Trigger:** ${{ github.event_name }}"
+            echo "**Runtime:** ${E2E_RUNTIME}"
+            echo "**Slug:** synth-${{ github.run_id }}"
+            echo ""
+            echo "### What this means"
+            echo ""
+            echo "Staging just regressed on a path that previously worked. Likely classes:"
+            echo "- Schema mismatch between sender and receiver (#2345 class)"
+            echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
+            echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
+            echo "- Staging-CP env var rotation"
+            echo ""
+            echo "### Next steps"
+            echo ""
+            echo "1. Check the script output above for the assertion that failed"
+            echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
+            echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
+            echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/e2e-api.yml

@@ -0,0 +1,307 @@
+name: E2E API Smoke Test
+# Extracted from ci.yml so workflow-level concurrency can protect this job
+# from run-level cancellation (issue #458).
+#
+# Trigger model (revised 2026-04-29):
+#
+# Always FIRES on push/pull_request to staging+main. Real work is gated
+# per-step on `needs.detect-changes.outputs.api` — when paths under
+# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
+# changed, the no-op step alone runs and emits SUCCESS for the
+# `E2E API Smoke Test` check, satisfying branch protection without
+# spending CI cycles. See the in-job comment on the `e2e-api` job for
+# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
+# PR #2264 incident that drove the consolidation.
+#
+# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
+# -------------------------------------------------------------------
+# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
+# Gitea act_runner runs with `container.network: host` (operator host
+# `/opt/molecule/runners/config.yaml`), which means:
+#
+#   * Two concurrent runs both try to bind their `-p 15432:5432` /
+#     `-p 16379:6379` host ports — the second postgres/redis FATALs
+#     with `Address in use` and `docker run` returns exit 125 with
+#     `Conflict. The container name "/molecule-ci-postgres" is already
+#     in use by container ...`. Verified in run a7/2727 on 2026-05-07.
+#   * The fixed container names `molecule-ci-postgres` / `-redis` (the
+#     pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
+#     `docker rm -f` at the start of the second job KILLS the first
+#     job's still-running postgres/redis.
+#
+# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
+# platform-server is a Go binary on the host, not a containerised
+# step):
+#
+#   1. Unique container names per run:
+#         pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
+#         redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
+#      `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
+#      same run_id.
+#   2. Ephemeral host port per run (`-p 0:5432`), then read the actual
+#      bound port via `docker port` and export DATABASE_URL/REDIS_URL
+#      pointing at it. No fixed host-port → no port collision.
+#   3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
+#      the original flake fixed in #92 and the script's still IPv6-
+#      enabled.
+#   4. `if: always()` cleanup so containers don't leak when test steps
+#      fail.
+#
+# Issue #94 items #2 + #3 (also fixed here):
+#   * Pre-pull `alpine:latest` so the platform-server's provisioner
+#     (`internal/handlers/container_files.go`) can stand up its
+#     ephemeral token-write helper without a daemon.io round-trip.
+#   * Create `molecule-core-net` bridge network if missing so the
+#     provisioner's container.HostConfig {NetworkMode: ...} attach
+#     succeeds.
+# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
+# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
+# they DO come up. Timeouts are not the bottleneck; not bumped.
+#
+# Item explicitly NOT fixed here: failing test `Status back online`
+# fails because the platform's langgraph workspace template image
+# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
+# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
+# template-registry resolution issue (ADR-002 / local-build mode) and
+# belongs in a separate change that touches workspace-server, not
+# this workflow file.
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  workflow_dispatch:
+
+concurrency:
+  # Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
+  # same auto-promote-staging brittleness as e2e-staging-canvas — back-
+  # to-back staging pushes share refs/heads/staging, so the older push's
+  # queued run gets cancelled when a newer push lands. Auto-promote-
+  # staging then sees `completed/cancelled` for the older SHA and stays
+  # put; the newer SHA's gates may eventually save the day, but if the
+  # newer push gets cancelled too, we deadlock.
+  #
+  # See e2e-staging-canvas.yml's identical concurrency block for the full
+  # rationale and the 2026-04-28 incident reference.
+  group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      api: ${{ steps.decide.outputs.api }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            api:
+              - 'workspace-server/**'
+              - 'tests/e2e/**'
+              - '.github/workflows/e2e-api.yml'
+      - id: decide
+        # Always run real work for manual dispatch — no diff context to
+        # filter against and ops dispatching this expects the suite to
+        # actually exercise the platform.
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "api=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "api=${{ steps.filter.outputs.api }}" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `E2E API Smoke Test`. Real work is gated per-step
+  # on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
+  # check run for every job that matches `name:`, and a job-level
+  # `if: false` produces a SKIPPED check run. Branch protection treats
+  # all check runs with a matching context name on the latest commit as a
+  # SET — any SKIPPED in the set fails the required-check eval, even with
+  # SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
+  # 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
+  # promotion despite all real work succeeding. Collapsing to a single
+  # always-running job with conditional steps emits exactly one SUCCESS
+  # check run regardless of paths filter — branch-protection-clean.
+  e2e-api:
+    needs: detect-changes
+    name: E2E API Smoke Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      # Unique per-run container names so concurrent runs on the host-
+      # network act_runner don't collide on name OR port.
+      # `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
+      # same run_id. PORT is set later (after docker port lookup) since
+      # we let Docker assign an ephemeral host port.
+      PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
+      REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
+      PORT: "8080"
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.api != 'true'
+        run: |
+          echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
+          echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.api == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.api == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+          cache: true
+          cache-dependency-path: workspace-server/go.sum
+      - name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          # Provisioner uses alpine:latest for ephemeral token-write
+          # containers (workspace-server/internal/handlers/container_files.go).
+          # Pre-pull so the first provision in test_api.sh doesn't race
+          # the daemon's pull cache. Idempotent — `docker pull` is a no-op
+          # when the image is already present.
+          docker pull alpine:latest >/dev/null
+          # Provisioner attaches workspace containers to
+          # molecule-core-net (workspace-server/internal/provisioner/
+          # provisioner.go::DefaultNetwork). The bridge already exists on
+          # the operator host's docker daemon — `network create` is
+          # idempotent via `|| true`.
+          docker network create molecule-core-net >/dev/null 2>&1 || true
+          echo "alpine:latest pre-pulled; molecule-core-net ensured."
+      - name: Start Postgres (docker)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          # Defensive cleanup — only matches THIS run's container name,
+          # so it cannot kill a sibling run's postgres. (Pre-fix the
+          # name was static and this rm hit other runs' containers.)
+          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          # `-p 0:5432` requests an ephemeral host port; we read it back
+          # below and export DATABASE_URL.
+          docker run -d --name "$PG_CONTAINER" \
+            -e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
+            -p 0:5432 postgres:16 >/dev/null
+          # Resolve the host-side port assignment. `docker port` prints
+          # `0.0.0.0:NNNN` (and on host-net runners may also print an
+          # IPv6 line — take the first IPv4 line).
+          PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
+          if [ -z "$PG_PORT" ]; then
+            # Fallback: any first line. Some Docker versions print only
+            # one line.
+            PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
+          fi
+          if [ -z "$PG_PORT" ]; then
+            echo "::error::Could not resolve host port for $PG_CONTAINER"
+            docker port "$PG_CONTAINER" 5432/tcp || true
+            docker logs "$PG_CONTAINER" || true
+            exit 1
+          fi
+          # 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
+          echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
+          echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
+          echo "Postgres host port: ${PG_PORT}"
+          for i in $(seq 1 30); do
+            if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
+              echo "Postgres ready after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::Postgres did not become ready in 30s"
+          docker logs "$PG_CONTAINER" || true
+          exit 1
+      - name: Start Redis (docker)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
+          docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
+          REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
+          if [ -z "$REDIS_PORT" ]; then
+            REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
+          fi
+          if [ -z "$REDIS_PORT" ]; then
+            echo "::error::Could not resolve host port for $REDIS_CONTAINER"
+            docker port "$REDIS_CONTAINER" 6379/tcp || true
+            docker logs "$REDIS_CONTAINER" || true
+            exit 1
+          fi
+          echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
+          echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
+          echo "Redis host port: ${REDIS_PORT}"
+          for i in $(seq 1 15); do
+            if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
+              echo "Redis ready after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::Redis did not become ready in 15s"
+          docker logs "$REDIS_CONTAINER" || true
+          exit 1
+      - name: Build platform
+        if: needs.detect-changes.outputs.api == 'true'
+        working-directory: workspace-server
+        run: go build -o platform-server ./cmd/server
+      - name: Start platform (background)
+        if: needs.detect-changes.outputs.api == 'true'
+        working-directory: workspace-server
+        run: |
+          # DATABASE_URL + REDIS_URL exported by the start-postgres /
+          # start-redis steps point at this run's per-run host ports.
+          ./platform-server > platform.log 2>&1 &
+          echo $! > platform.pid
+      - name: Wait for /health
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          for i in $(seq 1 30); do
+            if curl -sf http://127.0.0.1:8080/health > /dev/null; then
+              echo "Platform up after ${i}s"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "::error::Platform did not become healthy in 30s"
+          cat workspace-server/platform.log || true
+          exit 1
+      - name: Assert migrations applied
+        if: needs.detect-changes.outputs.api == 'true'
+        run: |
+          tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
+          if [ "$tables" != "1" ]; then
+            echo "::error::Migrations did not apply"
+            cat workspace-server/platform.log || true
+            exit 1
+          fi
+          echo "Migrations OK"
+      - name: Run E2E API tests
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_api.sh
+      - name: Run notify-with-attachments E2E
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_notify_attachments_e2e.sh
+      - name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_priority_runtimes_e2e.sh
+      - name: Run poll-mode + since_id cursor E2E (#2339)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_poll_mode_e2e.sh
+      - name: Run poll-mode chat upload E2E (RFC #2891)
+        if: needs.detect-changes.outputs.api == 'true'
+        run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
+      - name: Dump platform log on failure
+        if: failure() && needs.detect-changes.outputs.api == 'true'
+        run: cat workspace-server/platform.log || true
+      - name: Stop platform
+        if: always() && needs.detect-changes.outputs.api == 'true'
+        run: |
+          if [ -f workspace-server/platform.pid ]; then
+            kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
+          fi
+      - name: Stop service containers
+        # always() so containers don't leak when test steps fail. The
+        # cleanup is best-effort: if the container is already gone
+        # (e.g. concurrent rerun race), don't fail the job.
+        if: always() && needs.detect-changes.outputs.api == 'true'
+        run: |
+          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true

.github/workflows/e2e-staging-canvas.yml

@@ -0,0 +1,216 @@
+name: E2E Staging Canvas (Playwright)
+
+# Playwright test suite that provisions a fresh staging org per run and
+# verifies every workspace-panel tab renders without crashing. Complements
+# e2e-staging-saas.yml (which tests the API shape) by exercising the
+# actual browser + canvas bundle against live staging.
+#
+# Triggers: push to main/staging or PR touching canvas sources + this workflow,
+# manual dispatch, and weekly cron to catch browser/runtime drift even
+# when canvas is quiet.
+# Added staging to push/pull_request branches so the auto-promote gate
+# check (--event push --branch staging) can see a completed run for this
+# workflow — mirrors what PR #1891 does for e2e-api.yml.
+
+on:
+  # Trigger model (revised 2026-04-29):
+  #
+  # Always fires on push/pull_request; real work is gated per-step on
+  # `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
+  # changed, the no-op step alone runs and emits SUCCESS for the
+  # `Canvas tabs E2E` check, satisfying branch protection without
+  # spending CI cycles. See e2e-api.yml for the rationale on why this
+  # is a single job rather than two-jobs-sharing-name.
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+  schedule:
+    # Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
+    # release-note-shaped regressions that don't ride in with a PR.
+    - cron: '0 8 * * 0'
+
+concurrency:
+  # Per-SHA grouping (changed 2026-04-28 from a single global group). The
+  # global group made auto-promote-staging brittle: when a staging push
+  # queued behind an in-flight run and a third entrant (a PR run, a
+  # follow-on push) entered the group, the staging push got cancelled —
+  # leaving auto-promote-staging looking at `completed/cancelled` for a
+  # required gate and refusing to advance main. Observed 2026-04-28
+  # 23:51-23:53 on staging tip 3f99fede.
+  #
+  # The original intent of the global group was to throttle parallel
+  # E2E provisions (each spins a fresh EC2). At our scale that throttle
+  # isn't worth the correctness cost — fresh-org-per-run isolates the
+  # state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
+  # is rounding error vs. the cost of a stuck pipeline.
+  #
+  # Per-SHA still dedupes accidental double-triggers for the SAME SHA.
+  # It does NOT cancel obsolete-PR-version runs on force-push; that
+  # wasted CI is acceptable given the alternative is losing staging-tip
+  # data that auto-promote-staging needs.
+  group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      canvas: ${{ steps.decide.outputs.canvas }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            canvas:
+              - 'canvas/**'
+              - '.github/workflows/e2e-staging-canvas.yml'
+      - id: decide
+        # Always run real tests for manual dispatch and the weekly cron —
+        # both exist precisely to exercise the suite, regardless of diff.
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
+            echo "canvas=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "canvas=${{ steps.filter.outputs.canvas }}" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `Canvas tabs E2E`. Real work is gated per-step on
+  # `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
+  # rationale — same path-filter check-name parity issue blocked PR #2264
+  # (staging→main) on 2026-04-29 because branch protection treats matching-
+  # name check runs as a SET, and any SKIPPED member fails the eval.
+  playwright:
+    needs: detect-changes
+    name: Canvas tabs E2E
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+
+    env:
+      CANVAS_E2E_STAGING: '1'
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+
+    defaults:
+      run:
+        working-directory: canvas
+
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.canvas != 'true'
+        working-directory: .
+        run: |
+          echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
+          echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
+
+      - if: needs.detect-changes.outputs.canvas == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
+            exit 2
+          fi
+
+      - name: Set up Node
+        if: needs.detect-changes.outputs.canvas == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: canvas/package-lock.json
+
+      - name: Install canvas deps
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: npm ci
+
+      - name: Install Playwright browsers
+        if: needs.detect-changes.outputs.canvas == 'true'
+        timeout-minutes: 10
+        run: npx playwright install --with-deps chromium
+
+      - name: Run staging canvas E2E
+        if: needs.detect-changes.outputs.canvas == 'true'
+        run: npx playwright test --config=playwright.staging.config.ts
+
+      - name: Upload Playwright report on failure
+        if: failure() && needs.detect-changes.outputs.canvas == 'true'
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
+        # the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
+        # implement (see ci.yml upload step for the canonical error
+        # cite). Drop this pin when Gitea ships the v4 protocol.
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: playwright-report-staging
+          path: canvas/playwright-report-staging/
+          retention-days: 14
+
+      - name: Upload screenshots on failure
+        if: failure() && needs.detect-changes.outputs.canvas == 'true'
+        # Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
+        uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
+        with:
+          name: playwright-screenshots
+          path: canvas/test-results/
+          retention-days: 14
+
+      # Safety-net teardown — fires only when Playwright's globalTeardown
+      # didn't (worker crash, runner cancel). Reads the slug from
+      # canvas/.playwright-staging-state.json (written by staging-setup
+      # as its first action, before any CP call) and deletes only that
+      # slug.
+      #
+      # Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
+      # orgs to compensate for setup-crash-before-state-file-write. That
+      # over-aggressive cleanup raced concurrent canvas-E2E runs and
+      # poisoned each other's tenants — observed 2026-04-30 when three
+      # real-test runs killed each other mid-test, surfacing as
+      # `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
+      # DNS record. Pattern-sweep removed; setup now writes the state
+      # file before any CP work, so the slug is always recoverable.
+      - name: Teardown safety net
+        if: always() && needs.detect-changes.outputs.canvas == 'true'
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          STATE_FILE=".playwright-staging-state.json"
+          if [ ! -f "$STATE_FILE" ]; then
+            echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
+            exit 0
+          fi
+          slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
+          if [ -z "$slug" ]; then
+            echo "::warning::State file present but slug missing; nothing to clean up."
+            exit 0
+          fi
+          echo "Deleting orphan tenant: $slug"
+          # Verify HTTP 2xx instead of `>/dev/null || true` swallowing
+          # failures. A 5xx or timeout previously looked identical to
+          # success, leaving the tenant alive for up to ~45 min until
+          # sweep-stale-e2e-orgs caught it. Surface failures as
+          # workflow warnings naming the slug. Don't `exit 1` — a single
+          # cleanup miss shouldn't fail-flag the canvas test when the
+          # actual smoke check passed; the sweeper is the safety net.
+          # See molecule-controlplane#420.
+          # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+          # pollution of the captured status (lint-curl-status-capture.yml).
+          set +e
+          curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
+            -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
+          set -e
+          code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
+          if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+            echo "[teardown] deleted $slug (HTTP $code)"
+          else
+            echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
+          fi
+          exit 0

.github/workflows/e2e-staging-external.yml

@@ -0,0 +1,184 @@
+name: E2E Staging External Runtime
+
+# Regression for the four/five workspaces.status=awaiting_agent transitions
+# that silently failed in production for five days before migration 046
+# extended the workspace_status enum (see
+# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
+#
+# Why this is its own workflow (not folded into e2e-staging-saas.yml):
+#   - The full-saas harness defaults to runtime=hermes, never exercises
+#     external-runtime. Adding an `external` parameter to that script
+#     would force every push to staging through both lifecycles in
+#     series, doubling the EC2 cold-start budget.
+#   - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
+#     window, 90s default + sweep interval), which we wait through
+#     deliberately. Folding it into hermes would make the long path
+#     even longer.
+#   - It can run in parallel with the hermes E2E since both create
+#     fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
+#     `e2e-...`).
+#
+# Triggers:
+#   - Push to staging when any source affecting external runtime,
+#     hibernation, or the migration set changes.
+#   - PR review for the same set.
+#   - Manual workflow_dispatch.
+#   - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
+#     30 min after e2e-staging-saas.yml's 07:00 UTC cron).
+#
+# Concurrency: serialized so two staging pushes don't fight for the
+# same EC2 quota window. cancel-in-progress=false so a half-rolled
+# tenant always finishes its teardown.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_restart.go'
+      - 'workspace-server/internal/registry/healthsweep.go'
+      - 'workspace-server/internal/registry/liveness.go'
+      - 'workspace-server/migrations/**'
+      - 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
+      - 'tests/e2e/test_staging_external_runtime.sh'
+      - '.github/workflows/e2e-staging-external.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_restart.go'
+      - 'workspace-server/internal/registry/healthsweep.go'
+      - 'workspace-server/internal/registry/liveness.go'
+      - 'workspace-server/migrations/**'
+      - 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
+      - 'tests/e2e/test_staging_external_runtime.sh'
+      - '.github/workflows/e2e-staging-external.yml'
+  workflow_dispatch:
+    inputs:
+      keep_org:
+        description: "Skip teardown for debugging (only via manual dispatch)"
+        required: false
+        type: boolean
+        default: false
+      stale_wait_secs:
+        description: "Seconds to wait for the heartbeat-staleness sweep (default 180 = 90s window + 90s buffer)"
+        required: false
+        default: "180"
+  schedule:
+    - cron: '30 7 * * *'
+
+concurrency:
+  group: e2e-staging-external
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  e2e-staging-external:
+    name: E2E Staging External Runtime
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
+      E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            # Schedule + push triggers must hard-fail when the token is
+            # missing — silent skip would mask infra rot. Manual dispatch
+            # gets the same hard-fail; an operator running this on a fork
+            # without secrets configured needs to know up-front.
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: CP staging health preflight
+        run: |
+          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
+          if [ "$code" != "200" ]; then
+            echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
+            exit 1
+          fi
+          echo "Staging CP healthy ✓"
+
+      - name: Run external-runtime E2E
+        id: e2e
+        run: bash tests/e2e/test_staging_external_runtime.sh
+
+      # Mirror the e2e-staging-saas.yml safety net: if the runner is
+      # cancelled (e.g. concurrent staging push), the test script's
+      # EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
+      # *this* run id.
+      - name: Teardown safety net (runs on cancel/failure)
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
+          # so concurrent runs and unrelated dev probes are not touched.
+          # Sweep today AND yesterday so a midnight-crossing run still
+          # cleans up its own slug.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if not run_id:
+              # Without a run id we cannot scope safely; bail rather
+              # than risk deleting unrelated tenants.
+              sys.exit(0)
+          prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
+          for o in d.get('orgs', []):
+              s = o.get('slug', '')
+              if s.startswith(prefixes) and o.get('status') != 'purged':
+                  print(s)
+          " 2>/dev/null)
+          if [ -n "$orgs" ]; then
+            echo "Safety-net sweep: deleting leftover orgs:"
+            echo "$orgs"
+            # Per-slug verified DELETE — see molecule-controlplane#420.
+            # `>/dev/null 2>&1` previously hid every failure; surface
+            # non-2xx as workflow warnings so the run page names what
+            # leaked. Sweeper catches the rest within ~45 min.
+            leaks=()
+            for slug in $orgs; do
+              # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+              # pollution of the captured status (lint-curl-status-capture.yml).
+              set +e
+              curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
+                -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+                -H "Authorization: Bearer $ADMIN_TOKEN" \
+                -H "Content-Type: application/json" \
+                -d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
+              set -e
+              code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
+              if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+                echo "[teardown] deleted $slug (HTTP $code)"
+              else
+                echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
+                leaks+=("$slug")
+              fi
+            done
+            if [ ${#leaks[@]} -gt 0 ]; then
+              echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+            fi
+          else
+            echo "Safety-net sweep: no leftover orgs to clean."
+          fi

.github/workflows/e2e-staging-saas.yml

@@ -0,0 +1,246 @@
+name: E2E Staging SaaS (full lifecycle)
+
+# Dedicated workflow that provisions a fresh staging org per run, exercises
+# the full workspace lifecycle (register → heartbeat → A2A → delegation →
+# HMA memory → activity → peers), then tears down and asserts leak-free.
+#
+# Why a separate workflow (not folded into ci.yml):
+#   - The run takes ~25-35 min (EC2 boot + cloudflared DNS + provision sweeps +
+#     agent bootstrap), way too slow for every PR.
+#   - Needs its own concurrency group so two pushes don't fight over the
+#     same staging org slug prefix.
+#   - Has its own required secrets (session cookie, admin token) that most
+#     PRs don't need to read.
+#
+# Triggers:
+#   - Push to main (regression guard)
+#   - workflow_dispatch (manual re-run from UI)
+#   - Nightly cron (catches drift even when no pushes land)
+#   - Changes to any provisioning-critical file under PR review (opt-in
+#     via the same paths watcher that e2e-api.yml uses)
+
+on:
+  # Trunk-based (Phase 3 of internal#81): main is the only branch.
+  # Previously this fired on staging push too because staging was a
+  # superset of main and ran the gate ahead of auto-promote; with no
+  # staging branch, main is where E2E gates the deploy.
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_provision.go'
+      - 'workspace-server/internal/handlers/a2a_proxy.go'
+      - 'workspace-server/internal/middleware/**'
+      - 'workspace-server/internal/provisioner/**'
+      - 'tests/e2e/test_staging_full_saas.sh'
+      - '.github/workflows/e2e-staging-saas.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace_provision.go'
+      - 'workspace-server/internal/handlers/a2a_proxy.go'
+      - 'workspace-server/internal/middleware/**'
+      - 'workspace-server/internal/provisioner/**'
+      - 'tests/e2e/test_staging_full_saas.sh'
+      - '.github/workflows/e2e-staging-saas.yml'
+  workflow_dispatch:
+    inputs:
+      runtime:
+        description: "Runtime to test (claude-code [default, MiniMax] | hermes [OpenAI] | langgraph [OpenAI])"
+        required: false
+        default: "claude-code"
+      keep_org:
+        description: "Skip teardown for debugging (only use via manual dispatch!)"
+        required: false
+        type: boolean
+        default: false
+  schedule:
+    # 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
+    # Cloudflare API regressions, etc. even on quiet days.
+    - cron: '0 7 * * *'
+
+# Serialize: staging has a finite per-hour org creation quota. Two pushes
+# landing in quick succession should queue, not race. `cancel-in-progress:
+# false` mirrors e2e-api.yml — GitHub would otherwise cancel the running
+# teardown step and leave orphan EC2s.
+concurrency:
+  group: e2e-staging-saas
+  cancel-in-progress: false
+
+jobs:
+  e2e-staging-saas:
+    name: E2E Staging SaaS
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      # Single admin-bearer secret drives provision + tenant-token
+      # retrieval + teardown. Configure in
+      # Settings → Secrets and variables → Actions → Repository secrets.
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      # MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
+      # from hermes+OpenAI default after #2578 (the staging OpenAI key
+      # account went over quota and stayed dead for 36+ hours, taking
+      # the full-lifecycle E2E red on every provisioning-critical push).
+      # claude-code template's `minimax` provider routes
+      # ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
+      # MINIMAX_API_KEY at boot — separate billing account so an
+      # OpenAI quota collapse no longer wedges the gate. Mirrors the
+      # canary-staging.yml + continuous-synth-e2e.yml migrations.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      # Direct-Anthropic alternative for operators who don't want to
+      # set up a MiniMax account (priority below MiniMax — first
+      # non-empty wins in test_staging_full_saas.sh's secrets-injection
+      # block). See #2578 PR comment for the rationale.
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      # OpenAI fallback — kept wired so an operator-dispatched run with
+      # E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
+      # exercise the OpenAI path.
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
+      E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
+      # Pin the model when running on the default claude-code path —
+      # the per-runtime default ("sonnet") routes to direct Anthropic
+      # and defeats the cost saving. Operators can override via the
+      # workflow_dispatch flow (no input wired here yet — runtime
+      # override is enough for ad-hoc).
+      E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
+      E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
+      E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: Verify LLM key present
+        run: |
+          # Per-runtime key check — claude-code uses MiniMax; hermes /
+          # langgraph (operator-dispatched only) use OpenAI. Hard-fail
+          # rather than soft-skip per #2578's lesson — empty key
+          # silently falls through to the wrong SECRETS_JSON branch and
+          # produces a confusing auth error 5 min later instead of the
+          # clean "secret missing" message at the top.
+          case "${E2E_RUNTIME}" in
+            claude-code)
+              # Either MiniMax OR direct-Anthropic works — first
+              # non-empty wins in the test script's secrets-injection
+              # priority chain.
+              if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
+                required_secret_value="${E2E_MINIMAX_API_KEY}"
+              elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+                required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value="${E2E_ANTHROPIC_API_KEY}"
+              else
+                required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
+                required_secret_value=""
+              fi
+              ;;
+            langgraph|hermes)
+              required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
+              required_secret_value="${E2E_OPENAI_API_KEY:-}"
+              ;;
+            *)
+              echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
+              required_secret_name=""
+              required_secret_value="present"
+              ;;
+          esac
+          if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
+            echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
+            exit 2
+          fi
+          echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
+
+      - name: CP staging health preflight
+        run: |
+          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
+          if [ "$code" != "200" ]; then
+            echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
+            exit 1
+          fi
+          echo "Staging CP healthy ✓"
+
+      - name: Run full-lifecycle E2E
+        id: e2e
+        run: bash tests/e2e/test_staging_full_saas.sh
+
+      # Belt-and-braces teardown: the test script itself installs a trap
+      # for EXIT/INT/TERM, but if the GH runner itself is cancelled (e.g.
+      # someone pushes a new commit and workflow concurrency is set to
+      # cancel), the trap may not fire. This `always()` step runs even on
+      # cancellation and attempts the delete a second time. The admin
+      # DELETE endpoint is idempotent so double-invoking is safe.
+      - name: Teardown safety net (runs on cancel/failure)
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          # Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
+          # nuke them. Catches the case where the script died before
+          # exporting its slug.
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          d = json.load(sys.stdin)
+          # ONLY sweep slugs from *this* CI run. Previously the filter was
+          # f'e2e-{today}-' which stomped on parallel CI runs AND any manual
+          # E2E probes a dev was running against staging (incident 2026-04-21
+          # 15:02Z: this workflow's safety net deleted an unrelated manual
+          # run's tenant 1s after it hit 'running').
+          # Sweep both today AND yesterday's UTC dates so a run that crosses
+          # midnight still matches its own slug — see the 2026-04-26→27
+          # canvas-safety-net incident for the same bug class.
+          today = datetime.date.today()
+          yesterday = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
+          if run_id:
+              prefixes = tuple(f'e2e-{d}-{run_id}-' for d in dates)
+          else:
+              prefixes = tuple(f'e2e-{d}-' for d in dates)
+          candidates = [o['slug'] for o in d.get('orgs', [])
+                        if any(o.get('slug','').startswith(p) for p in prefixes)
+                        and o.get('instance_status') not in ('purged',)]
+          print('\n'.join(candidates))
+          " 2>/dev/null)
+          # Per-slug verified DELETE (was `>/dev/null || true` — see
+          # molecule-controlplane#420). Surface non-2xx as a workflow
+          # warning naming the leaked slug; don't exit 1 (sweeper is
+          # the safety net within ~45 min).
+          leaks=()
+          for slug in $orgs; do
+            echo "Safety-net teardown: $slug"
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
+            set -e
+            code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
+          done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
+          exit 0

.github/workflows/e2e-staging-sanity.yml

@@ -0,0 +1,171 @@
+name: E2E Staging Sanity (leak-detection self-check)
+
+# Periodic assertion that the teardown safety nets in e2e-staging-saas
+# and canary-staging actually work. Runs the E2E harness with
+# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
+# the org is provisioned. The workspace-provision step then fails, the
+# script exits non-zero, and the EXIT trap + workflow always()-step
+# must still tear down cleanly.
+#
+# A green run means:
+#   - The script exited non-zero (intentional failure caught)
+#   - The trap fired teardown
+#   - The leak-detection poll found zero orphan orgs
+#
+# A red run means the teardown path itself is broken — act on this the
+# same way you'd act on a canary failure (the whole E2E safety net is
+# compromised until it's fixed).
+#
+# Cadence: once a week, Monday 06:00 UTC. Drift-slow, not per-PR — the
+# teardown path rarely changes, and a weekly heartbeat is enough to
+# catch silent regressions in cleanup code paths.
+
+on:
+  schedule:
+    - cron: '0 6 * * 1'
+  workflow_dispatch:
+
+concurrency:
+  # Shares the group with canary + full so they don't collide on
+  # staging org-create quota.
+  group: e2e-staging-sanity
+  cancel-in-progress: false
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  sanity:
+    name: Intentional-failure teardown sanity
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      E2E_MODE: canary            # lean lifecycle; we only need the org to exist
+      E2E_RUNTIME: hermes
+      E2E_RUN_ID: "sanity-${{ github.run_id }}"
+      E2E_INTENTIONAL_FAILURE: "1"
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
+            exit 2
+          fi
+
+      # Inverted assertion: the run MUST fail. If it passes, the
+      # E2E_INTENTIONAL_FAILURE path is broken (token not being
+      # poisoned correctly, or the harness silently recovered).
+      - name: Run harness — expecting exit !=0
+        id: harness
+        run: |
+          set +e
+          bash tests/e2e/test_staging_full_saas.sh
+          rc=$?
+          echo "harness_rc=$rc" >> "$GITHUB_OUTPUT"
+          # The only acceptable outcomes:
+          #   1 — harness failed mid-run, teardown ran, leak-check passed
+          #   (exit 4 means teardown left a leak — that's the real bug
+          #    this sanity check exists to catch)
+          if [ "$rc" = "1" ]; then
+            echo "✓ Harness failed as expected (rc=1); teardown trap ran, leak-check passed"
+            exit 0
+          elif [ "$rc" = "0" ]; then
+            echo "::error::Harness succeeded under E2E_INTENTIONAL_FAILURE=1 — the poisoning path is broken"
+            exit 1
+          elif [ "$rc" = "4" ]; then
+            echo "::error::LEAK DETECTED (rc=4) — teardown failed to clean up the org. Safety net broken."
+            exit 4
+          else
+            echo "::error::Unexpected rc=$rc — neither clean-failure nor leak. Investigate harness."
+            exit 1
+          fi
+
+      - name: Open issue if safety net is broken
+        if: failure()
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const title = "🚨 E2E teardown safety net broken";
+            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const body =
+              `The weekly sanity run (E2E_INTENTIONAL_FAILURE=1) did not exit ` +
+              `as expected. This means one of:\n` +
+              `  - poisoning didn't actually cause failure (test harness regression), OR\n` +
+              `  - teardown left an orphan org (leak detection caught a real bug)\n\n` +
+              `Run: ${runURL}\n\n` +
+              `This is higher priority than a canary failure — the whole ` +
+              `E2E safety net can't be trusted until this is resolved.`;
+
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'e2e-safety-net',
+            });
+            const match = existing.find(i => i.title === title);
+            if (match) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: match.number,
+                body: `Still broken. ${runURL}`,
+              });
+            } else {
+              await github.rest.issues.create({
+                owner: context.repo.owner, repo: context.repo.repo,
+                title, body,
+                labels: ['e2e-safety-net', 'bug', 'priority-high'],
+              });
+            }
+
+      # Belt-and-braces: if teardown left anything behind, nuke it here
+      # so we don't bleed staging quota. Different label from the
+      # always()-steps in the other workflows so sanity-only orgs get
+      # cleaned up by sanity runs.
+      - name: Teardown safety net
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+        run: |
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys
+          d = json.load(sys.stdin)
+          today = __import__('datetime').date.today().strftime('%Y%m%d')
+          candidates = [o['slug'] for o in d.get('orgs', [])
+                        if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
+                        and o.get('status') not in ('purged',)]
+          print('\n'.join(candidates))
+          " 2>/dev/null)
+          # Per-slug verified DELETE — see molecule-controlplane#420.
+          # Failures surface as workflow warnings; the sweeper is the
+          # safety net within ~45 min.
+          leaks=()
+          for slug in $orgs; do
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
+            set -e
+            code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
+              leaks+=("$slug")
+            fi
+          done
+          if [ ${#leaks[@]} -gt 0 ]; then
+            echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
+          fi
+          exit 0

.github/workflows/handlers-postgres-integration.yml

@@ -0,0 +1,252 @@
+name: Handlers Postgres Integration
+
+# Real-Postgres integration tests for workspace-server/internal/handlers/.
+# Triggered on every PR/push that touches the handlers package.
+#
+# Why this workflow exists
+# ------------------------
+# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
+# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
+# depend on the row state AFTER the SQL runs. The result_preview-lost
+# bug shipped to staging in PR #2854 because every unit test was
+# satisfied with "an UPDATE statement fired" — none verified the row's
+# preview field actually landed. The local-postgres E2E that retrofit
+# self-review caught it took 2 minutes to set up and would have caught
+# the bug at PR-time.
+#
+# Why this workflow does NOT use `services: postgres:` (Class B fix)
+# ------------------------------------------------------------------
+# Our act_runner config has `container.network: host` (operator host
+# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
+# the job container AND every service container. With host-net, two
+# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
+# second postgres FATALs with `could not create any TCP/IP sockets:
+# Address in use`, and Docker auto-removes it (act_runner sets
+# AutoRemove:true on service containers). By the time the migrations
+# step runs `psql`, the postgres container is gone, hence
+# `Connection refused` then `failed to remove container: No such
+# container` at cleanup time.
+#
+# Per-job `container.network` override is silently ignored by
+# act_runner — `--network and --net in the options will be ignored.`
+# appears in the runner log. Documented constraint.
+#
+# So we sidestep `services:` entirely. The job container still uses
+# host-net (inherited from runner config; required for cache server
+# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
+# postgres on the existing `molecule-core-net` bridge with a
+# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
+# read its bridge IP via `docker inspect`. A host-net job container
+# can reach a bridge-net container directly via the bridge IP (verified
+# manually on operator host 2026-05-08).
+#
+# Trade-offs vs. the original `services:` shape:
+#   + No host-port collision; N parallel runs share the bridge cleanly
+#   + `if: always()` cleanup runs even on test-step failure
+#   - One more step in the workflow (+~3 lines)
+#   - Requires `molecule-core-net` to exist on the operator host
+#     (it does; declared in docker-compose.yml + docker-compose.infra.yml)
+#
+# Class B Hongming-owned CICD red sweep, 2026-05-08.
+#
+# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  merge_group:
+    types: [checks_requested]
+  workflow_dispatch:
+
+concurrency:
+  group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  detect-changes:
+    name: detect-changes
+    runs-on: ubuntu-latest
+    outputs:
+      handlers: ${{ steps.filter.outputs.handlers }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            handlers:
+              - 'workspace-server/internal/handlers/**'
+              - 'workspace-server/internal/wsauth/**'
+              - 'workspace-server/migrations/**'
+              - '.github/workflows/handlers-postgres-integration.yml'
+
+  # Single-job-with-per-step-if pattern: always runs to satisfy the
+  # required-check name on branch protection; real work gates on the
+  # paths filter. See ci.yml's Platform (Go) for the same shape.
+  integration:
+    name: Handlers Postgres Integration
+    needs: detect-changes
+    runs-on: ubuntu-latest
+    env:
+      # Unique name per run so concurrent jobs don't collide on the
+      # bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
+      # workflow_dispatch reruns of the same run_id.
+      PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
+      # Bridge network already exists on the operator host (declared
+      # in docker-compose.yml + docker-compose.infra.yml).
+      PG_NETWORK: molecule-core-net
+    defaults:
+      run:
+        working-directory: workspace-server
+    steps:
+      - if: needs.detect-changes.outputs.handlers != 'true'
+        working-directory: .
+        run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+        with:
+          go-version: 'stable'
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Start sibling Postgres on bridge network
+        working-directory: .
+        run: |
+          # Sanity: the bridge network must exist on the operator host.
+          # Hard-fail loud if it doesn't — easier to spot than a silent
+          # auto-create that diverges from the rest of the stack.
+          if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
+            echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
+            exit 1
+          fi
+
+          # If a stale container with the same name exists (rerun on
+          # the same run_id), wipe it first.
+          docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
+
+          docker run -d \
+            --name "${PG_NAME}" \
+            --network "${PG_NETWORK}" \
+            --health-cmd "pg_isready -U postgres" \
+            --health-interval 5s \
+            --health-timeout 5s \
+            --health-retries 10 \
+            -e POSTGRES_PASSWORD=test \
+            -e POSTGRES_DB=molecule \
+            postgres:15-alpine >/dev/null
+
+          # Read back the bridge IP. Always present immediately after
+          # `docker run -d` for bridge networks.
+          PG_HOST=$(docker inspect "${PG_NAME}" \
+            --format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
+          if [ -z "${PG_HOST}" ]; then
+            echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
+            docker logs "${PG_NAME}" || true
+            exit 1
+          fi
+          echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
+          echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
+          echo "Started ${PG_NAME} at ${PG_HOST}:5432"
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Apply migrations to Postgres service
+        env:
+          PGPASSWORD: test
+        run: |
+          # Wait for postgres to actually accept connections. Docker's
+          # health-cmd handles container-side readiness, but the wire
+          # to the bridge IP is best-tested with pg_isready directly.
+          for i in {1..15}; do
+            if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
+            echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
+          done
+
+          # Apply every .up.sql in lexicographic order with
+          # ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
+          # blocking the suite. This handles the current schema state
+          # where a few historical migrations (e.g. 017_memories_fts_*)
+          # depend on tables that were later renamed/dropped and so
+          # cannot replay from scratch. The migrations that DO succeed
+          # land their tables, which is sufficient for the integration
+          # tests in handlers/.
+          #
+          # Why not maintain a curated allowlist: every new migration
+          # touching a handlers/-tested table would have to update this
+          # workflow. With apply-all-or-skip, a future migration that
+          # adds a column to delegations runs automatically (its base
+          # table 049_delegations.up.sql already succeeded above it in
+          # the order). Operators only need to revisit this if the
+          # migration chain becomes legitimately replayable end-to-end.
+          #
+          # Per-migration result is logged so a failed migration that
+          # SHOULD have been replayable surfaces in the CI log instead
+          # of silently failing.
+          # Apply both *.sql (legacy, lives next to its module) and
+          # *.up.sql (newer up/down convention) in a single
+          # lexicographically-sorted pass. Excluding *.down.sql so the
+          # newest-naming-convention pairs don't undo themselves mid-run.
+          # Pre-#149-followup this loop only globbed *.up.sql, which
+          # silently skipped 001_workspaces.sql + 009_activity_logs.sql
+          # — fine while no integration test depended on those tables,
+          # not fine once a cross-table atomicity test came in.
+          set +e
+          for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
+            if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
+                  -f "$migration" >/dev/null 2>&1; then
+              echo "✓ $(basename "$migration")"
+            else
+              echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
+            fi
+          done
+          set -e
+
+          # Sanity: the delegations + workspaces + activity_logs tables
+          # MUST exist for the integration tests to be meaningful. Hard-
+          # fail if any didn't land — that would be a real regression we
+          # want loud.
+          for tbl in delegations workspaces activity_logs pending_uploads; do
+            if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
+                -c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
+                | grep -q 1; then
+              echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
+              exit 1
+            fi
+            echo "✓ $tbl table present"
+          done
+
+      - if: needs.detect-changes.outputs.handlers == 'true'
+        name: Run integration tests
+        run: |
+          # INTEGRATION_DB_URL is exported by the start-postgres step;
+          # points at the per-run bridge IP, not 127.0.0.1, so concurrent
+          # workflow runs don't fight over a host-net 5432 port.
+          go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
+
+      - if: failure() && needs.detect-changes.outputs.handlers == 'true'
+        name: Diagnostic dump on failure
+        env:
+          PGPASSWORD: test
+        run: |
+          echo "::group::postgres container status"
+          docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
+          docker logs "${PG_NAME}" 2>&1 | tail -50 || true
+          echo "::endgroup::"
+          echo "::group::delegations table state"
+          psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
+          echo "::endgroup::"
+
+      - if: always() && needs.detect-changes.outputs.handlers == 'true'
+        name: Stop sibling Postgres
+        working-directory: .
+        run: |
+          # always() so containers don't leak when migrations or tests
+          # fail. The cleanup is best-effort: if the container is
+          # already gone (e.g. concurrent rerun race), don't fail the job.
+          docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
+          echo "Cleaned up ${PG_NAME}"
+

.github/workflows/harness-replays.yml

@@ -0,0 +1,248 @@
+name: Harness Replays
+
+# Boots tests/harness (production-shape compose topology with TenantGuard,
+# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
+# every replay under tests/harness/replays/. Fails the PR if any replay
+# fails.
+#
+# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
+# a public route in router.go but forgot to add it to TenantGuard's
+# allowlist. The handler-level test in buildinfo_test.go constructed a
+# minimal gin engine without TenantGuard — green. The harness's
+# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
+# inject X-Molecule-Org-Id, so the curl path is identical to production's
+# redeploy verifier), but no one ran the harness pre-merge. The bug
+# shipped; the redeploy verifier silently soft-warned every tenant as
+# "unreachable" for ~1 day before being noticed.
+#
+# This gate makes "did you actually run the harness?" a CI invariant
+# instead of a memory-discipline thing.
+#
+# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
+# to staging+main, real work is gated per-step on detect-changes output.
+# One job → one check run → branch-protection-clean (the SKIPPED-in-set
+# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'tests/harness/**'
+      - '.github/workflows/harness-replays.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace-server/**'
+      - 'canvas/**'
+      - 'tests/harness/**'
+      - '.github/workflows/harness-replays.yml'
+  workflow_dispatch:
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  # Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
+  # cancellation deadlock — see e2e-api.yml's concurrency block for
+  # the 2026-04-28 incident that codified this pattern.
+  group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      run: ${{ steps.decide.outputs.run }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - id: decide
+        run: |
+          # workflow_dispatch: always run (manual trigger)
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Determine the base commit to diff against.
+          # For pull_request: use base.sha (the merge-base with main/staging).
+          # For push: use github.event.before (the previous tip of the branch).
+          # Fallback for new branches (all-zeros SHA): run everything.
+          if [ "${{ github.event_name }}" = "pull_request" ] && \
+             [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            BASE="${{ github.event.pull_request.base.sha }}"
+          elif [ -n "${{ github.event.before }}" ] && \
+               ! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
+            BASE="${{ github.event.before }}"
+          else
+            # New branch or github.event.before unavailable — run everything.
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # GitHub Actions and Gitea Actions both expose github.sha for HEAD.
+          DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
+          echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
+
+          if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.github/workflows/harness-replays\.yml$'; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job that always runs. Real work is gated per-step on
+  # detect-changes.outputs.run so an unrelated PR (e.g. doc-only
+  # change to molecule-controlplane wired here later) emits the
+  # required check without spending CI cycles. Single-job pattern
+  # matches e2e-api.yml — see that workflow's comment for why a
+  # job-level `if: false` would block branch protection via the
+  # SKIPPED-in-set bug.
+  harness-replays:
+    needs: detect-changes
+    name: Harness Replays
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.run != 'true'
+        run: |
+          echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
+          echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
+          echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
+
+      - if: needs.detect-changes.outputs.run == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      # Log what files were detected so future failures include the diff.
+      - name: Log detected changes
+        if: needs.detect-changes.outputs.run == 'true'
+        run: |
+          echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
+
+      # github-app-auth sibling-checkout removed 2026-05-07 (#157):
+      # the plugin was dropped + Dockerfile.tenant no longer COPYs it.
+
+      # Pre-clone manifest deps before docker compose builds the tenant
+      # image (Task #173 followup — same pattern as
+      # publish-workspace-server-image.yml's "Pre-clone manifest deps"
+      # step).
+      #
+      # Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
+      # and tenant-beta from workspace-server/Dockerfile.tenant with
+      # context=../.. (repo root). That Dockerfile expects
+      # .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
+      # to be present at build context root (post-#173 it COPYs from there
+      # instead of running an in-image clone — the in-image clone failed
+      # with "could not read Username for https://git.moleculesai.app"
+      # because there's no auth path inside the build sandbox).
+      #
+      # Without this step harness-replays fails before any replay runs,
+      # with `failed to calculate checksum of ref ...
+      # "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
+      # (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
+      # symptom, different root cause: staging still has the in-image
+      # clone path, hits the auth error directly).
+      #
+      # 2026-05-08 sub-finding (#192): the clone step ALSO fails when
+      # any referenced workspace-template repo is private and the
+      # AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
+      # access. Root cause: 5 of 9 workspace-template repos
+      # (openclaw, codex, crewai, deepagents, gemini-cli) had been
+      # marked private with no team grant. Resolution: flipped them
+      # to public per `feedback_oss_first_repo_visibility_default`
+      # (the OSS surface should be public). Layer-3 (customer-private +
+      # marketplace third-party repos) tracked separately in
+      # internal#102.
+      #
+      # Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
+      # is the devops-engineer persona PAT, NOT the founder PAT (per
+      # `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
+      # embeds it as basic-auth for the duration of the clones and strips
+      # .git directories — the token never enters the resulting image.
+      - name: Pre-clone manifest deps
+        if: needs.detect-changes.outputs.run == 'true'
+        env:
+          MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
+            echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
+            exit 1
+          fi
+          mkdir -p .tenant-bundle-deps
+          bash scripts/clone-manifest.sh \
+            manifest.json \
+            .tenant-bundle-deps/workspace-configs-templates \
+            .tenant-bundle-deps/org-templates \
+            .tenant-bundle-deps/plugins
+          # Sanity-check counts so a silent partial clone fails fast
+          # instead of producing a half-empty image.
+          ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
+          plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
+          echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
+
+      - name: Install Python deps for replays
+        # peer-discovery-404 (and future replays) eval Python against the
+        # running tenant — importing workspace/a2a_client.py pulls in
+        # httpx. tests/harness/requirements.txt holds just the HTTP-client
+        # surface to keep CI install fast (~3s) vs the full
+        # workspace/requirements.txt (~30s).
+        if: needs.detect-changes.outputs.run == 'true'
+        run: pip install -r tests/harness/requirements.txt
+
+      - name: Run all replays against the harness
+        # run-all-replays.sh: boot via up.sh → seed via seed.sh → run
+        # every replays/*.sh → tear down via down.sh on EXIT (trap).
+        # Non-zero exit on any replay failure.
+        #
+        # KEEP_UP=1: without this, the script's trap-on-EXIT tears
+        # down containers immediately on failure, leaving the dump
+        # step below with nothing to dump (verified on PR #2410's
+        # first run — tenant became unhealthy, trap fired, dump
+        # step saw empty containers). Keeping them up lets the
+        # failure path collect tenant/cp-stub/cf-proxy logs. The
+        # always-run "Force teardown" step does the actual cleanup.
+        if: needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        env:
+          KEEP_UP: "1"
+        run: ./run-all-replays.sh
+
+      - name: Dump compose logs on failure
+        # SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
+        # file even for read-only `logs` calls. up.sh generates a per-run key
+        # and exports it to its OWN shell — this step runs in a fresh shell
+        # that wouldn't see it, so without a placeholder the validate step
+        # errors before logs print (verified against PR #2492's first run:
+        # "required variable SECRETS_ENCRYPTION_KEY is missing a value").
+        # A placeholder is fine — we're only reading log streams, not booting.
+        if: failure() && needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        env:
+          SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
+        run: |
+          echo "=== docker compose ps ==="
+          docker compose -f compose.yml ps || true
+          echo "=== tenant-alpha logs ==="
+          docker compose -f compose.yml logs tenant-alpha || true
+          echo "=== tenant-beta logs ==="
+          docker compose -f compose.yml logs tenant-beta || true
+          echo "=== cp-stub logs ==="
+          docker compose -f compose.yml logs cp-stub || true
+          echo "=== cf-proxy logs ==="
+          docker compose -f compose.yml logs cf-proxy || true
+          echo "=== postgres-alpha logs (last 100) ==="
+          docker compose -f compose.yml logs --tail 100 postgres-alpha || true
+          echo "=== postgres-beta logs (last 100) ==="
+          docker compose -f compose.yml logs --tail 100 postgres-beta || true
+
+      - name: Force teardown
+        # We pass KEEP_UP=1 to run-all-replays.sh so the dump step
+        # above sees real containers — that means we own teardown
+        # explicitly here. Always run.
+        if: always() && needs.detect-changes.outputs.run == 'true'
+        working-directory: tests/harness
+        run: ./down.sh || true

.github/workflows/lint-curl-status-capture.yml

@@ -0,0 +1,94 @@
+name: Lint curl status-code capture
+
+# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
+# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
+#
+#   HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
+#
+# When curl exits non-zero (connection reset → 56, --fail-with-body 4xx/5xx
+# → 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
+# "000" for connection failures or the actual code for HTTP errors. The
+# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
+# stdout, producing values like "000000" or "409000" that fail string
+# comparisons against "200" while looking superficially right.
+#
+# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
+# #2797). Memory: feedback_curl_status_capture_pollution.md.
+#
+# Fix shape (route -w into a tempfile so curl's exit code can't pollute):
+#
+#   set +e
+#   curl ... -w '%{http_code}' >code.txt 2>/dev/null
+#   set -e
+#   HTTP_CODE=$(cat code.txt 2>/dev/null)
+#   [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+on:
+  pull_request:
+    paths: ['.github/workflows/**']
+  push:
+    branches: [main, staging]
+    paths: ['.github/workflows/**']
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  scan:
+    name: Scan workflows for curl status-capture pollution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
+        run: |
+          set -uo pipefail
+          # Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
+          # subshell where the entire command-substitution wraps a curl that
+          # ends with `|| echo "000"`. Must distinguish from the SAFE shape
+          # `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
+          # tempfile produces empty stdout, no pollution.
+          python3 <<'PY'
+          import os, re, sys, glob
+
+          BAD_FILES = []
+
+          # Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
+          # The `\\n` is the bash line-continuation that lets curl flags span lines.
+          # We collapse continuation lines first, then look for the single-line bad pattern.
+          PATTERN = re.compile(
+              r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
+              re.DOTALL,
+          )
+
+          # Self-skip: this lint workflow contains the literal anti-pattern in
+          # its own docstring — that's intentional, not a bug.
+          SELF = ".github/workflows/lint-curl-status-capture.yml"
+
+          for f in sorted(glob.glob(".github/workflows/*.yml")):
+              if f == SELF:
+                  continue
+              with open(f) as fh:
+                  content = fh.read()
+              # Collapse bash line-continuations (\\\n + leading whitespace)
+              # into a single logical line so the regex can see the full
+              # curl invocation as one chunk.
+              flat = re.sub(r'\\\s*\n\s*', ' ', content)
+              for m in PATTERN.finditer(flat):
+                  BAD_FILES.append((f, m.group(0)[:120]))
+
+          if not BAD_FILES:
+              print("✓ No curl-status-capture pollution patterns detected")
+              sys.exit(0)
+
+          print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
+          for f, snippet in BAD_FILES:
+              print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
+              print(f"   matched: {snippet}…")
+          print()
+          print("Fix template:")
+          print('  set +e')
+          print('  curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
+          print('  set -e')
+          print('  HTTP_CODE=$(cat code.txt 2>/dev/null)')
+          print('  [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
+          sys.exit(1)
+          PY

.github/workflows/publish-canvas-image.yml

@@ -0,0 +1,121 @@
+name: publish-canvas-image
+
+# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
+# on main that touches canvas code. Previously canvas changes were visible in
+# CI (npm run build passed) but the live container was never updated —
+# operators had to manually run `docker compose build canvas` each time.
+#
+# Mirror of publish-platform-image.yml, adapted for the Next.js canvas layer.
+# See that workflow for inline notes on macOS Keychain isolation and QEMU.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      # Only rebuild when canvas source changes — saves GHA minutes on
+      # platform-only / docs-only / MCP-only merges.
+      - 'canvas/**'
+      - '.github/workflows/publish-canvas-image.yml'
+  # Manual trigger: use after a non-canvas merge that still needs a fresh
+  # image (e.g. a Dockerfile change lives outside the canvas/ tree).
+  workflow_dispatch:
+    inputs:
+      platform_url:
+        description: 'NEXT_PUBLIC_PLATFORM_URL baked into the bundle (default: http://localhost:8080)'
+        required: false
+        default: ''
+      ws_url:
+        description: 'NEXT_PUBLIC_WS_URL baked into the bundle (default: ws://localhost:8080/ws)'
+        required: false
+        default: ''
+
+permissions:
+  contents: read
+  packages: write  # required to push to ghcr.io/${{ github.repository_owner }}/*
+
+env:
+  IMAGE_NAME: ghcr.io/molecule-ai/canvas
+
+jobs:
+  build-and-push:
+    name: Build & push canvas image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Log in to GHCR
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+
+      # Health check: verify Docker daemon is accessible before attempting any
+      # build steps. This fails loudly at step 1 when the runner's docker.sock
+      # is inaccessible rather than silently continuing to the build step
+      # where docker build fails deep in ECR auth with a cryptic error.
+      - name: Verify Docker daemon access
+        run: |
+          set -euo pipefail
+          echo "::group::Docker daemon health check"
+          docker info 2>&1 | head -5 || {
+            echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
+            echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
+            exit 1
+          }
+          echo "Docker daemon OK"
+          echo "::endgroup::"
+
+      - name: Compute tags
+        id: tags
+        shell: bash
+        run: |
+          echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve build args
+        id: build_args
+        # Priority: workflow_dispatch input > repo secret > hardcoded default.
+        # NEXT_PUBLIC_* env vars are baked into the JS bundle at build time by
+        # Next.js — they cannot be changed at runtime without a full rebuild.
+        # For local docker-compose deployments the defaults (localhost:8080)
+        # work as-is; production deployments should set CANVAS_PLATFORM_URL
+        # and CANVAS_WS_URL as repository secrets.
+        #
+        # Inputs are passed via env vars (not direct ${{ }} interpolation) to
+        # prevent shell injection from workflow_dispatch string inputs.
+        shell: bash
+        env:
+          INPUT_PLATFORM_URL: ${{ github.event.inputs.platform_url }}
+          SECRET_PLATFORM_URL: ${{ secrets.CANVAS_PLATFORM_URL }}
+          INPUT_WS_URL: ${{ github.event.inputs.ws_url }}
+          SECRET_WS_URL: ${{ secrets.CANVAS_WS_URL }}
+        run: |
+          PLATFORM_URL="${INPUT_PLATFORM_URL:-${SECRET_PLATFORM_URL:-http://localhost:8080}}"
+          WS_URL="${INPUT_WS_URL:-${SECRET_WS_URL:-ws://localhost:8080/ws}}"
+
+          echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
+          echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
+
+      - name: Build & push canvas image to GHCR
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
+        with:
+          context: ./canvas
+          file: ./canvas/Dockerfile
+          platforms: linux/amd64
+          push: true
+          build-args: |
+            NEXT_PUBLIC_PLATFORM_URL=${{ steps.build_args.outputs.platform_url }}
+            NEXT_PUBLIC_WS_URL=${{ steps.build_args.outputs.ws_url }}
+          tags: |
+            ${{ env.IMAGE_NAME }}:latest
+            ${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)

.github/workflows/railway-pin-audit.yml

@@ -0,0 +1,207 @@
+name: Railway pin audit (drift detection)
+
+# Daily audit of Railway env vars for drift-prone image-tag pins —
+# automation-cadence layer over the detection script + regression test
+# shipped in PR #2168 (#2001 closure).
+#
+# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
+# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
+# "every fix didn't propagate" — really the tenant image was so old it
+# didn't read the env vars those fixes produced. The audit script
+# (scripts/ops/audit-railway-sha-pins.sh) flags drift; this workflow
+# runs the same check unattended on a daily cron.
+#
+# Cadence: once a day, 13:00 UTC (06:00 PT). Daily is the right
+# cadence for variables-tier config — Railway env var changes are
+# deliberate operator actions, low-frequency. Hourly would risk
+# Railway API rate-limit surprises and is overkill for the change rate.
+#
+# Issue-on-failure: drift triggers a priority-high issue, mirroring
+# .github/workflows/e2e-staging-sanity.yml's pattern. Drift is
+# medium-priority "config slipped, fix at next ops window," not
+# active-outage paging.
+#
+# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
+# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN
+# (silent-success on schedule was the failure-mode class that bit the
+# team before; cron firing without checking anything is worse than no
+# cron). The workflow_dispatch trigger SOFT-SKIPS on missing secret so
+# an operator can dry-run the workflow shape during initial provisioning
+# without tripping a fake red.
+
+on:
+  schedule:
+    - cron: '0 13 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: railway-pin-audit
+  cancel-in-progress: false
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  audit:
+    name: Audit Railway env vars for drift-prone pins
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify RAILWAY_AUDIT_TOKEN present
+        # Schedule trigger: hard-fail when the secret is missing —
+        # otherwise the cron silently runs against the wrong scope (or
+        # exits 2 from the script and we issue-spam) without anyone
+        # noticing the token rot.
+        # Dispatch trigger: soft-skip — operator may be dry-running the
+        # workflow shape before provisioning the secret. Logged as a
+        # workflow notice, not a failure.
+        env:
+          RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+          EVENT_NAME: ${{ github.event_name }}
+        id: secret_check
+        run: |
+          set -euo pipefail
+          if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
+            echo "have_secret=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          echo "have_secret=false" >> "$GITHUB_OUTPUT"
+          if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
+            echo "::notice::RAILWAY_AUDIT_TOKEN not configured — soft-skipping (manual dispatch)"
+            exit 0
+          fi
+          echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
+          exit 1
+
+      - name: Install Railway CLI
+        if: steps.secret_check.outputs.have_secret == 'true'
+        # Pinned hash matching the public install instructions; bump in
+        # tandem with the audit-script's documented Railway CLI version.
+        run: |
+          set -euo pipefail
+          curl -fsSL https://railway.com/install.sh | sh
+          # The installer drops the binary in ~/.railway/bin
+          echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
+
+      - name: Verify Railway CLI authenticated
+        if: steps.secret_check.outputs.have_secret == 'true'
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set -euo pipefail
+          # `railway whoami` exits non-zero when the token is
+          # unauthenticated or doesn't have any project access.
+          if ! railway whoami >/dev/null 2>&1; then
+            echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
+            exit 2
+          fi
+
+      - name: Link molecule-platform project
+        if: steps.secret_check.outputs.have_secret == 'true'
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        # Project ID from reference_production_stack: molecule-platform
+        # / 7ccc8c68-61f4-42ab-9be5-586eeee11768. Linking is per-process,
+        # so we re-link in this CI shell (the audit script comment says
+        # it deliberately doesn't chdir for you because the linked
+        # project's identity matters).
+        run: |
+          set -euo pipefail
+          railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
+
+      - name: Run drift audit
+        if: steps.secret_check.outputs.have_secret == 'true'
+        id: audit
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
+        run: |
+          set +e
+          bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
+          rc=${PIPESTATUS[0]}
+          echo "rc=$rc" >> "$GITHUB_OUTPUT"
+          # Capture the audit log for the issue body.
+          {
+            echo 'log<<AUDIT_EOF'
+            cat /tmp/audit.log
+            echo 'AUDIT_EOF'
+          } >> "$GITHUB_OUTPUT"
+          # Exit codes from the script:
+          #   0 — no drift; workflow goes green
+          #   1 — drift detected; we'll file an issue and fail the run
+          #   2 — railway CLI unauthenticated / project unlinked; fail
+          # Anything else: also fail.
+          case "$rc" in
+            0) exit 0 ;;
+            1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
+            2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
+            *) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
+          esac
+
+      - name: Open / update drift issue
+        if: failure() && steps.audit.outputs.rc == '1'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          AUDIT_LOG: ${{ steps.audit.outputs.log }}
+        with:
+          script: |
+            const title = "🚨 Railway env-var drift detected";
+            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const body =
+              `Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n` +
+              `**What this means:** an env var (likely on \`controlplane\`) is pinned to a SHA-shaped or semver tag instead of a floating tag. ` +
+              `Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service doesn't pick them up.\n\n` +
+              `**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (\`:staging-latest\`, \`:main\`) unless the pin is intentional and documented in the ops runbook.\n\n` +
+              `**Audit output:**\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\`\n\n` +
+              `Run: ${runURL}\n\n` +
+              `Closes automatically when a subsequent daily run reports clean.`;
+
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'railway-drift',
+            });
+            const match = existing.find(i => i.title === title);
+            if (match) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: match.number,
+                body: `Still drifting. ${runURL}\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\``,
+              });
+            } else {
+              await github.rest.issues.create({
+                owner: context.repo.owner, repo: context.repo.repo,
+                title, body,
+                labels: ['railway-drift', 'bug', 'priority-high'],
+              });
+            }
+
+      - name: Close stale drift issue on clean run
+        # When a previously-flagged drift gets fixed by an operator,
+        # the next daily run goes green. Close any open `railway-drift`
+        # issue with a confirmation comment so the queue doesn't carry
+        # stale ones.
+        if: success() && steps.audit.outputs.rc == '0'
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner, repo: context.repo.repo,
+              state: 'open', labels: 'railway-drift',
+            });
+            for (const issue of existing) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: issue.number,
+                body: `Daily audit clean — drift resolved. ${runURL}`,
+              });
+              await github.rest.issues.update({
+                owner: context.repo.owner, repo: context.repo.repo,
+                issue_number: issue.number,
+                state: 'closed',
+                state_reason: 'completed',
+              });
+            }

.github/workflows/redeploy-tenants-on-main.yml

@@ -0,0 +1,400 @@
+name: redeploy-tenants-on-main
+
+# Auto-refresh prod tenant EC2s after every main merge.
+#
+# Why this workflow exists: publish-workspace-server-image builds and
+# pushes a new platform-tenant :<sha> to ECR on every merge to main,
+# but running tenants pulled their image once at boot and never re-pull.
+# Users see stale code indefinitely.
+#
+# This workflow closes the gap by calling the control-plane admin
+# endpoint that performs a canary-first, batched, health-gated rolling
+# redeploy across every live tenant. Implemented in molecule-ai/
+# molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
+# (feat/tenant-auto-redeploy, landing alongside this workflow).
+#
+# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
+# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
+# Gitea suspension migration. The canary-verify.yml promote step now
+# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
+#
+# Runtime ordering:
+#   1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
+#   2. This workflow fires via workflow_run, calls redeploy-fleet with
+#      target_tag=staging-<sha>. No CDN propagation wait needed —
+#      ECR image manifest is consistent immediately after push.
+#   3. Calls redeploy-fleet with canary_slug (if set) and a soak
+#      period. Canary proves the image boots; batches follow.
+#   4. Any failure aborts the rollout and leaves older tenants on the
+#      prior image — safer default than half-and-half state.
+#
+# Rollback path: re-run this workflow with a specific SHA pinned via
+# the workflow_dispatch input. That calls redeploy-fleet with
+# target_tag=<sha>, re-pulling the older image on every tenant.
+
+on:
+  workflow_run:
+    workflows: ['publish-workspace-server-image']
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      target_tag:
+        # Empty default → auto-trigger and dispatch-without-input both
+        # resolve to `staging-<short_head_sha>` (the digest publish-image
+        # just pushed). Pre-fix this defaulted to 'latest', which only
+        # gets retagged by canary-verify's promote-to-latest job — and
+        # that job soft-skips when CANARY_TENANT_URLS is unset (the
+        # current state, until Phase 2 canary fleet is live). Result:
+        # `:latest` had been pinned to a 4-day-old digest (2026-04-28)
+        # while every main push pushed fresh `staging-<sha>` images;
+        # every prod redeploy pulled the stale `:latest` and the verify
+        # step correctly flagged 3/3 tenants STALE. Pulling the
+        # just-published `staging-<sha>` directly skips the dead retag
+        # path. When canary fleet is real, this workflow should chain
+        # on canary-verify completion (workflow_run from canary-verify),
+        # not publish-image — separate, smaller PR.
+        description: 'Tenant image tag to deploy (e.g. "latest", "staging-a59f1a6c"). Empty = auto staging-<head_sha>.'
+        required: false
+        type: string
+        default: ''
+      canary_slug:
+        description: 'Tenant slug to deploy first + soak (empty = skip canary, fan out immediately).'
+        required: false
+        type: string
+        # Must be an actual prod tenant slug (current: hongming,
+        # chloe-dong, reno-stars). The previous default 'hongmingwang'
+        # didn't match any tenant — CP soft-skipped the missing canary
+        # and the fleet rolled out without the soak gate, defeating the
+        # whole point of canary-first.
+        default: 'hongming'
+      soak_seconds:
+        description: 'Seconds to wait after canary before fanning out.'
+        required: false
+        type: string
+        default: '60'
+      batch_size:
+        description: 'How many tenants SSM redeploys in parallel per batch.'
+        required: false
+        type: string
+        default: '3'
+      dry_run:
+        description: 'Plan only — do not actually redeploy.'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  # No write scopes needed — the workflow hits an external CP endpoint,
+  # not the GitHub API.
+
+# Serialize redeploys so two rapid main pushes' redeploys don't overlap
+# and cause confusing per-tenant SSM state. Without this, GitHub's
+# implicit workflow_run queueing would *probably* serialize them, but
+# the explicit block makes the invariant defensible. Mirrors the
+# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
+#
+# cancel-in-progress: false → aborting a half-rolled-out fleet would
+# leave tenants stuck on whatever image they happened to be on when
+# cancelled. Better to finish the in-flight rollout before starting
+# the next one.
+concurrency:
+  group: redeploy-tenants-on-main
+  cancel-in-progress: false
+
+jobs:
+  redeploy:
+    # Skip the auto-trigger if publish-workspace-server-image didn't
+    # actually succeed. workflow_run fires on any completion state; we
+    # don't want to redeploy against a half-built image.
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Note on ECR propagation
+        # ECR image manifests are consistent immediately after push — no
+        # CDN cache to wait for. The old GHCR-based workflow had a 30s
+        # sleep to avoid race conditions; ECR makes that unnecessary.
+        run: echo "ECR image available immediately after push — proceeding."
+
+      - name: Compute target tag
+        id: tag
+        # Resolution order:
+        #   1. Operator-supplied input (workflow_dispatch with explicit
+        #      tag) → used verbatim. Lets ops pin `latest` for emergency
+        #      rollback to last canary-verified digest, or pin a specific
+        #      `staging-<sha>` to roll back to a known-good build.
+        #   2. Default → `staging-<short_head_sha>`. The just-published
+        #      digest. Bypasses the `:latest` retag path that's currently
+        #      dead (canary-verify soft-skips without canary fleet, so
+        #      the only thing retagging `:latest` today is the manual
+        #      promote-latest.yml — last run 2026-04-28). Auto-trigger
+        #      from workflow_run uses workflow_run.head_sha; manual
+        #      dispatch with no input falls through to github.sha.
+        env:
+          INPUT_TAG: ${{ inputs.target_tag }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+        run: |
+          set -euo pipefail
+          if [ -n "${INPUT_TAG:-}" ]; then
+            echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
+            echo "Using operator-pinned tag: $INPUT_TAG"
+          else
+            SHORT="${HEAD_SHA:0:7}"
+            echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
+            echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
+          fi
+
+      - name: Call CP redeploy-fleet
+        # CP_ADMIN_API_TOKEN must be set as a repo/org secret on
+        # molecule-ai/molecule-core, matching the staging/prod CP's
+        # CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
+        # repo's secrets for CI.
+        env:
+          CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
+          CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
+          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
+          CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
+          SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
+          BATCH_SIZE: ${{ inputs.batch_size || '3' }}
+          DRY_RUN: ${{ inputs.dry_run || false }}
+        run: |
+          set -euo pipefail
+
+          if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
+            echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
+            echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
+            exit 1
+          fi
+
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --arg canary "$CANARY_SLUG" \
+            --argjson soak "$SOAK_SECONDS" \
+            --argjson batch "$BATCH_SIZE" \
+            --argjson dry "$DRY_RUN" \
+            '{
+              target_tag: $tag,
+              canary_slug: $canary,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          # Route -w into its own tempfile so curl's exit code (e.g. 56
+          # on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
+          # pollute the captured stdout. The previous inline-substitution
+          # shape produced "000000" on connection reset (curl wrote
+          # "000" via -w, then the inline echo-fallback appended another
+          # "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
+          # set +e/-e keeps the non-zero curl exit from tripping the
+          # outer pipeline. See lint-curl-status-capture.yml for the
+          # CI gate that pins this fix shape.
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          set -e
+          # Stderr from curl (e.g. dial errors with -sS) goes to the runner
+          # log so operators can see WHY a connection failed. Stdout is
+          # captured to $HTTP_CODE_FILE because that's where -w writes.
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          # Pretty-print per-tenant results in the job summary so
+          # ops can see which tenants were redeployed without drilling
+          # into the raw response.
+          {
+            echo "## Tenant redeploy fleet"
+            echo ""
+            echo "**Target tag:** \`$TARGET_TAG\`"
+            echo "**Canary:** \`$CANARY_SLUG\` (soak ${SOAK_SECONDS}s)"
+            echo "**Batch size:** $BATCH_SIZE"
+            echo "**Dry run:** $DRY_RUN"
+            echo "**HTTP:** $HTTP_CODE"
+            echo ""
+            echo "### Per-tenant result"
+            echo ""
+            echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
+            echo '|------|-------|------------|------|---------|-------|'
+            jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ "$HTTP_CODE" != "200" ]; then
+            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
+            exit 1
+          fi
+          OK=$(jq -r '.ok' "$HTTP_RESPONSE")
+          if [ "$OK" != "true" ]; then
+            echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
+            exit 1
+          fi
+          echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
+
+          # Stash the response for the verify step. $RUNNER_TEMP outlasts
+          # the step boundary; $HTTP_RESPONSE doesn't.
+          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
+
+      - name: Verify each tenant /buildinfo matches published SHA
+        # ROOT FIX FOR #2395.
+        #
+        # `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
+        # didn't error" — NOT "the new image is running on the tenant."
+        # `:latest` lives in the local Docker daemon's image cache; if
+        # the SSM document does `docker compose up -d` without an
+        # explicit `docker pull`, the daemon serves the previously-
+        # cached digest and the container restarts on stale code.
+        # 2026-04-30 incident: hongmingwang's tenant reported
+        # ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
+        # chat_files for 30+ min — the lazy-heal fix never reached the
+        # user despite green deploy + green redeploy.
+        #
+        # This step closes the gap by curling each tenant's /buildinfo
+        # endpoint (added in workspace-server/internal/buildinfo +
+        # /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
+        # returned git_sha to the SHA the workflow expects. Mismatches
+        # fail the workflow, which is what `ok=true` should have
+        # guaranteed all along.
+        #
+        # When the redeploy was triggered by workflow_dispatch with a
+        # specific tag (target_tag != "latest"), the expected SHA may
+        # not equal ${{ github.sha }} — in that case we resolve via
+        # GHCR's manifest. For workflow_run (default :latest) the
+        # workflow_run.head_sha is the SHA that just published.
+        env:
+          EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
+          # Tenant subdomain template — slugs from the response are
+          # appended. Production CP issues `<slug>.moleculesai.app`;
+          # staging CP issues `<slug>.staging.moleculesai.app`. This
+          # workflow runs on main → prod CP → no `staging.` infix.
+          TENANT_DOMAIN: 'moleculesai.app'
+        run: |
+          set -euo pipefail
+
+          EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
+          if [ "$TARGET_TAG" != "latest" ] \
+             && [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
+             && [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
+            # workflow_dispatch with a pinned tag that isn't the head
+            # SHA — operator is rolling back / pinning. Skip the
+            # verification because we don't have the expected SHA in
+            # this context (would need to crane-inspect the GHCR
+            # manifest, which is a follow-up). Failing-open here is
+            # safe: the operator chose the tag deliberately.
+            #
+            # `staging-<short_head_sha>` IS verified — it's the new
+            # auto-trigger default (see Compute target tag step) and
+            # the digest under that tag SHOULD match EXPECTED_SHA.
+            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
+            exit 0
+          fi
+
+          RESP="$RUNNER_TEMP/redeploy-response.json"
+          if [ ! -s "$RESP" ]; then
+            echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
+            exit 1
+          fi
+
+          # Pull only successfully-redeployed tenants. Any tenant that
+          # halted the rollout already failed the previous step, so we
+          # don't double-count them here.
+          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
+          if [ ${#SLUGS[@]} -eq 0 ]; then
+            echo "::warning::No tenants reported healthz_ok — nothing to verify"
+            exit 0
+          fi
+
+          echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
+
+          # Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
+          # vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
+          # comment for the full rationale; same logic applies on prod even
+          # though prod has fewer ephemeral tenants — the asymmetry would be a
+          # gratuitous fork.
+          STALE_COUNT=0
+          UNREACHABLE_COUNT=0
+          STALE_LINES=()
+          UNREACHABLE_LINES=()
+          for slug in "${SLUGS[@]}"; do
+            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
+            # 30s total: tenant just SSM-restarted, may still be coming
+            # up. Retry-on-empty rather than retry-on-status — we want
+            # to fail fast on "responded with wrong SHA", not "still
+            # warming up".
+            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
+            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
+            if [ -z "$ACTUAL_SHA" ]; then
+              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
+              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
+              continue
+            fi
+            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
+              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
+            else
+              STALE_COUNT=$((STALE_COUNT + 1))
+              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
+            fi
+          done
+
+          {
+            echo ""
+            echo "### Per-tenant /buildinfo verification"
+            echo ""
+            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
+            echo ""
+            if [ $STALE_COUNT -gt 0 ]; then
+              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${STALE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $UNREACHABLE_COUNT -gt 0 ]; then
+              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
+              echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ $UNREACHABLE_COUNT -gt 0 ]; then
+            echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
+          fi
+
+          # Belt-and-suspenders sanity floor: same logic as the staging
+          # variant — see that file's comment for the full rationale.
+          # Floor only applies when fleet >= 4; below that, canary-verify
+          # is the actual gate.
+          TOTAL_VERIFIED=${#SLUGS[@]}
+          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
+            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
+            exit 1
+          fi
+
+          if [ $STALE_COUNT -gt 0 ]; then
+            echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
+            exit 1
+          fi
+
+          echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."

.github/workflows/redeploy-tenants-on-staging.yml

@@ -0,0 +1,362 @@
+name: redeploy-tenants-on-staging
+
+# Auto-refresh staging tenant EC2s after every staging-branch merge.
+#
+# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
+# the :staging-latest tag. Sister workflow exists for prod (rolls
+# :latest after canary-verify). Both share the same shape — just
+# different CP_URL + target_tag + admin token secret.
+#
+# Why this workflow exists: publish-workspace-server-image now builds
+# on every staging-branch push (PR #2335), pushing
+# platform-tenant:staging-latest to GHCR. Existing tenants pulled
+# their image once at boot and never re-pull, so the new image just
+# sits unused until the tenant is reprovisioned.
+#
+# This workflow closes the gap by calling staging-CP's
+# /cp/admin/tenants/redeploy-fleet, which performs a canary-first,
+# batched, health-gated SSM redeploy across every live staging tenant.
+# Same endpoint shape as prod CP — only the host differs.
+#
+# Runtime ordering:
+#   1. publish-workspace-server-image completes on staging branch →
+#      new :staging-latest in GHCR.
+#   2. This workflow fires via workflow_run, waits 30s for GHCR's CDN
+#      to propagate the new tag.
+#   3. Calls redeploy-fleet with no canary (staging IS canary; we don't
+#      need a sub-canary inside it). Soak still applies to the first
+#      tenant in case of bad-deploy detection.
+#   4. Any failure aborts the rollout and leaves older tenants on the
+#      prior image — safer default than half-and-half state.
+#
+# Rollback path: re-run with workflow_dispatch + target_tag=staging-<sha>
+# of a known-good build.
+
+on:
+  workflow_run:
+    workflows: ['publish-workspace-server-image']
+    types: [completed]
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      target_tag:
+        description: 'Tenant image tag to deploy (e.g. "staging-latest" or "staging-a59f1a6c"). Defaults to staging-latest when empty.'
+        required: false
+        type: string
+        default: 'staging-latest'
+      canary_slug:
+        description: 'Tenant slug to deploy first + soak (empty = skip canary, fan out immediately). Default empty for staging since staging itself is the canary.'
+        required: false
+        type: string
+        default: ''
+      soak_seconds:
+        description: 'Seconds to wait after canary before fanning out. Only meaningful if canary_slug is set.'
+        required: false
+        type: string
+        default: '60'
+      batch_size:
+        description: 'How many tenants SSM redeploys in parallel per batch.'
+        required: false
+        type: string
+        default: '3'
+      dry_run:
+        description: 'Plan only — do not actually redeploy.'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  # No write scopes needed — the workflow hits an external CP endpoint,
+  # not the GitHub API.
+
+# Serialize per-branch so two rapid staging pushes' redeploys don't
+# overlap and cause confusing per-tenant SSM state. cancel-in-progress
+# is false because aborting a half-rolled-out fleet leaves tenants
+# stuck on whatever image they happened to be on when cancelled.
+concurrency:
+  group: redeploy-tenants-on-staging
+  cancel-in-progress: false
+
+jobs:
+  redeploy:
+    # Skip the auto-trigger if publish-workspace-server-image didn't
+    # actually succeed. workflow_run fires on any completion state; we
+    # don't want to redeploy against a half-built image.
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Wait for GHCR tag propagation
+        # GHCR's edge cache takes ~15-30s to consistently serve the new
+        # :staging-latest manifest after the registry accepts the push.
+        # Same rationale as redeploy-tenants-on-main.yml.
+        run: sleep 30
+
+      - name: Call staging-CP redeploy-fleet
+        # CP_STAGING_ADMIN_API_TOKEN must be set as a repo/org secret
+        # on molecule-ai/molecule-core, matching staging-CP's
+        # CP_ADMIN_API_TOKEN env var (visible in Railway controlplane
+        # / staging environment). Stored separately from the prod
+        # CP_ADMIN_API_TOKEN so a leak of one doesn't auth the other.
+        env:
+          CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
+          CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+          TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
+          CANARY_SLUG: ${{ inputs.canary_slug || '' }}
+          SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
+          BATCH_SIZE: ${{ inputs.batch_size || '3' }}
+          DRY_RUN: ${{ inputs.dry_run || false }}
+        run: |
+          set -euo pipefail
+
+          # Schedule-vs-dispatch hardening (mirrors sweep-cf-orphans
+          # and sweep-cf-tunnels): hard-fail on auto-trigger when the
+          # secret is missing so a misconfigured-repo doesn't silently
+          # serve stale staging tenants. Soft-skip on operator dispatch.
+          if [ -z "${CP_STAGING_ADMIN_API_TOKEN:-}" ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::CP_STAGING_ADMIN_API_TOKEN secret not set — skipping redeploy"
+              echo "::warning::Set CP_STAGING_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
+              echo "::notice::Pull the value from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+              exit 0
+            fi
+            echo "::error::staging redeploy cannot run — CP_STAGING_ADMIN_API_TOKEN secret missing"
+            echo "::error::set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
+            exit 1
+          fi
+
+          BODY=$(jq -nc \
+            --arg tag "$TARGET_TAG" \
+            --arg canary "$CANARY_SLUG" \
+            --argjson soak "$SOAK_SECONDS" \
+            --argjson batch "$BATCH_SIZE" \
+            --argjson dry "$DRY_RUN" \
+            '{
+              target_tag: $tag,
+              canary_slug: $canary,
+              soak_seconds: $soak,
+              batch_size: $batch,
+              dry_run: $dry
+            }')
+
+          echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
+          echo "  body: $BODY"
+
+          HTTP_RESPONSE=$(mktemp)
+          HTTP_CODE_FILE=$(mktemp)
+          # Route -w into its own tempfile so curl's exit code (e.g. 56
+          # on connection-reset) can't pollute the captured stdout. The
+          # previous inline-substitution shape produced "000000" on
+          # connection reset — caught on main variant 2026-05-04
+          # redeploying sha 2b862f6. Same fix shape as the synth-E2E
+          # §9c gate (PR #2797). See lint-curl-status-capture.yml for
+          # the CI gate that pins this fix shape.
+          set +e
+          curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
+            -m 1200 \
+            -H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
+            -H "Content-Type: application/json" \
+            -X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
+            -d "$BODY" >"$HTTP_CODE_FILE"
+          set -e
+          # Stderr from curl (-sS shows dial errors etc.) goes to the
+          # runner log so operators can see WHY a connection failed.
+          HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
+          [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
+
+          echo "HTTP $HTTP_CODE"
+          cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
+
+          {
+            echo "## Staging tenant redeploy fleet"
+            echo ""
+            echo "**Target tag:** \`$TARGET_TAG\`"
+            echo "**Canary:** \`${CANARY_SLUG:-(none — staging is itself the canary)}\` (soak ${SOAK_SECONDS}s)"
+            echo "**Batch size:** $BATCH_SIZE"
+            echo "**Dry run:** $DRY_RUN"
+            echo "**HTTP:** $HTTP_CODE"
+            echo ""
+            echo "### Per-tenant result"
+            echo ""
+            echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
+            echo '|------|-------|------------|------|---------|-------|'
+            jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          # Distinguish "real fleet failure" from "E2E teardown race".
+          #
+          # CP returns HTTP 500 + ok=false whenever ANY tenant in the
+          # fleet failed SSM or healthz. In practice the recurring source
+          # of these is ephemeral test tenants being torn down by their
+          # parent E2E run mid-redeploy: the EC2 dies → SSM exit=2 or
+          # healthz timeout → CP marks the fleet failed → this workflow
+          # goes red even though every operator-facing tenant rolled fine.
+          #
+          # Ephemeral slug prefixes (kept in sync with sweep-stale-e2e-orgs.yml
+          # — see that file for the source-of-truth list and rationale):
+          #   - e2e-*       — canvas/saas/ext E2E suites
+          #   - rt-e2e-*    — runtime-test harness fixtures (RFC #2251)
+          # Long-lived prefixes that are NOT ephemeral and MUST hard-fail:
+          # demo-prep, dryrun-*, dryrun2-*, plus all human tenant slugs.
+          #
+          # Filter: if HTTP=500/ok=false AND every failed slug matches an
+          # ephemeral prefix, treat as soft-warn and let the verify step
+          # downstream handle unreachable-vs-stale (#2402). Any non-ephemeral
+          # failure or a non-500 HTTP response remains a hard failure.
+          OK=$(jq -r '.ok // "false"' "$HTTP_RESPONSE")
+          FAILED_SLUGS=$(jq -r '
+            .results[]?
+            | select((.healthz_ok != true) or (.ssm_status != "Success"))
+            | .slug' "$HTTP_RESPONSE" 2>/dev/null || true)
+          EPHEMERAL_PREFIX_RE='^(e2e-|rt-e2e-)'
+          NON_EPHEMERAL_FAILED=$(printf '%s\n' "$FAILED_SLUGS" | grep -v '^$' | grep -Ev "$EPHEMERAL_PREFIX_RE" || true)
+
+          if [ "$HTTP_CODE" = "200" ] && [ "$OK" = "true" ]; then
+            : # happy path — fall through to verification
+          elif [ "$HTTP_CODE" = "500" ] && [ -z "$NON_EPHEMERAL_FAILED" ] && [ -n "$FAILED_SLUGS" ]; then
+            COUNT=$(printf '%s\n' "$FAILED_SLUGS" | grep -Ec "$EPHEMERAL_PREFIX_RE" || true)
+            echo "::warning::redeploy-fleet returned HTTP 500 but every failed tenant ($COUNT) is ephemeral (e2e-*/rt-e2e-*) — treating as teardown race, soft-warning."
+            printf '%s\n' "$FAILED_SLUGS" | sed 's/^/::warning::  failed: /'
+          elif [ "$HTTP_CODE" != "200" ]; then
+            echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
+            if [ -n "$NON_EPHEMERAL_FAILED" ]; then
+              echo "::error::non-ephemeral tenant(s) failed:"
+              printf '%s\n' "$NON_EPHEMERAL_FAILED" | sed 's/^/::error::  /'
+            fi
+            exit 1
+          else
+            # HTTP=200 but ok=false (shouldn't happen with current CP
+            # but keep the gate for completeness).
+            echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
+            exit 1
+          fi
+          echo "::notice::Staging tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
+
+          cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
+
+      - name: Verify each staging tenant /buildinfo matches published SHA
+        # Mirror of the verify step in redeploy-tenants-on-main.yml — see
+        # there for the rationale (#2395 root fix). Staging has the same
+        # ssm_status-success-but-stale-image hazard and benefits from the
+        # same gate. Diff: TENANT_DOMAIN includes the `staging.` infix.
+        env:
+          EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
+          TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
+          TENANT_DOMAIN: 'staging.moleculesai.app'
+        run: |
+          set -euo pipefail
+
+          # staging-latest is the staging-side moving tag; treat it the
+          # same way main treats `latest`. Operator-pinned SHAs skip
+          # verification (see main variant for why).
+          if [ "$TARGET_TAG" != "staging-latest" ] && [ "$TARGET_TAG" != "latest" ] && [ "$TARGET_TAG" != "$EXPECTED_SHA" ]; then
+            echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
+            exit 0
+          fi
+
+          RESP="$RUNNER_TEMP/redeploy-response.json"
+          if [ ! -s "$RESP" ]; then
+            echo "::error::redeploy-response.json missing or empty"
+            exit 1
+          fi
+
+          mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
+          if [ ${#SLUGS[@]} -eq 0 ]; then
+            echo "::warning::No staging tenants reported healthz_ok — nothing to verify"
+            exit 0
+          fi
+
+          echo "Verifying ${#SLUGS[@]} staging tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
+
+          # Two distinct failure modes here:
+          #   STALE_COUNT      — tenant returned a SHA that doesn't match. THIS is
+          #                      the #2395 bug class: tenant up + serving old code.
+          #                      Always hard-fail the workflow.
+          #   UNREACHABLE_COUNT — tenant didn't respond. Almost always a benign
+          #                      teardown race: redeploy-fleet snapshot says
+          #                      healthz_ok=true, then the E2E suite tears the
+          #                      ephemeral tenant down before this step runs (the
+          #                      e2e-* fixtures churn 5-10/hour on staging). Soft-
+          #                      warn so we don't block staging→main on cleanup.
+          #                      Real "tenant up but unreachable" is caught by CP's
+          #                      own healthz monitor + the post-redeploy alert; we
+          #                      don't need to double-count it here.
+          STALE_COUNT=0
+          UNREACHABLE_COUNT=0
+          STALE_LINES=()
+          UNREACHABLE_LINES=()
+          for slug in "${SLUGS[@]}"; do
+            URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
+            BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
+            ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
+            if [ -z "$ACTUAL_SHA" ]; then
+              UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
+              UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
+              continue
+            fi
+            if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
+              echo "  $slug: ${ACTUAL_SHA:0:7} ✓"
+            else
+              STALE_COUNT=$((STALE_COUNT + 1))
+              STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
+            fi
+          done
+
+          {
+            echo ""
+            echo "### Per-tenant /buildinfo verification (staging)"
+            echo ""
+            echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
+            echo ""
+            if [ $STALE_COUNT -gt 0 ]; then
+              echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${STALE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $UNREACHABLE_COUNT -gt 0 ]; then
+              echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely E2E teardown race (soft-warn, not failing):**"
+              echo ""
+              echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
+              echo "|------|----------------------|----------|--------|"
+              for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
+              echo ""
+            fi
+            if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
+              echo "All ${#SLUGS[@]} staging tenants returned matching SHA. ✓"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ $UNREACHABLE_COUNT -gt 0 ]; then
+            echo "::warning::$UNREACHABLE_COUNT staging tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
+          fi
+
+          # Belt-and-suspenders sanity floor: if MORE than half the fleet is
+          # unreachable AND the fleet is large enough that "half down" is
+          # statistically meaningful, this is a real outage (e.g. new image
+          # crashes on startup), not a teardown race. Hard-fail.
+          #
+          # Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
+          # canary-verify step is the actual gate for "all tenants down"
+          # detection (it runs against the canary first and aborts the
+          # rollout if the canary fails to come up). Without the >=4 gate,
+          # a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
+          # quiet staging push) would re-flake on the exact teardown-race
+          # condition #2402 fixed: 1 of 1 unreachable = 100% > 50% → fail.
+          TOTAL_VERIFIED=${#SLUGS[@]}
+          if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
+            echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED staging tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
+            exit 1
+          fi
+
+          if [ $STALE_COUNT -gt 0 ]; then
+            echo "::error::$STALE_COUNT staging tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
+            exit 1
+          fi
+
+          echo "::notice::Staging tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."

.github/workflows/runtime-pin-compat.yml

@@ -0,0 +1,91 @@
+name: Runtime Pin Compatibility
+
+# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
+# recurring (controlplane#253). The original failure mode:
+#   1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
+#      requires_dist metadata (incorrect — it actually imports
+#      a2a.server.routes which only exists in a2a-sdk 1.0+)
+#   2. `pip install molecule-ai-workspace-runtime` resolved cleanly
+#   3. `from molecule_runtime.main import main_sync` raised ImportError
+#   4. Every tenant workspace crashed; the canary tenant caught it but
+#      only after 5 hours of degraded staging
+#
+# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
+# top of `workspace/requirements.txt` and smoke-imports. Catches:
+#   - Upstream PyPI yanks
+#   - Bad re-releases of molecule-ai-workspace-runtime
+#   - Already-shipped wheels that stop importing because a transitive
+#     dep moved underneath
+#
+# This is the "PyPI artifact health" half of pin compatibility. The
+# companion workflow `runtime-prbuild-compat.yml` covers the
+# "PR-introduced breakage" half by building the wheel from THIS PR's
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter — the pypi-latest job no longer fires on doc-only
+# workspace/ edits whose content can't change what's currently on PyPI.
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      # Narrow filter: pypi-latest is sensitive only to changes that
+      # affect what we're INSTALLING (requirements.txt) or WHAT THE
+      # CHECK ITSELF DOES (this workflow file). Edits to workspace/
+      # source code don't change what's on PyPI right now, so they
+      # don't change this gate's verdict.
+      - 'workspace/requirements.txt'
+      - '.github/workflows/runtime-pin-compat.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'workspace/requirements.txt'
+      - '.github/workflows/runtime-pin-compat.yml'
+  # Daily catch for upstream PyPI publishes that break the pin combo
+  # without any change in our repo (e.g. someone re-yanks an a2a-sdk
+  # release or molecule-ai-workspace-runtime publishes a bad bump).
+  schedule:
+    - cron: '0 13 * * *'  # 06:00 PT
+  workflow_dispatch:
+  # Required-check support: when this becomes a branch-protection gate,
+  # merge_group runs let the queue green-check this in addition to PRs.
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pypi-latest-install:
+    name: PyPI-latest install + import smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install runtime + workspace requirements
+        # Install order is load-bearing: install the runtime FIRST so pip
+        # honors whatever a2a-sdk constraint the runtime metadata declares
+        # (this is the surface that broke in 2026-04-24 — runtime declared
+        # `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
+        # of workspace/requirements.txt then upgrades a2a-sdk to the
+        # constraint our runtime image actually pins. The import smoke
+        # below verifies the upgraded combination is consistent.
+        run: |
+          python -m venv /tmp/venv
+          /tmp/venv/bin/pip install --upgrade pip
+          /tmp/venv/bin/pip install molecule-ai-workspace-runtime
+          /tmp/venv/bin/pip install -r workspace/requirements.txt
+          /tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import — fail if metadata declares deps that don't satisfy real imports
+        # WORKSPACE_ID is validated at import time by platform_auth.py — EC2
+        # user-data sets it from the cloud-init template; set a placeholder
+        # here so the import smoke doesn't trip on the env-var guard.
+        env:
+          WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
+        run: |
+          /tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"

.github/workflows/runtime-prbuild-compat.yml

@@ -0,0 +1,152 @@
+name: Runtime PR-Built Compatibility
+
+# Companion to `runtime-pin-compat.yml`. That workflow tests what's
+# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
+# PUBLISHED if THIS PR merges.
+#
+# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
+# wheel" job to the original runtime-pin-compat.yml, but both jobs
+# shared a `paths:` filter that was the union of their needs
+# (`workspace/**`). That meant the PyPI-latest job ran on every doc
+# edit even though the upstream PyPI artifact can't change with our
+# workspace/ source. Splitting the two means each gets a narrow
+# `paths:` filter that matches the inputs it actually depends on.
+#
+# Catches the failure mode where a PR adds an import requiring a newer
+# SDK than `workspace/requirements.txt` pins:
+#   1. Pip resolves the existing PyPI wheel + the old SDK pin → smoke
+#      passes (it imports the OLD main.py from the wheel, not the PR's
+#      new main.py).
+#   2. Merge → publish-runtime.yml ships a wheel WITH the new import.
+#   3. Tenant images redeploy → all crash on first boot with
+#      ImportError.
+#
+# By building from the PR's source and smoke-importing THAT wheel, we
+# fail at PR-time instead of after publish.
+#
+# Required-check shape (2026-05-01): the workflow runs on EVERY push +
+# PR + merge_group event with no top-level `paths:` filter, then uses a
+# detect-changes job + per-step `if:` gates inside ONE always-running
+# job named `PR-built wheel + import smoke`. PRs that don't touch
+# wheel-relevant paths get a no-op SUCCESS check run, satisfying branch
+# protection without re-running the heavy build. Same pattern as
+# e2e-api.yml — see its comment for the full rationale + the 2026-04-29
+# PR #2264 incident that motivated the always-run-with-if-gates shape.
+
+on:
+  push:
+    branches: [main, staging]
+  pull_request:
+    branches: [main, staging]
+  workflow_dispatch:
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  # Include event_name so a PR sync (event=pull_request) and the
+  # subsequent staging push (event=push) on the SAME merge SHA don't
+  # collide in one group. Without event_name, both runs hashed to
+  # the same key and cancel-in-progress=true cancelled whichever
+  # arrived second — usually the push run, which staging branch-
+  # protection then sees as a CANCELLED required check and refuses
+  # to mark merged. Caught 2026-05-05 across PR #2869's runs (run
+  # ids 25371863455 / 25371811486 / 25371078157 / 25370403142 — every
+  # staging push run cancelled, every matching PR run green).
+  #
+  # Per memory `feedback_concurrency_group_per_sha.md` — same drift
+  # class that broke auto-promote-staging on 2026-04-28. Pin invariant:
+  # event_name + sha is the minimum unique key for these workflows.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      wheel: ${{ steps.decide.outputs.wheel }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            wheel:
+              - 'workspace/**'
+              - 'scripts/build_runtime_package.py'
+              - 'scripts/wheel_smoke.py'
+              - '.github/workflows/runtime-prbuild-compat.yml'
+      - id: decide
+        # Always run real work for manual dispatch + merge_group — no
+        # diff-against-base in those contexts, and the gate exists to
+        # validate the to-be-merged state regardless of which paths it
+        # touched (paths-filter would default to "no changes" which is
+        # the wrong answer when the queue is composing many PRs).
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "merge_group" ]; then
+            echo "wheel=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "wheel=${{ steps.filter.outputs.wheel }}" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ONE job (no job-level `if:`) that always runs and reports under the
+  # required-check name `PR-built wheel + import smoke`. Real work is
+  # gated per-step on `needs.detect-changes.outputs.wheel`. Same shape
+  # as e2e-api.yml's e2e-api job — see its comment block for the full
+  # rationale (SKIPPED check runs block branch protection even with
+  # SUCCESS siblings; collapsing to one always-run job emits exactly
+  # one SUCCESS check run).
+  local-build-install:
+    needs: detect-changes
+    name: PR-built wheel + import smoke
+    runs-on: ubuntu-latest
+    steps:
+      - name: No-op pass (paths filter excluded this commit)
+        if: needs.detect-changes.outputs.wheel != 'true'
+        run: |
+          echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
+          echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - if: needs.detect-changes.outputs.wheel == 'true'
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: workspace/requirements.txt
+      - name: Install build tooling
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: pip install build
+      - name: Build wheel from PR source (mirrors publish-runtime.yml)
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Use a fixed test version so the wheel filename is predictable.
+        # Doesn't reach PyPI — this build is local-only for the smoke.
+        # Use the SAME build script with the SAME args as
+        # publish-runtime.yml's build step. The temp dir path differs
+        # (`/tmp/runtime-build` here vs `${{ runner.temp }}/runtime-build`
+        # in publish-runtime.yml — they coincide on ubuntu-latest but
+        # the call sites are not byte-identical). The smoke import is
+        # also intentionally narrower than publish's: this gate exists
+        # to catch SDK-version-import drift specifically; full invariant
+        # coverage lives in publish-runtime.yml's own pre-PyPI smoke.
+        run: |
+          python scripts/build_runtime_package.py \
+            --version "0.0.0.dev0+pin-compat" \
+            --out /tmp/runtime-build
+          cd /tmp/runtime-build && python -m build
+      - name: Install built wheel + workspace requirements
+        if: needs.detect-changes.outputs.wheel == 'true'
+        run: |
+          python -m venv /tmp/venv-built
+          /tmp/venv-built/bin/pip install --upgrade pip
+          /tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
+          /tmp/venv-built/bin/pip install -r workspace/requirements.txt
+          /tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
+            | grep -E '^(Name|Version):'
+      - name: Smoke import the PR-built wheel
+        if: needs.detect-changes.outputs.wheel == 'true'
+        # Same script publish-runtime.yml runs against the to-be-PyPI wheel.
+        # Closes the PR-time vs publish-time gap: a PR adding a new SDK
+        # call-shape no longer passes here (narrow `import main_sync`) only
+        # to fail post-merge in publish-runtime's broader smoke.
+        run: |
+          /tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"

.github/workflows/secret-pattern-drift.yml

@@ -0,0 +1,58 @@
+name: SECRET_PATTERNS drift lint
+
+# Detects when the canonical SECRET_PATTERNS array in
+# .github/workflows/secret-scan.yml diverges from known consumer
+# mirrors (workspace-runtime's bundled pre-commit hook today; more
+# can be added as the consumer set grows).
+#
+# Why this exists: every side that scans for credentials has its own
+# copy of the pattern list. They drift — most recently the runtime
+# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
+# so a developer's local pre-commit would let a sk-cp- token through
+# while the org-wide CI scan would refuse it. The cost of that drift
+# is dev confusion + delayed feedback; the fix is automated detection.
+#
+# Triggers:
+#   - schedule: daily 05:00 UTC. Catches drift introduced by edits
+#     to a consumer copy that didn't update canonical here.
+#   - push to main/staging where the canonical or this lint changed:
+#     catches the inverse — canonical updated but consumers not yet
+#     bumped. The lint will fail the push; that's intentional, the
+#     person editing canonical is the right person to also update
+#     the consumer.
+#   - workflow_dispatch: ad-hoc operator runs.
+
+on:
+  schedule:
+    # 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
+    # email lands when humans are starting their day, not
+    # interrupting it.
+    - cron: "0 5 * * *"
+  push:
+    branches: [main, staging]
+    paths:
+      - ".github/workflows/secret-scan.yml"
+      - ".github/workflows/secret-pattern-drift.yml"
+      - ".github/scripts/lint_secret_pattern_drift.py"
+      - ".githooks/pre-commit"
+  workflow_dispatch:
+
+# GITHUB_TOKEN scoped to read-only. The lint only does git checkout
+# + HTTPS GETs to public consumer files; no writes to anything.
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Detect SECRET_PATTERNS drift
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Run drift lint
+        run: python3 .github/scripts/lint_secret_pattern_drift.py

.github/workflows/sweep-aws-secrets.yml

@@ -0,0 +1,129 @@
+name: Sweep stale AWS Secrets Manager secrets
+
+# Janitor for per-tenant AWS Secrets Manager secrets
+# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
+# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
+# sweep-cf-orphans.yml — different cloud, same justification.
+#
+# Why this exists separately from a long-term reconciler integration:
+#   - molecule-controlplane's tenant_resources audit table (mig 024)
+#     currently tracks four resource kinds: CloudflareTunnel,
+#     CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
+#     not in the list, so the existing reconciler doesn't catch
+#     orphan secrets.
+#   - At ~$0.40/secret/month the cost grew to ~$19/month before this
+#     sweeper was written, indicating ~45+ orphan secrets from
+#     crashed provisions and incomplete deprovision flows.
+#   - The proper fix (KindSecretsManagerSecret + recorder hook +
+#     reconciler enumerator) is filed as a separate controlplane
+#     issue. This sweeper is the immediate cost-relief stopgap.
+#
+# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
+# This is a DEDICATED principal — the production `molecule-cp` IAM
+# user lacks `secretsmanager:ListSecrets` (it only has
+# Get/Create/Update/Delete on specific resources, scoped to its
+# operational needs). The janitor needs ListSecrets across the
+# `molecule/tenant/*` prefix, which warrants a separate principal so
+# we don't broaden the prod-CP policy.
+#
+# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
+# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
+# the mostly-orphan tunnels) refuses to nuke past the threshold.
+
+on:
+  schedule:
+    # Hourly at :30 — offsets from sweep-cf-orphans (:15) and
+    # sweep-cf-tunnels (:45) so the three janitors don't burst the
+    # CP admin endpoints at the same minute.
+    - cron: '30 * * * *'
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run only — list what would be deleted, no deletion"
+        required: false
+        type: boolean
+        default: true
+      max_delete_pct:
+        description: "Override safety gate (default 50, set higher only for major cleanup)"
+        required: false
+        default: "50"
+      grace_hours:
+        description: "Skip secrets created within this many hours (default 24)"
+        required: false
+        default: "24"
+
+# Don't let two sweeps race the same AWS account.
+concurrency:
+  group: sweep-aws-secrets
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    name: Sweep AWS Secrets Manager
+    runs-on: ubuntu-latest
+    # 30 min cap, mirroring the other janitors. AWS DeleteSecret is
+    # fast (~0.3s/call) so even a 100+ backlog drains in seconds
+    # under the 8-way xargs parallelism, but the cap is set generously
+    # to leave headroom for any actual API hang.
+    timeout-minutes: 30
+    env:
+      AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
+      CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
+      CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
+      GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
+        # and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
+        #   - schedule → exit 1 on missing secrets (red CI surfaces it)
+        #   - workflow_dispatch → exit 0 with warning (operator-driven,
+        #     they already accepted the repo state)
+        run: |
+          missing=()
+          for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-aws-secrets.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-aws-secrets.sh --execute
+          fi

.github/workflows/sweep-cf-orphans.yml

@@ -0,0 +1,146 @@
+name: Sweep stale Cloudflare DNS records
+
+# Janitor for Cloudflare DNS records whose backing tenant/workspace no
+# longer exists. Without this loop, every short-lived E2E or canary
+# leaves a CF record on the moleculesai.app zone — the zone has a
+# 200-record quota (controlplane#239 hit it 2026-04-23+) and provisions
+# start failing with code 81045 once exhausted.
+#
+# Why a separate workflow vs sweep-stale-e2e-orgs.yml:
+#   - That workflow operates at the CP layer (DELETE /cp/admin/tenants/:slug
+#     drives the cascade). It assumes CP has the org row to drive the
+#     deprovision from. It doesn't catch records left behind when CP
+#     itself never knew about the tenant (canary scratch, manual ops
+#     experiments) or when the cascade's CF-delete branch failed.
+#   - sweep-cf-orphans.sh enumerates the CF zone directly and matches
+#     each record against live CP slugs + AWS EC2 names. It catches
+#     leaks the CP-driven sweep can't.
+#
+# Safety: the script's own MAX_DELETE_PCT gate refuses to nuke more
+# than 50% of records in a single run. If something has gone weird
+# (CP admin endpoint returns no orgs → every tenant looks orphan) the
+# gate halts before damage. Decision-function unit tests in
+# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
+# classifier.
+
+on:
+  schedule:
+    # Hourly. Mirrors sweep-stale-e2e-orgs cadence so the two janitors
+    # converge on the same tick. CF API rate budget is generous (1200
+    # req/5min); a single sweep makes ~1 list + N deletes (N<=quota/2).
+    - cron: '15 * * * *'  # offset from sweep-stale-e2e-orgs (top of hour)
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run only — list what would be deleted, no deletion"
+        required: false
+        type: boolean
+        default: true
+      max_delete_pct:
+        description: "Override safety gate (default 50, set higher only for major cleanup)"
+        required: false
+        default: "50"
+  # No `merge_group:` trigger on purpose. This is a janitor — it doesn't
+  # need to gate merges, and including it as written before #2088 fired
+  # the full sweep job (or its secret-check) on every PR going through
+  # the merge queue, generating one red CI run per merge-queue eval. If
+  # this workflow is ever wired up as a required check, re-add
+  #   merge_group: { types: [checks_requested] }
+  # AND gate the sweep step with `if: github.event_name != 'merge_group'`
+  # so merge-queue evals report success without actually running.
+
+# Don't let two sweeps race the same zone. workflow_dispatch during a
+# scheduled run would otherwise issue duplicate DELETE calls.
+concurrency:
+  group: sweep-cf-orphans
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    name: Sweep CF orphans
+    runs-on: ubuntu-latest
+    # 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
+    # within one cron interval instead of burning a full tick. Realistic
+    # worst case is ~2 min: 4 sequential curls + 1 aws + N×CF-DELETE
+    # each individually capped at 10s by the script's curl -m flag.
+    timeout-minutes: 3
+    env:
+      CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+      CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
+      CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
+      CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split (hardened 2026-04-28
+        # after the silent-no-op incident below):
+        #
+        # The earlier soft-skip-on-schedule policy hid a real leak. All
+        # six secrets were unset on this repo for an unknown duration;
+        # every hourly run printed a yellow ::warning:: and exited 0,
+        # so the workflow registered as "passing" while doing nothing.
+        # CF orphans accumulated to 152/200 (~76% of the zone quota
+        # gone) before a manual `dig`-driven audit caught it. Anything
+        # that runs as a janitor and reports green while idle is
+        # indistinguishable from "the janitor is healthy" — so we now
+        # treat schedule (and any future workflow_run/push triggers)
+        # as a hard-fail when secrets are missing.
+        #
+        #   - schedule / workflow_run / push → exit 1 (red CI run
+        #     surfaces the misconfiguration the next tick)
+        #   - workflow_dispatch              → exit 0 with a warning
+        #     (an operator ran this ad-hoc; they already accepted the
+        #     state of the repo and want the workflow to short-circuit
+        #     so they can rerun after fixing the secret)
+        run: |
+          missing=()
+          for var in CF_API_TOKEN CF_ZONE_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry (intentional):
+        #   - Scheduled runs: github.event.inputs.dry_run is empty →
+        #     defaults to "false" below → script runs with --execute
+        #     (the whole point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default is true (line 38)
+        #     so an ad-hoc operator-triggered run is dry-run by default;
+        #     they have to flip the toggle to actually delete.
+        # The script's MAX_DELETE_PCT gate (default 50%) is the second
+        # line of defense regardless of mode.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-cf-orphans.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-cf-orphans.sh --execute
+          fi

.github/workflows/sweep-cf-tunnels.yml

@@ -0,0 +1,124 @@
+name: Sweep stale Cloudflare Tunnels
+
+# Janitor for Cloudflare Tunnels whose backing tenant no longer
+# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
+# records); same justification, different CF resource.
+#
+# Why this exists separately from sweep-cf-orphans:
+#   - DNS records live on the zone (`/zones/<id>/dns_records`).
+#   - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
+#   - Different CF API surface, different scopes; the existing CF
+#     token might not have `account:cloudflare_tunnel:edit`. Splitting
+#     the workflows keeps each one's secret-presence gate independent
+#     so neither silent-skips when the other's secret is missing.
+#   - Cleaner blast radius — operators can disable one without the
+#     other if a regression surfaces.
+#
+# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
+# the DNS sweep's 50% because tenant-shaped tunnels are mostly
+# orphans by design) refuses to nuke past the threshold.
+
+on:
+  schedule:
+    # Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
+    # janitors don't issue parallel CF API bursts at the same minute.
+    - cron: '45 * * * *'
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run only — list what would be deleted, no deletion"
+        required: false
+        type: boolean
+        default: true
+      max_delete_pct:
+        description: "Override safety gate (default 90, set higher only for major cleanup)"
+        required: false
+        default: "90"
+
+# Don't let two sweeps race the same account.
+concurrency:
+  group: sweep-cf-tunnels
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    name: Sweep CF tunnels
+    runs-on: ubuntu-latest
+    # 30 min cap. Was 5 min on the theory that the only thing that
+    # could take >5min is a CF-API hang — but on 2026-05-02 a backlog
+    # of 672 stale tunnels accumulated (large staging E2E run + delayed
+    # sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
+    # ~7-8min to drain. The 5-min cap killed the run mid-sweep
+    # (cancelled at 424/672, see run 25248788312); a manual rerun
+    # finished the remainder fine.
+    #
+    # The fix is two-part: parallelize the delete loop (8-way xargs in
+    # the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
+    # cap so a one-off backlog doesn't trip a hangs-detector that
+    # turned out to be a real-job-too-slow detector. With 8-way
+    # parallelism, 600+ tunnels drains in ~60s; 30 min is generous
+    # headroom for actual hangs to still surface (and is in line with
+    # the sweep-cf-orphans companion job).
+    timeout-minutes: 30
+    env:
+      CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+      CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
+      CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
+      CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
+      MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify required secrets present
+        id: verify
+        # Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
+        # (hardened 2026-04-28 after the silent-no-op incident: the
+        # janitor reported green while doing nothing because secrets
+        # were unset, masking a 152/200 zone-record leak). Same
+        # principle applies here:
+        #   - schedule → exit 1 on missing secrets (red CI surfaces it)
+        #   - workflow_dispatch → exit 0 with warning (operator-driven,
+        #     they already accepted the repo state)
+        run: |
+          missing=()
+          for var in CF_API_TOKEN CF_ACCOUNT_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
+            if [ -z "${!var:-}" ]; then
+              missing+=("$var")
+            fi
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
+              echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
+              echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
+            echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
+            echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
+            exit 1
+          fi
+          echo "All required secrets present ✓"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Run sweep
+        if: steps.verify.outputs.skip != 'true'
+        # Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
+        #   - Scheduled: input empty → "false" → --execute (the whole
+        #     point of an hourly janitor).
+        #   - Manual workflow_dispatch: input default true → dry-run;
+        #     operator must flip it to actually delete.
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
+            echo "Running in dry-run mode — no deletions"
+            bash scripts/ops/sweep-cf-tunnels.sh
+          else
+            echo "Running with --execute — will delete identified orphans"
+            bash scripts/ops/sweep-cf-tunnels.sh --execute
+          fi

.github/workflows/sweep-stale-e2e-orgs.yml

@@ -0,0 +1,239 @@
+name: Sweep stale e2e-* orgs (staging)
+
+# Janitor for staging tenants left behind when E2E cleanup didn't run:
+# CI cancellations, runner crashes, transient AWS errors mid-cascade,
+# bash trap missed (signal 9), etc. Without this loop, every failed
+# teardown leaks an EC2 + DNS + DB row until manual ops cleanup —
+# 2026-04-23 staging hit the 64 vCPU AWS quota from ~27 such orphans.
+#
+# Why not rely on per-test-run teardown:
+#   - Per-run teardown is best-effort by definition. Any process death
+#     after the test starts but before the trap fires leaves debris.
+#   - GH Actions cancellation kills the runner without grace period.
+#     The workflow's `if: always()` step usually catches this, but it
+#     too can fail (CP transient 5xx, runner network issue at the
+#     wrong moment).
+#   - Even when teardown runs, the CP cascade is best-effort in places
+#     (cascadeTerminateWorkspaces logs+continues; DNS deletion same).
+#   - This sweep is the catch-all that converges staging back to clean
+#     regardless of which specific path leaked.
+#
+# The PROPER fix is making CP cleanup transactional + verify-after-
+# terminate (filed separately as cleanup-correctness work). This
+# workflow is the safety net that catches everything else AND any
+# future leak source we haven't yet identified.
+
+on:
+  schedule:
+    # Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
+    # create to teardown — canary is ~8 min, full SaaS ~25 min). The
+    # previous hourly + 120-min stale threshold meant a leaked tenant
+    # could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
+    # leak. Tightening the cadence + threshold reduces the worst-case
+    # leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
+    # threshold) without risk of catching in-progress runs (the longest
+    # e2e run is the 25-min canary, well under the 30-min threshold).
+    # See molecule-controlplane#420 for the leak-class accounting that
+    # motivated this tightening.
+    - cron: '*/15 * * * *'
+  workflow_dispatch:
+    inputs:
+      max_age_minutes:
+        description: "Delete e2e-* orgs older than N minutes (default 30)"
+        required: false
+        default: "30"
+      dry_run:
+        description: "Dry run only — list what would be deleted"
+        required: false
+        type: boolean
+        default: false
+
+# Don't let two sweeps fight. Cron + workflow_dispatch could overlap
+# on a manual trigger; queue rather than parallel-delete.
+concurrency:
+  group: sweep-stale-e2e-orgs
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sweep:
+    name: Sweep e2e orgs
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
+      MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
+      DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
+      # Refuse to delete more than this many orgs in one tick. If the
+      # CP DB is briefly empty (or the admin endpoint goes weird and
+      # returns no created_at), every e2e- org would look stale.
+      # Bailing protects against runaway nukes.
+      SAFETY_CAP: 50
+
+    steps:
+      - name: Verify admin token present
+        run: |
+          if [ -z "$ADMIN_TOKEN" ]; then
+            echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
+            exit 2
+          fi
+          echo "Admin token present ✓"
+
+      - name: Identify stale e2e orgs
+        id: identify
+        run: |
+          set -euo pipefail
+          # Fetch into a file so the python step reads it via stdin —
+          # cleaner than embedding $(curl ...) into a heredoc.
+          curl -sS --fail-with-body --max-time 30 \
+            "$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" \
+            > orgs.json
+
+          # Filter:
+          #   1. slug starts with one of the ephemeral test prefixes:
+          #        - 'e2e-'    — covers e2e-canary-, e2e-canvas-*, etc.
+          #        - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
+          #                      missing this prefix left two such tenants
+          #                      orphaned 8h on staging (2026-05-03), then
+          #                      hard-failed redeploy-tenants-on-staging
+          #                      and broke the staging→main auto-promote
+          #                      chain. Kept in sync with the EPHEMERAL_PREFIX_RE
+          #                      regex in redeploy-tenants-on-staging.yml.
+          #   2. created_at is older than MAX_AGE_MINUTES ago
+          # Output one slug per line to a file the next step reads.
+          python3 > stale_slugs.txt <<'PY'
+          import json, os
+          from datetime import datetime, timezone, timedelta
+          # SSOT for this list lives in the controlplane Go code:
+          # molecule-controlplane/internal/slugs/ephemeral.go
+          # (var EphemeralPrefixes). The redeploy-fleet auto-rollout
+          # also reads from there to SKIP these slugs — without that
+          # filter, fleet redeploy SSM-failed in-flight E2E tenants
+          # whose containers were still booting, breaking the test
+          # that just spun them up (molecule-controlplane#493).
+          # Update both files together.
+          EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
+          with open("orgs.json") as f:
+              data = json.load(f)
+          max_age = int(os.environ["MAX_AGE_MINUTES"])
+          cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
+          for o in data.get("orgs", []):
+              slug = o.get("slug", "")
+              if not slug.startswith(EPHEMERAL_PREFIXES):
+                  continue
+              created = o.get("created_at")
+              if not created:
+                  # Defensively skip rows without created_at — better
+                  # to leave one orphan than nuke a brand-new row
+                  # whose timestamp didn't render.
+                  continue
+              # Python 3.11+ handles RFC3339 with Z directly via
+              # fromisoformat; older runners need the trailing Z swap.
+              created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
+              if created_dt < cutoff:
+                  print(slug)
+          PY
+
+          count=$(wc -l < stale_slugs.txt | tr -d ' ')
+          echo "Found $count stale e2e org(s) older than ${MAX_AGE_MINUTES}m"
+          if [ "$count" -gt 0 ]; then
+            echo "First 20:"
+            head -20 stale_slugs.txt | sed 's/^/  /'
+          fi
+          echo "count=$count" >> "$GITHUB_OUTPUT"
+
+      - name: Safety gate
+        if: steps.identify.outputs.count != '0'
+        run: |
+          count="${{ steps.identify.outputs.count }}"
+          if [ "$count" -gt "$SAFETY_CAP" ]; then
+            echo "::error::Refusing to delete $count orgs in one sweep (cap=$SAFETY_CAP). Investigate manually — this usually means the CP admin API returned no created_at or returned a degraded result. Re-run with workflow_dispatch + max_age_minutes if intentional."
+            exit 1
+          fi
+          echo "Within safety cap ($count ≤ $SAFETY_CAP) ✓"
+
+      - name: Delete stale orgs
+        if: steps.identify.outputs.count != '0' && env.DRY_RUN != 'true'
+        run: |
+          set -uo pipefail
+          deleted=0
+          failed=0
+          while IFS= read -r slug; do
+            [ -z "$slug" ] && continue
+            # The DELETE handler requires {"confirm": "<slug>"} matching
+            # the URL slug — fat-finger guard. Idempotent: re-issuing
+            # picks up via org_purges.last_step.
+            # Tempfile-routed -w + set +e/-e prevents curl-exit-code
+            # pollution of the captured status (lint-curl-status-capture.yml).
+            set +e
+            curl -sS -o /tmp/del_resp -w "%{http_code}" \
+              --max-time 60 \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/del_code
+            set -e
+            # Stderr from curl (-sS shows dial errors etc.) goes to runner log.
+            http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
+            if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
+              deleted=$((deleted+1))
+              echo "  deleted: $slug"
+            else
+              failed=$((failed+1))
+              echo "  FAILED ($http_code): $slug — $(cat /tmp/del_resp 2>/dev/null | head -c 200)"
+            fi
+          done < stale_slugs.txt
+          echo ""
+          echo "Sweep summary: deleted=$deleted failed=$failed"
+          # Don't fail the workflow on per-org delete errors — the
+          # sweeper is best-effort. Next hourly tick re-attempts. We
+          # only fail loud at the safety-cap gate above.
+
+      - name: Sweep orphan tunnels
+        # Stale-org cleanup deletes the org (which cascades to tunnel
+        # delete inside the CP). But when that cascade fails partway —
+        # CP transient 5xx after the org row is deleted but before the
+        # CF tunnel delete completes — the tunnel persists with no
+        # matching org row. The reconciler in internal/sweep flags this
+        # as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
+        #
+        # `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
+        # reaper. Calling it here at the end of every sweep tick
+        # converges the staging CF account to clean even when CP
+        # cascades half-fail.
+        #
+        # PR #492 made the underlying DeleteTunnel actually check
+        # status — pre-fix it silent-succeeded on CF code 1022
+        # ("active connections"), so this step would have been a no-op
+        # against stuck connectors. Post-fix the cleanup invokes
+        # CleanupTunnelConnections + retry, which actually clears the
+        # 1022 case. (#2987)
+        #
+        # Best-effort. Failure here doesn't fail the workflow — next
+        # tick re-attempts. Errors flow to step output for ops review.
+        if: env.DRY_RUN != 'true'
+        run: |
+          set +e
+          curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
+            --max-time 60 \
+            -X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
+          set -e
+          http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
+          body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
+          if [ "$http_code" = "200" ]; then
+            count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
+            failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
+            echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
+          else
+            echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
+          fi
+
+      - name: Dry-run summary
+        if: env.DRY_RUN == 'true'
+        run: |
+          echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."

.github/workflows/test-ops-scripts.yml

@@ -0,0 +1,52 @@
+name: Ops Scripts Tests
+
+# Runs the unittest suite for scripts/ on every PR + push that touches
+# anything under scripts/. Kept separate from the main CI so a script-only
+# change doesn't trigger the heavier Go/Canvas/Python pipelines.
+#
+# Discovery layout: tests sit alongside the code they test (see
+# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
+# test_build_runtime_package.py for the rewriter coverage). The job
+# below runs `unittest discover` TWICE — once from `scripts/`, once
+# from `scripts/ops/` — because neither dir has an `__init__.py`, so
+# a single discover from `scripts/` doesn't recurse into the ops
+# subdir. Two passes is simpler than retrofitting namespace packages.
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      - 'scripts/**'
+      - '.github/workflows/test-ops-scripts.yml'
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'scripts/**'
+      - '.github/workflows/test-ops-scripts.yml'
+  merge_group:
+    types: [checks_requested]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Ops scripts (unittest)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.11'
+      - name: Run scripts/ unittests (build_runtime_package, …)
+        # Top-level scripts/ tests live alongside their target file
+        # (e.g. scripts/test_build_runtime_package.py exercises
+        # scripts/build_runtime_package.py). discover from scripts/
+        # picks up only top-level test_*.py because scripts/ops/ has
+        # no __init__.py — that's intentional, so we run two passes.
+        working-directory: scripts
+        run: python -m unittest discover -t . -p 'test_*.py' -v
+      - name: Run scripts/ops/ unittests (sweep_cf_decide, …)
+        working-directory: scripts/ops
+        run: python -m unittest discover -p 'test_*.py' -v

CONTRIBUTING.md

@@ -57,24 +57,24 @@ See `CLAUDE.md` for a full list of environment variables and their purposes.
 
 This repo is scoped to **code** (canvas, workspace, workspace-server, related
 infra). Public content (blog posts, marketing copy, OG images, SEO briefs,
-DevRel demos) lives in [`molecule-ai/docs`](https://git.moleculesai.app/molecule-ai/docs).
+DevRel demos) lives in [`Molecule-AI/docs`](https://git.moleculesai.app/molecule-ai/docs).
 The `Block forbidden paths` CI gate fails any PR that writes to `marketing/`
-or other removed paths — open against `molecule-ai/docs` instead.
+or other removed paths — open against `Molecule-AI/docs` instead.
 
 | Content type | Target |
 |---|---|
-| Blog posts | `molecule-ai/docs` → `content/blog/<YYYY-MM-DD-slug>/` |
-| Doc pages | `molecule-ai/docs` → `content/docs/` |
-| Marketing copy / PMM positioning | `molecule-ai/docs` → `marketing/` |
-| OG images, visual assets | `molecule-ai/docs` → `app/` or `marketing/` |
-| SEO briefs | `molecule-ai/docs` → `marketing/` |
-| DevRel demos (runnable code) | Standalone repo under `molecule-ai/`, OR embedded in `molecule-ai/docs` |
-| Launch checklists, internal tracking | Gitea Issues — **not** committed files |
+| Blog posts | `Molecule-AI/docs` → `content/blog/<YYYY-MM-DD-slug>/` |
+| Doc pages | `Molecule-AI/docs` → `content/docs/` |
+| Marketing copy / PMM positioning | `Molecule-AI/docs` → `marketing/` |
+| OG images, visual assets | `Molecule-AI/docs` → `app/` or `marketing/` |
+| SEO briefs | `Molecule-AI/docs` → `marketing/` |
+| DevRel demos (runnable code) | Standalone repo under `Molecule-AI/`, OR embedded in `Molecule-AI/docs` |
+| Launch checklists, internal tracking | GitHub Issues — **not** committed files |
 | Engineering docs (`docs/adr/`, `docs/architecture/`, `docs/incidents/`) | This repo (internal, not published) |
 | Live product pages (e.g. `canvas/src/app/pricing/page.tsx`) | This repo (these are app code, not marketing copy) |
 
 If a PR fails the `Block forbidden paths` check, the contents belong in
-`molecule-ai/docs`. No CI drag, no Canvas E2E, content lands in minutes.
+`Molecule-AI/docs`. No CI drag, no Canvas E2E, content lands in minutes.
 
 ## Development Workflow
 
@@ -106,7 +106,7 @@ causing a render loop when any node position changed.
 
 #### Auto-merge & the "extra commit" trap
 
-**Two system guards protect against pushing commits after auto-merge has been enabled.** Don't try to work around them — they exist because we shipped a half-merged PR on 2026-04-27 (`#2174` merged with only its first commit; the second was orphaned on a branch the host had already deleted).
+**Two system guards protect against pushing commits after auto-merge has been enabled.** Don't try to work around them — they exist because we shipped a half-merged PR on 2026-04-27 (`#2174` merged with only its first commit; the second was orphaned on a branch GitHub had already deleted).
 
 1. **Repo-wide:** "Automatically delete head branches" is on. Once a PR merges, the branch is deleted server-side. Any subsequent `git push` to that branch fails with `remote rejected — no such branch`.
 
@@ -127,11 +127,7 @@ cd workspace-server && go test -race ./...
 cd canvas && npm test
 
 # Workspace runtime (Python)
-# Runtime code is SSOT in molecule-ai-workspace-runtime, not molecule-core/workspace.
-cd ../molecule-ai-workspace-runtime
-python -m venv .venv && source .venv/bin/activate
-pip install --index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ -e . pytest pytest-asyncio
-pytest -q
+cd workspace && python -m pytest -v
 
 # E2E API tests (requires running platform)
 bash tests/e2e/test_api.sh
@@ -149,7 +145,7 @@ Fix violations before committing — the hook will reject the commit.
 
 ### CI Pipeline
 
-CI runs on Gitea Actions with self-hosted runners. External contributors:
+CI runs on GitHub Actions with a self-hosted runner. External contributors:
 PRs from forks will not trigger CI automatically. A maintainer will review
 and run CI manually.
 
@@ -163,19 +159,6 @@ and run CI manually.
 | review-check-tests | `review-check.sh` evaluator regression suite (13 scenarios) |
 | ops-scripts | Python unittest suite for `scripts/*.py` |
 
-### Workspace runtime SSOT
-
-Runtime code lives in
-[`molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime).
-Do not reintroduce `molecule-core/workspace/` or vendored `molecule_runtime/`
-copies in consumers. Core and templates consume the published runtime package
-from the Gitea package registry.
-
-For local external MCP agents, multi-workspace config is
-`MOLECULE_WORKSPACES=[{"id":"...","token":"...","platform_url":"..."}]`.
-`platform_url` selects the tenant; `org_id` is not part of this config.
-Workspace IDs can differ across orgs.
-
 ## Local Testing
 
 ### review-check.sh
@@ -207,9 +190,9 @@ Runs the full regression suite against a fixture HTTP server. No network access
 Code in this repo lands in molecule-core. Some related runtime artifacts
 live in their own repos:
 
-- [`molecule-ai/molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime) — Python adapter SDK (`molecule_runtime`) that runs inside containerized Molecule workspaces. Bridges Claude Code SDK / hermes / langgraph / etc. → A2A queue.
-- [`molecule-ai/molecule-sdk-python`](https://git.moleculesai.app/molecule-ai/molecule-sdk-python) — `A2AServer` + `RemoteAgentClient` for external agents that register over the public `/registry/register` flow.
-- [`molecule-ai/molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel) — Claude Code channel plugin. Bridges A2A traffic into a running Claude Code session via MCP `notifications/claude/channel`. Polling-based (no tunnel required); install inside Claude Code via `/plugin marketplace add https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel.git` → `/plugin install molecule@molecule-channel`, then launch with `claude --dangerously-load-development-channels=plugin:molecule@molecule-channel`.
+- [`Molecule-AI/molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime) — Python adapter SDK (`molecule_runtime`) that runs inside containerized Molecule workspaces. Bridges Claude Code SDK / hermes / langgraph / etc. → A2A queue.
+- [`Molecule-AI/molecule-sdk-python`](https://git.moleculesai.app/molecule-ai/molecule-sdk-python) — `A2AServer` + `RemoteAgentClient` for external agents that register over the public `/registry/register` flow.
+- [`Molecule-AI/molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel) — Claude Code channel plugin. Bridges A2A traffic into a running Claude Code session via MCP `notifications/claude/channel`. Polling-based (no tunnel required); install with `claude --channels plugin:molecule@Molecule-AI/molecule-mcp-claude-channel`.
 
 When extending the **A2A surface** in molecule-core (`workspace-server/internal/handlers/a2a_proxy.go` etc.), consider whether the change has a downstream impact on the runtime SDK or the channel plugin — they're versioned independently but share the wire shape.
 
@@ -223,7 +206,7 @@ See `CLAUDE.md` for detailed architecture documentation, including:
 
 ## Reporting Issues
 
-Use Gitea Issues with a clear title and reproduction steps. Include:
+Use GitHub Issues with a clear title and reproduction steps. Include:
 - What you expected
 - What actually happened
 - Platform/OS version
@@ -231,9 +214,8 @@ Use Gitea Issues with a clear title and reproduction steps. Include:
 
 ## Security
 
-If you discover a security vulnerability, please report it privately by
-opening an issue against `molecule-ai/internal` (a private repo only
-maintainers can see) rather than filing a public issue here.
+If you discover a security vulnerability, please report it privately via
+GitHub Security Advisories rather than opening a public issue.
 
 ## License
 

Makefile

@@ -4,10 +4,10 @@
 # use this Makefile; CI calls docker compose / go test directly so the
 # Makefile can evolve without breaking the build.
 
-.PHONY: help dev up down logs build test e2e-peer-visibility openapi-spec openapi-spec-check
+.PHONY: help dev up down logs build test
 
 help: ## Show this help.
-	@grep -E '^[a-zA-Z0-9_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-12s\033[0m %s\n", $$1, $$2}'
 
 dev: ## Start the full stack with air hot-reload for the platform service.
 	docker compose -f docker-compose.yml -f docker-compose.dev.yml up
@@ -26,33 +26,3 @@ build: ## Force a fresh build of the platform image (no cache).
 
 test: ## Run Go unit tests in workspace-server/.
 	cd workspace-server && go test -race ./...
-
-# ─── Local prod-mimic E2E gates ────────────────────────────────────────
-# Run the LITERAL peer-visibility MCP list_peers gate against the
-# already-running local stack (`make up` or `make dev`). Same byte-
-# identical assertion as the staging gate — only provisioning differs.
-# Skips any runtime whose provider key is absent (partially-keyed env
-# is fine). See tests/e2e/test_peer_visibility_mcp_local.sh for the
-# env contract (CLAUDE_CODE_OAUTH_TOKEN / E2E_MINIMAX_API_KEY / etc).
-e2e-peer-visibility: ## Run the LOCAL peer-visibility MCP gate vs the running stack (needs `make up` first).
-	bash tests/e2e/test_peer_visibility_mcp_local.sh
-
-# ─── OpenAPI spec generation (RFC #1706, Phase 1) ─────────────────────
-# Regenerate workspace-server/docs/openapi/swagger.{yaml,json} from
-# swaggo annotations on the gin handlers. Commit the output. CI runs
-# `make openapi-spec-check` to assert no drift between annotations and
-# the committed file — if a PR changes a handler but forgets to
-# regenerate, CI fails with a diff.
-openapi-spec: ## Regenerate OpenAPI spec from workspace-server handler annotations.
-	@command -v swag >/dev/null 2>&1 || go install github.com/swaggo/swag/cmd/swag@v1.16.4
-	cd workspace-server && swag init \
-	  --generalInfo cmd/server/main.go \
-	  --output docs/openapi \
-	  --outputTypes yaml,json \
-	  --dir . \
-	  --parseDependency=false \
-	  --parseInternal=true
-
-openapi-spec-check: openapi-spec ## CI gate — fail if openapi-spec produces a diff vs the committed file.
-	@git diff --exit-code -- workspace-server/docs/openapi/ \
-	  || (echo "openapi-spec is stale — run 'make openapi-spec' and commit the result" && exit 1)

README.md

@@ -46,18 +46,6 @@
 
 ---
 
-## Quick Start
-
-```bash
-git clone https://git.moleculesai.app/molecule-ai/molecule-monorepo.git
-cd molecule-monorepo
-./scripts/dev-start.sh
-```
-
-Then open [http://localhost:3000](http://localhost:3000), add your model API key in **Config → Secrets & API Keys → Global**, and create a workspace from a template.
-
-See the full [Quickstart Guide](./docs/quickstart.md) for prerequisites, manual setup, and troubleshooting.
-
 ## The Pitch
 
 Molecule AI is the most powerful way to govern an AI agent organization in production.
@@ -65,7 +53,7 @@ Molecule AI is the most powerful way to govern an AI agent organization in produ
 It combines the parts that are usually scattered across demos, internal glue code, and framework-specific tooling into one product:
 
 - one org-native control plane for teams, roles, hierarchy, and lifecycle
-- one runtime layer that lets **four** maintained agent runtimes — Claude Code, Codex, **Hermes**, and OpenClaw — run side by side behind one workspace contract
+- one runtime layer that lets **eight** agent runtimes — LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, **Hermes**, **Gemini CLI**, and OpenClaw — run side by side behind one workspace contract
 - one memory model that keeps recall, sharing, and skill evolution aligned with organizational boundaries (Memory v2 backed by pgvector for semantic recall)
 - one operational surface for observing, pausing, restarting, inspecting, and improving live workspaces
 
@@ -87,11 +75,11 @@ You do not wire collaboration paths by hand. Hierarchy defines the default commu
 
 ### 3. Runtime choice stops being a dead-end decision
 
-Claude Code, Codex, Hermes, and OpenClaw can all plug into the same workspace abstraction. Teams can standardize governance without forcing every group onto one runtime.
+LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, Hermes, Gemini CLI, and OpenClaw can all plug into the same workspace abstraction. Teams can standardize governance without forcing every group onto one runtime.
 
 ### 4. Memory is treated like infrastructure
 
-Molecule AI's HMA approach is designed around organizational boundaries, not just "store more context somewhere." Durable recall, scoped sharing through the v2 memory plugin, and skill promotion are all part of one coherent system.
+Molecule AI's HMA approach is designed around organizational boundaries, not just “store more context somewhere.” Durable recall, scoped sharing, awareness namespaces, and skill promotion are all part of one coherent system.
 
 ### 5. It comes with a real control plane
 
@@ -113,7 +101,7 @@ Registry, heartbeats, restart, pause/resume, activity logs, approvals, terminal
 | **Role-native workspace abstraction** | Your org structure survives model swaps, framework changes, and team expansion |
 | **Fractal team expansion** | A single specialist can become a managed department without breaking upstream integrations |
 | **Heterogeneous runtime compatibility** | Different teams can keep their preferred agent architecture while sharing one control plane |
-| **HMA + v2 memory plugin** | Memory sharing follows hierarchy instead of leaking across the whole system; one plugin per tenant, namespace-scoped per workspace |
+| **HMA + awareness namespaces** | Memory sharing follows hierarchy instead of leaking across the whole system |
 | **Skill evolution loop** | Durable successful workflows can graduate from memory into reusable, hot-reloadable skills |
 | **WebSocket-first operational UX** | The canvas reflects task state, structure changes, and A2A responses in near real time |
 | **Global secrets with local override** | Centralize provider access, then override only where a workspace needs specialized credentials |
@@ -124,9 +112,13 @@ Molecule AI is not trying to replace the frameworks below. It is the system that
 
 | Runtime / architecture | Status in current repo | Native strength | What Molecule AI adds |
 |---|---|---|---|
+| **LangGraph** | Shipping on `main` | Graph control, tool use, Python extensibility | Canvas orchestration, hierarchy routing, A2A, memory scopes, operational lifecycle |
+| **DeepAgents** | Shipping on `main` | Deeper planning and decomposition | Same workspace contract, team topology, activity stream, restart behavior |
 | **Claude Code** | Shipping on `main` | Real coding workflows, CLI-native continuity | Secure workspace abstraction, A2A delegation, org boundaries, shared control plane |
-| **Codex** | Shipping on `main` | OpenAI Codex CLI workflows | Secure workspace abstraction, A2A delegation, org boundaries, shared control plane |
+| **CrewAI** | Shipping on `main` | Role-based crews | Persistent workspace identity, policy consistency, shared canvas and registry |
+| **AutoGen** | Shipping on `main` | Assistant/tool orchestration | Standardized deployment, hierarchy-aware collaboration, shared ops plane |
 | **Hermes 4** | Shipping on `main` | Hybrid reasoning, native tools, json_schema (NousResearch/hermes-agent) | Option B upstream hook, A2A bridge to OpenAI-compat API, multi-provider provider derivation |
+| **Gemini CLI** | Shipping on `main` | Google Gemini CLI continuity | Workspace lifecycle, A2A, hierarchy-aware collaboration, shared ops plane |
 | **OpenClaw** | Shipping on `main` | CLI-native runtime with its own session model | Workspace lifecycle, templates, activity logs, topology-aware collaboration |
 | **NemoClaw** | WIP on `feat/nemoclaw-t4-docker` | NVIDIA-oriented runtime path | Planned to join the same abstraction once merged; not yet part of `main` |
 
@@ -141,7 +133,7 @@ Most projects stop at “we added memory.” Molecule AI pushes further:
 | Flat store or weak namespaces | Hierarchy-aligned `LOCAL`, `TEAM`, `GLOBAL` scopes |
 | Sharing is easy to overexpose | Sharing is explicit and structure-aware |
 | Memory and procedure get mixed together | Memory stores durable facts; skills store repeatable procedure |
-| Every agent can become over-privileged | Per-workspace namespaces in the v2 memory plugin reduce blast radius |
+| Every agent can become over-privileged | Workspace awareness namespaces reduce blast radius |
 | UI memory and runtime memory blur together | Separate surfaces for scoped agent memory, key/value workspace memory, and recall |
 
 ### The flywheel
@@ -171,16 +163,16 @@ Most agent systems stop at "a smart runtime." Molecule AI pushes further: it giv
 
 | Core mechanism | Molecule AI module(s) | Why it matters |
 |---|---|---|
-| **Durable memory that survives sessions** | `molecule-ai-workspace-runtime/molecule_runtime/builtin_tools/`, `workspace-server/internal/handlers/memories.go`, `workspace-server/internal/memory/` (v2 plugin client + namespace resolver) | Memory is not just durable, it is **workspace-scoped** — every write lands in the workspace's own `workspace:<id>` namespace, with `team:<root>` and `org:<root>` available for cross-workspace shares via the platform's namespace ACL when an agent explicitly promotes a memory |
+| **Durable memory that survives sessions** | `workspace/builtin_tools/memory.py`, `workspace/builtin_tools/awareness_client.py`, `workspace-server/internal/handlers/memories.go` | Memory is not just durable, it is **workspace-scoped** and can route into awareness namespaces tied to the org structure |
 | **Cross-session recall** | `workspace-server/internal/handlers/activity.go` (`/workspaces/:id/session-search`) | Recall spans both activity history and memory rows, so the system can search what happened and what was learned without inventing a separate hidden store |
-| **Skills built from experience** | `molecule-ai-workspace-runtime/molecule_runtime/builtin_tools/memory.py` (`_maybe_log_skill_promotion`) | Promotion from memory into a skill candidate is surfaced as an explicit platform activity, not a silent internal side effect |
-| **Skill improvement during use** | `molecule-ai-workspace-runtime/molecule_runtime/skill_loader/`, `molecule-ai-workspace-runtime/molecule_runtime/main.py` | Skills hot-reload into the live runtime, so improvements become available on the next A2A task without restarting the workspace |
-| **Persistent skill lifecycle** | `workspace-server/cmd/cli/cmd_agent_skill.go`, `molecule-ai-workspace-runtime/molecule_runtime/plugins.py` | Skills are not just generated once; they can be audited, installed, published, shared, mounted by plugins, and governed as reusable operational assets |
+| **Skills built from experience** | `workspace/builtin_tools/memory.py` (`_maybe_log_skill_promotion`) | Promotion from memory into a skill candidate is surfaced as an explicit platform activity, not a silent internal side effect |
+| **Skill improvement during use** | `workspace/skill_loader/watcher.py`, `workspace/skill_loader/loader.py`, `workspace/main.py` | Skills hot-reload into the live runtime, so improvements become available on the next A2A task without restarting the workspace |
+| **Persistent skill lifecycle** | `workspace-server/cmd/cli/cmd_agent_skill.go`, `workspace/plugins.py` | Skills are not just generated once; they can be audited, installed, published, shared, mounted by plugins, and governed as reusable operational assets |
 
 ### Why this matters in Molecule AI
 
 1. **The learning loop is org-aware, not just session-aware.**
-   Memory can live at `LOCAL`, `TEAM`, or `GLOBAL` scope, and the v2 plugin's namespace ACL gives each workspace a durable identity boundary.
+   Memory can live at `LOCAL`, `TEAM`, or `GLOBAL` scope, and awareness namespaces give each workspace a durable identity boundary.
 
 2. **The learning loop is visible to operators.**
    Promotion events, activity logs, current-task updates, traces, and WebSocket fanout mean self-improvement is part of the control plane, not a hidden black box.
@@ -216,10 +208,10 @@ The result is not just “an agent that learns.” It is **an organization that
 
 ### Runtime
 
-- standalone workspace-template images that install `molecule-ai-workspace-runtime` from the Gitea package registry; thin AMI in production (us-east-2)
-- adapter-driven execution across **4 maintained runtimes** (Claude Code, Codex, Hermes, OpenClaw)
+- unified `workspace/` image; thin AMI in production (us-east-2)
+- adapter-driven execution across **8 runtimes** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw)
 - Agent Card registration
-- **Memory v2 backed by pgvector** — per-tenant plugin sidecar serving HMA namespaces with FTS + semantic recall
+- awareness-backed memory integration; **Memory v2 backed by pgvector** for semantic recall
 - plugin-mounted shared rules/skills
 - hot-reloadable local skills
 - coordinator-only delegation path
@@ -246,14 +238,14 @@ The result is not just “an agent that learns.” It is **an organization that
 - subscribe to one or more workspaces; peer messages surface as conversation turns; replies route back through Molecule's A2A
 - no tunnel, no public endpoint — the plugin self-registers each watched workspace as `delivery_mode=poll` and long-polls `/activity?since_id=…`
 - multi-tenant friendly: one plugin install can watch workspaces across multiple Molecule tenants (`MOLECULE_PLATFORM_URLS` per-workspace)
-- install via the standard marketplace flow: `/plugin marketplace add https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel.git` → `/plugin install molecule@molecule-channel`, then launch with `claude --dangerously-load-development-channels=plugin:molecule@molecule-channel`
+- install via the standard marketplace flow: `/plugin marketplace add Molecule-AI/molecule-mcp-claude-channel` → `/plugin install molecule-channel@molecule-mcp-claude-channel`
 
 ## Built For Teams That Need More Than A Demo
 
 Molecule AI is especially strong when you need to run:
 
 - AI engineering teams with PM / Dev Lead / QA / Research / Ops roles
-- mixed runtime organizations where one team prefers Hermes and another prefers Claude Code
+- mixed runtime organizations where one team prefers LangGraph and another prefers Claude Code
 - long-lived agent organizations that need memory boundaries and reusable procedures
 - internal platforms that want to expose agent teams as structured infrastructure, not ad hoc scripts
 
@@ -268,9 +260,9 @@ Canvas (Next.js 15, warm-paper :3000)  <--HTTP / WS-->  Platform (Go 1.25 :8080)
          +------------------------- shows ------------------------> workspaces, teams, tasks, traces, events
 
 Workspace Runtime (Python ≥3.11, image with adapters)
-  - 4 adapters: Claude Code / Codex / Hermes / OpenClaw
+  - 8 adapters: LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / Hermes / Gemini CLI / OpenClaw
   - Agent Card + A2A server (typed-SSOT response path, RFC #2967)
-  - heartbeat + activity + Memory v2 (pgvector semantic recall via per-tenant plugin sidecar)
+  - heartbeat + activity + awareness-backed memory (Memory v2 — pgvector semantic recall)
   - skills + plugins + hot reload
 
 SaaS Control Plane (molecule-controlplane, private)
@@ -336,7 +328,7 @@ Then open `http://localhost:3000`:
 
 ## Current Scope
 
-The current `main` branch ships the core platform, Canvas v4 (warm-paper themed), Memory v2 (pgvector semantic recall), the typed-SSOT A2A response path (RFC #2967), **four maintained production adapters** (Claude Code, Codex, Hermes, OpenClaw), skill lifecycle, and operational surfaces.
+The current `main` branch ships the core platform, Canvas v4 (warm-paper themed), Memory v2 (pgvector semantic recall), the typed-SSOT A2A response path (RFC #2967), **eight production adapters** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw), skill lifecycle, and operational surfaces.
 
 The companion private repo [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) provides the SaaS surface — multi-tenant orchestration on EC2 + Neon + Cloudflare Tunnels, KMS envelope encryption, WorkOS auth, Stripe billing, and a `tenant_resources` audit table with a 30-min reconciler.
 

README.zh-CN.md

@@ -52,7 +52,7 @@ Molecule AI 是目前最强的 AI Agent 组织治理方案之一，用来把 age
 它把过去分散在 demo、内部胶水代码和各类 framework 私有工具里的关键能力，收敛成一个产品：
 
 - 一套组织原生 control plane，管理团队、角色、层级和生命周期
-- 一套 runtime abstraction，让 **4 个**维护中的 agent runtime —— Claude Code、Codex、**Hermes**、OpenClaw —— 共用一套 workspace 契约
+- 一套 runtime abstraction，让 **8 个** agent runtime —— LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、**Hermes**、**Gemini CLI**、OpenClaw —— 共用一套 workspace 契约
 - 一套与组织边界对齐的 memory 模型，把 recall、sharing 和 skill evolution 放进同一体系（Memory v2 由 pgvector 支撑语义召回）
 - 一套面向线上 workspace 的运维面，统一完成观测、暂停、重启、检查和持续改进
 
@@ -74,11 +74,11 @@ Molecule AI 填的就是这个空白。
 
 ### 3. Runtime 选择不再是死路
 
-Claude Code、Codex、Hermes、OpenClaw 都可以挂到同一个 workspace abstraction 下。团队可以统一治理方式，而不必统一到底层 runtime。
+LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、Hermes、Gemini CLI、OpenClaw 都可以挂到同一个 workspace abstraction 下。团队可以统一治理方式，而不必统一到底层 runtime。
 
 ### 4. Memory 被当成基础设施来做
 
-Molecule AI 的 HMA 不是“多存一点上下文”而已。它关注组织边界、durable recall、scope sharing、v2 memory plugin、skill promotion，把这些放在一个完整体系里。
+Molecule AI 的 HMA 不是“多存一点上下文”而已。它关注组织边界、durable recall、scope sharing、awareness namespace、skill promotion，把这些放在一个完整体系里。
 
 ### 5. 它自带真正的 control plane
 
@@ -100,7 +100,7 @@ Registry、heartbeat、restart、pause/resume、activity、approval、terminal
 | **角色原生 workspace 抽象** | 模型切换、框架切换、团队扩容都不会打碎你的组织结构 |
 | **分形式团队扩展** | 一个 specialist 可以平滑升级成一个部门，而不影响上游集成 |
 | **异构 runtime 兼容** | 不同团队可以保留偏好的 agent 架构，但共用一套平台规则 |
-| **HMA + v2 memory plugin** | Memory 分享沿组织边界走，而不是全局乱穿透；每个 tenant 一个 plugin，按 workspace namespace 隔离 |
+| **HMA + awareness namespace** | Memory 分享沿组织边界走，而不是全局乱穿透 |
 | **Skill 演化闭环** | 成功工作流可以从 memory 逐步提升成可热加载的 skill |
 | **WebSocket-first 运维体验** | Canvas 能即时反映任务状态、结构变更和 A2A 响应 |
 | **Global secrets + local override** | 统一管理 provider 凭据，只在需要时做 workspace 级覆写 |
@@ -111,9 +111,13 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 
 | Runtime / 架构 | 当前仓库状态 | 原生优势 | Molecule AI 额外补上的能力 |
 |---|---|---|---|
+| **LangGraph** | `main` 已支持 | 图控制强、工具调用成熟、Python 扩展性好 | Canvas orchestration、层级路由、A2A、memory scope、operational lifecycle |
+| **DeepAgents** | `main` 已支持 | 规划和任务拆解更强 | 同一套 workspace contract、团队拓扑、activity、restart 行为 |
 | **Claude Code** | `main` 已支持 | 真实编码工作流、CLI-native continuity | 安全 workspace 抽象、A2A delegation、组织边界、共享 control plane |
-| **Codex** | `main` 已支持 | OpenAI Codex CLI 工作流 | 安全 workspace 抽象、A2A delegation、组织边界、共享 control plane |
+| **CrewAI** | `main` 已支持 | 角色型 crew 模式清晰 | 持久 workspace 身份、统一策略、共享 Canvas 和 registry |
+| **AutoGen** | `main` 已支持 | assistant/tool orchestration | 统一部署、层级协作、共享运维平面 |
 | **Hermes 4** | `main` 已支持 | 混合推理、原生工具调用、json_schema 输出（NousResearch/hermes-agent） | Option B 上游 hook、A2A 桥接 OpenAI 兼容 API、多 provider 自动派生 |
+| **Gemini CLI** | `main` 已支持 | Google Gemini CLI 持续会话 | workspace 生命周期、A2A、层级感知协作、共享运维平面 |
 | **OpenClaw** | `main` 已支持 | CLI-native runtime，自有 session 模型 | workspace 生命周期、templates、activity logs、拓扑感知协作 |
 | **NemoClaw** | `feat/nemoclaw-t4-docker` 分支 WIP | NVIDIA 方向 runtime 路线 | 计划并入同一抽象层，但当前还不是 `main` 已合并能力 |
 
@@ -128,7 +132,7 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 | 扁平 store 或弱命名空间隔离 | 与层级对齐的 `LOCAL`、`TEAM`、`GLOBAL` scope |
 | 分享很容易越界 | 分享是显式且结构感知的 |
 | Memory 和 procedure 混成一团 | Memory 存 durable facts，skills 存 repeatable procedure |
-| 任意 agent 容易过权 | v2 memory plugin 的 per-workspace namespace 缩小 blast radius |
+| 任意 agent 容易过权 | workspace awareness namespace 缩小 blast radius |
 | UI memory 和 runtime memory 混在一起 | scoped agent memory、key/value workspace memory、recall surface 分层清晰 |
 
 ### 这套飞轮怎么转
@@ -158,7 +162,7 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 
 | 核心机制 | Molecule AI 对应模块 | 为什么重要 |
 |---|---|---|
-| **跨 session 的 durable memory** | `workspace/builtin_tools/memory.py`、`workspace-server/internal/handlers/memories.go`、`workspace-server/internal/memory/`（v2 plugin client + namespace resolver）| 不只是持久化，而且是**按 workspace 隔离**的 —— 每次写入都落在 workspace 自己的 `workspace:<id>` namespace 里；当 agent 显式升级到跨 workspace 共享时，可以通过平台 namespace ACL 写到 `team:<root>` 和 `org:<root>` |
+| **跨 session 的 durable memory** | `workspace/builtin_tools/memory.py`、`workspace/builtin_tools/awareness_client.py`、`workspace-server/internal/handlers/memories.go` | 不只是持久化，而且是**按 workspace 隔离**的，可进一步路由到和组织结构绑定的 awareness namespace |
 | **Cross-session recall** | `workspace-server/internal/handlers/activity.go` 中的 `/workspaces/:id/session-search` | Recall 同时覆盖 activity history 和 memory rows，不需要再造一个隐蔽的新存储层 |
 | **从经验里长出技能** | `workspace/builtin_tools/memory.py` 里的 `_maybe_log_skill_promotion` | 从 memory 到 skill candidate 的提升会被显式记录成平台 activity，而不是默默发生在黑盒里 |
 | **技能在使用中持续改进** | `workspace/skill_loader/watcher.py`、`workspace/skill_loader/loader.py`、`workspace/main.py` | Skill 改动可以热加载进 live runtime，下一次 A2A 任务就能直接使用，不需要重启 workspace |
@@ -167,7 +171,7 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 ### 为什么这在 Molecule AI 里更适合团队级系统
 
 1. **学习闭环是 org-aware 的，而不只是 session-aware。**
-   Memory 可以按 `LOCAL`、`TEAM`、`GLOBAL` scope 运作，v2 plugin 的 namespace ACL 让每个 workspace 都有清晰的持久边界。
+   Memory 可以按 `LOCAL`、`TEAM`、`GLOBAL` scope 运作，awareness namespace 让每个 workspace 都有清晰的持久边界。
 
 2. **学习闭环是对运维可见的。**
    Promotion events、activity logs、current-task updates、traces、WebSocket fanout 让自我进化进入 control plane，而不是藏在黑盒内部。
@@ -204,9 +208,9 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 ### Runtime
 
 - 统一 `workspace/` 镜像；生产环境采用 thin AMI（us-east-2）
-- adapter 驱动执行，覆盖 **4 个维护中的 runtime**（Claude Code、Codex、Hermes、OpenClaw）
+- adapter 驱动执行，覆盖 **8 个 runtime**（Claude Code、Hermes、Gemini CLI、LangGraph、DeepAgents、CrewAI、AutoGen、OpenClaw）
 - Agent Card 注册
-- **Memory v2 由 pgvector 支撑** —— 每个 tenant 一个 plugin sidecar，承载 HMA namespace、FTS 与语义召回
+- awareness-backed memory；**Memory v2 由 pgvector 支撑**语义召回
 - plugin 挂载共享 rules/skills
 - 本地 skills 热加载
 - coordinator-only delegation 路径
@@ -233,7 +237,7 @@ Molecule AI 并不是要替代下面这些 framework，而是把它们纳入更
 - 订阅一个或多个 workspace；peer 的消息会以 user-turn 出现，回复会经 Molecule A2A 路由出去
 - 无需公网隧道、无需公开端点 —— 插件启动时自动把每个 watched workspace 注册成 `delivery_mode=poll`，长轮询 `/activity?since_id=…`
 - 多租户友好：单次安装即可同时 watch 跨多个 Molecule 租户的 workspace（`MOLECULE_PLATFORM_URLS` 按 workspace 配置）
-- 通过标准 marketplace 流程安装：`/plugin marketplace add https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel.git` → `/plugin install molecule@molecule-channel`，然后用 `claude --dangerously-load-development-channels=plugin:molecule@molecule-channel` 启动
+- 通过标准 marketplace 流程安装：`/plugin marketplace add Molecule-AI/molecule-mcp-claude-channel` → `/plugin install molecule-channel@molecule-mcp-claude-channel`
 
 ## 适合什么团队
 
@@ -255,9 +259,9 @@ Canvas (Next.js 15, warm-paper :3000)  <--HTTP / WS-->  Platform (Go 1.25 :8080)
          +------------------------- 展示 ------------------------> workspaces, teams, tasks, traces, events
 
 Workspace Runtime (Python ≥3.11，含 adapter 集合的镜像)
-  - 4 个 adapter: Claude Code / Codex / Hermes / OpenClaw
+  - 8 个 adapter: LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / Hermes / Gemini CLI / OpenClaw
   - Agent Card + A2A server（typed-SSOT 响应路径，RFC #2967）
-  - heartbeat + activity + Memory v2（pgvector 语义召回，per-tenant plugin sidecar）
+  - heartbeat + activity + awareness-backed memory（Memory v2 —— pgvector 语义召回）
   - skills + plugins + hot reload
 
 SaaS Control Plane (molecule-controlplane，私有)
@@ -317,7 +321,7 @@ npm run dev
 
 ## 当前范围说明
 
-当前 `main` 已经包含核心平台、Canvas v4（warm-paper 主题）、Memory v2（pgvector 语义召回）、typed-SSOT A2A 响应路径（RFC #2967）、**4 个维护中的正式 adapter**（Claude Code、Codex、Hermes、OpenClaw）、skill lifecycle，以及主要运维面。
+当前 `main` 已经包含核心平台、Canvas v4（warm-paper 主题）、Memory v2（pgvector 语义召回）、typed-SSOT A2A 响应路径（RFC #2967）、**8 个正式 adapter**（Claude Code、Hermes、Gemini CLI、LangGraph、DeepAgents、CrewAI、AutoGen、OpenClaw）、skill lifecycle，以及主要运维面。
 
 配套的私有仓库 [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) 提供 SaaS 层 —— 多租户编排（EC2 + Neon + Cloudflare Tunnels）、KMS 信封加密、WorkOS 鉴权、Stripe 计费，以及 `tenant_resources` 审计表加 30 分钟 reconciler。
 

_ci_trigger.txt

@@ -1,2 +1 @@
-trigger
-retrigger 2026-05-20T04:09Z after op-config#110 (HOME=/home/runner) deploy to fleet — internal#603
+trigger

canvas/e2e/chat-desktop.spec.ts

@@ -55,7 +55,7 @@ test.describe("Desktop ChatTab", () => {
     await textarea.fill("What is the weather?");
     await page.getByRole("button", { name: /Send/ }).first().click();
 
-    await expect(page.getByText("What is the weather?", { exact: true })).toBeVisible({ timeout: 5_000 });
+    await expect(page.getByText("What is the weather?")).toBeVisible({ timeout: 5_000 });
     await expect(page.getByText("Echo: What is the weather?")).toBeVisible({ timeout: 15_000 });
   });