Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6ba9424196 | |||
| 531d98efea | |||
| 0b17567891 | |||
| 59d699b61c |
@@ -1,174 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared path-filter helper for Gitea Actions workflows.
|
||||
|
||||
Computes changed files against the PR base SHA or push-before SHA and writes
|
||||
boolean outputs to GITHUB_OUTPUT. If the diff base is missing or untrusted, the
|
||||
helper fails open by setting every output in the selected profile to true.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROFILES: dict[str, dict[str, str]] = {
|
||||
"ci": {
|
||||
"platform": r"^workspace-server/",
|
||||
"canvas": r"^canvas/",
|
||||
"python": r"^workspace/",
|
||||
"scripts": r"^tests/e2e/|^scripts/|^infra/scripts/",
|
||||
},
|
||||
"handlers-postgres": {
|
||||
"handlers": (
|
||||
r"^workspace-server/internal/handlers/"
|
||||
r"|^workspace-server/internal/wsauth/"
|
||||
r"|^workspace-server/migrations/"
|
||||
r"|^\.gitea/workflows/handlers-postgres-integration\.yml$"
|
||||
),
|
||||
},
|
||||
"e2e-api": {
|
||||
"api": r"^workspace-server/|^tests/e2e/|^\.gitea/workflows/e2e-api\.yml$",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def classify(profile: str, paths: list[str]) -> dict[str, bool]:
|
||||
patterns = PROFILES[profile]
|
||||
return {
|
||||
name: any(re.search(pattern, path) for path in paths)
|
||||
for name, pattern in patterns.items()
|
||||
}
|
||||
|
||||
|
||||
def all_true(profile: str) -> dict[str, bool]:
|
||||
return {name: True for name in PROFILES[profile]}
|
||||
|
||||
|
||||
def resolve_base(event_name: str, pr_base_sha: str, push_before: str) -> str:
|
||||
if event_name == "pull_request" and pr_base_sha:
|
||||
return pr_base_sha
|
||||
return push_before
|
||||
|
||||
|
||||
def is_zero_sha(value: str) -> bool:
|
||||
return not value or bool(re.fullmatch(r"0+", value))
|
||||
|
||||
|
||||
def run_git(args: list[str], *, timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def base_exists(base: str) -> bool:
|
||||
return run_git(["cat-file", "-e", base]).returncode == 0
|
||||
|
||||
|
||||
def fetch_base(base: str, base_ref: str) -> None:
|
||||
# Gitea may reject fetching an arbitrary unadvertised SHA from a shallow
|
||||
# PR checkout. Fetch the advertised base branch first, then fall back to
|
||||
# the SHA for hosts that allow it.
|
||||
if base_ref:
|
||||
run_git(["fetch", "--depth=1", "origin", base_ref])
|
||||
if not base_exists(base):
|
||||
run_git(["fetch", "--depth=1", "origin", base])
|
||||
|
||||
|
||||
def deepen_base_ref(base_ref: str) -> None:
|
||||
if base_ref:
|
||||
run_git(["fetch", "--deepen=200", "origin", base_ref], timeout=60)
|
||||
|
||||
|
||||
def merge_base(base: str) -> str | None:
|
||||
proc = run_git(["merge-base", base, "HEAD"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
value = proc.stdout.strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def changed_paths(base: str, *, use_merge_base: bool) -> list[str] | None:
|
||||
compare_base = base
|
||||
if use_merge_base:
|
||||
compare_base = merge_base(base) or ""
|
||||
if not compare_base:
|
||||
return None
|
||||
|
||||
proc = run_git(["diff", "--name-only", compare_base, "HEAD"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return [line for line in proc.stdout.splitlines() if line]
|
||||
|
||||
|
||||
def write_outputs(values: dict[str, bool], output_path: str | None) -> None:
|
||||
lines = [f"{name}={'true' if value else 'false'}" for name, value in values.items()]
|
||||
if output_path:
|
||||
with Path(output_path).open("a", encoding="utf-8") as fh:
|
||||
for line in lines:
|
||||
fh.write(line + "\n")
|
||||
else:
|
||||
for line in lines:
|
||||
print(line)
|
||||
|
||||
|
||||
def detect(
|
||||
profile: str,
|
||||
event_name: str,
|
||||
pr_base_sha: str,
|
||||
push_before: str,
|
||||
base_ref: str = "",
|
||||
) -> dict[str, bool]:
|
||||
base = resolve_base(event_name, pr_base_sha, push_before)
|
||||
if is_zero_sha(base):
|
||||
return all_true(profile)
|
||||
|
||||
if not base_exists(base):
|
||||
fetch_base(base, base_ref)
|
||||
if not base_exists(base):
|
||||
return all_true(profile)
|
||||
|
||||
use_merge_base = event_name == "pull_request"
|
||||
if use_merge_base and base_ref and merge_base(base) is None:
|
||||
deepen_base_ref(base_ref)
|
||||
|
||||
paths = changed_paths(base, use_merge_base=use_merge_base)
|
||||
if paths is None:
|
||||
return all_true(profile)
|
||||
return classify(profile, paths)
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--profile", required=True, choices=sorted(PROFILES))
|
||||
parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME", ""))
|
||||
parser.add_argument("--pr-base-sha", default="")
|
||||
parser.add_argument("--base-ref", default="")
|
||||
parser.add_argument("--push-before", default=os.environ.get("GITHUB_EVENT_BEFORE", ""))
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
values = detect(
|
||||
args.profile,
|
||||
args.event_name,
|
||||
args.pr_base_sha,
|
||||
args.push_before,
|
||||
args.base_ref,
|
||||
)
|
||||
write_outputs(values, os.environ.get("GITHUB_OUTPUT"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -61,7 +61,6 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -90,19 +89,6 @@ API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
# match by exact title without parsing.
|
||||
TITLE_PREFIX = "[main-red]"
|
||||
|
||||
# Settling window (seconds) between initial red detection and the
|
||||
# pre-file recheck. The recheck filters out the two largest false-
|
||||
# positive classes seen in mc#1597..1630 (task #394, 2026-05-21):
|
||||
# 1. HEAD moved on (a new commit landed mid-tick) — the prior red SHA
|
||||
# is no longer authoritative; let the next cron tick re-evaluate.
|
||||
# 2. Combined status recovered on the SAME SHA (transient
|
||||
# cancel-cascade rolled forward to success on retry).
|
||||
# 90s is well below the hourly cron cadence; a real failure that
|
||||
# persists past it is the one we want surfaced.
|
||||
# Override with WATCHDOG_RECHECK_DELAY_SECS for tests / local probes
|
||||
# (the test suite stubs time.sleep to a no-op).
|
||||
RECHECK_DELAY_SECS = int(_env("WATCHDOG_RECHECK_DELAY_SECS", default="90"))
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only.
|
||||
@@ -186,49 +172,6 @@ def api(
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# action_run.status resolver — extensibility hook for task #394.
|
||||
# --------------------------------------------------------------------------
|
||||
def _resolve_action_run_status(target_url: str) -> int | None:
|
||||
"""Resolve the underlying Gitea `action_run.status` integer for the
|
||||
run referenced by `target_url`, returning None if the resolver
|
||||
cannot reach an authoritative source from the runner.
|
||||
|
||||
Canonical Gitea 1.22.6 enum (per `models/actions/status.go` +
|
||||
`reference_gitea_action_status_enum_corrected_2026_05_19`):
|
||||
1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
|
||||
5=Waiting, 6=Running, 7=Blocked
|
||||
Only `status == 2` is a real defect; status=3 is cancel-cascade and
|
||||
status=1 is an emission artifact (Gitea wrote a 'failure' commit_status
|
||||
row for a run that actually succeeded — observed empirically on
|
||||
`publish-canvas-image` jobs at SHAs in mc#1597..1630).
|
||||
|
||||
CURRENT STATE (2026-05-20, verified): Gitea 1.22.6 exposes NO REST
|
||||
endpoint for `action_run.status`. Probed:
|
||||
/api/v1/repos/{o}/{r}/actions/runs/{id} → HTTP 404
|
||||
/api/v1/repos/{o}/{r}/actions/jobs/{id} → HTTP 404
|
||||
/api/v1/repos/{o}/{r}/actions/tasks/{id} → HTTP 404
|
||||
/swagger.v1.json paths containing 'actions' → secrets+variables+runners only
|
||||
The SPA backend (`/{repo}/actions/runs/{id}/jobs/{idx}` POST) requires
|
||||
a session CSRF token, unreachable from a runner. The only authoritative
|
||||
source today is direct DB access (`mol_action_status` on op-host,
|
||||
`docker exec molecule-postgres-1 psql ...`), which the runner cannot
|
||||
reach.
|
||||
|
||||
Therefore: this hook returns None on every call. Callers MUST fall
|
||||
back to the description-string filter (existing) plus the HEAD
|
||||
recheck (this PR). When a future Gitea release (>=1.23 expected) or
|
||||
an op-host proxy exposes the endpoint, replace the body of this
|
||||
function with an `api(...)` call — the caller contract is stable.
|
||||
|
||||
See also:
|
||||
- `reference_chronic_red_sweep_cancelled_vs_failed_filter`
|
||||
- `feedback_gitea_status_enum_use_helper_not_raw_int`
|
||||
"""
|
||||
_ = target_url # noqa: F841 — intentional placeholder
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea reads
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -671,56 +614,6 @@ def run_once(*, dry_run: bool = False) -> int:
|
||||
}
|
||||
|
||||
if red:
|
||||
# HEAD recheck (task #394 — guards mc#1597..1630 false-positive
|
||||
# cluster). After the initial detection, wait RECHECK_DELAY_SECS
|
||||
# (default 90s; tests stub time.sleep) and re-evaluate:
|
||||
#
|
||||
# 1. Re-fetch HEAD SHA. If HEAD moved, a new commit landed
|
||||
# mid-tick — the prior red SHA is no longer authoritative
|
||||
# and the next cron run will re-evaluate against the new
|
||||
# HEAD. Skip-file.
|
||||
#
|
||||
# 2. If HEAD unchanged, re-fetch the combined status. If it
|
||||
# recovered (combined state no longer in {failure,error}
|
||||
# after the cancel-cascade filter), a transient retry
|
||||
# rolled the run forward. Skip-file.
|
||||
#
|
||||
# Both paths emit a Loki event distinguishable from the real
|
||||
# `main_red_detected` so obs queries can track filter activity.
|
||||
# The settling window is well below the hourly cron cadence —
|
||||
# genuine failures persist past it and are surfaced normally.
|
||||
time.sleep(RECHECK_DELAY_SECS)
|
||||
|
||||
recheck_sha = get_head_sha(WATCH_BRANCH)
|
||||
if recheck_sha != sha:
|
||||
emit_loki_event("main_red_skipped_head_drift", sha, [])
|
||||
print(
|
||||
f"::notice::skip-file (HEAD moved): initial red at "
|
||||
f"{sha[:10]} but HEAD is now {recheck_sha[:10]} on "
|
||||
f"{WATCH_BRANCH}; next cron tick will re-evaluate."
|
||||
)
|
||||
return 0
|
||||
|
||||
recheck_status = get_combined_status(sha)
|
||||
recheck_red, recheck_failed = is_red(recheck_status)
|
||||
if not recheck_red:
|
||||
emit_loki_event("main_red_skipped_recovered", sha, [])
|
||||
print(
|
||||
f"::notice::skip-file (recovered after settling): "
|
||||
f"combined state at {sha[:10]} flipped to "
|
||||
f"{recheck_status.get('state')!r} on recheck; "
|
||||
f"initial red was a transient cancel-cascade."
|
||||
)
|
||||
return 0
|
||||
|
||||
# Still red after settling — file/update. Use the recheck data
|
||||
# as authoritative so the issue body reflects the latest state.
|
||||
failed = recheck_failed
|
||||
debug["recheck_combined_state"] = recheck_status.get("state")
|
||||
debug["recheck_failed_contexts"] = [
|
||||
s.get("context") for s in failed
|
||||
]
|
||||
|
||||
failed_ctxs = [s.get("context") for s in failed if s.get("context")]
|
||||
emit_loki_event("main_red_detected", sha, failed_ctxs)
|
||||
print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
|
||||
|
||||
+12
-149
@@ -64,41 +64,11 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Callable, Iterator
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Address-space guardrail (RFC#369 / task #369 follow-up to mc#1242-class OOM).
|
||||
#
|
||||
# `get_issue_comments` paginates the full comment history of a PR. On
|
||||
# bot-relay-heavy PRs (e.g. mc#291, mc#1242) this can balloon past the
|
||||
# runner's cgroup memory limit and 137 the job. Cap virtual-address-space
|
||||
# at 2 GiB so the script OOMs as a `MemoryError` (catchable / surfaceable)
|
||||
# rather than a SIGKILL we can't post a status for.
|
||||
#
|
||||
# 2 GiB is generous — a 5000-comment PR with 1 KiB minimal-dicts (see
|
||||
# get_issue_comments below) fits in ~10 MiB, leaving plenty of headroom
|
||||
# for the Python runtime + urllib + json buffers.
|
||||
#
|
||||
# Skipped under pytest / dry-run where RLIMIT_AS would interfere with
|
||||
# test runner memory needs (set SOP_CHECKLIST_NO_RLIMIT=1 to opt out).
|
||||
if not os.environ.get("SOP_CHECKLIST_NO_RLIMIT"):
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_AS, (2 * 1024**3, 2 * 1024**3))
|
||||
except (ValueError, OSError):
|
||||
# macOS sometimes refuses RLIMIT_AS; not fatal — the Linux runner
|
||||
# is the only place this matters for the OOM-prevention goal.
|
||||
pass
|
||||
|
||||
# Per-comment body cap (task #369). The directive parser walks the body
|
||||
# line-by-line looking for ^/sop-ack ^/sop-revoke ^/sop-n/a markers — only
|
||||
# the first few KiB matter for that. Cap each comment body so a single
|
||||
# pasted-log comment can't push us past the cgroup limit.
|
||||
_MAX_BODY_BYTES = int(os.environ.get("SOP_CHECKLIST_MAX_BODY_BYTES") or 8 * 1024)
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -490,35 +460,16 @@ class GiteaClient:
|
||||
raise RuntimeError(f"GET pulls/{pr} → HTTP {code}: {data!r}")
|
||||
return data
|
||||
|
||||
def iter_issue_comments(
|
||||
self, owner: str, repo: str, issue: int, page_size: int = 50
|
||||
) -> Iterator[dict[str, Any]]:
|
||||
"""Stream comments page-by-page, yielding ONE minimal-dict per comment.
|
||||
|
||||
Each yielded comment carries ONLY the fields the gate actually reads
|
||||
— `{"user": {"login": str}, "body": str}` — and DROPS the much
|
||||
larger Gitea-API extras (html_url, pull_request_url, issue_url,
|
||||
assets, created_at, updated_at, id, original_author_*).
|
||||
|
||||
Memory motivation (task #369 / mc#1242-class OOM): full Gitea
|
||||
comment dicts are ~2 KiB median + ~3 KiB p95. On PRs with several
|
||||
thousand bot-relay comments the eager `list[full_dict]` shape used
|
||||
previously pushed runner anon-rss past the cgroup limit. The
|
||||
minimal-dict shape is ~10-20x smaller (typically ~50-100B Python
|
||||
overhead + the body string).
|
||||
|
||||
The two downstream consumers (`compute_ack_state`,
|
||||
`compute_na_state`) each iterate the comment list exactly once and
|
||||
read only `body` + `user.login`, so dropping every other field is
|
||||
safe. They still receive `list[dict[str, Any]]`-shaped objects so
|
||||
the test fixtures (which already used the minimal shape) keep
|
||||
working with no fixture changes.
|
||||
"""
|
||||
def get_issue_comments(
|
||||
self, owner: str, repo: str, issue: int
|
||||
) -> list[dict[str, Any]]:
|
||||
# Paginate. Gitea default page size 50.
|
||||
out: list[dict[str, Any]] = []
|
||||
page = 1
|
||||
while True:
|
||||
code, data = self._req(
|
||||
"GET",
|
||||
f"/repos/{owner}/{repo}/issues/{issue}/comments?limit={page_size}&page={page}",
|
||||
f"/repos/{owner}/{repo}/issues/{issue}/comments?limit=50&page={page}",
|
||||
)
|
||||
if code != 200:
|
||||
raise RuntimeError(
|
||||
@@ -526,41 +477,10 @@ class GiteaClient:
|
||||
)
|
||||
if not data:
|
||||
break
|
||||
for c in data:
|
||||
# Minimal projection — drop ALL fields the gate doesn't read.
|
||||
user_login = ((c.get("user") or {}).get("login") or "") if isinstance(c, dict) else ""
|
||||
body = (c.get("body") if isinstance(c, dict) else "") or ""
|
||||
# Body-size guardrail: huge comments (e.g. pasted CI logs) can
|
||||
# individually be MiBs. The directive parser only needs the
|
||||
# first ~8 KiB to find /sop-ack /sop-revoke /sop-n/a markers
|
||||
# — anything past that is filler. Truncate at 8 KiB so a
|
||||
# single oversized comment can't OOM the runner.
|
||||
if len(body) > _MAX_BODY_BYTES:
|
||||
body = body[:_MAX_BODY_BYTES]
|
||||
yield {"user": {"login": user_login}, "body": body}
|
||||
if len(data) < page_size:
|
||||
out.extend(data)
|
||||
if len(data) < 50:
|
||||
break
|
||||
page += 1
|
||||
|
||||
def get_issue_comments(
|
||||
self,
|
||||
owner: str,
|
||||
repo: str,
|
||||
issue: int,
|
||||
max_comments: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Paginate + collect minimal comment dicts. See `iter_issue_comments`
|
||||
for the per-comment shape and the OOM-prevention rationale.
|
||||
|
||||
`max_comments` (optional, default unbounded): hard cap. When the cap
|
||||
is hit we stop fetching further pages and the caller surfaces a
|
||||
soft 'skipping due to volume' status (see main()).
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for c in self.iter_issue_comments(owner, repo, issue):
|
||||
out.append(c)
|
||||
if max_comments is not None and len(out) >= max_comments:
|
||||
break
|
||||
return out
|
||||
|
||||
def resolve_team_id(self, org: str, team_name: str) -> int | None:
|
||||
@@ -912,17 +832,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
"thing BP sees is the POSTed status. Useful for local debugging."
|
||||
),
|
||||
)
|
||||
p.add_argument(
|
||||
"--max-comments",
|
||||
type=int,
|
||||
default=int(os.environ.get("SOP_CHECKLIST_MAX_COMMENTS") or 5000),
|
||||
help=(
|
||||
"Hard cap on comments fetched from the PR. Above this we post "
|
||||
"a SOFT-pending status with a 'skipping due to volume' note "
|
||||
"instead of OOM'ing the runner (task #369). Override with the "
|
||||
"SOP_CHECKLIST_MAX_COMMENTS env var. Set 0 to disable the cap."
|
||||
),
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
token = os.environ.get("GITEA_TOKEN", "")
|
||||
@@ -956,18 +865,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
print("::error::PR payload missing user.login or head.sha", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
max_comments_cap = args.max_comments if args.max_comments and args.max_comments > 0 else None
|
||||
comments = client.get_issue_comments(
|
||||
args.owner, args.repo, args.pr, max_comments=max_comments_cap
|
||||
)
|
||||
|
||||
# Volume short-circuit: PRs with thousands of bot-relay comments
|
||||
# (the mc#1242-class OOM source) get a soft 'volume-skipped' status
|
||||
# so the gate doesn't churn the runner; reviewers can re-trigger by
|
||||
# editing the PR or filing a fresh PR with the housekeeping comments
|
||||
# split off. Cap-hit means we couldn't see the WHOLE history, so we
|
||||
# can't fairly post failure — pending is the safe default.
|
||||
volume_skipped = bool(max_comments_cap and len(comments) >= max_comments_cap)
|
||||
comments = client.get_issue_comments(args.owner, args.repo, args.pr)
|
||||
|
||||
# High-risk classification (RFC#450 Option C, governance fix for
|
||||
# internal#442). Computed ONCE per PR — used by both the probe
|
||||
@@ -981,34 +879,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
team_member_cache: dict[tuple[str, int], bool | None] = {}
|
||||
|
||||
def probe(slug: str, users: list[str]) -> list[str]:
|
||||
# `slug` may be either an items-key (compute_ack_state caller) OR
|
||||
# an n/a-gate key (compute_na_state caller). Previously this hard
|
||||
# KeyError'd on the n/a-gate path when slug was e.g. "security-review"
|
||||
# — that's a config gate, not an item — so the gate would crash
|
||||
# instead of falling back to the gate's own required_teams. Fix
|
||||
# task #369 follow-up to issue #355.
|
||||
if slug in items_by_slug:
|
||||
item = items_by_slug[slug]
|
||||
team_names: list[str] = resolve_required_teams(item, high_risk)
|
||||
elif slug in na_gates:
|
||||
# n/a-gate configs carry `required_teams` directly (see
|
||||
# sop-checklist-config.yaml: n/a_gates.<gate>.required_teams).
|
||||
gate_cfg = na_gates[slug] or {}
|
||||
team_names = list(gate_cfg.get("required_teams") or [])
|
||||
if not team_names:
|
||||
print(
|
||||
f"::warning::n/a-gate '{slug}' has no required_teams; "
|
||||
"fail-closed (no users will be approved)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
# Unknown slug — fail closed, log so we can find config drift.
|
||||
print(
|
||||
f"::warning::probe() called with slug '{slug}' which is "
|
||||
f"neither an items entry nor an n/a-gate; fail-closed",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return []
|
||||
item = items_by_slug[slug]
|
||||
team_names: list[str] = resolve_required_teams(item, high_risk)
|
||||
# Resolve names → ids. NOTE: orgs/{org}/teams/search may not be
|
||||
# available — fall back to the list endpoint.
|
||||
team_ids: list[int] = []
|
||||
@@ -1066,15 +938,6 @@ def main(argv: list[str] | None = None) -> int:
|
||||
# were not required (vs a tier:medium+ PR that truly passed all acks).
|
||||
state = "success"
|
||||
description = f"[info tier:low] {description}"
|
||||
if volume_skipped:
|
||||
# Above the comment-cap — we may have a partial view. Soft-pend
|
||||
# so neither BP nor the author gets stuck; surface the cap so
|
||||
# reviewers know what's up. No-block at the gate level.
|
||||
state = "pending"
|
||||
description = (
|
||||
f"[volume-skipped] comment-cap={max_comments_cap} hit; please file "
|
||||
f"a fresh PR with bot-relay history split off (#369). {description}"
|
||||
)
|
||||
|
||||
# Diagnostics to job log.
|
||||
print(
|
||||
|
||||
@@ -104,13 +104,10 @@ if [ "${SOP_REFIRE_DISABLE_RATE_LIMIT:-}" != "1" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects.
|
||||
# The canonical workflow intentionally fail-opens the job conclusion
|
||||
# (`bash .gitea/scripts/sop-tier-check.sh || true`) while Gitea branch
|
||||
# protection enforces reviewer approvals separately. Keep the refire path
|
||||
# aligned with that workflow status behavior; otherwise /refire-tier-check can
|
||||
# post a hard failure that the canonical pull_request_target workflow would
|
||||
# not publish.
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects. Capture exit code.
|
||||
# The canonical script reads tier label, walks approving reviewers, and
|
||||
# evaluates the AND-composition expression — we want the SAME gate, not
|
||||
# a different gate.
|
||||
#
|
||||
# SOP_REFIRE_TIER_CHECK_SCRIPT env var lets tests substitute a mock —
|
||||
# sop-tier-check.sh uses bash 4+ associative arrays which trigger a known
|
||||
@@ -126,6 +123,7 @@ fi
|
||||
|
||||
# Re-invoke. Pipe stdout/stderr through so the runner log shows the
|
||||
# tier-check decision inline.
|
||||
set +e
|
||||
GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
GITEA_HOST="$GITEA_HOST" \
|
||||
REPO="$REPO" \
|
||||
@@ -133,8 +131,9 @@ GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
PR_AUTHOR="$PR_AUTHOR" \
|
||||
SOP_DEBUG="${SOP_DEBUG:-0}" \
|
||||
SOP_LEGACY_CHECK="${SOP_LEGACY_CHECK:-0}" \
|
||||
bash "$SCRIPT" || true
|
||||
TIER_EXIT=0
|
||||
bash "$SCRIPT"
|
||||
TIER_EXIT=$?
|
||||
set -e
|
||||
debug "sop-tier-check.sh exit=$TIER_EXIT"
|
||||
|
||||
# 4. POST the resulting status.
|
||||
|
||||
@@ -47,9 +47,7 @@ What this script does, per `.gitea/workflows/status-reaper.yml` invocation:
|
||||
Parse context as `<workflow_name> / <job_name> (push)`.
|
||||
Look up workflow_name in the trigger map:
|
||||
- missing → log ::notice:: and skip (conservative).
|
||||
- has_push_trigger=True and description == "Has been cancelled"
|
||||
→ compensate cancelled/superseded push noise.
|
||||
- has_push_trigger=True otherwise → preserve (real defect signal).
|
||||
- has_push_trigger=True → preserve (real defect signal).
|
||||
- has_push_trigger=False → POST a compensating
|
||||
`state=success` status to /statuses/{sha} with the same
|
||||
context (Gitea de-dups by context) and a description
|
||||
@@ -143,11 +141,6 @@ PR_SHADOW_COMPENSATION_DESCRIPTION = (
|
||||
"shadowed by successful push status on same SHA; see "
|
||||
".gitea/scripts/status-reaper.py)"
|
||||
)
|
||||
CANCELLED_PUSH_COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (push run was cancelled/superseded; "
|
||||
"Gitea 1.22.6 reports cancelled runs as failure statuses)"
|
||||
)
|
||||
CANCELLED_DESCRIPTION = "Has been cancelled"
|
||||
|
||||
# Context suffix the reaper acts on. Gitea hardcodes this for ALL
|
||||
# default-branch workflow runs.
|
||||
@@ -483,7 +476,7 @@ def reap(
|
||||
{compensated, preserved_real_push, preserved_unknown,
|
||||
preserved_non_failure, preserved_non_push_suffix,
|
||||
preserved_unparseable, compensated_pr_shadowed_by_push_success,
|
||||
preserved_pr_without_push_success, compensated_cancelled_push,
|
||||
preserved_pr_without_push_success,
|
||||
compensated_contexts: [<context>, ...]}
|
||||
|
||||
`compensated_contexts` is rev2-added so `reap_branch` can build
|
||||
@@ -497,7 +490,6 @@ def reap(
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_contexts": [],
|
||||
}
|
||||
@@ -575,27 +567,8 @@ def reap(
|
||||
counters["preserved_unknown"] += 1
|
||||
continue
|
||||
|
||||
if (s.get("description") or "").strip() == CANCELLED_DESCRIPTION:
|
||||
# Gitea 1.22.6 maps cancelled action runs to failure commit
|
||||
# statuses. During merge bursts, older push runs can be
|
||||
# superseded and cancelled even though a newer run for the
|
||||
# same branch is the real signal. Compensate only the exact
|
||||
# Gitea cancellation description; real push failures remain red.
|
||||
post_compensating_status(
|
||||
sha,
|
||||
context,
|
||||
s.get("target_url"),
|
||||
description=CANCELLED_PUSH_COMPENSATION_DESCRIPTION,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
counters["compensated"] += 1
|
||||
counters["compensated_cancelled_push"] += 1
|
||||
counters["compensated_contexts"].append(context)
|
||||
continue
|
||||
|
||||
if workflow_trigger_map[workflow_name]:
|
||||
# Real push trigger with a non-cancelled failure description
|
||||
# remains a defect signal. Preserve.
|
||||
# Real push trigger → real defect signal. Preserve.
|
||||
counters["preserved_real_push"] += 1
|
||||
continue
|
||||
|
||||
@@ -701,7 +674,6 @@ def reap_branch(
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_per_sha": {},
|
||||
"skipped": True,
|
||||
@@ -717,7 +689,6 @@ def reap_branch(
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_per_sha": {},
|
||||
}
|
||||
@@ -757,7 +728,6 @@ def reap_branch(
|
||||
"preserved_non_push_suffix",
|
||||
"preserved_unparseable",
|
||||
"compensated_pr_shadowed_by_push_success",
|
||||
"compensated_cancelled_push",
|
||||
"preserved_pr_without_push_success",
|
||||
):
|
||||
aggregate[key] += per_sha[key]
|
||||
|
||||
@@ -815,192 +815,3 @@ class TestHighRiskClassUsesElevatedListInConfig(unittest.TestCase):
|
||||
sop.resolve_required_teams(items[slug], high_risk=True),
|
||||
f"item {slug} should not be affected by risk-class",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_issue_comments — streaming + minimal-dict shape (task #369 / OOM fix)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeReq:
|
||||
"""Stand-in for GiteaClient._req that serves canned pages."""
|
||||
|
||||
def __init__(self, pages):
|
||||
# pages: list[list[dict]]; one page per call, exhausted in order.
|
||||
self._pages = list(pages)
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, method, path, body=None, ok_codes=(200, 201, 204)):
|
||||
self.calls.append((method, path))
|
||||
if not self._pages:
|
||||
return 200, []
|
||||
return 200, self._pages.pop(0)
|
||||
|
||||
|
||||
class TestGetIssueCommentsStreaming(unittest.TestCase):
|
||||
"""Verify the OOM-fix invariants — minimal-dict shape + page break."""
|
||||
|
||||
def _client_with_pages(self, pages):
|
||||
client = sop.GiteaClient("git.example.com", "tok")
|
||||
client._req = _FakeReq(pages) # type: ignore[method-assign]
|
||||
return client
|
||||
|
||||
def test_minimal_dict_shape_drops_large_fields(self):
|
||||
"""get_issue_comments must DROP html_url/assets/timestamps/etc. and
|
||||
keep ONLY {user.login, body} — that's the whole OOM-prevention."""
|
||||
full_page = [
|
||||
{
|
||||
"id": 1234,
|
||||
"html_url": "https://example.com/some-huge-url",
|
||||
"pull_request_url": "https://example.com/some-other-huge-url",
|
||||
"issue_url": "https://example.com/yet-another-url",
|
||||
"user": {"login": "bob", "avatar_url": "x" * 4000, "id": 99},
|
||||
"original_author": "",
|
||||
"original_author_id": 0,
|
||||
"body": "/sop-ack comprehensive-testing\n\nlooks good",
|
||||
"assets": ["x" * 1000, "y" * 1000],
|
||||
"created_at": "2026-05-19T01:02:03Z",
|
||||
"updated_at": "2026-05-19T01:02:03Z",
|
||||
}
|
||||
]
|
||||
client = self._client_with_pages([full_page])
|
||||
out = client.get_issue_comments("o", "r", 1)
|
||||
self.assertEqual(len(out), 1)
|
||||
# Only the two whitelisted keys + nested user.login
|
||||
self.assertEqual(set(out[0].keys()), {"user", "body"})
|
||||
self.assertEqual(set(out[0]["user"].keys()), {"login"})
|
||||
self.assertEqual(out[0]["user"]["login"], "bob")
|
||||
self.assertEqual(out[0]["body"], "/sop-ack comprehensive-testing\n\nlooks good")
|
||||
# Critical: avatar/assets/timestamps/etc. must be gone (~4KB+ each).
|
||||
self.assertNotIn("html_url", out[0])
|
||||
self.assertNotIn("assets", out[0])
|
||||
self.assertNotIn("created_at", out[0])
|
||||
|
||||
def test_pagination_break_on_short_page(self):
|
||||
# Page-size 50; a page of <50 means no more pages.
|
||||
page1 = [{"user": {"login": "u"}, "body": "x"}] * 7
|
||||
client = self._client_with_pages([page1])
|
||||
out = client.get_issue_comments("o", "r", 2)
|
||||
self.assertEqual(len(out), 7)
|
||||
# Should have made exactly 1 _req call (no page-2 probe).
|
||||
self.assertEqual(len(client._req.calls), 1)
|
||||
|
||||
def test_pagination_continues_until_empty(self):
|
||||
# Two full pages + one short page.
|
||||
page1 = [{"user": {"login": "u"}, "body": "x"}] * 50
|
||||
page2 = [{"user": {"login": "u"}, "body": "y"}] * 50
|
||||
page3 = [{"user": {"login": "u"}, "body": "z"}] * 3
|
||||
client = self._client_with_pages([page1, page2, page3])
|
||||
out = client.get_issue_comments("o", "r", 3)
|
||||
self.assertEqual(len(out), 103)
|
||||
self.assertEqual(len(client._req.calls), 3)
|
||||
|
||||
def test_max_comments_caps_collection(self):
|
||||
page1 = [{"user": {"login": "u"}, "body": "x"}] * 50
|
||||
page2 = [{"user": {"login": "u"}, "body": "y"}] * 50
|
||||
page3 = [{"user": {"login": "u"}, "body": "z"}] * 50
|
||||
client = self._client_with_pages([page1, page2, page3])
|
||||
out = client.get_issue_comments("o", "r", 4, max_comments=75)
|
||||
self.assertEqual(len(out), 75)
|
||||
# Stops short: shouldn't have requested page-3.
|
||||
self.assertLessEqual(len(client._req.calls), 2)
|
||||
|
||||
def test_oversized_body_truncated(self):
|
||||
# An individual comment with a multi-MiB body (e.g. pasted CI log)
|
||||
# must NOT pull the whole thing into memory. The directive parser
|
||||
# only needs the first ~8 KiB to find /sop-* markers.
|
||||
huge_body = "/sop-ack comprehensive-testing\n" + ("X" * (4 * 1024 * 1024))
|
||||
page = [{"user": {"login": "bob"}, "body": huge_body}]
|
||||
client = self._client_with_pages([page])
|
||||
out = client.get_issue_comments("o", "r", 99)
|
||||
self.assertEqual(len(out), 1)
|
||||
# Cap is 8 KiB; comment body must be <= 8 KiB after streaming.
|
||||
self.assertLessEqual(len(out[0]["body"]), 8 * 1024)
|
||||
# Marker still discoverable at the start.
|
||||
self.assertTrue(out[0]["body"].startswith("/sop-ack comprehensive-testing"))
|
||||
|
||||
def test_iter_handles_missing_user_or_body(self):
|
||||
# Defensive: Gitea has been seen to return user=null on deleted users.
|
||||
page = [
|
||||
{"user": None, "body": "abandoned-author"},
|
||||
{"user": {"login": "alice"}, "body": None},
|
||||
{"body": "no-user-key"},
|
||||
{"user": {"login": "bob"}, "body": "ok"},
|
||||
]
|
||||
client = self._client_with_pages([page])
|
||||
out = client.get_issue_comments("o", "r", 5)
|
||||
self.assertEqual(len(out), 4)
|
||||
self.assertEqual(out[0]["user"]["login"], "")
|
||||
self.assertEqual(out[0]["body"], "abandoned-author")
|
||||
self.assertEqual(out[1]["user"]["login"], "alice")
|
||||
self.assertEqual(out[1]["body"], "")
|
||||
self.assertEqual(out[2]["user"]["login"], "")
|
||||
self.assertEqual(out[3]["user"]["login"], "bob")
|
||||
|
||||
def test_minimal_dicts_work_with_compute_ack_state(self):
|
||||
"""Round-trip: minimal dicts feed back through compute_ack_state."""
|
||||
page = [{"user": {"login": "bob"}, "body": "/sop-ack comprehensive-testing"}]
|
||||
client = self._client_with_pages([page])
|
||||
comments = client.get_issue_comments("o", "r", 6)
|
||||
items = _items_by_slug()
|
||||
aliases = _numeric_aliases()
|
||||
state = sop.compute_ack_state(
|
||||
comments, "alice", items, aliases, lambda slug, users: list(users)
|
||||
)
|
||||
self.assertEqual(state["comprehensive-testing"]["ackers"], ["bob"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# probe() na-gate fallback — fix for #355-class KeyError 'security-review'
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestComputeNaStateAcceptsGateNotInItems(unittest.TestCase):
|
||||
"""compute_na_state passes the gate NAME to probe(); when the gate is
|
||||
NOT also an items entry (the common case for `security-review`,
|
||||
`qa-review`), probe must fall back to the gate's own required_teams
|
||||
instead of KeyError'ing on items_by_slug[slug].
|
||||
|
||||
This test exercises the public surface (compute_na_state) rather than
|
||||
the inline `probe` closure, because the closure is built inside main().
|
||||
We simulate the fallback by passing a probe that mirrors the production
|
||||
contract — slug may be either an item OR an n/a-gate key, both are valid.
|
||||
"""
|
||||
|
||||
def test_na_gate_with_required_teams_resolves_without_keyerror(self):
|
||||
na_gates = {
|
||||
"security-review": {
|
||||
"required_teams": ["security", "managers", "ceo"],
|
||||
"description": "security N/A",
|
||||
},
|
||||
}
|
||||
comments = [
|
||||
{"user": {"login": "carol"}, "body": "/sop-n/a security-review docs-only"},
|
||||
]
|
||||
# Probe approves any user in the security team; importantly it does
|
||||
# NOT try items_by_slug[slug] for the gate name.
|
||||
called_with = []
|
||||
|
||||
def probe(slug, users):
|
||||
called_with.append(slug)
|
||||
# production probe accepts gate-name OR item-slug; for this test
|
||||
# we just approve everyone.
|
||||
return list(users)
|
||||
|
||||
na_state = sop.compute_na_state(comments, "alice", na_gates, probe)
|
||||
self.assertTrue(na_state["security-review"]["declared"])
|
||||
self.assertEqual(na_state["security-review"]["decl_ackers"], ["carol"])
|
||||
# probe must have been called with the GATE name, not an item slug.
|
||||
self.assertEqual(called_with, ["security-review"])
|
||||
|
||||
def test_na_gate_self_declaration_rejected(self):
|
||||
# Author cannot self-declare N/A — pre-existing invariant; pin it
|
||||
# so the new probe-fallback doesn't regress this.
|
||||
na_gates = {"security-review": {"required_teams": ["security"]}}
|
||||
comments = [
|
||||
{"user": {"login": "alice"}, "body": "/sop-n/a security-review"},
|
||||
]
|
||||
na_state = sop.compute_na_state(
|
||||
comments, "alice", na_gates, lambda *_: ["alice"]
|
||||
)
|
||||
self.assertFalse(na_state["security-review"]["declared"])
|
||||
|
||||
@@ -6,10 +6,9 @@
|
||||
# T1: PR open + APPROVED via tier:low → script invokes sop-tier-check
|
||||
# and POSTs status=success.
|
||||
# T2: PR open + missing tier label → sop-tier-check exits non-zero;
|
||||
# refire still POSTs status=success, matching the canonical
|
||||
# pull_request_target workflow's fail-open job conclusion.
|
||||
# refire POSTs status=failure (description mentions failure).
|
||||
# T3: PR open + tier:low but NO approving reviews → sop-tier-check
|
||||
# exits non-zero; refire still POSTs status=success for the same reason.
|
||||
# exits non-zero; refire POSTs status=failure.
|
||||
# T4: PR CLOSED → refire exits 0 with no status POST (no-op on closed).
|
||||
# T5: Rate-limit — recent status update within 30s → refire skips,
|
||||
# no new POST.
|
||||
@@ -33,7 +32,7 @@ THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
|
||||
WORKFLOW_DIR="$(cd "$THIS_DIR/../../workflows" && pwd)"
|
||||
WORKFLOW="$WORKFLOW_DIR/sop-tier-refire.yml"
|
||||
DISPATCH_WORKFLOW="$WORKFLOW_DIR/sop-checklist.yml"
|
||||
DISPATCH_WORKFLOW="$WORKFLOW_DIR/review-refire-comments.yml"
|
||||
SCRIPT="$SCRIPT_DIR/sop-tier-refire.sh"
|
||||
|
||||
PASS=0
|
||||
@@ -89,7 +88,7 @@ assert_file_exists() {
|
||||
echo
|
||||
echo "== existence =="
|
||||
assert_file_exists "workflow file exists" "$WORKFLOW"
|
||||
assert_file_exists "SSOT dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
|
||||
assert_file_exists "dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
|
||||
assert_file_exists "script file exists" "$SCRIPT"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo
|
||||
@@ -134,15 +133,15 @@ else
|
||||
fi
|
||||
|
||||
DISPATCH_PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$DISPATCH_WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T6e SSOT dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
|
||||
assert_eq "T6e dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
|
||||
DISPATCH_CONTENT=$(cat "$DISPATCH_WORKFLOW")
|
||||
assert_contains "T6f SSOT dispatcher listens on issue_comment" \
|
||||
assert_contains "T6f dispatcher listens on issue_comment" \
|
||||
"issue_comment" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6g SSOT dispatcher handles /qa-recheck" \
|
||||
assert_contains "T6g dispatcher handles /qa-recheck" \
|
||||
"/qa-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6h SSOT dispatcher handles /security-recheck" \
|
||||
assert_contains "T6h dispatcher handles /security-recheck" \
|
||||
"/security-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6i SSOT dispatcher handles /refire-tier-check" \
|
||||
assert_contains "T6i dispatcher handles /refire-tier-check" \
|
||||
"/refire-tier-check" "$DISPATCH_CONTENT"
|
||||
|
||||
# T1-T5 — script behavior against a local Gitea-fixture
|
||||
@@ -246,21 +245,34 @@ assert_contains "T1 POST context is sop-tier-check / tier-check" \
|
||||
'"context": "sop-tier-check / tier-check (pull_request)"' "$POSTED"
|
||||
assert_contains "T1 description names commenter" "test-runner" "$POSTED"
|
||||
|
||||
# T2: missing tier label → tier-check fails internally, but refire status
|
||||
# matches the canonical workflow's fail-open job conclusion.
|
||||
# T2: missing tier label → tier-check fails → failure status POSTed
|
||||
run_scenario "T2_no_tier_label" "fail_no_label"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T2 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T2 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
# tier-check.sh exits 1; refire script forwards that exit, so RC != 0
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T2 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T2 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T2_rc"
|
||||
fi
|
||||
assert_contains "T2 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T3: tier:low present but ZERO approving reviews → internal tier check fails,
|
||||
# refire status remains aligned with the canonical workflow.
|
||||
# T3: tier:low present but ZERO approving reviews → failure
|
||||
run_scenario "T3_no_approvals" "fail_no_approvals"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T3 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T3 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T3 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T3 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T3_rc"
|
||||
fi
|
||||
assert_contains "T3 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T4: closed PR — refire is a no-op (no POST, exit 0)
|
||||
run_scenario "T4_closed" "pass"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths reference .gitea/workflows/publish-runtime.yml (the active
|
||||
# Gitea workflow file) instead of .github/workflows/publish-runtime.yml
|
||||
# (which Category A of this sweep deletes).
|
||||
# - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
|
||||
# .gitea/ workflow (the script's default is still .github/... which
|
||||
# will not exist post-Cat-A).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .gitea/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# bp-exempt: drift visibility gate; CI / all-required remains the required aggregate.
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
# Pass the .gitea/ workflow path explicitly — the script's
|
||||
# default still points at .github/... which Category A of this
|
||||
# sweep removes.
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml
|
||||
@@ -1,186 +0,0 @@
|
||||
# ci-arm64-advisory — Mac arm64 self-hosted ADVISORY fast-check lane.
|
||||
#
|
||||
# === WHY ===
|
||||
#
|
||||
# The amd64 Gitea runner pool (molecule-runner-1..20) is queue-contended
|
||||
# (internal#418). This lane offloads the *genuinely container-independent*
|
||||
# fast checks (Go build/vet/lint, shellcheck, Python lint) onto the Mac
|
||||
# arm64 self-hosted runner so developers get a fast arm64 signal WITHOUT
|
||||
# adding load to the starved amd64 pool — capability-honestly, as an
|
||||
# additive pilot. Pilot ② of the Mac-CI strategy (CTO-delegated 2026-05-17).
|
||||
#
|
||||
# === NON-NEGOTIABLE SAFETY CONTRACT (the prime directive) ===
|
||||
#
|
||||
# This lane is **ADVISORY ONLY**. It is provably incapable of hanging a
|
||||
# merge. Concretely:
|
||||
#
|
||||
# 1. It is a SEPARATE workflow file. `ci.yml` is byte-for-byte
|
||||
# untouched by this PR. The `CI / all-required` aggregator sentinel
|
||||
# and the five contexts it polls
|
||||
# (`CI / Detect changes|Platform (Go)|Canvas (Next.js)|
|
||||
# Shellcheck (E2E scripts)|Python Lint & Test (pull_request)`)
|
||||
# are unchanged. The canonical required gate stays 100% on the
|
||||
# existing amd64 pool.
|
||||
#
|
||||
# 2. The context this workflow emits is
|
||||
# `ci-arm64-advisory / fast-checks (pull_request)`. That string is
|
||||
# DELIBERATELY NOT present in, and this PR does NOT add it to:
|
||||
# - branch_protections/{main,staging}.status_check_contexts
|
||||
# (DB-verified pb 86/75 = exactly
|
||||
# ["CI / all-required (pull_request)",
|
||||
# "sop-checklist / all-items-acked (pull_request)"])
|
||||
# - audit-force-merge.yml REQUIRED_CHECKS env
|
||||
# - ci.yml `all-required` sentinel's hardcoded `required[]` list
|
||||
# Branch protection therefore never waits on this context. If the
|
||||
# Mac runner is absent / offline / removed, this workflow's status
|
||||
# simply never appears — and because nothing requires it, every
|
||||
# merge proceeds exactly as it does today. There is no path by
|
||||
# which a missing/red arm64 status blocks a merge.
|
||||
#
|
||||
# 3. `continue-on-error: true` on the job — even a genuine arm64-only
|
||||
# failure (toolchain drift, arch-specific test flake) is surfaced
|
||||
# as information, never as a merge blocker, for the duration of
|
||||
# the pilot.
|
||||
#
|
||||
# 4. The job carries a `github.event_name` `if:` gate. Beyond its
|
||||
# functional purpose this also keeps the job OUT of
|
||||
# `ci-required-drift.py:ci_job_names()` (which excludes
|
||||
# `github.event_name`/`github.ref`-gated jobs), so the hourly
|
||||
# ci-required-drift sentinel's F1 ("job not under sentinel needs")
|
||||
# cannot ever flag this advisory job. F2/F3 are untouched because
|
||||
# this context is absent from BP and from REQUIRED_CHECKS.
|
||||
# `lint-bp-context-emit-match` only fails on BP→emitter gaps; an
|
||||
# emitter without a BP context is explicitly informational there.
|
||||
#
|
||||
# === RUNNER TARGETING ===
|
||||
#
|
||||
# The Mac runner is `hongming-pc-runner-1`. The bare `self-hosted`
|
||||
# label is POLLUTED in this Gitea instance: molecule-runner-1..20
|
||||
# (the contended amd64 pool) also advertise `self-hosted`. Targeting
|
||||
# bare `self-hosted` would route back onto the very pool we are trying
|
||||
# to relieve — and onto amd64 hardware. We therefore require an
|
||||
# AND-set of labels that ONLY the Mac satisfies. `macos-self-hosted`
|
||||
# is Mac-exclusive (the amd64 pool does not carry it). Until the
|
||||
# label-install burst (a10862b2) lands `self-hosted`+`macos-self-hosted`
|
||||
# on the Mac, the runner's current unique label `hongming-pc-laptop`
|
||||
# is also listed; AND-semantics over the labels a runner advertises
|
||||
# means a job requiring [self-hosted, macos-self-hosted] can ONLY be
|
||||
# claimed once the Mac advertises both. If neither label set is yet
|
||||
# present on the Mac, the workflow stays queued harmlessly and is
|
||||
# garbage-collected by the normal stale-run reaper — it blocks nothing
|
||||
# (see safety contract point 2).
|
||||
#
|
||||
# === ROLLBACK ===
|
||||
#
|
||||
# Delete this single file (`git rm .gitea/workflows/ci-arm64-advisory.yml`)
|
||||
# and merge. No branch-protection edit, no ci.yml edit, no
|
||||
# REQUIRED_CHECKS edit is required to roll back, because none were made
|
||||
# to roll forward. Zero blast radius either direction.
|
||||
|
||||
name: ci-arm64-advisory
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
# Per-ref cancel: a newer commit on the same ref supersedes the older
|
||||
# advisory run. Distinct from ci.yml's `ci-${ref}` group so this lane
|
||||
# never cancels (or is cancelled by) the canonical required CI.
|
||||
concurrency:
|
||||
group: ci-arm64-advisory-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
fast-checks:
|
||||
name: fast-checks
|
||||
# AND-set: only the Mac arm64 runner advertises macos-self-hosted.
|
||||
# See "RUNNER TARGETING" header note for why bare self-hosted is unsafe.
|
||||
runs-on: [self-hosted, macos-self-hosted]
|
||||
# ADVISORY: never blocks. See safety contract point 3.
|
||||
continue-on-error: true
|
||||
# event_name gate: functional (only meaningful on push/PR) AND keeps
|
||||
# this job out of ci-required-drift.py:ci_job_names() so F1 can never
|
||||
# flag it. See safety contract point 4.
|
||||
if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Provenance — advisory lane, non-gating
|
||||
run: |
|
||||
echo "This is the arm64 ADVISORY fast-check lane."
|
||||
echo "It does NOT gate merges. Canonical required CI is ci.yml"
|
||||
echo "on the amd64 pool. Arch: $(uname -m) on $(uname -s)."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# ---- Go: build + vet + lint (container-independent: needs only the
|
||||
# Go toolchain; no amd64 ECR image, no docker-in-job). Race-detector
|
||||
# unit-test + coverage gates are deliberately NOT duplicated here —
|
||||
# those stay authoritative on amd64 ci.yml `Platform (Go)`. This lane
|
||||
# is fast-feedback for the compile/vet/lint surface only. ----
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- name: Go build + vet (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go mod download
|
||||
go build ./cmd/server
|
||||
go vet ./...
|
||||
- name: golangci-lint (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
"$(go env GOPATH)/bin/golangci-lint" run --timeout 3m ./...
|
||||
|
||||
# ---- Shellcheck (container-independent: shellcheck binary only).
|
||||
# Mirrors ci.yml `Shellcheck (E2E scripts)` bulk pass scope. ----
|
||||
- name: Install shellcheck (arm64)
|
||||
run: |
|
||||
if ! command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "shellcheck not preinstalled on this self-hosted runner."
|
||||
echo "Attempting Homebrew install (Mac arm64)."
|
||||
brew install shellcheck || {
|
||||
echo "::warning::shellcheck unavailable on runner; advisory shellcheck skipped."
|
||||
exit 0
|
||||
}
|
||||
fi
|
||||
shellcheck --version
|
||||
- name: Shellcheck tests/e2e + infra/scripts
|
||||
run: |
|
||||
command -v shellcheck >/dev/null 2>&1 || { echo "skip"; exit 0; }
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
# ---- Python lint/compile (container-independent: CPython only).
|
||||
# Lint + import-compile surface; the authoritative pytest + coverage
|
||||
# floors stay on amd64 ci.yml `Python Lint & Test`. ----
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Python byte-compile (workspace)
|
||||
working-directory: workspace
|
||||
run: |
|
||||
python -m pip install --quiet ruff || true
|
||||
python -m compileall -q .
|
||||
if command -v ruff >/dev/null 2>&1; then
|
||||
ruff check . || echo "::warning::ruff findings (advisory only)"
|
||||
fi
|
||||
|
||||
- name: Advisory summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## arm64 advisory fast-checks complete"
|
||||
echo ""
|
||||
echo "This lane is **advisory** — it does not gate merges."
|
||||
echo "Authoritative required CI remains \`CI / all-required\`"
|
||||
echo "on the amd64 pool (\`ci.yml\`, unchanged by this PR)."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,225 @@
|
||||
name: MCP Stdio Transport Regression
|
||||
|
||||
# Regression test for molecule-ai-workspace-runtime#61:
|
||||
# asyncio.connect_read_pipe / connect_write_pipe fail with
|
||||
# ValueError: "Pipe transport is only for pipes, sockets and character devices"
|
||||
# when stdout is a regular file (openclaw capture, CI tee, debugging).
|
||||
#
|
||||
# This workflow reproduces the exact failure mode and verifies the
|
||||
# fallback to direct buffer I/O works. It runs on every PR that
|
||||
# touches the MCP server or this workflow, plus nightly cron.
|
||||
#
|
||||
# Why a separate workflow (not folded into ci.yml python-lint):
|
||||
# - The test needs to spawn the MCP server with stdout redirected
|
||||
# to a regular file (not a TTY/pipe), which conflicts with
|
||||
# pytest's own capture mechanism.
|
||||
# - It exercises the actual process spawn path (python a2a_mcp_server.py)
|
||||
# not just unit-test mocks — closer to the real openclaw integration.
|
||||
# - A dedicated workflow surfaces stdio-specific regressions without
|
||||
# coupling to the broader Python test suite's coverage gate.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/a2a_mcp_server.py'
|
||||
- 'workspace/mcp_cli.py'
|
||||
- 'workspace/tests/test_a2a_mcp_server.py'
|
||||
- '.gitea/workflows/ci-mcp-stdio-transport.yml'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/a2a_mcp_server.py'
|
||||
- 'workspace/mcp_cli.py'
|
||||
- 'workspace/tests/test_a2a_mcp_server.py'
|
||||
- '.gitea/workflows/ci-mcp-stdio-transport.yml'
|
||||
schedule:
|
||||
# Nightly at 04:00 UTC — catches drift from dependency updates
|
||||
# (e.g. asyncio behavior changes in new Python patch releases).
|
||||
- cron: '0 4 * * *'
|
||||
|
||||
concurrency:
|
||||
group: mcp-stdio-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: regression canary for runtime#61; not a merge gate — informational only until promoted to required.
|
||||
# mc#774: continue-on-error mask — new workflow, flip to false once it's green on ≥3 consecutive main runs.
|
||||
mcp-stdio-regular-file:
|
||||
name: MCP stdio with regular-file stdout
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # mc#774
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
WORKSPACE_ID: "00000000-0000-0000-0000-000000000001"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov
|
||||
|
||||
- name: Reproduce runtime#61 — stdout as regular file
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "=== Reproducing molecule-ai-workspace-runtime#61 ==="
|
||||
echo ""
|
||||
echo "Before the fix, this command would fail with:"
|
||||
echo ' ValueError: Pipe transport is only for pipes, sockets and character devices'
|
||||
echo ""
|
||||
|
||||
# Spawn the MCP server with stdout redirected to a regular file.
|
||||
# This is exactly what openclaw does when capturing MCP output.
|
||||
OUTPUT=$(mktemp)
|
||||
trap 'rm -f "$OUTPUT"' EXIT
|
||||
|
||||
# Send initialize request, then tools/list, then exit
|
||||
{
|
||||
echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}'
|
||||
echo '{"jsonrpc":"2.0","id":2,"method":"tools/list"}'
|
||||
} | python a2a_mcp_server.py > "$OUTPUT" 2>&1 || {
|
||||
RC=$?
|
||||
echo "FAIL: MCP server exited with code $RC"
|
||||
echo "--- stdout+stderr ---"
|
||||
cat "$OUTPUT"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "PASS: MCP server handled regular-file stdout without crashing"
|
||||
echo ""
|
||||
echo "--- Output (first 20 lines) ---"
|
||||
head -20 "$OUTPUT"
|
||||
echo ""
|
||||
|
||||
# Verify we got valid JSON-RPC responses
|
||||
if grep -q '"result"' "$OUTPUT"; then
|
||||
echo "PASS: JSON-RPC responses found in output"
|
||||
else
|
||||
echo "FAIL: No JSON-RPC responses in output"
|
||||
cat "$OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Reproduce runtime#61 — stdin from regular file
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "=== stdin as regular file (CI tee / capture pattern) ==="
|
||||
|
||||
INPUT=$(mktemp)
|
||||
OUTPUT=$(mktemp)
|
||||
trap 'rm -f "$INPUT" "$OUTPUT"' EXIT
|
||||
|
||||
cat > "$INPUT" <<'EOF'
|
||||
{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}
|
||||
{"jsonrpc":"2.0","id":2,"method":"tools/list"}
|
||||
EOF
|
||||
|
||||
python a2a_mcp_server.py < "$INPUT" > "$OUTPUT" 2>&1 || {
|
||||
RC=$?
|
||||
echo "FAIL: MCP server exited with code $RC"
|
||||
cat "$OUTPUT"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "PASS: MCP server handled regular-file stdin without crashing"
|
||||
|
||||
if grep -q '"result"' "$OUTPUT"; then
|
||||
echo "PASS: JSON-RPC responses found in output"
|
||||
else
|
||||
echo "FAIL: No JSON-RPC responses in output"
|
||||
cat "$OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify warning is emitted for non-pipe stdio
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "=== Verify diagnostic warning ==="
|
||||
|
||||
OUTPUT=$(mktemp)
|
||||
trap 'rm -f "$OUTPUT"' EXIT
|
||||
|
||||
{
|
||||
echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}'
|
||||
} | python a2a_mcp_server.py > "$OUTPUT" 2>&1
|
||||
|
||||
# The warning should mention "not a pipe" for operator visibility
|
||||
if grep -qi "not a pipe" "$OUTPUT"; then
|
||||
echo "PASS: Diagnostic warning emitted for non-pipe stdio"
|
||||
else
|
||||
echo "NOTE: No warning in output (may be suppressed by log level)"
|
||||
fi
|
||||
|
||||
- name: Reproduce openclaw failure — pipe held OPEN, no EOF
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "=== keep-stdin-open pipe (the real openclaw / Claude Code case) ==="
|
||||
echo ""
|
||||
echo "Before the readline() fix this HANGS: main() did"
|
||||
echo " stdin.read(65536) -> on a pipe, blocks until 64KB OR EOF."
|
||||
echo "An MCP client sends one ~150B initialize and keeps stdin"
|
||||
echo "open waiting for the response, so the server never parsed"
|
||||
echo "the request and the client timed out (openclaw: 'MCP error"
|
||||
echo "-32000: Connection closed'). The earlier regular-file /"
|
||||
echo "heredoc-pipe steps PASSED through this bug because a file"
|
||||
echo "(or a closing heredoc) yields EOF immediately."
|
||||
echo ""
|
||||
|
||||
# Drive the server through a real pipe that stays OPEN: write
|
||||
# one initialize, do NOT close stdin, and require a response
|
||||
# within a hard timeout. read(65536) -> no output -> timeout
|
||||
# kills it -> FAIL. readline() -> immediate response -> PASS.
|
||||
python - <<'PYEOF'
|
||||
import json, subprocess, sys, time, select
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "a2a_mcp_server.py"],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env={**__import__("os").environ},
|
||||
)
|
||||
req = json.dumps({
|
||||
"jsonrpc": "2.0", "id": 1, "method": "initialize",
|
||||
"params": {"protocolVersion": "2024-11-05",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "keepopen", "version": "1"}},
|
||||
}) + "\n"
|
||||
proc.stdin.write(req.encode())
|
||||
proc.stdin.flush()
|
||||
# Deliberately DO NOT close proc.stdin — mirror a live MCP client.
|
||||
|
||||
deadline = time.time() + 15
|
||||
line = b""
|
||||
while time.time() < deadline:
|
||||
r, _, _ = select.select([proc.stdout], [], [], 1)
|
||||
if r:
|
||||
line = proc.stdout.readline()
|
||||
if line:
|
||||
break
|
||||
proc.kill()
|
||||
|
||||
if not line:
|
||||
print("FAIL: no response within 15s on an open pipe — "
|
||||
"stdin.read(65536) regression is back")
|
||||
sys.exit(1)
|
||||
resp = json.loads(line.decode())
|
||||
assert resp.get("id") == 1 and "result" in resp, \
|
||||
f"unexpected response: {line[:200]!r}"
|
||||
assert resp["result"]["serverInfo"]["name"] == "molecule", \
|
||||
f"wrong serverInfo: {line[:200]!r}"
|
||||
print("PASS: server answered initialize on a still-open pipe")
|
||||
PYEOF
|
||||
|
||||
- name: Run unit tests for stdio transport
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "=== Running stdio transport unit tests ==="
|
||||
python -m pytest tests/test_a2a_mcp_server.py::TestStdioPipeAssertion tests/test_a2a_mcp_server.py::TestStdioKeepOpenPipe -v --no-cov
|
||||
+144
-63
@@ -86,25 +86,53 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
env:
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile ci \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "$PR_BASE_SHA" \
|
||||
--base-ref "$PR_BASE_REF" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}"
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Workflow-only edits are covered by the workflow lint family
|
||||
# and by this workflow's always-present required jobs. Do not fan
|
||||
# those edits out into Go/Canvas/Python/shellcheck work; the
|
||||
# downstream jobs still emit their required contexts via no-op
|
||||
# steps when their surface flag is false.
|
||||
#
|
||||
# If the diff itself cannot be trusted, fail open by running every
|
||||
# surface instead of silently under-testing the PR.
|
||||
if ! DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null); then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The job always
|
||||
# emits the required context, but expensive steps are path-scoped on every
|
||||
# event so docs/E2E/Canvas-only main pushes do not block deploy on unrelated
|
||||
# Go bootstrap work.
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
|
||||
# + per-step gating shape preserves the GitHub-side required-check name
|
||||
# contract (so when this Gitea port becomes a required check in Phase 4,
|
||||
# the name match works on PRs that don't touch workspace-server/).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
# mc#774 (closed 2026-05-14): Phase 4 flip of the platform-build job.
|
||||
# Phase 4 (#656) originally flipped this to continue-on-error: false based on
|
||||
@@ -125,29 +153,29 @@ jobs:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.platform != 'true' }}
|
||||
- if: false
|
||||
working-directory: .
|
||||
run: echo "No workspace-server/** changes — Platform (Go) gate satisfied without running Go build/test/lint."
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: always()
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
run: go mod download
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
run: go vet ./...
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
name: Install golangci-lint
|
||||
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
name: Run golangci-lint
|
||||
run: $(go env GOPATH)/bin/golangci-lint run --timeout 3m ./...
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
name: Diagnostic — per-package verbose 60s
|
||||
run: |
|
||||
set +e
|
||||
@@ -163,7 +191,7 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
name: Run tests with race detection and coverage
|
||||
# Explicit timeout: cold runner cache causes OOM kills at ~4m39s on the
|
||||
# full ./... suite with race detection + coverage. A 10m per-step timeout
|
||||
@@ -171,7 +199,7 @@ jobs:
|
||||
# instead of OOM-killing. The job-level timeout (15m) is a backstop.
|
||||
run: go test -race -timeout 10m -coverprofile=coverage.out ./...
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
@@ -185,7 +213,7 @@ jobs:
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: always()
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
@@ -273,7 +301,6 @@ jobs:
|
||||
# siblings — verified empirically on PR #2314).
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
@@ -282,20 +309,20 @@ jobs:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.canvas != 'true' }}
|
||||
- if: false
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — Canvas (Next.js) gate satisfied without running npm build/test."
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: always()
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: always()
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: npm ci --include=optional --prefer-offline
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: always()
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: always()
|
||||
run: npm run build
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: always()
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
@@ -304,7 +331,7 @@ jobs:
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
if: always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
@@ -318,19 +345,18 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# Shellcheck (E2E scripts) — required context, path-scoped heavy steps.
|
||||
# Shellcheck (E2E scripts) — required check, always runs.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.scripts != 'true' }}
|
||||
run: echo "No tests/e2e, scripts, or infra/scripts changes — Shellcheck gate satisfied without running script checks."
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: false
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: always()
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: always()
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
@@ -341,16 +367,16 @@ jobs:
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: always()
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: always()
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: always()
|
||||
name: Test ECR promote-tenant-image script (mock-driven, no live infra)
|
||||
# Covers scripts/promote-tenant-image.sh — the codified
|
||||
# :staging-latest → :latest ECR promote + tenant fleet redeploy
|
||||
@@ -360,7 +386,7 @@ jobs:
|
||||
run: |
|
||||
bash scripts/test-promote-tenant-image.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: always()
|
||||
name: Shellcheck promote-tenant-image script
|
||||
# scripts/ is excluded from the bulk shellcheck pass above (legacy
|
||||
# SC3040/SC3043 cleanup pending). Run shellcheck explicitly on
|
||||
@@ -430,29 +456,84 @@ jobs:
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs.
|
||||
# Runtime Python moved to molecule-ai-workspace-runtime. Keep this context as
|
||||
# a guard so branch protection still catches attempts to reintroduce an
|
||||
# editable runtime copy under molecule-core/workspace/.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Runtime SSOT guard
|
||||
- if: false
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: always()
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: always()
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: always()
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: always()
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: always()
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. See issue #2790 for full rationale.
|
||||
run: |
|
||||
set -eu
|
||||
if [ -d workspace ]; then
|
||||
echo "::error file=workspace::Runtime source must live in molecule-ai-workspace-runtime, not molecule-core/workspace."
|
||||
exit 1
|
||||
fi
|
||||
for f in scripts/build_runtime_package.py scripts/test_build_runtime_package.py; do
|
||||
if [ -e "$f" ]; then
|
||||
echo "::error file=$f::Legacy build-from-workspace packaging script must not be restored."
|
||||
exit 1
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text.
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
echo "Runtime SSOT guard passed; core consumes the standalone runtime package."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
all-required:
|
||||
# Aggregator sentinel — RFC internal#219 §2 (Phase 4 — closes internal#286).
|
||||
|
||||
@@ -43,18 +43,6 @@ name: Continuous synthetic E2E (staging)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 minutes, on :02 and :32. This keeps a recurring SaaS
|
||||
# behavior probe while cutting runner occupancy from this workflow by
|
||||
# roughly two thirds; fast liveness belongs in the lighter smoke/heartbeat
|
||||
# probes, not in a full tenant/workspace synth every 10 minutes.
|
||||
#
|
||||
# Previous cadence was every 10 minutes (:02 :12 :22 :32 :42 :52).
|
||||
# The current operator-host runner pool is the bottleneck, so full
|
||||
# synth E2E is deliberately lower-cadence until it moves to a dedicated
|
||||
# runner host or warm-runtime pool.
|
||||
#
|
||||
# Historical notes from the 10-minute shape:
|
||||
#
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
@@ -78,7 +66,7 @@ on:
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,32 * * * *'
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
@@ -118,7 +106,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2 via the template's third-party-
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
@@ -131,9 +119,9 @@ jobs:
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2' }}
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
@@ -145,11 +133,6 @@ jobs:
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
@@ -190,12 +173,6 @@ jobs:
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var secret missing — EC2 leak verification cannot run"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
|
||||
@@ -132,13 +132,31 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — same pattern PR#372's
|
||||
# ci.yml port used. Diffs against the PR base or push BEFORE SHA,
|
||||
# then matches against the api-relevant path set.
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile e2e-api \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "${{ github.event.pull_request.base.sha }}" \
|
||||
--base-ref "${{ github.event.pull_request.base.ref }}" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-${{ github.event.before }}}"
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
@@ -348,9 +366,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run today's-PR-coverage E2E (mc#1525/1535/1536/1539/1542 fix-specific assertions)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_today_pr_coverage_e2e.sh
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
@@ -360,18 +375,15 @@ jobs:
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Install standalone runtime parser from Gitea registry
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
python3 -m pip install --no-deps \
|
||||
--index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ \
|
||||
molecule-ai-workspace-runtime
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
- name: Run poll-mode chat upload E2E (RFC #2891)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
|
||||
- name: Run today's-PR-coverage E2E (mc#1525/1535/1536/1539/1542 fix-specific assertions)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_today_pr_coverage_e2e.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.api == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
@@ -389,3 +401,4 @@ jobs:
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
name: E2E Chat
|
||||
|
||||
# Comprehensive Playwright E2E for the unified chat stack (desktop
|
||||
# ChatTab + mobile MobileChat). Heavy browser execution is intentionally
|
||||
# outside the normal required PR path: PRs run it only after entering the
|
||||
# `merge-queue`, while push/main, nightly, and manual dispatch preserve
|
||||
# coverage without making every PR pay the full runtime/browser cost.
|
||||
# ChatTab + mobile MobileChat). Runs on every PR that touches canvas,
|
||||
# workspace-server, or this workflow file.
|
||||
#
|
||||
# Architecture:
|
||||
# 1. Ephemeral Postgres + Redis (docker, unique container names)
|
||||
@@ -24,11 +22,6 @@ on:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
schedule:
|
||||
# Nightly at 09:00 UTC. Keeps coverage for the currently non-required
|
||||
# heavy browser lane without spending runner time on every PR.
|
||||
- cron: '0 9 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-chat-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
@@ -57,14 +50,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
@@ -81,26 +67,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if ! echo "$CHANGED" | grep -qE '^(canvas/|workspace-server/|\.gitea/workflows/e2e-chat\.yml$)'; then
|
||||
echo "chat=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" != "pull_request" ]; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
labels=$(curl -fsS -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
|
||||
| python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
|
||||
rm -f "$authfile"
|
||||
if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
|
||||
if echo "$CHANGED" | grep -qE '^(canvas/|workspace-server/|\.gitea/workflows/e2e-chat\.yml$)'; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "PR is not in merge-queue; skipping heavy E2E Chat for normal PR path."
|
||||
echo "chat=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -261,14 +230,7 @@ jobs:
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
PREBAKED_PLAYWRIGHT=/ms-playwright
|
||||
if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
|
||||
echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
|
||||
echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
npx playwright install --with-deps chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Start canvas dev server (background)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
|
||||
@@ -44,8 +44,6 @@ name: E2E Peer Visibility (literal MCP list_peers)
|
||||
# - No cross-repo `uses:` (feedback_gitea_cross_repo_uses_blocked). The
|
||||
# actions/checkout SHA is the one e2e-staging-canvas.yml already uses
|
||||
# successfully (a mirrored SHA — see #1277/PR#1292 root-cause).
|
||||
# - 2026-05-21 retrigger: verify fresh platform-tenant image after the
|
||||
# publish Buildx DOCKER_CONFIG fix restored staging-latest image updates.
|
||||
# - Per-SHA concurrency, not global (feedback_concurrency_group_per_sha).
|
||||
# - Workflow-level GITHUB_SERVER_URL pinned
|
||||
# (feedback_act_runner_github_server_url).
|
||||
@@ -70,11 +68,14 @@ name: E2E Peer Visibility (literal MCP list_peers)
|
||||
# minutes, not the 30+ min cold-EC2 path), so peer-visibility is part of
|
||||
# the local gate that fires before the staging E2E.
|
||||
#
|
||||
# It is its OWN non-required status context `E2E Peer Visibility (local)`.
|
||||
# The local backend uses external-mode workspaces by default so it tests
|
||||
# the literal platform MCP list_peers path without depending on local
|
||||
# template container boot/heartbeat. Container-mode runtime boot remains
|
||||
# available via PV_LOCAL_PROVISION_MODE=container for targeted debugging.
|
||||
# It is its OWN non-required status context `E2E Peer Visibility (local)`
|
||||
# — same non-required-by-design decision as the staging job (red until
|
||||
# Hermes-401 #162 / OpenClaw-never-online #165 land; flip-to-required
|
||||
# tracked at molecule-core#1296). It is an HONEST gate: NO
|
||||
# continue-on-error mask (feedback_fix_root_not_symptom). It is kept a
|
||||
# distinct context (not folded into e2e-api.yml's required `E2E API
|
||||
# Smoke Test`) precisely so a deliberately-RED-today gate cannot wedge
|
||||
# the required local-E2E job or any unrelated merge.
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -85,8 +86,9 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace/a2a_mcp_server.py'
|
||||
- 'workspace/platform_tools/registry.py'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_local.sh'
|
||||
- 'tests/e2e/lib/peer_visibility_assert.sh'
|
||||
- '.gitea/workflows/e2e-peer-visibility.yml'
|
||||
@@ -98,8 +100,9 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace/a2a_mcp_server.py'
|
||||
- 'workspace/platform_tools/registry.py'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_local.sh'
|
||||
- 'tests/e2e/lib/peer_visibility_assert.sh'
|
||||
- '.gitea/workflows/e2e-peer-visibility.yml'
|
||||
@@ -139,14 +142,8 @@ jobs:
|
||||
echo "lib/peer_visibility_assert.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_mcp_staging.sh
|
||||
echo "test_peer_visibility_mcp_staging.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_token_mint_staging.sh
|
||||
echo "test_peer_visibility_token_mint_staging.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
echo "test_peer_visibility_mcp_local.sh — bash syntax OK"
|
||||
if rg -n '/admin/workspaces/.*/test-token|test-token' tests/e2e/test_*staging*.sh; then
|
||||
echo "::error::staging E2E must not use dev-only /admin/workspaces/:id/test-token; use production-safe admin token minting instead"
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging fresh-provision MCP list_peers E2E runs on push to"
|
||||
echo "main / workflow_dispatch / daily cron (30+ min EC2 boot)."
|
||||
echo "The LOCAL backend runs in the peer-visibility-local job"
|
||||
@@ -160,9 +157,9 @@ jobs:
|
||||
# ephemeral host ports so concurrent host-network act_runner runs don't
|
||||
# collide; go build; background platform-server). Its OWN non-required
|
||||
# status context `E2E Peer Visibility (local)` — non-required-by-design
|
||||
# exactly like the staging job (flip-to-required tracked at
|
||||
# molecule-core#1296). HONEST gate, NO continue-on-error mask
|
||||
# (feedback_fix_root_not_symptom). Runs on PR +
|
||||
# exactly like the staging job (red until #162/#165 land;
|
||||
# flip-to-required tracked at molecule-core#1296). HONEST gate, NO
|
||||
# continue-on-error mask (feedback_fix_root_not_symptom). Runs on PR +
|
||||
# push (local boot is minutes, not the 30+ min cold-EC2 path).
|
||||
# bp-required: pending #1296
|
||||
peer-visibility-local:
|
||||
@@ -182,9 +179,6 @@ jobs:
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
PV_RUNTIMES: "hermes openclaw claude-code"
|
||||
PV_LOCAL_PROVISION_MODE: external
|
||||
ADMIN_TOKEN: local-e2e-admin-token
|
||||
MOLECULE_ADMIN_TOKEN: local-e2e-admin-token
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
@@ -273,9 +267,10 @@ jobs:
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true; exit 1
|
||||
- name: Run LOCAL fresh-provision peer-visibility E2E (literal MCP list_peers)
|
||||
# HONEST gate — NO continue-on-error. The local backend uses
|
||||
# external-mode workspaces so this context tests the literal MCP
|
||||
# peer-visibility path without coupling to template container boot.
|
||||
# HONEST gate — NO continue-on-error. Red today (Hermes-401 #162 /
|
||||
# OpenClaw-never-online #165 not yet fixed); green when they land.
|
||||
# Non-required-by-design via its distinct status context until the
|
||||
# molecule-core#1296 flip-to-required.
|
||||
run: bash tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure()
|
||||
|
||||
@@ -16,9 +16,9 @@ name: E2E Staging Canvas (Playwright)
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main, PR touching canvas sources + this workflow only
|
||||
# after the PR enters `merge-queue`, manual dispatch, and scheduled cron to
|
||||
# catch browser/runtime drift even when canvas is quiet.
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
@@ -37,10 +37,9 @@ on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Nightly at 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * *'
|
||||
workflow_dispatch:
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
@@ -80,13 +79,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
# Cron and manual triggers always run real work (no diff context).
|
||||
# Cron triggers always run real work (no diff context).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
if [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
@@ -106,26 +102,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if ! echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" != "pull_request" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
labels=$(curl -fsS -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
|
||||
| python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
|
||||
rm -f "$authfile"
|
||||
if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
|
||||
if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "PR is not in merge-queue; skipping heavy E2E Staging Canvas for normal PR path."
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -190,14 +169,7 @@ jobs:
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
PREBAKED_PLAYWRIGHT=/ms-playwright
|
||||
if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
|
||||
echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
|
||||
echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
npx playwright install --with-deps chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
|
||||
@@ -49,8 +49,6 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
@@ -61,8 +59,6 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
@@ -131,11 +127,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -161,7 +152,7 @@ jobs:
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2' }}
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
@@ -174,12 +165,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
|
||||
@@ -47,11 +47,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
E2E_MODE: smoke
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
@@ -66,12 +61,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken.
|
||||
|
||||
@@ -13,12 +13,8 @@ name: gitea-merge-queue
|
||||
# - add `merge-queue-hold` to pause a queued PR without removing it
|
||||
|
||||
on:
|
||||
# Schedule moved to operator-config:
|
||||
# /etc/cron.d/molecule-core-merge-queue ->
|
||||
# /usr/local/bin/molecule-core-cron-bot.sh merge-queue
|
||||
#
|
||||
# The queue bot still processes one PR per tick, but no longer occupies
|
||||
# one of the shared Actions runners just to poll.
|
||||
schedule:
|
||||
- cron: '*/5 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -101,13 +101,36 @@ jobs:
|
||||
# not present in the shallow checkout.
|
||||
fetch-depth: 2
|
||||
- id: filter
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile handlers-postgres \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "${{ github.event.pull_request.base.sha }}" \
|
||||
--base-ref "${{ github.event.pull_request.base.ref }}" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-}"
|
||||
# Gitea Actions evaluates github.event.before to empty string in shell
|
||||
# scripts. Use GITHUB_EVENT_BEFORE shell env var instead (Gitea
|
||||
# correctly populates it for push events). PR case uses template var.
|
||||
BASE=""
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "$GITHUB_EVENT_BEFORE" ]; then
|
||||
BASE="$GITHUB_EVENT_BEFORE"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# timeout 30 guards against the case where BASE points to a ref that
|
||||
# git can resolve but cat-file hangs (rare on corrupted objects).
|
||||
if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "handlers=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
|
||||
@@ -28,10 +28,12 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.github/workflows/**'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.github/workflows/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -73,11 +75,8 @@ jobs:
|
||||
fails = []
|
||||
warnings = []
|
||||
|
||||
# Gitea is SSOT for molecule-core CI per task #347 / memory
|
||||
# reference_molecule_core_actions_gitea_only. The legacy
|
||||
# .github/workflows/ tree was deleted in SSOT-Instance-4 (#331).
|
||||
roots = []
|
||||
for root in ('.gitea/workflows',):
|
||||
for root in ('.gitea/workflows', '.github/workflows'):
|
||||
if os.path.isdir(root):
|
||||
roots.append(root)
|
||||
|
||||
|
||||
@@ -42,13 +42,7 @@ permissions:
|
||||
packages: write
|
||||
|
||||
env:
|
||||
# SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
|
||||
# sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
|
||||
# bootstrap fallback. When the org var is set, the fallback becomes dead code and
|
||||
# switching accounts/regions is a one-line change at the org level (instead of
|
||||
# touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
|
||||
# use below in this repo's staging-verify.yml.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/canvas
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
name: publish-runtime-autobump
|
||||
|
||||
# Auto-bump-on-workspace-edit half of the publish pipeline.
|
||||
#
|
||||
# Why this file exists (issue #351):
|
||||
# Gitea Actions does not correctly disambiguate `paths:` from `tags:`
|
||||
# when both are bundled under a single `on.push` key. The result is
|
||||
# that tag pushes get filtered out and `publish-runtime.yml` never
|
||||
# fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
|
||||
# because PYPI_TOKEN was absent (publishes would have failed anyway).
|
||||
#
|
||||
# Split design:
|
||||
# - publish-runtime.yml : on.push.tags only (the publisher)
|
||||
# - publish-runtime-autobump.yml: on.push.branches+paths (this file — the version-bumper)
|
||||
#
|
||||
# This file computes the next version from PyPI's latest, pushes a
|
||||
# `runtime-v$VERSION` tag, and exits. The tag push then triggers
|
||||
# publish-runtime.yml via its tags-only trigger.
|
||||
#
|
||||
# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
|
||||
# so concurrent workspace pushes serialize at the bump step. Without
|
||||
# this, two pushes minutes apart could both read PyPI latest=0.1.129
|
||||
# and try to tag 0.1.130 simultaneously, only one of which would land.
|
||||
|
||||
on:
|
||||
# Run on PR pushes to post a success status so Gitea can merge the PR.
|
||||
# All steps use continue-on-error: true so operational failures
|
||||
# (PyPI unreachable, DISPATCH_TOKEN missing) do not block merge.
|
||||
pull_request:
|
||||
paths:
|
||||
- "workspace/**"
|
||||
# mc#1578 / a05add29 cure: build_runtime_package.py owns PYPROJECT_TEMPLATE
|
||||
# (deps, classifiers, project metadata). A change there is publish-affecting
|
||||
# even when workspace/** is untouched, so the autobump must fire to claim
|
||||
# the next runtime-v$VERSION tag. Without this, manual tagging races PyPI
|
||||
# (e.g. runtime-v0.1.18 collided with the 2026-04-27 PyPI 0.1.18 publish,
|
||||
# blocking the python-multipart pin from reaching prod).
|
||||
- "scripts/build_runtime_package.py"
|
||||
- "scripts/test_build_runtime_package.py"
|
||||
# Bump-and-tag on main/staging push (the actual operational trigger).
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- staging
|
||||
paths:
|
||||
- "workspace/**"
|
||||
- "scripts/build_runtime_package.py"
|
||||
- "scripts/test_build_runtime_package.py"
|
||||
# Manual dispatch — useful when Gitea Actions API (/actions/*) is
|
||||
# unreachable (e.g. act_runner 404 on Gitea 1.22.6) and we cannot
|
||||
# re-trigger via curl.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write # required to push tags back
|
||||
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# PR-validation path: always succeeds so Gitea can merge workflow-only PRs.
|
||||
# Operational failures (PyPI unreachable, missing DISPATCH_TOKEN) are
|
||||
# surfaced via continue-on-error: true rather than blocking the merge.
|
||||
# The actual bump work happens on the main/staging push after merge.
|
||||
# bp-exempt: advisory validation for runtime publication; not a branch-protection gate.
|
||||
pr-validate:
|
||||
runs-on: ubuntu-latest
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true # do not block PR merge on operational failures
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Validate PyPI connectivity (best-effort)
|
||||
run: |
|
||||
set -eu
|
||||
echo "=== Checking PyPI accessibility ==="
|
||||
LATEST=$(curl -fsS --retry 3 --max-time 10 \
|
||||
https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])" \
|
||||
|| echo "PyPI unreachable (non-blocking for PR validation)")
|
||||
echo "Latest: ${LATEST:-unknown}"
|
||||
|
||||
# Actual bump-and-tag: runs on main/staging pushes, posts real success/failure.
|
||||
# No continue-on-error — operational failures here trip the main-red
|
||||
# watchdog, which is the desired signal for infrastructure degradation.
|
||||
# bp-exempt: post-merge tag publication side effect; CI / all-required gates source changes.
|
||||
bump-and-tag:
|
||||
runs-on: ubuntu-latest
|
||||
# Only fire on push events (main/staging after PR merge). Pull_request
|
||||
# events are handled by pr-validate above; we do NOT bump on every
|
||||
# push-synchronize because that would race with the PR head.
|
||||
#
|
||||
# NOTE: the prior condition `github.event.pull_request.base.ref == ''`
|
||||
# was broken — on a PR-merge push in Gitea Actions, the pull_request
|
||||
# context is still attached (base.ref='main'), so the condition always
|
||||
# evaluated to false and bump-and-tag was permanently skipped.
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Fetch tags for collision check
|
||||
run: git fetch origin --tags --depth=1
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Compute next version from PyPI latest and existing tags
|
||||
id: bump
|
||||
run: |
|
||||
set -eu
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
TAG_LATEST=$(git tag --list "runtime-v${MAJOR}.${MINOR}.*" \
|
||||
| sed -E 's/^runtime-v//' \
|
||||
| grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \
|
||||
| sort -V \
|
||||
| tail -1 || true)
|
||||
VERSION=$(PYPI_LATEST="$LATEST" TAG_LATEST="$TAG_LATEST" python - <<'PY'
|
||||
import os
|
||||
|
||||
def parse(v):
|
||||
return tuple(int(part) for part in v.split("."))
|
||||
|
||||
pypi = os.environ["PYPI_LATEST"]
|
||||
tag = os.environ.get("TAG_LATEST") or pypi
|
||||
base = max(parse(pypi), parse(tag))
|
||||
print(f"{base[0]}.{base[1]}.{base[2] + 1}")
|
||||
PY
|
||||
)
|
||||
echo "PyPI latest=$LATEST, latest runtime tag=${TAG_LATEST:-none} -> next=$VERSION"
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
|
||||
exit 1
|
||||
fi
|
||||
if git tag --list | grep -qx "runtime-v$VERSION"; then
|
||||
echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Push runtime-v$VERSION tag
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
VERSION: ${{ steps.bump.outputs.version }}
|
||||
GITEA_URL: https://git.moleculesai.app
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
|
||||
exit 1
|
||||
fi
|
||||
git config user.name "publish-runtime autobump"
|
||||
git config user.email "publish-runtime@moleculesai.app"
|
||||
git tag -a "runtime-v$VERSION" \
|
||||
-m "Auto-bump on workspace/** edit on $GITHUB_REF" \
|
||||
-m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
|
||||
-m "publish-runtime.yml will pick up this tag and upload to PyPI"
|
||||
# Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
|
||||
# ensures the resulting tag-push event is dispatched to
|
||||
# publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
|
||||
# trigger downstream workflows.
|
||||
git remote set-url origin "${GITEA_URL#https://}"
|
||||
git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
|
||||
git push origin "runtime-v$VERSION"
|
||||
echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"
|
||||
@@ -0,0 +1,345 @@
|
||||
name: publish-runtime
|
||||
|
||||
# Gitea Actions port of .github/workflows/publish-runtime.yml.
|
||||
#
|
||||
# Ported 2026-05-10 (issue #206). Key differences from the GitHub version:
|
||||
# - Gitea Actions reads .gitea/workflows/, not .github/workflows/
|
||||
# - Dropped `environment: pypi-publish` — Gitea Actions does not support
|
||||
# named environments or OIDC trusted publishers
|
||||
# - Replaced `pypa/gh-action-pypi-publish@release/v1` (OIDC) with
|
||||
# `twine upload` using PYPI_TOKEN secret — same mechanism as a local
|
||||
# `python -m twine upload` with a PyPI token
|
||||
# - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
|
||||
# — Gitea Actions exposes github.ref (the full ref) but not ref_name
|
||||
# - Dropped `merge_group` trigger (Gitea has no merge queue)
|
||||
#
|
||||
# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
|
||||
# `workspace/**` path-filter trigger in PR #349.
|
||||
#
|
||||
# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
|
||||
# file. Bundling `paths` with `tags` under a single `on.push` key caused
|
||||
# Gitea Actions to never dispatch the workflow for tag-push events (0
|
||||
# runs in `action_run` for workflow_id='publish-runtime.yml' since the
|
||||
# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
|
||||
# 0.1.129 despite a v1.0.0 Gitea tag existing).
|
||||
#
|
||||
# The auto-bump-on-workspace-edit trigger now lives in
|
||||
# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
|
||||
# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
|
||||
# which THIS file then picks up via the tags-only trigger below.
|
||||
#
|
||||
# This decoupling means Gitea's path-vs-tag evaluator never has to
|
||||
# disambiguate — each file has a single unambiguous trigger shape.
|
||||
#
|
||||
# PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
|
||||
# Set via: repo Settings → Actions → Variables and Secrets → New Secret.
|
||||
# The token should be a PyPI API token scoped to molecule-ai-workspace-runtime.
|
||||
#
|
||||
# The DISPATCH_TOKEN cascade (git push to template repos) is unchanged —
|
||||
# it uses the Gitea API directly and was already Gitea-compatible.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "runtime-v*"
|
||||
workflow_dispatch:
|
||||
# 2026-05-11 (root cause of #351 / 0 runs ever):
|
||||
# Gitea 1.22.6's workflow parser rejects `workflow_dispatch.inputs.version`
|
||||
# with "unknown on type" — it mis-treats the inputs sub-keys as top-level
|
||||
# `on:` event types. Log line:
|
||||
# actions/workflows.go:DetectWorkflows() [W] ignore invalid workflow
|
||||
# "publish-runtime.yml": unknown on type: map["version": {...}]
|
||||
# That `[W] ignore invalid workflow` is silent UX — the workflow never
|
||||
# registers, so it never fires for ANY event (push.tags included).
|
||||
# Removing the inputs block restores parsing. Manual dispatch from the
|
||||
# Gitea UI now triggers the PyPI auto-bump fallback in `Derive version`
|
||||
# below (no `inputs.version` to read).
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Serialize publishes so two concurrent tag pushes don't both compute
|
||||
# "latest+1" and race on PyPI upload. The second one waits.
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399). Ship
|
||||
# path (on: push tag runtime-v*) — reserved capacity, never FIFO
|
||||
# behind PR-CI. `publish` resolves only to molecule-runner-publish-*.
|
||||
runs-on: publish
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
|
||||
- name: Derive version (tag or PyPI auto-bump)
|
||||
id: version
|
||||
run: |
|
||||
if echo "$GITHUB_REF" | grep -q "^refs/tags/runtime-v"; then
|
||||
# Tag is `runtime-vX.Y.Z` — strip the prefix.
|
||||
VERSION="${GITHUB_REF#refs/tags/runtime-v}"
|
||||
else
|
||||
# workflow_dispatch path (no inputs supported on Gitea 1.22.6) or
|
||||
# any other non-tag trigger: derive from PyPI latest + patch bump.
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
|
||||
fi
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
|
||||
echo "::error::version $VERSION does not match PEP 440"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Publishing molecule-ai-workspace-runtime $VERSION"
|
||||
|
||||
- name: Install build tooling
|
||||
run: pip install build twine
|
||||
|
||||
- name: Build package from workspace/
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "${{ steps.version.outputs.version }}" \
|
||||
--out "${{ runner.temp }}/runtime-build"
|
||||
|
||||
- name: Build wheel + sdist
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: python -m build
|
||||
|
||||
- name: Capture wheel SHA256 for cascade content-verification
|
||||
id: wheel_hash
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
set -eu
|
||||
WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
|
||||
if [ -z "$WHEEL" ]; then
|
||||
echo "::error::No .whl in dist/ — \`python -m build\` must have failed silently"
|
||||
exit 1
|
||||
fi
|
||||
HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
|
||||
echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
|
||||
echo "Local wheel SHA256 (pre-upload): ${HASH}"
|
||||
echo "Wheel filename: $(basename "$WHEEL")"
|
||||
|
||||
- name: Verify package contents (sanity)
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
python -m twine check dist/*
|
||||
python -m venv /tmp/smoke
|
||||
/tmp/smoke/bin/pip install --quiet dist/*.whl
|
||||
/tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
|
||||
- name: Publish to PyPI
|
||||
# working-directory matches the preceding Build/Verify steps. Without
|
||||
# this, twine runs from the default workspace checkout dir where
|
||||
# `dist/` doesn't exist and fails with:
|
||||
# ERROR InvalidDistribution: Cannot find file (or expand pattern): 'dist/*'
|
||||
# Caught on the first-ever successful dispatch of this workflow
|
||||
# (run 5097, 2026-05-11 02:08Z) — every other step in the publish
|
||||
# job already had this working-directory; Publish was missing it.
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
env:
|
||||
# PYPI_TOKEN: repository secret scoped to molecule-ai-workspace-runtime.
|
||||
# Set via: Settings → Actions → Variables and Secrets → New Secret.
|
||||
# Format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$PYPI_TOKEN" ]; then
|
||||
echo "::error::PYPI_TOKEN secret is not set — set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
echo "::error::Required format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
|
||||
exit 1
|
||||
fi
|
||||
python -m twine upload \
|
||||
--verbose \
|
||||
--repository pypi \
|
||||
--username __token__ \
|
||||
--password "$PYPI_TOKEN" \
|
||||
dist/*
|
||||
|
||||
cascade:
|
||||
needs: publish
|
||||
# Publish/release lane (internal#462) — downstream of the runtime
|
||||
# publish ship job; keep it on the reserved lane too.
|
||||
runs-on: publish
|
||||
steps:
|
||||
- name: Wait for PyPI to propagate the new version
|
||||
env:
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
python -m venv /tmp/propagation-probe
|
||||
PROBE=/tmp/propagation-probe/bin
|
||||
$PROBE/pip install --upgrade --quiet pip
|
||||
for i in $(seq 1 30); do
|
||||
if $PROBE/pip install \
|
||||
--quiet \
|
||||
--no-cache-dir \
|
||||
--force-reinstall \
|
||||
--no-deps \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1; then
|
||||
INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
|
||||
| awk -F': ' '/^Version:/{print $2}')
|
||||
if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
|
||||
echo "✓ PyPI resolved $RUNTIME_VERSION (install check)"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "::error::pip install --no-cache-dir molecule-ai-workspace-runtime==${RUNTIME_VERSION} never resolved within ~5 min."
|
||||
echo "::error::Refusing to fan out cascade against a potentially stale PyPI index."
|
||||
exit 1
|
||||
fi
|
||||
echo " [$i/30] waiting for PyPI to propagate ${RUNTIME_VERSION}..."
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Stage (b): download wheel + SHA256 compare against what we built.
|
||||
# Catches Fastly stale-content serving old bytes under a new version URL.
|
||||
#
|
||||
# Caught run 5196 (first-ever successful publish, 2026-05-11): the
|
||||
# previous one-liner `HASH=$(pip download ... && sha256sum ...)`
|
||||
# captured pip's stdout (`Collecting molecule-ai-workspace-runtime
|
||||
# ==X.Y.Z`) into HASH, then the SHA comparison failed against the
|
||||
# leaked `Collecting...` string. `2>/dev/null` silences stderr but
|
||||
# NOT stdout; pip writes its progress to stdout by default.
|
||||
# Fix: split into two steps, silence pip's stdout explicitly, capture
|
||||
# only sha256sum's output into HASH.
|
||||
python -m pip download \
|
||||
--no-deps \
|
||||
--no-cache-dir \
|
||||
--dest /tmp/wheel-probe \
|
||||
--quiet \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1
|
||||
HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
|
||||
if [ "$HASH" != "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
|
||||
echo "::error::Expected: $EXPECTED_SHA256"
|
||||
echo "::error::Got: $HASH"
|
||||
echo "::error::Fastly may be serving stale content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ PyPI CDN verified (SHA256 match)"
|
||||
|
||||
- name: Fan out via push to .runtime-version
|
||||
env:
|
||||
# Gitea PAT with write:repository scope on the 8 cascade-active
|
||||
# template repos. Used for git push to each template repo's main
|
||||
# branch, which trips their `on: push: branches: [main]` trigger
|
||||
# on publish-image.yml.
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
run: |
|
||||
set +e # don't abort on a single repo failure — collect them all
|
||||
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
|
||||
echo "::warning::set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
|
||||
echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version."
|
||||
exit 1
|
||||
fi
|
||||
VERSION="$RUNTIME_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::publish job did not expose a version output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
|
||||
TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
|
||||
FAILED=""
|
||||
SKIPPED=""
|
||||
|
||||
git config --global user.name "publish-runtime cascade"
|
||||
git config --global user.email "publish-runtime@moleculesai.app"
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
for tpl in $TEMPLATES; do
|
||||
REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
|
||||
CLONE="$WORKDIR/$tpl"
|
||||
|
||||
HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $DISPATCH_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "↷ $tpl has no publish-image.yml — soft-skip"
|
||||
SKIPPED="$SKIPPED $tpl"
|
||||
continue
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
success=false
|
||||
while [ $attempt -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
rm -rf "$CLONE"
|
||||
if ! git clone --depth=1 \
|
||||
"https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
|
||||
"$CLONE" >/tmp/clone.log 2>&1; then
|
||||
echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
|
||||
cd "$CLONE"
|
||||
echo "$VERSION" > .runtime-version
|
||||
|
||||
if git diff --quiet -- .runtime-version; then
|
||||
echo "✓ $tpl already at $VERSION — no commit needed"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
git add .runtime-version
|
||||
git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
|
||||
-m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
|
||||
>/dev/null
|
||||
|
||||
if git push origin HEAD:main >/tmp/push.log 2>&1; then
|
||||
echo "✓ $tpl pushed $VERSION on attempt $attempt"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
echo "::warning::push $tpl attempt $attempt failed, pull-rebasing"
|
||||
git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
|
||||
cd - >/dev/null
|
||||
done
|
||||
|
||||
if [ "$success" != "true" ]; then
|
||||
FAILED="$FAILED $tpl"
|
||||
fi
|
||||
done
|
||||
rm -rf "$WORKDIR"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "::error::Cascade incomplete after 3 retries each. Failed:$FAILED"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$SKIPPED" ]; then
|
||||
echo "Cascade complete: pinned $VERSION. Soft-skipped (no publish-image.yml):$SKIPPED"
|
||||
else
|
||||
echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
|
||||
fi
|
||||
@@ -25,12 +25,8 @@ name: publish-workspace-server-image
|
||||
# staging-<sha>. Set repo variable or secret PROD_AUTO_DEPLOY_DISABLED=true
|
||||
# to stop production rollout while keeping image publishing enabled.
|
||||
#
|
||||
# Primary ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Optional staging tenant mirror target:
|
||||
# 004947743811.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Required secrets: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AUTO_SYNC_TOKEN
|
||||
# Staging ECR grants the primary SSOT-managed publisher principal repository
|
||||
# policy access, so no persistent staging AWS access keys are required.
|
||||
#
|
||||
# mc#711: Docker daemon not accessible on ubuntu-latest runner (molecule-canonical-1
|
||||
# shows client-only in `docker info` — daemon not running). DinD mount is present but
|
||||
@@ -47,29 +43,14 @@ on:
|
||||
# `cancel-in-progress: false`; that is not acceptable for a workflow with a
|
||||
# production deploy job. Per-SHA image tags are immutable, and staging-latest is
|
||||
# best-effort last-writer-wins metadata.
|
||||
#
|
||||
# 2026-05-20 retrigger: run #86994 on mc#1589 merge sha 0f0f1ba2 failed at
|
||||
# setup-buildx-action with EACCES on PC2 WSL publish runner — the runner's
|
||||
# DOCKER_CONFIG=/home/hongming/.docker-ecr/ dir didn't have a buildx/certs
|
||||
# subdir writable by the container's UID 1001. Hot-patched the dir perms;
|
||||
# this chore push retriggers the workflow. Proper fix (per-runner
|
||||
# DOCKER_CONFIG owned by 1001, internal#597 --env HOME=/home/runner pattern)
|
||||
# is tracked as a CI-hygiene follow-up — not in scope here.
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
# SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
|
||||
# sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
|
||||
# bootstrap fallback. When the org var is set, the fallback becomes dead code and
|
||||
# switching accounts/regions is a one-line change at the org level (instead of
|
||||
# touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
|
||||
# use below in this repo's staging-verify.yml.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
STAGING_TENANT_IMAGE_NAME: ${{ vars.STAGING_ECR_REGISTRY || '004947743811.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
@@ -140,18 +121,6 @@ jobs:
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Keep Buildx state inside the job temp dir. The publish runner's
|
||||
# inherited DOCKER_CONFIG can point at a host-owned ECR config path
|
||||
# (/home/hongming/.docker-ecr), which caused setup-buildx-action to
|
||||
# fail before image build with EACCES creating buildx/certs.
|
||||
- name: Prepare writable Docker config
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export DOCKER_CONFIG="$RUNNER_TEMP/docker-config"
|
||||
mkdir -p "$DOCKER_CONFIG/buildx/certs"
|
||||
echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
|
||||
docker buildx version
|
||||
|
||||
# Build + push platform image (inline ECR auth — mirrors the operator-host
|
||||
# approach; credentials come from GITHUB_SECRET_AWS_ACCESS_KEY_ID /
|
||||
# GITHUB_SECRET_AWS_SECRET_ACCESS_KEY in Gitea Actions).
|
||||
@@ -187,14 +156,9 @@ jobs:
|
||||
--push .
|
||||
|
||||
# Build + push tenant image (Go platform + Next.js canvas in one image).
|
||||
# Push the same build to the staging account too so fresh staging/E2E
|
||||
# tenants can pull without cross-account ECR reads. The staging ECR repo
|
||||
# policy trusts the primary SSOT-managed publisher principal; do not add
|
||||
# separate persistent staging AWS access keys here.
|
||||
- name: Build & push tenant image to ECR (staging-<sha> + staging-latest)
|
||||
env:
|
||||
TENANT_IMAGE_NAME: ${{ env.TENANT_IMAGE_NAME }}
|
||||
STAGING_TENANT_IMAGE_NAME: ${{ env.STAGING_TENANT_IMAGE_NAME }}
|
||||
TAG_SHA: staging-${{ steps.tags.outputs.sha }}
|
||||
TAG_LATEST: staging-latest
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
@@ -205,19 +169,8 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${TENANT_IMAGE_NAME%%/*}"
|
||||
STAGING_ECR_REGISTRY="${STAGING_TENANT_IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${STAGING_ECR_REGISTRY}"
|
||||
|
||||
build_tags=(
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
--tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
--tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
)
|
||||
|
||||
docker buildx build \
|
||||
--file ./workspace-server/Dockerfile.tenant \
|
||||
--build-arg NEXT_PUBLIC_PLATFORM_URL= \
|
||||
@@ -226,7 +179,8 @@ jobs:
|
||||
--label "org.opencontainers.image.revision=${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--label "molecule.workflow.run_id=${GITHUB_RUN_ID}" \
|
||||
"${build_tags[@]}" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}" \
|
||||
--push .
|
||||
|
||||
# bp-exempt: production deploy side-effect; merge is gated by CI / all-required and this job waits for push CI before acting.
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and
|
||||
# `workflow_dispatch:` (no inputs, but the trigger itself is
|
||||
# parser-rejected when inputs are absent in some Gitea 1.22.x
|
||||
# builds; safest to drop entirely — manual runs go via cron-trigger
|
||||
# bump or push-with-paths-filter).
|
||||
# - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -0,0 +1,150 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
|
||||
# (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
|
||||
# finicky without them).
|
||||
# - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
|
||||
# pattern for ci.yml port).
|
||||
# - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on every job (RFC §1 contract).
|
||||
#
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy -> all crash on first boot with ImportError.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
# event_name + sha keeps PR sync and the subsequent staging push on the
|
||||
# same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
run: |
|
||||
# Inline replacement for dorny/paths-filter — same pattern
|
||||
# PR#372's ci.yml port used. Diffs against the PR base or the
|
||||
# previous push SHA, then matches against the wheel-relevant
|
||||
# path set.
|
||||
#
|
||||
# NOTE: Gitea Actions does not expose github.event.before as a
|
||||
# shell environment variable. The ${{ github.event.before }} template
|
||||
# expression works inside YAML run: blocks but is evaluated to an
|
||||
# empty string for push events, making the ${VAR:-fallback} always
|
||||
# use the fallback. Use GITHUB_EVENT_BEFORE instead — it IS set in
|
||||
# the runner's shell environment for push events.
|
||||
BASE=""
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "$GITHUB_EVENT_BEFORE" ]; then
|
||||
BASE="$GITHUB_EVENT_BEFORE"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
# New branch or no previous SHA: treat as wheel-relevant.
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! timeout 30 git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`.
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -81,11 +81,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the smoke's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -112,9 +107,9 @@ jobs:
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the smoke to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: MiniMax-M2
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "smoke-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the smoke script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
@@ -134,12 +129,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
|
||||
@@ -75,12 +75,8 @@ permissions:
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
# SSOT-Instance-10 (#333): triplet sourced from org/repo var `ECR_REGISTRY` with
|
||||
# the current prod-account literal as bootstrap fallback. When the org var is set,
|
||||
# the fallback becomes dead code and switching accounts/regions is a one-line
|
||||
# change at the org level. Pattern mirrors `vars.CP_URL || 'literal'` below.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
@@ -53,12 +53,19 @@ name: status-reaper
|
||||
# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
|
||||
# "unknown on type" when `workflow_dispatch.inputs.X` is present.
|
||||
on:
|
||||
# Schedule moved to operator-config:
|
||||
# /etc/cron.d/molecule-core-status-reaper ->
|
||||
# /usr/local/bin/molecule-core-cron-bot.sh status-reaper
|
||||
#
|
||||
# This keeps the 5-minute compensation cadence but stops a maintenance
|
||||
# bot from consuming Gitea Actions runner slots during PR merge waves.
|
||||
# SCHEDULE RE-ENABLED 2026-05-12 rev3 — interim disable (mc#645) reverted now that
|
||||
# rev3 widens DEFAULT_SWEEP_LIMIT 10 → 30 (covers retroactive-failure timing window).
|
||||
# Sibling watchdog re-enabled in the same PR with timeout-minutes raised 5 → 15.
|
||||
schedule:
|
||||
# Every 5 minutes. Off-zero alignment with sibling cron workflows:
|
||||
# ci-required-drift (`:17`), main-red-watchdog (`:05`),
|
||||
# railway-pin-audit (`:23`). 5-min cadence gives a tight enough
|
||||
# close on schedule-triggered false-reds that main-red-watchdog
|
||||
# (hourly :05) almost never files an issue on the false case.
|
||||
# rev3 keeps `*/5` unchanged per hongming-pc2 03:25Z review:
|
||||
# "trades window-width-cheap for cadence-loady" — N=30 widens
|
||||
# the lookback cheaply without doubling runner load via `*/2`.
|
||||
- cron: '*/5 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# Compensating-status POST needs write on repo statuses; no other
|
||||
|
||||
@@ -40,12 +40,14 @@ name: Sweep stale AWS Secrets Manager secrets
|
||||
# the mostly-orphan tunnels) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30, offset from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45). This janitor is intentionally schedule-only
|
||||
# for deletes; manual dispatch is forced to dry-run below because Gitea
|
||||
# 1.22.6 rejects workflow_dispatch.inputs.
|
||||
- cron: '30 * * * *'
|
||||
# Disabled as an hourly schedule until the dedicated
|
||||
# AWS_SECRETS_JANITOR_* key exists in the key-management SSOT and is
|
||||
# mirrored into Gitea. Falling back to the molecule-cp app principal is
|
||||
# intentionally not allowed: it lacks account-wide ListSecrets, and
|
||||
# granting that to an application credential would weaken least privilege.
|
||||
#
|
||||
# Keep the manual trigger so operators can validate the workflow immediately
|
||||
# after provisioning the janitor key, then restore the hourly :30 schedule.
|
||||
workflow_dispatch:
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
@@ -62,24 +64,22 @@ jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# This is a cost/leak janitor. A scheduled failure must be red so
|
||||
# operators know tenant bootstrap secrets may be leaking.
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# Keep this literal. Gitea/act_runner 1.22.6 can mis-render
|
||||
# secret-backed expressions with `||`, which produced an invalid
|
||||
# Secrets Manager endpoint in the scheduled janitor.
|
||||
AWS_REGION: us-east-2
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_SECRETS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: 50
|
||||
GRACE_HOURS: 24
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -114,25 +114,17 @@ jobs:
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry:
|
||||
# - schedule: execute (the whole point of an hourly janitor).
|
||||
# - workflow_dispatch: dry-run. Gitea 1.22.6 rejects
|
||||
# workflow_dispatch.inputs, so there is no safe manual
|
||||
# "flip it to execute" toggle in this workflow.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) remains the
|
||||
# second line of defense regardless of trigger.
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
|
||||
- name: Notify on sweep failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "::error::sweep-aws-secrets FAILED — AWS tenant bootstrap secrets may be leaking. Check missing Gitea secrets, staging/prod CP admin tokens, AWS janitor IAM permissions, or the script safety gate."
|
||||
exit 1
|
||||
|
||||
@@ -58,20 +58,14 @@ jobs:
|
||||
python-version: '3.11'
|
||||
- name: Install .gitea script test dependencies
|
||||
run: python -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2'
|
||||
- name: Run scripts/ unittests, if any
|
||||
# Top-level scripts/ tests live alongside their target file. The
|
||||
# runtime packaging tests moved to molecule-ai-workspace-runtime, so
|
||||
# this pass may legitimately find no tests.
|
||||
- name: Run scripts/ unittests (build_runtime_package, ...)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: |
|
||||
set +e
|
||||
python -m unittest discover -t . -p 'test_*.py' -v
|
||||
rc=$?
|
||||
if [ "$rc" -eq 5 ]; then
|
||||
echo "No top-level scripts/ unittest files found; skipping."
|
||||
exit 0
|
||||
fi
|
||||
exit "$rc"
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
name: Block internal-flavored paths
|
||||
|
||||
# Hard CI gate. Internal content (positioning, competitive briefs, sales
|
||||
# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
|
||||
# this public monorepo must never re-acquire those paths. CEO directive
|
||||
# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
|
||||
#
|
||||
# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
|
||||
# briefs into the easiest path their cwd resolves to (root /research,
|
||||
# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
|
||||
# or a stale gitignore line. This workflow is the mechanical backstop.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
# Required for GitHub merge queue: the queue's pre-merge CI run on
|
||||
# `gh-readonly-queue/...` refs needs this check to fire so the queue
|
||||
# gets a real result instead of stalling forever AWAITING_CHECKS.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Block forbidden paths
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base is github.event.pull_request.base.sha,
|
||||
# which may be many commits behind HEAD and therefore absent from the
|
||||
# shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
# For merge_group events the queue's pre-merge ref is a commit on
|
||||
# `gh-readonly-queue/...` whose parent is the queue's base_sha.
|
||||
# That parent isn't part of the queue branch's shallow clone, so
|
||||
# we fetch it explicitly. Mirrors the equivalent step in
|
||||
# secret-scan.yml (#2120) — same shallow-clone bug class.
|
||||
- name: Fetch merge_group base SHA (merge_group events only)
|
||||
if: github.event_name == 'merge_group'
|
||||
run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
|
||||
|
||||
- name: Refuse if forbidden paths appear
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# merge_group has its own base_sha/head_sha; pull_request has
|
||||
# pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
|
||||
MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Paths that must NEVER live in the public monorepo. Add to this
|
||||
# list narrowly — broader patterns belong in .gitignore so day-to-day
|
||||
# docs work isn't accidentally blocked.
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"^research/"
|
||||
"^marketing/"
|
||||
"^docs/marketing/"
|
||||
"^comment-[0-9]+\.json$"
|
||||
"^test-pmm.*\.(txt|md)$"
|
||||
"^tick-reflections.*\.(txt|md)$"
|
||||
".*-temp\.(md|txt)$"
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
merge_group)
|
||||
BASE="$MG_BASE_SHA"
|
||||
HEAD="$MG_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
# Same recovery shape as secret-scan.yml (#2120 — incident
|
||||
# 2026-04-27 06:50Z block-internal-paths exit 128 with
|
||||
# "fatal: bad object <sha>" on staging push).
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check
|
||||
# the entire tree as if every file were new. Slower but
|
||||
# correct on first push or post-fetch-failure recovery.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFFENDING=""
|
||||
for path in $CHANGED; do
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
if echo "$path" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Forbidden internal-flavored paths detected:"
|
||||
printf "$OFFENDING"
|
||||
echo ""
|
||||
echo "These paths belong in molecule-ai/internal, not this public repo."
|
||||
echo "See docs/internal-content-policy.md for canonical locations."
|
||||
echo ""
|
||||
echo "If your file is genuinely public-facing (e.g. a blog post"
|
||||
echo "ready to ship), use one of these alternatives instead:"
|
||||
echo " • Public-bound blog posts: docs/blog/<slug>.md"
|
||||
echo " • Public-bound tutorials: docs/tutorials/<slug>.md"
|
||||
echo " • Public devrel content: docs/devrel/<slug>.md"
|
||||
echo ""
|
||||
echo "If you legitimately need to add a new top-level path that"
|
||||
echo "happens to match a forbidden pattern, edit"
|
||||
echo ".github/workflows/block-internal-paths.yml and update the"
|
||||
echo "FORBIDDEN_PATTERNS list with reviewer signoff."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No forbidden paths in this change."
|
||||
@@ -0,0 +1,320 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
|
||||
# Minimum viable health check: provisions one Hermes workspace on a fresh
|
||||
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
|
||||
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
|
||||
# issue on the next green run.
|
||||
#
|
||||
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
|
||||
# but runs only on provisioning-critical pushes + nightly — this one
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
- cron: '*/30 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_on_failure:
|
||||
description: >-
|
||||
Skip teardown when the canary fails (debugging only). The
|
||||
tenant org + EC2 + CF tunnel + DNS stay alive so an operator
|
||||
can SSM into the workspace EC2 and capture docker logs of the
|
||||
failing claude-code container. REMEMBER to manually delete
|
||||
via DELETE /cp/admin/tenants/<slug> when done so the org
|
||||
doesn't accumulate cost. Only honored on workflow_dispatch;
|
||||
cron runs always tear down (we don't want unattended cron
|
||||
to leak resources).
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
# Needed to open / close the alerting issue.
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
runs-on: ubuntu-latest
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
# failure mode #1 — capturing the actual exception requires
|
||||
# docker logs from the live container.
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain. Operators only need to set ONE of these
|
||||
# secrets; we don't force a choice between them.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
# subsequent failures; auto-close on next green. Comment-on-existing
|
||||
# de-duplicates so a single open issue accumulates the streak —
|
||||
# ops sees one issue with N comments rather than N issues.
|
||||
#
|
||||
# Why no consecutive-failures threshold (e.g., wait 3 runs before
|
||||
# filing): the prior threshold check used
|
||||
# `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
|
||||
# not expose (returns 404). On Gitea Actions the threshold call
|
||||
# ALWAYS failed, breaking the entire alerting step and going days
|
||||
# silent on real regressions (38h+ chronic red on 2026-05-07/08
|
||||
# before this fix; tracked in molecule-core#129). Filing on first
|
||||
# failure is also better UX — we want to know about the first red,
|
||||
# not wait 90 min for it to "count." Real flakes get one issue +
|
||||
# a quick close-on-green; persistent reds accumulate comments.
|
||||
- name: Open issue on failure
|
||||
if: failure()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const runURL = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
// Find an existing open canary issue (stable title match).
|
||||
// If one exists, this isn't a "first failure" — comment and exit.
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary still failing. ${runURL}`,
|
||||
});
|
||||
core.info(`Commented on existing issue #${match.number}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// No open issue yet — file one on this first failure. The
|
||||
// comment-on-existing branch above means subsequent failures
|
||||
// accumulate as comments on this same issue, so we don't
|
||||
// spam new issues per run.
|
||||
const body =
|
||||
`Canary run failed at ${new Date().toISOString()}.\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`This issue auto-closes on the next green canary run. ` +
|
||||
`Consecutive failures add a comment here rather than a new issue.`;
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['canary-staging', 'bug'],
|
||||
});
|
||||
core.info('Opened canary failure issue (first red)');
|
||||
|
||||
- name: Auto-close canary issue on success
|
||||
if: success()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const { data: open } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = open.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary recovered at ${new Date().toISOString()}. Closing.`,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
state: 'closed',
|
||||
});
|
||||
core.info(`Closed recovered canary issue #${match.number}`);
|
||||
}
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
# crosses midnight still cleans up its own slug — see the
|
||||
# 2026-04-26→27 canvas-safety-net incident.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug DELETE with HTTP-code verification. The previous
|
||||
# `... >/dev/null || true` swallowed every failure, so a 5xx
|
||||
# or timeout from CP looked identical to "successfully cleaned
|
||||
# up" and the tenant kept eating ~2 vCPU until the hourly
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,39 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Why a gate, not just discipline: PR #2536 pruned the manifest, but the
|
||||
# cascade list wasn't updated for ~weeks before someone (PR #2556)
|
||||
# noticed during an unrelated audit. During that window, codex never
|
||||
# rebuilt on a runtime publish. A structural gate catches the drift
|
||||
# the same day either file changes.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .github/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh
|
||||
@@ -0,0 +1,58 @@
|
||||
name: Check migration collisions
|
||||
|
||||
# Hard gate (#2341): fails a PR that adds a migration prefix already
|
||||
# claimed by the base branch or another open PR. Caught manually 2026-04-30
|
||||
# during PR #2276 rebase: 044_runtime_image_pins collided with
|
||||
# 044_platform_inbound_secret from RFC #2312. This workflow makes that
|
||||
# check automatic.
|
||||
#
|
||||
# Trigger model: pull_request only — there's no value running this on
|
||||
# pushes to staging or main (those are post-merge; the gate must fire
|
||||
# pre-merge to be useful). Path filter scopes to PRs that actually touch
|
||||
# migrations.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'scripts/ops/check_migration_collisions.py'
|
||||
- '.github/workflows/check-migration-collisions.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# gh pr list/diff need read access to other PRs
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Need history to diff against base ref
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect collisions
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: origin/${{ github.event.pull_request.base.ref }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
# gh CLI uses GH_TOKEN from env
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Ensure the named base ref exists locally. checkout@v4 with
|
||||
# fetch-depth=0 pulls full history, but the explicit fetch is
|
||||
# cheap insurance against form-of-ref differences across runs.
|
||||
#
|
||||
# IMPORTANT: do NOT pass --depth=1 here. The script below uses
|
||||
# `git diff origin/<base>...<head>` (three-dot, merge-base form),
|
||||
# which fails with "fatal: no merge base" if the base ref is
|
||||
# shallow. The auto-promote staging→main PR (#2361) was blocked
|
||||
# by exactly this for ~5h on 2026-04-30 — the depth=1 fetch
|
||||
# overwrote checkout@v4's full-history clone with a shallow tip.
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
python3 scripts/ops/check_migration_collisions.py
|
||||
@@ -0,0 +1,442 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# GitHub merge queue fires `merge_group` for the queue's pre-merge CI run.
|
||||
# Required so the queue gets a real check result instead of a false-green
|
||||
# from the absence of a triggered workflow. Safe to add unconditionally —
|
||||
# the event simply doesn't fire until the queue is enabled on the branch.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
# Cancel in-progress CI runs when a new commit arrives on the same ref.
|
||||
# This prevents stale runs from queuing behind each other. The merge_group
|
||||
# refs (refs/heads/gh-readonly-queue/...) get their own concurrency group
|
||||
# automatically because github.ref differs from the PR ref.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Detect which paths changed so downstream jobs can skip when only
|
||||
# docs/markdown files were modified.
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
platform: ${{ steps.check.outputs.platform }}
|
||||
canvas: ${{ steps.check.outputs.canvas }}
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
scripts: ${{ steps.check.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
run: |
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set by GitHub for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".github/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) is a required check on staging. Always-run + per-step
|
||||
# gating (see Canvas (Next.js) for the rationale and the failure mode
|
||||
# this avoids).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: github.com/molecule-ai/molecule-cli
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
# offenders float to the top. Does NOT fail the build; the hard
|
||||
# gate is the threshold check below. (#1823)
|
||||
run: |
|
||||
echo "=== Per-file coverage (worst first) ==="
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
# 2. Per-file floor — non-test .go files in security-critical
|
||||
# paths with coverage <10% fail the build, UNLESS the file
|
||||
# path is listed in .coverage-allowlist.txt (acknowledged
|
||||
# historical debt with a tracking issue + expiry).
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
# Security-critical paths where a 0%-coverage file is a real risk.
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
|
||||
> /tmp/perfile.txt
|
||||
|
||||
# Build allowlist — paths relative to workspace-server, one per line.
|
||||
# Lines starting with # are comments.
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
WARNED=0
|
||||
for path in "${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !($pct < 10)}" || continue
|
||||
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
WARNED=$((WARNED+1))
|
||||
else
|
||||
echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done < /tmp/perfile.txt
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED security-critical file(s) have <10% test coverage and are"
|
||||
echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
|
||||
echo "workspace provisioning — a 0% file here is the exact gap that let"
|
||||
echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
|
||||
echo " (a) add tests to raise coverage above 10%, or"
|
||||
echo " (b) add the path to .coverage-allowlist.txt with an expiry date"
|
||||
echo " and a tracking issue reference."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canvas (Next.js) — required check, always runs. See platform-build
|
||||
# comment above for the rationale.
|
||||
#
|
||||
# Supersedes the canvas-build-noop pattern attempted in PR #2321: two
|
||||
# jobs sharing `name:` doesn't actually satisfy branch protection
|
||||
# because the SKIPPED check run sibling is treated as not-passed
|
||||
# regardless of how many SUCCESS siblings it has. Verified empirically
|
||||
# on PR #2314 — mergeStateStatus stayed BLOCKED until I collapsed to
|
||||
# a single-job-with-conditional-steps shape.
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
# #1815 — wires coverage into CI so we get a baseline visible on
|
||||
# every PR. No threshold gate yet; thresholds dial in (Step 3, also
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
# Per the inline comment in vitest.config.ts: "first land
|
||||
# observability so we can see the baseline, then dial in
|
||||
# thresholds + a hard gate" — this PR ships the observability half.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
# v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
|
||||
# currently supported on GHES`. Drop this pin when Gitea ships
|
||||
# the v4 protocol (tracked: post-Gitea-1.23 followup).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: canvas-coverage-${{ github.run_id }}
|
||||
path: canvas/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# MCP Server + SDK removed from CI — now in standalone repos:
|
||||
# - github.com/molecule-ai/molecule-mcp-server (npm CI)
|
||||
# - github.com/molecule-ai/molecule-sdk-python (PyPI CI)
|
||||
|
||||
# e2e-api job moved to .github/workflows/e2e-api.yml (issue #458).
|
||||
# It now has workflow-level concurrency (cancel-in-progress: false) so
|
||||
# new pushes queue the E2E run rather than cancelling it at the run level.
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs. See
|
||||
# platform-build for the rationale.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
run: |
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
# Asserts every shell E2E test that calls `mktemp` also installs
|
||||
# an EXIT trap. Catches the /tmp-leak class — a missing trap
|
||||
# silently leaks scratch into CI runners (~10-100KB per run).
|
||||
# See tests/e2e/lint_cleanup_traps.sh for the rule + fix pattern.
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
# Pure-bash unit tests for E2E helper libs (lib/*.sh). These pin
|
||||
# behavior of dispatch logic that — when broken — silently masks as
|
||||
# "Could not resolve authentication method" only after a successful
|
||||
# tenant + workspace provision (PR #2571 incident, 2026-05-03). Add
|
||||
# new self-contained unit tests here as the lib/ directory grows;
|
||||
# tests requiring live CP/tenant credentials belong in the dedicated
|
||||
# e2e-staging-* workflows, not this job.
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: docker-host
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed ✅ — deploy required
|
||||
|
||||
The `publish-canvas-image` workflow is now building a fresh Docker image
|
||||
(`ghcr.io/molecule-ai/canvas:latest`) in the background.
|
||||
|
||||
Once it completes (~3–5 min), apply on the host machine with:
|
||||
```bash
|
||||
cd <runner-workspace>
|
||||
git pull origin main
|
||||
docker compose pull canvas && docker compose up -d canvas
|
||||
```
|
||||
|
||||
If you need to rebuild from local source instead (e.g. testing unreleased
|
||||
changes or a new `NEXT_PUBLIC_*` URL), use:
|
||||
```bash
|
||||
docker compose build canvas && docker compose up -d canvas
|
||||
```
|
||||
BODY
|
||||
printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
|
||||
"$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
|
||||
|
||||
# Gitea has no commit-comments API (no equivalent of
|
||||
# POST /repos/{owner}/{repo}/commits/{commit_sha}/comments).
|
||||
# Write to GITHUB_STEP_SUMMARY instead — both GitHub Actions and
|
||||
# Gitea Actions render this as the workflow run's summary page,
|
||||
# which is where operators look for post-deploy action items.
|
||||
# (#75 / PR-D)
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs. See platform-build
|
||||
# for the rationale.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. Rationale (issue #2790, after
|
||||
# the PR #2766 → PR #2771 cycle): the total floor averages ~6000
|
||||
# lines, so a single MCP file could regress to ~50% with no
|
||||
# complaint as long as other modules compensate. These five
|
||||
# files handle multi-tenant routing + auth + inbox dispatch —
|
||||
# a coverage drop here is the same risk shape as a Go-side
|
||||
# workspace-server token/secrets file dropping below 10%.
|
||||
#
|
||||
# Floor 75% sits below current actuals (80-96%) so this gate is
|
||||
# strictly additive — no existing PR fails. Ratchet plan in
|
||||
# COVERAGE_FLOOR.md.
|
||||
run: |
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text. --include uses fnmatch, and
|
||||
# the leading "*" allows the file to live anywhere under the
|
||||
# workspace root (today they sit at workspace/<name>.py).
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
# Match by top-level path key (e.g. "a2a_tools.py", not
|
||||
# "builtin_tools/a2a_tools.py" — different file at 100%).
|
||||
# The keys in coverage.json are paths relative to the run
|
||||
# cwd (workspace/), so the critical-path entry sits at the
|
||||
# bare basename.
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# SDK + plugin validation moved to standalone repo:
|
||||
# github.com/molecule-ai/molecule-sdk-python
|
||||
@@ -0,0 +1,257 @@
|
||||
name: Continuous synthetic E2E (staging)
|
||||
|
||||
# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
|
||||
# regressions visible only at runtime — schema drift, deployment-pipeline
|
||||
# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
|
||||
#
|
||||
# Why this gate exists:
|
||||
# PR-time CI catches code-level regressions but not deployment-time or
|
||||
# integration-time ones. Today's empirical data:
|
||||
# • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
|
||||
# JSON-RPC parse layer between sender and receiver. Visible only
|
||||
# to a sender exercising the full path.
|
||||
# • RFC #2312 chat upload — landed on staging-branch but never
|
||||
# reached staging tenants because publish-workspace-server-image
|
||||
# was main-only. Caught by manual dogfooding hours after deploy.
|
||||
# Both would have surfaced within 15-20 min of regression if a
|
||||
# continuous synth-E2E was running.
|
||||
#
|
||||
# Cadence: every 20 min (3x/hour). The script is conservatively
|
||||
# bounded at 10 min wall-clock; even on degraded staging it should
|
||||
# finish before the next firing. cron-overlap is guarded by the
|
||||
# concurrency group below.
|
||||
#
|
||||
# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
|
||||
# Plus a fresh tenant provisioned + torn down each run (Railway +
|
||||
# AWS pennies). Negligible.
|
||||
#
|
||||
# Failure handling: when the run fails, the workflow exits non-zero
|
||||
# and GitHub's standard email/notification path fires. Operators
|
||||
# can subscribe to this workflow's failure channel for paging-grade
|
||||
# alerting.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
|
||||
# Prior history: cron was '0,20,40' (2026-05-02) — only :00
|
||||
# ever survived. Bumped to '10,30,50' (2026-05-03) on the
|
||||
# theory that further-from-:00 wins. Empirically 2026-05-04
|
||||
# that ALSO dropped to ~60 min effective cadence (only ~1
|
||||
# schedule fire per hour — see molecule-core#2726). Detection
|
||||
# latency was claimed 20 min, actual 60 min.
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
# Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
|
||||
# lanes (1-3 min away from any other cron). Even with empirically-
|
||||
# observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runtime:
|
||||
description: "Runtime to provision (claude-code = default + cheapest via MiniMax; langgraph = OpenAI-only; hermes = SDK-native path, slower)"
|
||||
required: false
|
||||
default: "claude-code"
|
||||
type: string
|
||||
model_slug:
|
||||
description: "Model id to provision the workspace with (default MiniMax-M2.7-highspeed; e.g. 'sonnet' to test direct Anthropic, 'openai/gpt-4o' for hermes)"
|
||||
required: false
|
||||
default: "MiniMax-M2.7-highspeed"
|
||||
type: string
|
||||
keep_org:
|
||||
description: "Skip teardown for post-mortem debugging (only manual dispatch — never set this for cron runs)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
# history. If you want auto-issue-on-fail, add a follow-up step that
|
||||
# uses gh issue create gated on `if: failure()`. Keeping the surface
|
||||
# minimal until that's actually wanted.
|
||||
|
||||
# Serialize so two firings can never overlap. Cron firing every 20 min
|
||||
# but scripts conservatively bounded at 10 min — overlap shouldn't
|
||||
# happen in steady state, but if a run hangs we don't want N more
|
||||
# stacking up.
|
||||
concurrency:
|
||||
group: continuous-synth-e2e
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
synth:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
# ssm-agent) runs from raw Ubuntu on every boot — none of it is
|
||||
# pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
|
||||
# across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
|
||||
# latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
|
||||
# A 12min budget leaves only ~2min for the workspace (which needs
|
||||
# ~3.5min for claude-code cold boot) on slow-apt days, blowing the
|
||||
# budget. 20min absorbs the worst tenant tail so the workspace probe
|
||||
# gets the full ~7min it needs even on a slow apt day. Real fix:
|
||||
# pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
# exhaustion class that took the canary down 2026-05-03 (#265).
|
||||
# Operators can pick langgraph / hermes via workflow_dispatch
|
||||
# when they specifically need to exercise the OpenAI or SDK-
|
||||
# native paths.
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
E2E_PROVISION_TIMEOUT_SECS: '600'
|
||||
# Slug suffix — namespaced "synth-" so these runs are
|
||||
# distinguishable from PR-driven runs in CP admin.
|
||||
E2E_RUN_ID: synth-${{ github.run_id }}
|
||||
# Forced false for cron; respected for manual dispatch
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
# tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
|
||||
# which key is present — MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so operators can dispatch with
|
||||
# E2E_RUNTIME=langgraph or =hermes and still have a working
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
run: |
|
||||
# Hard-fail on missing secret REGARDLESS of trigger. Previously
|
||||
# this step soft-skipped on workflow_dispatch via `exit 0`, but
|
||||
# `exit 0` only ends the STEP — subsequent steps still ran with
|
||||
# the empty secret, the synth script fell through to the wrong
|
||||
# SECRETS_JSON branch, and the canary failed 5 min later with a
|
||||
# confusing "Agent error (Exception)" instead of the clean
|
||||
# "secret missing" message at the top. Caught 2026-05-04 by
|
||||
# dispatched run 25296530706: claude-code + missing MINIMAX
|
||||
# silently used OpenAI keys but kept model=MiniMax-M2.7, then
|
||||
# the workspace 401'd against MiniMax once it tried to call.
|
||||
# Fix: exit 1 in both cron and dispatch paths. Operators who
|
||||
# want to verify a YAML change without setting up the secret
|
||||
# can read the verify-secrets step's stderr — the failure is
|
||||
# itself the verification signal.
|
||||
if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
# langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_KEY).
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install required tools
|
||||
run: |
|
||||
# The script depends on jq + curl (already on ubuntu-latest)
|
||||
# and python3 (likewise). Verify they're all present so we
|
||||
# fail fast on a runner image regression rather than mid-script.
|
||||
for cmd in jq curl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "::error::required tool '$cmd' not on PATH — runner image regression?"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
- name: Run synthetic E2E
|
||||
# The script handles its own teardown via EXIT trap; even on
|
||||
# failure (timeout, assertion), the org is deprovisioned and
|
||||
# leaks are reported. Exit code propagates from the script.
|
||||
run: |
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
- name: Failure summary
|
||||
# Runs only on failure. Adds a job summary so the workflow run
|
||||
# page shows a quick "what happened" instead of forcing readers
|
||||
# to scroll through script output.
|
||||
if: failure()
|
||||
run: |
|
||||
{
|
||||
echo "## Continuous synth E2E failed"
|
||||
echo ""
|
||||
echo "**Run ID:** ${{ github.run_id }}"
|
||||
echo "**Trigger:** ${{ github.event_name }}"
|
||||
echo "**Runtime:** ${E2E_RUNTIME}"
|
||||
echo "**Slug:** synth-${{ github.run_id }}"
|
||||
echo ""
|
||||
echo "### What this means"
|
||||
echo ""
|
||||
echo "Staging just regressed on a path that previously worked. Likely classes:"
|
||||
echo "- Schema mismatch between sender and receiver (#2345 class)"
|
||||
echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
|
||||
echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
|
||||
echo "- Staging-CP env var rotation"
|
||||
echo ""
|
||||
echo "### Next steps"
|
||||
echo ""
|
||||
echo "1. Check the script output above for the assertion that failed"
|
||||
echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
|
||||
echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
|
||||
echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,307 @@
|
||||
name: E2E API Smoke Test
|
||||
# Extracted from ci.yml so workflow-level concurrency can protect this job
|
||||
# from run-level cancellation (issue #458).
|
||||
#
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always FIRES on push/pull_request to staging+main. Real work is gated
|
||||
# per-step on `needs.detect-changes.outputs.api` — when paths under
|
||||
# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `E2E API Smoke Test` check, satisfying branch protection without
|
||||
# spending CI cycles. See the in-job comment on the `e2e-api` job for
|
||||
# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
|
||||
# PR #2264 incident that drove the consolidation.
|
||||
#
|
||||
# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
|
||||
# -------------------------------------------------------------------
|
||||
# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
|
||||
# Gitea act_runner runs with `container.network: host` (operator host
|
||||
# `/opt/molecule/runners/config.yaml`), which means:
|
||||
#
|
||||
# * Two concurrent runs both try to bind their `-p 15432:5432` /
|
||||
# `-p 16379:6379` host ports — the second postgres/redis FATALs
|
||||
# with `Address in use` and `docker run` returns exit 125 with
|
||||
# `Conflict. The container name "/molecule-ci-postgres" is already
|
||||
# in use by container ...`. Verified in run a7/2727 on 2026-05-07.
|
||||
# * The fixed container names `molecule-ci-postgres` / `-redis` (the
|
||||
# pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
|
||||
# `docker rm -f` at the start of the second job KILLS the first
|
||||
# job's still-running postgres/redis.
|
||||
#
|
||||
# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
|
||||
# platform-server is a Go binary on the host, not a containerised
|
||||
# step):
|
||||
#
|
||||
# 1. Unique container names per run:
|
||||
# pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
|
||||
# same run_id.
|
||||
# 2. Ephemeral host port per run (`-p 0:5432`), then read the actual
|
||||
# bound port via `docker port` and export DATABASE_URL/REDIS_URL
|
||||
# pointing at it. No fixed host-port → no port collision.
|
||||
# 3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
|
||||
# the original flake fixed in #92 and the script's still IPv6-
|
||||
# enabled.
|
||||
# 4. `if: always()` cleanup so containers don't leak when test steps
|
||||
# fail.
|
||||
#
|
||||
# Issue #94 items #2 + #3 (also fixed here):
|
||||
# * Pre-pull `alpine:latest` so the platform-server's provisioner
|
||||
# (`internal/handlers/container_files.go`) can stand up its
|
||||
# ephemeral token-write helper without a daemon.io round-trip.
|
||||
# * Create `molecule-core-net` bridge network if missing so the
|
||||
# provisioner's container.HostConfig {NetworkMode: ...} attach
|
||||
# succeeds.
|
||||
# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
|
||||
# template-registry resolution issue (ADR-002 / local-build mode) and
|
||||
# belongs in a separate change that touches workspace-server, not
|
||||
# this workflow file.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
|
||||
# same auto-promote-staging brittleness as e2e-staging-canvas — back-
|
||||
# to-back staging pushes share refs/heads/staging, so the older push's
|
||||
# queued run gets cancelled when a newer push lands. Auto-promote-
|
||||
# staging then sees `completed/cancelled` for the older SHA and stays
|
||||
# put; the newer SHA's gates may eventually save the day, but if the
|
||||
# newer push gets cancelled too, we deadlock.
|
||||
#
|
||||
# See e2e-staging-canvas.yml's identical concurrency block for the full
|
||||
# rationale and the 2026-04-28 incident reference.
|
||||
group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
api:
|
||||
- 'workspace-server/**'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/e2e-api.yml'
|
||||
- id: decide
|
||||
# Always run real work for manual dispatch — no diff context to
|
||||
# filter against and ops dispatching this expects the suite to
|
||||
# actually exercise the platform.
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=${{ steps.filter.outputs.api }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
# on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
|
||||
# check run for every job that matches `name:`, and a job-level
|
||||
# `if: false` produces a SKIPPED check run. Branch protection treats
|
||||
# all check runs with a matching context name on the latest commit as a
|
||||
# SET — any SKIPPED in the set fails the required-check eval, even with
|
||||
# SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
|
||||
# 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
|
||||
# promotion despite all real work succeeding. Collapsing to a single
|
||||
# always-running job with conditional steps emits exactly one SUCCESS
|
||||
# check run regardless of paths filter — branch-protection-clean.
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
# Unique per-run container names so concurrent runs on the host-
|
||||
# network act_runner don't collide on name OR port.
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
|
||||
# same run_id. PORT is set later (after docker port lookup) since
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
PORT: "8080"
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
|
||||
echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Provisioner uses alpine:latest for ephemeral token-write
|
||||
# containers (workspace-server/internal/handlers/container_files.go).
|
||||
# Pre-pull so the first provision in test_api.sh doesn't race
|
||||
# the daemon's pull cache. Idempotent — `docker pull` is a no-op
|
||||
# when the image is already present.
|
||||
docker pull alpine:latest >/dev/null
|
||||
# Provisioner attaches workspace containers to
|
||||
# molecule-core-net (workspace-server/internal/provisioner/
|
||||
# provisioner.go::DefaultNetwork). The bridge already exists on
|
||||
# the operator host's docker daemon — `network create` is
|
||||
# idempotent via `|| true`.
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Defensive cleanup — only matches THIS run's container name,
|
||||
# so it cannot kill a sibling run's postgres. (Pre-fix the
|
||||
# name was static and this rm hit other runs' containers.)
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
# `-p 0:5432` requests an ephemeral host port; we read it back
|
||||
# below and export DATABASE_URL.
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
# Resolve the host-side port assignment. `docker port` prints
|
||||
# `0.0.0.0:NNNN` (and on host-net runners may also print an
|
||||
# IPv6 line — take the first IPv4 line).
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
# Fallback: any first line. Some Docker versions print only
|
||||
# one line.
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker port "$PG_CONTAINER" 5432/tcp || true
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
# 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Postgres host port: ${PG_PORT}"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker port "$REDIS_CONTAINER" 6379/tcp || true
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Redis host port: ${REDIS_PORT}"
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
# DATABASE_URL + REDIS_URL exported by the start-postgres /
|
||||
# start-redis steps point at this run's per-run host ports.
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
- name: Assert migrations applied
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
|
||||
if [ "$tables" != "1" ]; then
|
||||
echo "::error::Migrations did not apply"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
- name: Run notify-with-attachments E2E
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_notify_attachments_e2e.sh
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
- name: Run poll-mode chat upload E2E (RFC #2891)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.api == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
# always() so containers don't leak when test steps fail. The
|
||||
# cleanup is best-effort: if the container is already gone
|
||||
# (e.g. concurrent rerun race), don't fail the job.
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -0,0 +1,216 @@
|
||||
name: E2E Staging Canvas (Playwright)
|
||||
|
||||
# Playwright test suite that provisions a fresh staging org per run and
|
||||
# verifies every workspace-panel tab renders without crashing. Complements
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
|
||||
on:
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always fires on push/pull_request; real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `Canvas tabs E2E` check, satisfying branch protection without
|
||||
# spending CI cycles. See e2e-api.yml for the rationale on why this
|
||||
# is a single job rather than two-jobs-sharing-name.
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
# global group made auto-promote-staging brittle: when a staging push
|
||||
# queued behind an in-flight run and a third entrant (a PR run, a
|
||||
# follow-on push) entered the group, the staging push got cancelled —
|
||||
# leaving auto-promote-staging looking at `completed/cancelled` for a
|
||||
# required gate and refusing to advance main. Observed 2026-04-28
|
||||
# 23:51-23:53 on staging tip 3f99fede.
|
||||
#
|
||||
# The original intent of the global group was to throttle parallel
|
||||
# E2E provisions (each spins a fresh EC2). At our scale that throttle
|
||||
# isn't worth the correctness cost — fresh-org-per-run isolates the
|
||||
# state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
|
||||
# is rounding error vs. the cost of a stuck pipeline.
|
||||
#
|
||||
# Per-SHA still dedupes accidental double-triggers for the SAME SHA.
|
||||
# It does NOT cancel obsolete-PR-version runs on force-push; that
|
||||
# wasted CI is acceptable given the alternative is losing staging-tip
|
||||
# data that auto-promote-staging needs.
|
||||
group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
canvas:
|
||||
- 'canvas/**'
|
||||
- '.github/workflows/e2e-staging-canvas.yml'
|
||||
- id: decide
|
||||
# Always run real tests for manual dispatch and the weekly cron —
|
||||
# both exist precisely to exercise the suite, regardless of diff.
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "canvas=${{ steps.filter.outputs.canvas }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `Canvas tabs E2E`. Real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
|
||||
# rationale — same path-filter check-name parity issue blocked PR #2264
|
||||
# (staging→main) on 2026-04-29 because branch protection treats matching-
|
||||
# name check runs as a SET, and any SKIPPED member fails the eval.
|
||||
playwright:
|
||||
needs: detect-changes
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
|
||||
env:
|
||||
CANVAS_E2E_STAGING: '1'
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: |
|
||||
echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
|
||||
echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Set up Node
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Install canvas deps
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
timeout-minutes: 10
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright test --config=playwright.staging.config.ts
|
||||
|
||||
- name: Upload Playwright report on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement (see ci.yml upload step for the canonical error
|
||||
# cite). Drop this pin when Gitea ships the v4 protocol.
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-report-staging
|
||||
path: canvas/playwright-report-staging/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload screenshots on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-screenshots
|
||||
path: canvas/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Safety-net teardown — fires only when Playwright's globalTeardown
|
||||
# didn't (worker crash, runner cancel). Reads the slug from
|
||||
# canvas/.playwright-staging-state.json (written by staging-setup
|
||||
# as its first action, before any CP call) and deletes only that
|
||||
# slug.
|
||||
#
|
||||
# Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
|
||||
# orgs to compensate for setup-crash-before-state-file-write. That
|
||||
# over-aggressive cleanup raced concurrent canvas-E2E runs and
|
||||
# poisoned each other's tenants — observed 2026-04-30 when three
|
||||
# real-test runs killed each other mid-test, surfacing as
|
||||
# `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
|
||||
# DNS record. Pattern-sweep removed; setup now writes the state
|
||||
# file before any CP work, so the slug is always recoverable.
|
||||
- name: Teardown safety net
|
||||
if: always() && needs.detect-changes.outputs.canvas == 'true'
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
STATE_FILE=".playwright-staging-state.json"
|
||||
if [ ! -f "$STATE_FILE" ]; then
|
||||
echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
|
||||
exit 0
|
||||
fi
|
||||
slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
|
||||
if [ -z "$slug" ]; then
|
||||
echo "::warning::State file present but slug missing; nothing to clean up."
|
||||
exit 0
|
||||
fi
|
||||
echo "Deleting orphan tenant: $slug"
|
||||
# Verify HTTP 2xx instead of `>/dev/null || true` swallowing
|
||||
# failures. A 5xx or timeout previously looked identical to
|
||||
# success, leaving the tenant alive for up to ~45 min until
|
||||
# sweep-stale-e2e-orgs caught it. Surface failures as
|
||||
# workflow warnings naming the slug. Don't `exit 1` — a single
|
||||
# cleanup miss shouldn't fail-flag the canvas test when the
|
||||
# actual smoke check passed; the sweeper is the safety net.
|
||||
# See molecule-controlplane#420.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,184 @@
|
||||
name: E2E Staging External Runtime
|
||||
|
||||
# Regression for the four/five workspaces.status=awaiting_agent transitions
|
||||
# that silently failed in production for five days before migration 046
|
||||
# extended the workspace_status enum (see
|
||||
# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
|
||||
#
|
||||
# Why this is its own workflow (not folded into e2e-staging-saas.yml):
|
||||
# - The full-saas harness defaults to runtime=hermes, never exercises
|
||||
# external-runtime. Adding an `external` parameter to that script
|
||||
# would force every push to staging through both lifecycles in
|
||||
# series, doubling the EC2 cold-start budget.
|
||||
# - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
|
||||
# window, 90s default + sweep interval), which we wait through
|
||||
# deliberately. Folding it into hermes would make the long path
|
||||
# even longer.
|
||||
# - It can run in parallel with the hermes E2E since both create
|
||||
# fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
|
||||
# `e2e-...`).
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to staging when any source affecting external runtime,
|
||||
# hibernation, or the migration set changes.
|
||||
# - PR review for the same set.
|
||||
# - Manual workflow_dispatch.
|
||||
# - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
|
||||
# 30 min after e2e-staging-saas.yml's 07:00 UTC cron).
|
||||
#
|
||||
# Concurrency: serialized so two staging pushes don't fight for the
|
||||
# same EC2 quota window. cancel-in-progress=false so a half-rolled
|
||||
# tenant always finishes its teardown.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.github/workflows/e2e-staging-external.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.github/workflows/e2e-staging-external.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_org:
|
||||
description: "Skip teardown for debugging (only via manual dispatch)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
stale_wait_secs:
|
||||
description: "Seconds to wait for the heartbeat-staleness sweep (default 180 = 90s window + 90s buffer)"
|
||||
required: false
|
||||
default: "180"
|
||||
schedule:
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-external
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
e2e-staging-external:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
# Schedule + push triggers must hard-fail when the token is
|
||||
# missing — silent skip would mask infra rot. Manual dispatch
|
||||
# gets the same hard-fail; an operator running this on a fork
|
||||
# without secrets configured needs to know up-front.
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run external-runtime E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_external_runtime.sh
|
||||
|
||||
# Mirror the e2e-staging-saas.yml safety net: if the runner is
|
||||
# cancelled (e.g. concurrent staging push), the test script's
|
||||
# EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
|
||||
# *this* run id.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
|
||||
# so concurrent runs and unrelated dev probes are not touched.
|
||||
# Sweep today AND yesterday so a midnight-crossing run still
|
||||
# cleans up its own slug.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if not run_id:
|
||||
# Without a run id we cannot scope safely; bail rather
|
||||
# than risk deleting unrelated tenants.
|
||||
sys.exit(0)
|
||||
prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
|
||||
for o in d.get('orgs', []):
|
||||
s = o.get('slug', '')
|
||||
if s.startswith(prefixes) and o.get('status') != 'purged':
|
||||
print(s)
|
||||
" 2>/dev/null)
|
||||
if [ -n "$orgs" ]; then
|
||||
echo "Safety-net sweep: deleting leftover orgs:"
|
||||
echo "$orgs"
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# `>/dev/null 2>&1` previously hid every failure; surface
|
||||
# non-2xx as workflow warnings so the run page names what
|
||||
# leaked. Sweeper catches the rest within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
else
|
||||
echo "Safety-net sweep: no leftover orgs to clean."
|
||||
fi
|
||||
@@ -0,0 +1,246 @@
|
||||
name: E2E Staging SaaS (full lifecycle)
|
||||
|
||||
# Dedicated workflow that provisions a fresh staging org per run, exercises
|
||||
# the full workspace lifecycle (register → heartbeat → A2A → delegation →
|
||||
# HMA memory → activity → peers), then tears down and asserts leak-free.
|
||||
#
|
||||
# Why a separate workflow (not folded into ci.yml):
|
||||
# - The run takes ~25-35 min (EC2 boot + cloudflared DNS + provision sweeps +
|
||||
# agent bootstrap), way too slow for every PR.
|
||||
# - Needs its own concurrency group so two pushes don't fight over the
|
||||
# same staging org slug prefix.
|
||||
# - Has its own required secrets (session cookie, admin token) that most
|
||||
# PRs don't need to read.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to main (regression guard)
|
||||
# - workflow_dispatch (manual re-run from UI)
|
||||
# - Nightly cron (catches drift even when no pushes land)
|
||||
# - Changes to any provisioning-critical file under PR review (opt-in
|
||||
# via the same paths watcher that e2e-api.yml uses)
|
||||
|
||||
on:
|
||||
# Trunk-based (Phase 3 of internal#81): main is the only branch.
|
||||
# Previously this fired on staging push too because staging was a
|
||||
# superset of main and ran the gate ahead of auto-promote; with no
|
||||
# staging branch, main is where E2E gates the deploy.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.github/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.github/workflows/e2e-staging-saas.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runtime:
|
||||
description: "Runtime to test (claude-code [default, MiniMax] | hermes [OpenAI] | langgraph [OpenAI])"
|
||||
required: false
|
||||
default: "claude-code"
|
||||
keep_org:
|
||||
description: "Skip teardown for debugging (only use via manual dispatch!)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
|
||||
# Cloudflare API regressions, etc. even on quiet days.
|
||||
- cron: '0 7 * * *'
|
||||
|
||||
# Serialize: staging has a finite per-hour org creation quota. Two pushes
|
||||
# landing in quick succession should queue, not race. `cancel-in-progress:
|
||||
# false` mirrors e2e-api.yml — GitHub would otherwise cancel the running
|
||||
# teardown step and leave orphan EC2s.
|
||||
concurrency:
|
||||
group: e2e-staging-saas
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
e2e-staging-saas:
|
||||
name: E2E Staging SaaS
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
# Single admin-bearer secret drives provision + tenant-token
|
||||
# retrieval + teardown. Configure in
|
||||
# Settings → Secrets and variables → Actions → Repository secrets.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the full-lifecycle E2E red on every provisioning-critical push).
|
||||
# claude-code template's `minimax` provider routes
|
||||
# ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
|
||||
# MINIMAX_API_KEY at boot — separate billing account so an
|
||||
# OpenAI quota collapse no longer wedges the gate. Mirrors the
|
||||
# canary-staging.yml + continuous-synth-e2e.yml migrations.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
|
||||
# exercise the OpenAI path.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the model when running on the default claude-code path —
|
||||
# the per-runtime default ("sonnet") routes to direct Anthropic
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per #2578's lesson — empty key
|
||||
# silently falls through to the wrong SECRETS_JSON branch and
|
||||
# produces a confusing auth error 5 min later instead of the
|
||||
# clean "secret missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run full-lifecycle E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Belt-and-braces teardown: the test script itself installs a trap
|
||||
# for EXIT/INT/TERM, but if the GH runner itself is cancelled (e.g.
|
||||
# someone pushes a new commit and workflow concurrency is set to
|
||||
# cancel), the trap may not fire. This `always()` step runs even on
|
||||
# cancellation and attempts the delete a second time. The admin
|
||||
# DELETE endpoint is idempotent so double-invoking is safe.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
# Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
|
||||
# nuke them. Catches the case where the script died before
|
||||
# exporting its slug.
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# ONLY sweep slugs from *this* CI run. Previously the filter was
|
||||
# f'e2e-{today}-' which stomped on parallel CI runs AND any manual
|
||||
# E2E probes a dev was running against staging (incident 2026-04-21
|
||||
# 15:02Z: this workflow's safety net deleted an unrelated manual
|
||||
# run's tenant 1s after it hit 'running').
|
||||
# Sweep both today AND yesterday's UTC dates so a run that crosses
|
||||
# midnight still matches its own slug — see the 2026-04-26→27
|
||||
# canvas-safety-net incident for the same bug class.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-{d}-{run_id}-' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('instance_status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug verified DELETE (was `>/dev/null || true` — see
|
||||
# molecule-controlplane#420). Surface non-2xx as a workflow
|
||||
# warning naming the leaked slug; don't exit 1 (sweeper is
|
||||
# the safety net within ~45 min).
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
echo "Safety-net teardown: $slug"
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,171 @@
|
||||
name: E2E Staging Sanity (leak-detection self-check)
|
||||
|
||||
# Periodic assertion that the teardown safety nets in e2e-staging-saas
|
||||
# and canary-staging actually work. Runs the E2E harness with
|
||||
# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
|
||||
# the org is provisioned. The workspace-provision step then fails, the
|
||||
# script exits non-zero, and the EXIT trap + workflow always()-step
|
||||
# must still tear down cleanly.
|
||||
#
|
||||
# A green run means:
|
||||
# - The script exited non-zero (intentional failure caught)
|
||||
# - The trap fired teardown
|
||||
# - The leak-detection poll found zero orphan orgs
|
||||
#
|
||||
# A red run means the teardown path itself is broken — act on this the
|
||||
# same way you'd act on a canary failure (the whole E2E safety net is
|
||||
# compromised until it's fixed).
|
||||
#
|
||||
# Cadence: once a week, Monday 06:00 UTC. Drift-slow, not per-PR — the
|
||||
# teardown path rarely changes, and a weekly heartbeat is enough to
|
||||
# catch silent regressions in cleanup code paths.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
# Shares the group with canary + full so they don't collide on
|
||||
# staging org-create quota.
|
||||
group: e2e-staging-sanity
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sanity:
|
||||
name: Intentional-failure teardown sanity
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_MODE: canary # lean lifecycle; we only need the org to exist
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
E2E_INTENTIONAL_FAILURE: "1"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken (token not being
|
||||
# poisoned correctly, or the harness silently recovered).
|
||||
- name: Run harness — expecting exit !=0
|
||||
id: harness
|
||||
run: |
|
||||
set +e
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
rc=$?
|
||||
echo "harness_rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# The only acceptable outcomes:
|
||||
# 1 — harness failed mid-run, teardown ran, leak-check passed
|
||||
# (exit 4 means teardown left a leak — that's the real bug
|
||||
# this sanity check exists to catch)
|
||||
if [ "$rc" = "1" ]; then
|
||||
echo "✓ Harness failed as expected (rc=1); teardown trap ran, leak-check passed"
|
||||
exit 0
|
||||
elif [ "$rc" = "0" ]; then
|
||||
echo "::error::Harness succeeded under E2E_INTENTIONAL_FAILURE=1 — the poisoning path is broken"
|
||||
exit 1
|
||||
elif [ "$rc" = "4" ]; then
|
||||
echo "::error::LEAK DETECTED (rc=4) — teardown failed to clean up the org. Safety net broken."
|
||||
exit 4
|
||||
else
|
||||
echo "::error::Unexpected rc=$rc — neither clean-failure nor leak. Investigate harness."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Open issue if safety net is broken
|
||||
if: failure()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = "🚨 E2E teardown safety net broken";
|
||||
const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const body =
|
||||
`The weekly sanity run (E2E_INTENTIONAL_FAILURE=1) did not exit ` +
|
||||
`as expected. This means one of:\n` +
|
||||
` - poisoning didn't actually cause failure (test harness regression), OR\n` +
|
||||
` - teardown left an orphan org (leak detection caught a real bug)\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`This is higher priority than a canary failure — the whole ` +
|
||||
`E2E safety net can't be trusted until this is resolved.`;
|
||||
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'e2e-safety-net',
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Still broken. ${runURL}`,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['e2e-safety-net', 'bug', 'priority-high'],
|
||||
});
|
||||
}
|
||||
|
||||
# Belt-and-braces: if teardown left anything behind, nuke it here
|
||||
# so we don't bleed staging quota. Different label from the
|
||||
# always()-steps in the other workflows so sanity-only orgs get
|
||||
# cleaned up by sanity runs.
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
today = __import__('datetime').date.today().strftime('%Y%m%d')
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# Failures surface as workflow warnings; the sweeper is the
|
||||
# safety net within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,251 @@
|
||||
name: Handlers Postgres Integration
|
||||
|
||||
# Real-Postgres integration tests for workspace-server/internal/handlers/.
|
||||
# Triggered on every PR/push that touches the handlers package.
|
||||
#
|
||||
# Why this workflow exists
|
||||
# ------------------------
|
||||
# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
|
||||
# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
|
||||
# depend on the row state AFTER the SQL runs. The result_preview-lost
|
||||
# bug shipped to staging in PR #2854 because every unit test was
|
||||
# satisfied with "an UPDATE statement fired" — none verified the row's
|
||||
# preview field actually landed. The local-postgres E2E that retrofit
|
||||
# self-review caught it took 2 minutes to set up and would have caught
|
||||
# the bug at PR-time.
|
||||
#
|
||||
# Why this workflow does NOT use `services: postgres:` (Class B fix)
|
||||
# ------------------------------------------------------------------
|
||||
# Our act_runner config has `container.network: host` (operator host
|
||||
# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
|
||||
# the job container AND every service container. With host-net, two
|
||||
# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
|
||||
# second postgres FATALs with `could not create any TCP/IP sockets:
|
||||
# Address in use`, and Docker auto-removes it (act_runner sets
|
||||
# AutoRemove:true on service containers). By the time the migrations
|
||||
# step runs `psql`, the postgres container is gone, hence
|
||||
# `Connection refused` then `failed to remove container: No such
|
||||
# container` at cleanup time.
|
||||
#
|
||||
# Per-job `container.network` override is silently ignored by
|
||||
# act_runner — `--network and --net in the options will be ignored.`
|
||||
# appears in the runner log. Documented constraint.
|
||||
#
|
||||
# So we sidestep `services:` entirely. The job container still uses
|
||||
# host-net (inherited from runner config; required for cache server
|
||||
# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
|
||||
# postgres on the existing `molecule-core-net` bridge with a
|
||||
# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
|
||||
# read its bridge IP via `docker inspect`. A host-net job container
|
||||
# can reach a bridge-net container directly via the bridge IP (verified
|
||||
# manually on operator host 2026-05-08).
|
||||
#
|
||||
# Trade-offs vs. the original `services:` shape:
|
||||
# + No host-port collision; N parallel runs share the bridge cleanly
|
||||
# + `if: always()` cleanup runs even on test-step failure
|
||||
# - One more step in the workflow (+~3 lines)
|
||||
# - Requires `molecule-core-net` to exist on the operator host
|
||||
# (it does; declared in docker-compose.yml + docker-compose.infra.yml)
|
||||
#
|
||||
# Class B Hongming-owned CICD red sweep, 2026-05-08.
|
||||
#
|
||||
# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
handlers: ${{ steps.filter.outputs.handlers }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
handlers:
|
||||
- 'workspace-server/internal/handlers/**'
|
||||
- 'workspace-server/internal/wsauth/**'
|
||||
- 'workspace-server/migrations/**'
|
||||
- '.github/workflows/handlers-postgres-integration.yml'
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
# paths filter. See ci.yml's Platform (Go) for the same shape.
|
||||
integration:
|
||||
name: Handlers Postgres Integration
|
||||
needs: detect-changes
|
||||
runs-on: docker-host
|
||||
env:
|
||||
# Unique name per run so concurrent jobs don't collide on the
|
||||
# bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
|
||||
# workflow_dispatch reruns of the same run_id.
|
||||
PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# Bridge network already exists on the operator host (declared
|
||||
# in docker-compose.yml + docker-compose.infra.yml).
|
||||
PG_NETWORK: molecule-core-net
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.detect-changes.outputs.handlers != 'true'
|
||||
working-directory: .
|
||||
run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Start sibling Postgres on bridge network
|
||||
working-directory: .
|
||||
run: |
|
||||
# Sanity: the bridge network must exist on the operator host.
|
||||
# Hard-fail loud if it doesn't — easier to spot than a silent
|
||||
# auto-create that diverges from the rest of the stack.
|
||||
if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
|
||||
echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If a stale container with the same name exists (rerun on
|
||||
# the same run_id), wipe it first.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
|
||||
docker run -d \
|
||||
--name "${PG_NAME}" \
|
||||
--network "${PG_NETWORK}" \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 5s \
|
||||
--health-timeout 5s \
|
||||
--health-retries 10 \
|
||||
-e POSTGRES_PASSWORD=test \
|
||||
-e POSTGRES_DB=molecule \
|
||||
postgres:15-alpine >/dev/null
|
||||
|
||||
# Read back the bridge IP. Always present immediately after
|
||||
# `docker run -d` for bridge networks.
|
||||
PG_HOST=$(docker inspect "${PG_NAME}" \
|
||||
--format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
|
||||
if [ -z "${PG_HOST}" ]; then
|
||||
echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
|
||||
docker logs "${PG_NAME}" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
|
||||
echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Started ${PG_NAME} at ${PG_HOST}:5432"
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Apply migrations to Postgres service
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
# Wait for postgres to actually accept connections. Docker's
|
||||
# health-cmd handles container-side readiness, but the wire
|
||||
# to the bridge IP is best-tested with pg_isready directly.
|
||||
for i in {1..15}; do
|
||||
if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
|
||||
echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
|
||||
done
|
||||
|
||||
# Apply every .up.sql in lexicographic order with
|
||||
# ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
|
||||
# blocking the suite. This handles the current schema state
|
||||
# where a few historical migrations (e.g. 017_memories_fts_*)
|
||||
# depend on tables that were later renamed/dropped and so
|
||||
# cannot replay from scratch. The migrations that DO succeed
|
||||
# land their tables, which is sufficient for the integration
|
||||
# tests in handlers/.
|
||||
#
|
||||
# Why not maintain a curated allowlist: every new migration
|
||||
# touching a handlers/-tested table would have to update this
|
||||
# workflow. With apply-all-or-skip, a future migration that
|
||||
# adds a column to delegations runs automatically (its base
|
||||
# table 049_delegations.up.sql already succeeded above it in
|
||||
# the order). Operators only need to revisit this if the
|
||||
# migration chain becomes legitimately replayable end-to-end.
|
||||
#
|
||||
# Per-migration result is logged so a failed migration that
|
||||
# SHOULD have been replayable surfaces in the CI log instead
|
||||
# of silently failing.
|
||||
# Apply both *.sql (legacy, lives next to its module) and
|
||||
# *.up.sql (newer up/down convention) in a single
|
||||
# lexicographically-sorted pass. Excluding *.down.sql so the
|
||||
# newest-naming-convention pairs don't undo themselves mid-run.
|
||||
# Pre-#149-followup this loop only globbed *.up.sql, which
|
||||
# silently skipped 001_workspaces.sql + 009_activity_logs.sql
|
||||
# — fine while no integration test depended on those tables,
|
||||
# not fine once a cross-table atomicity test came in.
|
||||
set +e
|
||||
for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
|
||||
if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
|
||||
-f "$migration" >/dev/null 2>&1; then
|
||||
echo "✓ $(basename "$migration")"
|
||||
else
|
||||
echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
|
||||
# Sanity: the delegations + workspaces + activity_logs tables
|
||||
# MUST exist for the integration tests to be meaningful. Hard-
|
||||
# fail if any didn't land — that would be a real regression we
|
||||
# want loud.
|
||||
for tbl in delegations workspaces activity_logs pending_uploads; do
|
||||
if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
|
||||
-c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
|
||||
| grep -q 1; then
|
||||
echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ $tbl table present"
|
||||
done
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Run integration tests
|
||||
run: |
|
||||
# INTEGRATION_DB_URL is exported by the start-postgres step;
|
||||
# points at the per-run bridge IP, not 127.0.0.1, so concurrent
|
||||
# workflow runs don't fight over a host-net 5432 port.
|
||||
go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
|
||||
|
||||
- if: failure() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Diagnostic dump on failure
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
echo "::group::postgres container status"
|
||||
docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
|
||||
docker logs "${PG_NAME}" 2>&1 | tail -50 || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::delegations table state"
|
||||
psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
|
||||
echo "::endgroup::"
|
||||
|
||||
- if: always() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Stop sibling Postgres
|
||||
working-directory: .
|
||||
run: |
|
||||
# always() so containers don't leak when migrations or tests
|
||||
# fail. The cleanup is best-effort: if the container is
|
||||
# already gone (e.g. concurrent rerun race), don't fail the job.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
echo "Cleaned up ${PG_NAME}"
|
||||
@@ -0,0 +1,248 @@
|
||||
name: Harness Replays
|
||||
|
||||
# Boots tests/harness (production-shape compose topology with TenantGuard,
|
||||
# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
|
||||
# every replay under tests/harness/replays/. Fails the PR if any replay
|
||||
# fails.
|
||||
#
|
||||
# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
|
||||
# a public route in router.go but forgot to add it to TenantGuard's
|
||||
# allowlist. The handler-level test in buildinfo_test.go constructed a
|
||||
# minimal gin engine without TenantGuard — green. The harness's
|
||||
# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
|
||||
# inject X-Molecule-Org-Id, so the curl path is identical to production's
|
||||
# redeploy verifier), but no one ran the harness pre-merge. The bug
|
||||
# shipped; the redeploy verifier silently soft-warned every tenant as
|
||||
# "unreachable" for ~1 day before being noticed.
|
||||
#
|
||||
# This gate makes "did you actually run the harness?" a CI invariant
|
||||
# instead of a memory-discipline thing.
|
||||
#
|
||||
# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
|
||||
# to staging+main, real work is gated per-step on detect-changes output.
|
||||
# One job → one check run → branch-protection-clean (the SKIPPED-in-set
|
||||
# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.github/workflows/harness-replays.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.github/workflows/harness-replays.yml'
|
||||
workflow_dispatch:
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
|
||||
# cancellation deadlock — see e2e-api.yml's concurrency block for
|
||||
# the 2026-04-28 incident that codified this pattern.
|
||||
group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- id: decide
|
||||
run: |
|
||||
# workflow_dispatch: always run (manual trigger)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine the base commit to diff against.
|
||||
# For pull_request: use base.sha (the merge-base with main/staging).
|
||||
# For push: use github.event.before (the previous tip of the branch).
|
||||
# Fallback for new branches (all-zeros SHA): run everything.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && \
|
||||
[ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "${{ github.event.before }}" ] && \
|
||||
! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
|
||||
BASE="${{ github.event.before }}"
|
||||
else
|
||||
# New branch or github.event.before unavailable — run everything.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# GitHub Actions and Gitea Actions both expose github.sha for HEAD.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.github/workflows/harness-replays\.yml$'; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job that always runs. Real work is gated per-step on
|
||||
# detect-changes.outputs.run so an unrelated PR (e.g. doc-only
|
||||
# change to molecule-controlplane wired here later) emits the
|
||||
# required check without spending CI cycles. Single-job pattern
|
||||
# matches e2e-api.yml — see that workflow's comment for why a
|
||||
# job-level `if: false` would block branch protection via the
|
||||
# SKIPPED-in-set bug.
|
||||
harness-replays:
|
||||
needs: detect-changes
|
||||
name: Harness Replays
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.run != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
|
||||
echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
|
||||
echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
- if: needs.detect-changes.outputs.run == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Log what files were detected so future failures include the diff.
|
||||
- name: Log detected changes
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: |
|
||||
echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
# github-app-auth sibling-checkout removed 2026-05-07 (#157):
|
||||
# the plugin was dropped + Dockerfile.tenant no longer COPYs it.
|
||||
|
||||
# Pre-clone manifest deps before docker compose builds the tenant
|
||||
# image (Task #173 followup — same pattern as
|
||||
# publish-workspace-server-image.yml's "Pre-clone manifest deps"
|
||||
# step).
|
||||
#
|
||||
# Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
|
||||
# and tenant-beta from workspace-server/Dockerfile.tenant with
|
||||
# context=../.. (repo root). That Dockerfile expects
|
||||
# .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
|
||||
# to be present at build context root (post-#173 it COPYs from there
|
||||
# instead of running an in-image clone — the in-image clone failed
|
||||
# with "could not read Username for https://git.moleculesai.app"
|
||||
# because there's no auth path inside the build sandbox).
|
||||
#
|
||||
# Without this step harness-replays fails before any replay runs,
|
||||
# with `failed to calculate checksum of ref ...
|
||||
# "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
|
||||
# (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
|
||||
# symptom, different root cause: staging still has the in-image
|
||||
# clone path, hits the auth error directly).
|
||||
#
|
||||
# 2026-05-08 sub-finding (#192): the clone step ALSO fails when
|
||||
# any referenced workspace-template repo is private and the
|
||||
# AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
|
||||
# access. Root cause: 5 of 9 workspace-template repos
|
||||
# (openclaw, codex, crewai, deepagents, gemini-cli) had been
|
||||
# marked private with no team grant. Resolution: flipped them
|
||||
# to public per `feedback_oss_first_repo_visibility_default`
|
||||
# (the OSS surface should be public). Layer-3 (customer-private +
|
||||
# marketplace third-party repos) tracked separately in
|
||||
# internal#102.
|
||||
#
|
||||
# Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
|
||||
# is the devops-engineer persona PAT, NOT the founder PAT (per
|
||||
# `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
|
||||
# embeds it as basic-auth for the duration of the clones and strips
|
||||
# .git directories — the token never enters the resulting image.
|
||||
- name: Pre-clone manifest deps
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
env:
|
||||
MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
bash scripts/clone-manifest.sh \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
# Sanity-check counts so a silent partial clone fails fast
|
||||
# instead of producing a half-empty image.
|
||||
ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
|
||||
|
||||
- name: Install Python deps for replays
|
||||
# peer-discovery-404 (and future replays) eval Python against the
|
||||
# running tenant — importing workspace/a2a_client.py pulls in
|
||||
# httpx. tests/harness/requirements.txt holds just the HTTP-client
|
||||
# surface to keep CI install fast (~3s) vs the full
|
||||
# workspace/requirements.txt (~30s).
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: pip install -r tests/harness/requirements.txt
|
||||
|
||||
- name: Run all replays against the harness
|
||||
# run-all-replays.sh: boot via up.sh → seed via seed.sh → run
|
||||
# every replays/*.sh → tear down via down.sh on EXIT (trap).
|
||||
# Non-zero exit on any replay failure.
|
||||
#
|
||||
# KEEP_UP=1: without this, the script's trap-on-EXIT tears
|
||||
# down containers immediately on failure, leaving the dump
|
||||
# step below with nothing to dump (verified on PR #2410's
|
||||
# first run — tenant became unhealthy, trap fired, dump
|
||||
# step saw empty containers). Keeping them up lets the
|
||||
# failure path collect tenant/cp-stub/cf-proxy logs. The
|
||||
# always-run "Force teardown" step does the actual cleanup.
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
KEEP_UP: "1"
|
||||
run: ./run-all-replays.sh
|
||||
|
||||
- name: Dump compose logs on failure
|
||||
# SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
|
||||
# file even for read-only `logs` calls. up.sh generates a per-run key
|
||||
# and exports it to its OWN shell — this step runs in a fresh shell
|
||||
# that wouldn't see it, so without a placeholder the validate step
|
||||
# errors before logs print (verified against PR #2492's first run:
|
||||
# "required variable SECRETS_ENCRYPTION_KEY is missing a value").
|
||||
# A placeholder is fine — we're only reading log streams, not booting.
|
||||
if: failure() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
|
||||
run: |
|
||||
echo "=== docker compose ps ==="
|
||||
docker compose -f compose.yml ps || true
|
||||
echo "=== tenant-alpha logs ==="
|
||||
docker compose -f compose.yml logs tenant-alpha || true
|
||||
echo "=== tenant-beta logs ==="
|
||||
docker compose -f compose.yml logs tenant-beta || true
|
||||
echo "=== cp-stub logs ==="
|
||||
docker compose -f compose.yml logs cp-stub || true
|
||||
echo "=== cf-proxy logs ==="
|
||||
docker compose -f compose.yml logs cf-proxy || true
|
||||
echo "=== postgres-alpha logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-alpha || true
|
||||
echo "=== postgres-beta logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-beta || true
|
||||
|
||||
- name: Force teardown
|
||||
# We pass KEEP_UP=1 to run-all-replays.sh so the dump step
|
||||
# above sees real containers — that means we own teardown
|
||||
# explicitly here. Always run.
|
||||
if: always() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
run: ./down.sh || true
|
||||
@@ -0,0 +1,94 @@
|
||||
name: Lint curl status-code capture
|
||||
|
||||
# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
|
||||
# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
|
||||
#
|
||||
# HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
|
||||
#
|
||||
# When curl exits non-zero (connection reset → 56, --fail-with-body 4xx/5xx
|
||||
# → 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
|
||||
# "000" for connection failures or the actual code for HTTP errors. The
|
||||
# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
|
||||
# stdout, producing values like "000000" or "409000" that fail string
|
||||
# comparisons against "200" while looking superficially right.
|
||||
#
|
||||
# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
|
||||
# #2797). Memory: feedback_curl_status_capture_pollution.md.
|
||||
#
|
||||
# Fix shape (route -w into a tempfile so curl's exit code can't pollute):
|
||||
#
|
||||
# set +e
|
||||
# curl ... -w '%{http_code}' >code.txt 2>/dev/null
|
||||
# set -e
|
||||
# HTTP_CODE=$(cat code.txt 2>/dev/null)
|
||||
# [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths: ['.github/workflows/**']
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths: ['.github/workflows/**']
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan workflows for curl status-capture pollution
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
|
||||
# subshell where the entire command-substitution wraps a curl that
|
||||
# ends with `|| echo "000"`. Must distinguish from the SAFE shape
|
||||
# `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
|
||||
# tempfile produces empty stdout, no pollution.
|
||||
python3 <<'PY'
|
||||
import os, re, sys, glob
|
||||
|
||||
BAD_FILES = []
|
||||
|
||||
# Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
|
||||
# The `\\n` is the bash line-continuation that lets curl flags span lines.
|
||||
# We collapse continuation lines first, then look for the single-line bad pattern.
|
||||
PATTERN = re.compile(
|
||||
r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Self-skip: this lint workflow contains the literal anti-pattern in
|
||||
# its own docstring — that's intentional, not a bug.
|
||||
SELF = ".github/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
for f in sorted(glob.glob(".github/workflows/*.yml")):
|
||||
if f == SELF:
|
||||
continue
|
||||
with open(f) as fh:
|
||||
content = fh.read()
|
||||
# Collapse bash line-continuations (\\\n + leading whitespace)
|
||||
# into a single logical line so the regex can see the full
|
||||
# curl invocation as one chunk.
|
||||
flat = re.sub(r'\\\s*\n\s*', ' ', content)
|
||||
for m in PATTERN.finditer(flat):
|
||||
BAD_FILES.append((f, m.group(0)[:120]))
|
||||
|
||||
if not BAD_FILES:
|
||||
print("✓ No curl-status-capture pollution patterns detected")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
|
||||
for f, snippet in BAD_FILES:
|
||||
print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
|
||||
print(f" matched: {snippet}…")
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(' set +e')
|
||||
print(' curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
|
||||
print(' set -e')
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
sys.exit(1)
|
||||
PY
|
||||
@@ -0,0 +1,121 @@
|
||||
name: publish-canvas-image
|
||||
|
||||
# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
|
||||
# on main that touches canvas code. Previously canvas changes were visible in
|
||||
# CI (npm run build passed) but the live container was never updated —
|
||||
# operators had to manually run `docker compose build canvas` each time.
|
||||
#
|
||||
# Mirror of publish-platform-image.yml, adapted for the Next.js canvas layer.
|
||||
# See that workflow for inline notes on macOS Keychain isolation and QEMU.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
# Only rebuild when canvas source changes — saves GHA minutes on
|
||||
# platform-only / docs-only / MCP-only merges.
|
||||
- 'canvas/**'
|
||||
- '.github/workflows/publish-canvas-image.yml'
|
||||
# Manual trigger: use after a non-canvas merge that still needs a fresh
|
||||
# image (e.g. a Dockerfile change lives outside the canvas/ tree).
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
platform_url:
|
||||
description: 'NEXT_PUBLIC_PLATFORM_URL baked into the bundle (default: http://localhost:8080)'
|
||||
required: false
|
||||
default: ''
|
||||
ws_url:
|
||||
description: 'NEXT_PUBLIC_WS_URL baked into the bundle (default: ws://localhost:8080/ws)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write # required to push to ghcr.io/${{ github.repository_owner }}/*
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/canvas
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build & push canvas image
|
||||
runs-on: publish
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing to the build step
|
||||
# where docker build fails deep in ECR auth with a cryptic error.
|
||||
- name: Verify Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
|
||||
exit 1
|
||||
}
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Compute tags
|
||||
id: tags
|
||||
shell: bash
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve build args
|
||||
id: build_args
|
||||
# Priority: workflow_dispatch input > repo secret > hardcoded default.
|
||||
# NEXT_PUBLIC_* env vars are baked into the JS bundle at build time by
|
||||
# Next.js — they cannot be changed at runtime without a full rebuild.
|
||||
# For local docker-compose deployments the defaults (localhost:8080)
|
||||
# work as-is; production deployments should set CANVAS_PLATFORM_URL
|
||||
# and CANVAS_WS_URL as repository secrets.
|
||||
#
|
||||
# Inputs are passed via env vars (not direct ${{ }} interpolation) to
|
||||
# prevent shell injection from workflow_dispatch string inputs.
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_PLATFORM_URL: ${{ github.event.inputs.platform_url }}
|
||||
SECRET_PLATFORM_URL: ${{ secrets.CANVAS_PLATFORM_URL }}
|
||||
INPUT_WS_URL: ${{ github.event.inputs.ws_url }}
|
||||
SECRET_WS_URL: ${{ secrets.CANVAS_WS_URL }}
|
||||
run: |
|
||||
PLATFORM_URL="${INPUT_PLATFORM_URL:-${SECRET_PLATFORM_URL:-http://localhost:8080}}"
|
||||
WS_URL="${INPUT_WS_URL:-${SECRET_WS_URL:-ws://localhost:8080/ws}}"
|
||||
|
||||
echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build & push canvas image to GHCR
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ./canvas
|
||||
file: ./canvas/Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PLATFORM_URL=${{ steps.build_args.outputs.platform_url }}
|
||||
NEXT_PUBLIC_WS_URL=${{ steps.build_args.outputs.ws_url }}
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)
|
||||
@@ -0,0 +1,207 @@
|
||||
name: Railway pin audit (drift detection)
|
||||
|
||||
# Daily audit of Railway env vars for drift-prone image-tag pins —
|
||||
# automation-cadence layer over the detection script + regression test
|
||||
# shipped in PR #2168 (#2001 closure).
|
||||
#
|
||||
# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
|
||||
# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
|
||||
# "every fix didn't propagate" — really the tenant image was so old it
|
||||
# didn't read the env vars those fixes produced. The audit script
|
||||
# (scripts/ops/audit-railway-sha-pins.sh) flags drift; this workflow
|
||||
# runs the same check unattended on a daily cron.
|
||||
#
|
||||
# Cadence: once a day, 13:00 UTC (06:00 PT). Daily is the right
|
||||
# cadence for variables-tier config — Railway env var changes are
|
||||
# deliberate operator actions, low-frequency. Hourly would risk
|
||||
# Railway API rate-limit surprises and is overkill for the change rate.
|
||||
#
|
||||
# Issue-on-failure: drift triggers a priority-high issue, mirroring
|
||||
# .github/workflows/e2e-staging-sanity.yml's pattern. Drift is
|
||||
# medium-priority "config slipped, fix at next ops window," not
|
||||
# active-outage paging.
|
||||
#
|
||||
# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
|
||||
# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN
|
||||
# (silent-success on schedule was the failure-mode class that bit the
|
||||
# team before; cron firing without checking anything is worse than no
|
||||
# cron). The workflow_dispatch trigger SOFT-SKIPS on missing secret so
|
||||
# an operator can dry-run the workflow shape during initial provisioning
|
||||
# without tripping a fake red.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 13 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: railway-pin-audit
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
name: Audit Railway env vars for drift-prone pins
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify RAILWAY_AUDIT_TOKEN present
|
||||
# Schedule trigger: hard-fail when the secret is missing —
|
||||
# otherwise the cron silently runs against the wrong scope (or
|
||||
# exits 2 from the script and we issue-spam) without anyone
|
||||
# noticing the token rot.
|
||||
# Dispatch trigger: soft-skip — operator may be dry-running the
|
||||
# workflow shape before provisioning the secret. Logged as a
|
||||
# workflow notice, not a failure.
|
||||
env:
|
||||
RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
id: secret_check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
|
||||
echo "have_secret=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "have_secret=false" >> "$GITHUB_OUTPUT"
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
echo "::notice::RAILWAY_AUDIT_TOKEN not configured — soft-skipping (manual dispatch)"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
|
||||
exit 1
|
||||
|
||||
- name: Install Railway CLI
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
# Pinned hash matching the public install instructions; bump in
|
||||
# tandem with the audit-script's documented Railway CLI version.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL https://railway.com/install.sh | sh
|
||||
# The installer drops the binary in ~/.railway/bin
|
||||
echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify Railway CLI authenticated
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# `railway whoami` exits non-zero when the token is
|
||||
# unauthenticated or doesn't have any project access.
|
||||
if ! railway whoami >/dev/null 2>&1; then
|
||||
echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Link molecule-platform project
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
# Project ID from reference_production_stack: molecule-platform
|
||||
# / 7ccc8c68-61f4-42ab-9be5-586eeee11768. Linking is per-process,
|
||||
# so we re-link in this CI shell (the audit script comment says
|
||||
# it deliberately doesn't chdir for you because the linked
|
||||
# project's identity matters).
|
||||
run: |
|
||||
set -euo pipefail
|
||||
railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
|
||||
|
||||
- name: Run drift audit
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
id: audit
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# Capture the audit log for the issue body.
|
||||
{
|
||||
echo 'log<<AUDIT_EOF'
|
||||
cat /tmp/audit.log
|
||||
echo 'AUDIT_EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
# Exit codes from the script:
|
||||
# 0 — no drift; workflow goes green
|
||||
# 1 — drift detected; we'll file an issue and fail the run
|
||||
# 2 — railway CLI unauthenticated / project unlinked; fail
|
||||
# Anything else: also fail.
|
||||
case "$rc" in
|
||||
0) exit 0 ;;
|
||||
1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
|
||||
2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
|
||||
*) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Open / update drift issue
|
||||
if: failure() && steps.audit.outputs.rc == '1'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
AUDIT_LOG: ${{ steps.audit.outputs.log }}
|
||||
with:
|
||||
script: |
|
||||
const title = "🚨 Railway env-var drift detected";
|
||||
const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const body =
|
||||
`Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n` +
|
||||
`**What this means:** an env var (likely on \`controlplane\`) is pinned to a SHA-shaped or semver tag instead of a floating tag. ` +
|
||||
`Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service doesn't pick them up.\n\n` +
|
||||
`**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (\`:staging-latest\`, \`:main\`) unless the pin is intentional and documented in the ops runbook.\n\n` +
|
||||
`**Audit output:**\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\`\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`Closes automatically when a subsequent daily run reports clean.`;
|
||||
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'railway-drift',
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Still drifting. ${runURL}\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\``,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['railway-drift', 'bug', 'priority-high'],
|
||||
});
|
||||
}
|
||||
|
||||
- name: Close stale drift issue on clean run
|
||||
# When a previously-flagged drift gets fixed by an operator,
|
||||
# the next daily run goes green. Close any open `railway-drift`
|
||||
# issue with a confirmation comment so the queue doesn't carry
|
||||
# stale ones.
|
||||
if: success() && steps.audit.outputs.rc == '0'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'railway-drift',
|
||||
});
|
||||
for (const issue of existing) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
body: `Daily audit clean — drift resolved. ${runURL}`,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
state: 'closed',
|
||||
state_reason: 'completed',
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
#
|
||||
# This is the "PyPI artifact health" half of pin compatibility. The
|
||||
# companion workflow `runtime-prbuild-compat.yml` covers the
|
||||
# "PR-introduced breakage" half by building the wheel from THIS PR's
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter — the pypi-latest job no longer fires on doc-only
|
||||
# workspace/ edits whose content can't change what's currently on PyPI.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.github/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.github/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
workflow_dispatch:
|
||||
# Required-check support: when this becomes a branch-protection gate,
|
||||
# merge_group runs let the queue green-check this in addition to PRs.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -0,0 +1,152 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin → smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge → publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy → all crash on first boot with
|
||||
# ImportError.
|
||||
#
|
||||
# By building from the PR's source and smoke-importing THAT wheel, we
|
||||
# fail at PR-time instead of after publish.
|
||||
#
|
||||
# Required-check shape (2026-05-01): the workflow runs on EVERY push +
|
||||
# PR + merge_group event with no top-level `paths:` filter, then uses a
|
||||
# detect-changes job + per-step `if:` gates inside ONE always-running
|
||||
# job named `PR-built wheel + import smoke`. PRs that don't touch
|
||||
# wheel-relevant paths get a no-op SUCCESS check run, satisfying branch
|
||||
# protection without re-running the heavy build. Same pattern as
|
||||
# e2e-api.yml — see its comment for the full rationale + the 2026-04-29
|
||||
# PR #2264 incident that motivated the always-run-with-if-gates shape.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
workflow_dispatch:
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
# Include event_name so a PR sync (event=pull_request) and the
|
||||
# subsequent staging push (event=push) on the SAME merge SHA don't
|
||||
# collide in one group. Without event_name, both runs hashed to
|
||||
# the same key and cancel-in-progress=true cancelled whichever
|
||||
# arrived second — usually the push run, which staging branch-
|
||||
# protection then sees as a CANCELLED required check and refuses
|
||||
# to mark merged. Caught 2026-05-05 across PR #2869's runs (run
|
||||
# ids 25371863455 / 25371811486 / 25371078157 / 25370403142 — every
|
||||
# staging push run cancelled, every matching PR run green).
|
||||
#
|
||||
# Per memory `feedback_concurrency_group_per_sha.md` — same drift
|
||||
# class that broke auto-promote-staging on 2026-04-28. Pin invariant:
|
||||
# event_name + sha is the minimum unique key for these workflows.
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
wheel:
|
||||
- 'workspace/**'
|
||||
- 'scripts/build_runtime_package.py'
|
||||
- 'scripts/wheel_smoke.py'
|
||||
- '.github/workflows/runtime-prbuild-compat.yml'
|
||||
- id: decide
|
||||
# Always run real work for manual dispatch + merge_group — no
|
||||
# diff-against-base in those contexts, and the gate exists to
|
||||
# validate the to-be-merged state regardless of which paths it
|
||||
# touched (paths-filter would default to "no changes" which is
|
||||
# the wrong answer when the queue is composing many PRs).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "merge_group" ]; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=${{ steps.filter.outputs.wheel }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`. Same shape
|
||||
# as e2e-api.yml's e2e-api job — see its comment block for the full
|
||||
# rationale (SKIPPED check runs block branch protection even with
|
||||
# SUCCESS siblings; collapsing to one always-run job emits exactly
|
||||
# one SUCCESS check run).
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
# Use the SAME build script with the SAME args as
|
||||
# publish-runtime.yml's build step. The temp dir path differs
|
||||
# (`/tmp/runtime-build` here vs `${{ runner.temp }}/runtime-build`
|
||||
# in publish-runtime.yml — they coincide on ubuntu-latest but
|
||||
# the call sites are not byte-identical). The smoke import is
|
||||
# also intentionally narrower than publish's: this gate exists
|
||||
# to catch SDK-version-import drift specifically; full invariant
|
||||
# coverage lives in publish-runtime.yml's own pre-PyPI smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
# Closes the PR-time vs publish-time gap: a PR adding a new SDK
|
||||
# call-shape no longer passes here (narrow `import main_sync`) only
|
||||
# to fail post-merge in publish-runtime's broader smoke.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -0,0 +1,58 @@
|
||||
name: SECRET_PATTERNS drift lint
|
||||
|
||||
# Detects when the canonical SECRET_PATTERNS array in
|
||||
# .github/workflows/secret-scan.yml diverges from known consumer
|
||||
# mirrors (workspace-runtime's bundled pre-commit hook today; more
|
||||
# can be added as the consumer set grows).
|
||||
#
|
||||
# Why this exists: every side that scans for credentials has its own
|
||||
# copy of the pattern list. They drift — most recently the runtime
|
||||
# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
|
||||
# so a developer's local pre-commit would let a sk-cp- token through
|
||||
# while the org-wide CI scan would refuse it. The cost of that drift
|
||||
# is dev confusion + delayed feedback; the fix is automated detection.
|
||||
#
|
||||
# Triggers:
|
||||
# - schedule: daily 05:00 UTC. Catches drift introduced by edits
|
||||
# to a consumer copy that didn't update canonical here.
|
||||
# - push to main/staging where the canonical or this lint changed:
|
||||
# catches the inverse — canonical updated but consumers not yet
|
||||
# bumped. The lint will fail the push; that's intentional, the
|
||||
# person editing canonical is the right person to also update
|
||||
# the consumer.
|
||||
# - workflow_dispatch: ad-hoc operator runs.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
|
||||
# email lands when humans are starting their day, not
|
||||
# interrupting it.
|
||||
- cron: "0 5 * * *"
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- ".github/workflows/secret-scan.yml"
|
||||
- ".github/workflows/secret-pattern-drift.yml"
|
||||
- ".github/scripts/lint_secret_pattern_drift.py"
|
||||
- ".githooks/pre-commit"
|
||||
workflow_dispatch:
|
||||
|
||||
# GITHUB_TOKEN scoped to read-only. The lint only does git checkout
|
||||
# + HTTPS GETs to public consumer files; no writes to anything.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Detect SECRET_PATTERNS drift
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Run drift lint
|
||||
run: python3 .github/scripts/lint_secret_pattern_drift.py
|
||||
@@ -0,0 +1,129 @@
|
||||
name: Sweep stale AWS Secrets Manager secrets
|
||||
|
||||
# Janitor for per-tenant AWS Secrets Manager secrets
|
||||
# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
|
||||
# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
|
||||
# sweep-cf-orphans.yml — different cloud, same justification.
|
||||
#
|
||||
# Why this exists separately from a long-term reconciler integration:
|
||||
# - molecule-controlplane's tenant_resources audit table (mig 024)
|
||||
# currently tracks four resource kinds: CloudflareTunnel,
|
||||
# CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
|
||||
# not in the list, so the existing reconciler doesn't catch
|
||||
# orphan secrets.
|
||||
# - At ~$0.40/secret/month the cost grew to ~$19/month before this
|
||||
# sweeper was written, indicating ~45+ orphan secrets from
|
||||
# crashed provisions and incomplete deprovision flows.
|
||||
# - The proper fix (KindSecretsManagerSecret + recorder hook +
|
||||
# reconciler enumerator) is filed as a separate controlplane
|
||||
# issue. This sweeper is the immediate cost-relief stopgap.
|
||||
#
|
||||
# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
|
||||
# This is a DEDICATED principal — the production `molecule-cp` IAM
|
||||
# user lacks `secretsmanager:ListSecrets` (it only has
|
||||
# Get/Create/Update/Delete on specific resources, scoped to its
|
||||
# operational needs). The janitor needs ListSecrets across the
|
||||
# `molecule/tenant/*` prefix, which warrants a separate principal so
|
||||
# we don't broaden the prod-CP policy.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
|
||||
# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
|
||||
# the mostly-orphan tunnels) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30 — offsets from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45) so the three janitors don't burst the
|
||||
# CP admin endpoints at the same minute.
|
||||
- cron: '30 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted, no deletion"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
max_delete_pct:
|
||||
description: "Override safety gate (default 50, set higher only for major cleanup)"
|
||||
required: false
|
||||
default: "50"
|
||||
grace_hours:
|
||||
description: "Skip secrets created within this many hours (default 24)"
|
||||
required: false
|
||||
default: "24"
|
||||
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
group: sweep-aws-secrets
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
|
||||
CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
@@ -0,0 +1,146 @@
|
||||
name: Sweep stale Cloudflare DNS records
|
||||
|
||||
# Janitor for Cloudflare DNS records whose backing tenant/workspace no
|
||||
# longer exists. Without this loop, every short-lived E2E or canary
|
||||
# leaves a CF record on the moleculesai.app zone — the zone has a
|
||||
# 200-record quota (controlplane#239 hit it 2026-04-23+) and provisions
|
||||
# start failing with code 81045 once exhausted.
|
||||
#
|
||||
# Why a separate workflow vs sweep-stale-e2e-orgs.yml:
|
||||
# - That workflow operates at the CP layer (DELETE /cp/admin/tenants/:slug
|
||||
# drives the cascade). It assumes CP has the org row to drive the
|
||||
# deprovision from. It doesn't catch records left behind when CP
|
||||
# itself never knew about the tenant (canary scratch, manual ops
|
||||
# experiments) or when the cascade's CF-delete branch failed.
|
||||
# - sweep-cf-orphans.sh enumerates the CF zone directly and matches
|
||||
# each record against live CP slugs + AWS EC2 names. It catches
|
||||
# leaks the CP-driven sweep can't.
|
||||
#
|
||||
# Safety: the script's own MAX_DELETE_PCT gate refuses to nuke more
|
||||
# than 50% of records in a single run. If something has gone weird
|
||||
# (CP admin endpoint returns no orgs → every tenant looks orphan) the
|
||||
# gate halts before damage. Decision-function unit tests in
|
||||
# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
|
||||
# classifier.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly. Mirrors sweep-stale-e2e-orgs cadence so the two janitors
|
||||
# converge on the same tick. CF API rate budget is generous (1200
|
||||
# req/5min); a single sweep makes ~1 list + N deletes (N<=quota/2).
|
||||
- cron: '15 * * * *' # offset from sweep-stale-e2e-orgs (top of hour)
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted, no deletion"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
max_delete_pct:
|
||||
description: "Override safety gate (default 50, set higher only for major cleanup)"
|
||||
required: false
|
||||
default: "50"
|
||||
# No `merge_group:` trigger on purpose. This is a janitor — it doesn't
|
||||
# need to gate merges, and including it as written before #2088 fired
|
||||
# the full sweep job (or its secret-check) on every PR going through
|
||||
# the merge queue, generating one red CI run per merge-queue eval. If
|
||||
# this workflow is ever wired up as a required check, re-add
|
||||
# merge_group: { types: [checks_requested] }
|
||||
# AND gate the sweep step with `if: github.event_name != 'merge_group'`
|
||||
# so merge-queue evals report success without actually running.
|
||||
|
||||
# Don't let two sweeps race the same zone. workflow_dispatch during a
|
||||
# scheduled run would otherwise issue duplicate DELETE calls.
|
||||
concurrency:
|
||||
group: sweep-cf-orphans
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF orphans
|
||||
runs-on: ubuntu-latest
|
||||
# 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
|
||||
# within one cron interval instead of burning a full tick. Realistic
|
||||
# worst case is ~2 min: 4 sequential curls + 1 aws + N×CF-DELETE
|
||||
# each individually capped at 10s by the script's curl -m flag.
|
||||
timeout-minutes: 3
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
|
||||
CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
|
||||
CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split (hardened 2026-04-28
|
||||
# after the silent-no-op incident below):
|
||||
#
|
||||
# The earlier soft-skip-on-schedule policy hid a real leak. All
|
||||
# six secrets were unset on this repo for an unknown duration;
|
||||
# every hourly run printed a yellow ::warning:: and exited 0,
|
||||
# so the workflow registered as "passing" while doing nothing.
|
||||
# CF orphans accumulated to 152/200 (~76% of the zone quota
|
||||
# gone) before a manual `dig`-driven audit caught it. Anything
|
||||
# that runs as a janitor and reports green while idle is
|
||||
# indistinguishable from "the janitor is healthy" — so we now
|
||||
# treat schedule (and any future workflow_run/push triggers)
|
||||
# as a hard-fail when secrets are missing.
|
||||
#
|
||||
# - schedule / workflow_run / push → exit 1 (red CI run
|
||||
# surfaces the misconfiguration the next tick)
|
||||
# - workflow_dispatch → exit 0 with a warning
|
||||
# (an operator ran this ad-hoc; they already accepted the
|
||||
# state of the repo and want the workflow to short-circuit
|
||||
# so they can rerun after fixing the secret)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ZONE_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry (intentional):
|
||||
# - Scheduled runs: github.event.inputs.dry_run is empty →
|
||||
# defaults to "false" below → script runs with --execute
|
||||
# (the whole point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default is true (line 38)
|
||||
# so an ad-hoc operator-triggered run is dry-run by default;
|
||||
# they have to flip the toggle to actually delete.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) is the second
|
||||
# line of defense regardless of mode.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-orphans.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-orphans.sh --execute
|
||||
fi
|
||||
@@ -0,0 +1,124 @@
|
||||
name: Sweep stale Cloudflare Tunnels
|
||||
|
||||
# Janitor for Cloudflare Tunnels whose backing tenant no longer
|
||||
# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
|
||||
# records); same justification, different CF resource.
|
||||
#
|
||||
# Why this exists separately from sweep-cf-orphans:
|
||||
# - DNS records live on the zone (`/zones/<id>/dns_records`).
|
||||
# - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
|
||||
# - Different CF API surface, different scopes; the existing CF
|
||||
# token might not have `account:cloudflare_tunnel:edit`. Splitting
|
||||
# the workflows keeps each one's secret-presence gate independent
|
||||
# so neither silent-skips when the other's secret is missing.
|
||||
# - Cleaner blast radius — operators can disable one without the
|
||||
# other if a regression surfaces.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
|
||||
# the DNS sweep's 50% because tenant-shaped tunnels are mostly
|
||||
# orphans by design) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
|
||||
# janitors don't issue parallel CF API bursts at the same minute.
|
||||
- cron: '45 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted, no deletion"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
max_delete_pct:
|
||||
description: "Override safety gate (default 90, set higher only for major cleanup)"
|
||||
required: false
|
||||
default: "90"
|
||||
|
||||
# Don't let two sweeps race the same account.
|
||||
concurrency:
|
||||
group: sweep-cf-tunnels
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF tunnels
|
||||
runs-on: ubuntu-latest
|
||||
# 30 min cap. Was 5 min on the theory that the only thing that
|
||||
# could take >5min is a CF-API hang — but on 2026-05-02 a backlog
|
||||
# of 672 stale tunnels accumulated (large staging E2E run + delayed
|
||||
# sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
|
||||
# ~7-8min to drain. The 5-min cap killed the run mid-sweep
|
||||
# (cancelled at 424/672, see run 25248788312); a manual rerun
|
||||
# finished the remainder fine.
|
||||
#
|
||||
# The fix is two-part: parallelize the delete loop (8-way xargs in
|
||||
# the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
|
||||
# cap so a one-off backlog doesn't trip a hangs-detector that
|
||||
# turned out to be a real-job-too-slow detector. With 8-way
|
||||
# parallelism, 600+ tunnels drains in ~60s; 30 min is generous
|
||||
# headroom for actual hangs to still surface (and is in line with
|
||||
# the sweep-cf-orphans companion job).
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
|
||||
CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# (hardened 2026-04-28 after the silent-no-op incident: the
|
||||
# janitor reported green while doing nothing because secrets
|
||||
# were unset, masking a 152/200 zone-record leak). Same
|
||||
# principle applies here:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ACCOUNT_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh --execute
|
||||
fi
|
||||
@@ -0,0 +1,239 @@
|
||||
name: Sweep stale e2e-* orgs (staging)
|
||||
|
||||
# Janitor for staging tenants left behind when E2E cleanup didn't run:
|
||||
# CI cancellations, runner crashes, transient AWS errors mid-cascade,
|
||||
# bash trap missed (signal 9), etc. Without this loop, every failed
|
||||
# teardown leaks an EC2 + DNS + DB row until manual ops cleanup —
|
||||
# 2026-04-23 staging hit the 64 vCPU AWS quota from ~27 such orphans.
|
||||
#
|
||||
# Why not rely on per-test-run teardown:
|
||||
# - Per-run teardown is best-effort by definition. Any process death
|
||||
# after the test starts but before the trap fires leaves debris.
|
||||
# - GH Actions cancellation kills the runner without grace period.
|
||||
# The workflow's `if: always()` step usually catches this, but it
|
||||
# too can fail (CP transient 5xx, runner network issue at the
|
||||
# wrong moment).
|
||||
# - Even when teardown runs, the CP cascade is best-effort in places
|
||||
# (cascadeTerminateWorkspaces logs+continues; DNS deletion same).
|
||||
# - This sweep is the catch-all that converges staging back to clean
|
||||
# regardless of which specific path leaked.
|
||||
#
|
||||
# The PROPER fix is making CP cleanup transactional + verify-after-
|
||||
# terminate (filed separately as cleanup-correctness work). This
|
||||
# workflow is the safety net that catches everything else AND any
|
||||
# future leak source we haven't yet identified.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
|
||||
# create to teardown — canary is ~8 min, full SaaS ~25 min). The
|
||||
# previous hourly + 120-min stale threshold meant a leaked tenant
|
||||
# could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
|
||||
# leak. Tightening the cadence + threshold reduces the worst-case
|
||||
# leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
|
||||
# threshold) without risk of catching in-progress runs (the longest
|
||||
# e2e run is the 25-min canary, well under the 30-min threshold).
|
||||
# See molecule-controlplane#420 for the leak-class accounting that
|
||||
# motivated this tightening.
|
||||
- cron: '*/15 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
max_age_minutes:
|
||||
description: "Delete e2e-* orgs older than N minutes (default 30)"
|
||||
required: false
|
||||
default: "30"
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Don't let two sweeps fight. Cron + workflow_dispatch could overlap
|
||||
# on a manual trigger; queue rather than parallel-delete.
|
||||
concurrency:
|
||||
group: sweep-stale-e2e-orgs
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep e2e orgs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
# Refuse to delete more than this many orgs in one tick. If the
|
||||
# CP DB is briefly empty (or the admin endpoint goes weird and
|
||||
# returns no created_at), every e2e- org would look stale.
|
||||
# Bailing protects against runaway nukes.
|
||||
SAFETY_CAP: 50
|
||||
|
||||
steps:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Identify stale e2e orgs
|
||||
id: identify
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch into a file so the python step reads it via stdin —
|
||||
# cleaner than embedding $(curl ...) into a heredoc.
|
||||
curl -sS --fail-with-body --max-time 30 \
|
||||
"$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
> orgs.json
|
||||
|
||||
# Filter:
|
||||
# 1. slug starts with one of the ephemeral test prefixes:
|
||||
# - 'e2e-' — covers e2e-canary-, e2e-canvas-*, etc.
|
||||
# - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
|
||||
# missing this prefix left two such tenants
|
||||
# orphaned 8h on staging (2026-05-03), then
|
||||
# hard-failed redeploy-tenants-on-staging
|
||||
# and broke the staging→main auto-promote
|
||||
# chain. Kept in sync with the EPHEMERAL_PREFIX_RE
|
||||
# regex in redeploy-tenants-on-staging.yml.
|
||||
# 2. created_at is older than MAX_AGE_MINUTES ago
|
||||
# Output one slug per line to a file the next step reads.
|
||||
python3 > stale_slugs.txt <<'PY'
|
||||
import json, os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
# SSOT for this list lives in the controlplane Go code:
|
||||
# molecule-controlplane/internal/slugs/ephemeral.go
|
||||
# (var EphemeralPrefixes). The redeploy-fleet auto-rollout
|
||||
# also reads from there to SKIP these slugs — without that
|
||||
# filter, fleet redeploy SSM-failed in-flight E2E tenants
|
||||
# whose containers were still booting, breaking the test
|
||||
# that just spun them up (molecule-controlplane#493).
|
||||
# Update both files together.
|
||||
EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
|
||||
with open("orgs.json") as f:
|
||||
data = json.load(f)
|
||||
max_age = int(os.environ["MAX_AGE_MINUTES"])
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
|
||||
for o in data.get("orgs", []):
|
||||
slug = o.get("slug", "")
|
||||
if not slug.startswith(EPHEMERAL_PREFIXES):
|
||||
continue
|
||||
created = o.get("created_at")
|
||||
if not created:
|
||||
# Defensively skip rows without created_at — better
|
||||
# to leave one orphan than nuke a brand-new row
|
||||
# whose timestamp didn't render.
|
||||
continue
|
||||
# Python 3.11+ handles RFC3339 with Z directly via
|
||||
# fromisoformat; older runners need the trailing Z swap.
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
if created_dt < cutoff:
|
||||
print(slug)
|
||||
PY
|
||||
|
||||
count=$(wc -l < stale_slugs.txt | tr -d ' ')
|
||||
echo "Found $count stale e2e org(s) older than ${MAX_AGE_MINUTES}m"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo "First 20:"
|
||||
head -20 stale_slugs.txt | sed 's/^/ /'
|
||||
fi
|
||||
echo "count=$count" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Safety gate
|
||||
if: steps.identify.outputs.count != '0'
|
||||
run: |
|
||||
count="${{ steps.identify.outputs.count }}"
|
||||
if [ "$count" -gt "$SAFETY_CAP" ]; then
|
||||
echo "::error::Refusing to delete $count orgs in one sweep (cap=$SAFETY_CAP). Investigate manually — this usually means the CP admin API returned no created_at or returned a degraded result. Re-run with workflow_dispatch + max_age_minutes if intentional."
|
||||
exit 1
|
||||
fi
|
||||
echo "Within safety cap ($count ≤ $SAFETY_CAP) ✓"
|
||||
|
||||
- name: Delete stale orgs
|
||||
if: steps.identify.outputs.count != '0' && env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set -uo pipefail
|
||||
deleted=0
|
||||
failed=0
|
||||
while IFS= read -r slug; do
|
||||
[ -z "$slug" ] && continue
|
||||
# The DELETE handler requires {"confirm": "<slug>"} matching
|
||||
# the URL slug — fat-finger guard. Idempotent: re-issuing
|
||||
# picks up via org_purges.last_step.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/del_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/del_code
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to runner log.
|
||||
http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
||||
deleted=$((deleted+1))
|
||||
echo " deleted: $slug"
|
||||
else
|
||||
failed=$((failed+1))
|
||||
echo " FAILED ($http_code): $slug — $(cat /tmp/del_resp 2>/dev/null | head -c 200)"
|
||||
fi
|
||||
done < stale_slugs.txt
|
||||
echo ""
|
||||
echo "Sweep summary: deleted=$deleted failed=$failed"
|
||||
# Don't fail the workflow on per-org delete errors — the
|
||||
# sweeper is best-effort. Next hourly tick re-attempts. We
|
||||
# only fail loud at the safety-cap gate above.
|
||||
|
||||
- name: Sweep orphan tunnels
|
||||
# Stale-org cleanup deletes the org (which cascades to tunnel
|
||||
# delete inside the CP). But when that cascade fails partway —
|
||||
# CP transient 5xx after the org row is deleted but before the
|
||||
# CF tunnel delete completes — the tunnel persists with no
|
||||
# matching org row. The reconciler in internal/sweep flags this
|
||||
# as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
|
||||
#
|
||||
# `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
|
||||
# reaper. Calling it here at the end of every sweep tick
|
||||
# converges the staging CF account to clean even when CP
|
||||
# cascades half-fail.
|
||||
#
|
||||
# PR #492 made the underlying DeleteTunnel actually check
|
||||
# status — pre-fix it silent-succeeded on CF code 1022
|
||||
# ("active connections"), so this step would have been a no-op
|
||||
# against stuck connectors. Post-fix the cleanup invokes
|
||||
# CleanupTunnelConnections + retry, which actually clears the
|
||||
# 1022 case. (#2987)
|
||||
#
|
||||
# Best-effort. Failure here doesn't fail the workflow — next
|
||||
# tick re-attempts. Errors flow to step output for ops review.
|
||||
if: env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set +e
|
||||
curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
|
||||
set -e
|
||||
http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
|
||||
body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
|
||||
if [ "$http_code" = "200" ]; then
|
||||
count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
|
||||
failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
|
||||
echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
|
||||
else
|
||||
echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
|
||||
fi
|
||||
|
||||
- name: Dry-run summary
|
||||
if: env.DRY_RUN == 'true'
|
||||
run: |
|
||||
echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."
|
||||
@@ -0,0 +1,52 @@
|
||||
name: Ops Scripts Tests
|
||||
|
||||
# Runs the unittest suite for scripts/ on every PR + push that touches
|
||||
# anything under scripts/. Kept separate from the main CI so a script-only
|
||||
# change doesn't trigger the heavier Go/Canvas/Python pipelines.
|
||||
#
|
||||
# Discovery layout: tests sit alongside the code they test (see
|
||||
# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
|
||||
# test_build_runtime_package.py for the rewriter coverage). The job
|
||||
# below runs `unittest discover` TWICE — once from `scripts/`, once
|
||||
# from `scripts/ops/` — because neither dir has an `__init__.py`, so
|
||||
# a single discover from `scripts/` doesn't recurse into the ops
|
||||
# subdir. Two passes is simpler than retrofitting namespace packages.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.github/workflows/test-ops-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.github/workflows/test-ops-scripts.yml'
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Ops scripts (unittest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Run scripts/ unittests (build_runtime_package, …)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, …)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
+1
-18
@@ -127,11 +127,7 @@ cd workspace-server && go test -race ./...
|
||||
cd canvas && npm test
|
||||
|
||||
# Workspace runtime (Python)
|
||||
# Runtime code is SSOT in molecule-ai-workspace-runtime, not molecule-core/workspace.
|
||||
cd ../molecule-ai-workspace-runtime
|
||||
python -m venv .venv && source .venv/bin/activate
|
||||
pip install --index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ -e . pytest pytest-asyncio
|
||||
pytest -q
|
||||
cd workspace && python -m pytest -v
|
||||
|
||||
# E2E API tests (requires running platform)
|
||||
bash tests/e2e/test_api.sh
|
||||
@@ -163,19 +159,6 @@ and run CI manually.
|
||||
| review-check-tests | `review-check.sh` evaluator regression suite (13 scenarios) |
|
||||
| ops-scripts | Python unittest suite for `scripts/*.py` |
|
||||
|
||||
### Workspace runtime SSOT
|
||||
|
||||
Runtime code lives in
|
||||
[`molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime).
|
||||
Do not reintroduce `molecule-core/workspace/` or vendored `molecule_runtime/`
|
||||
copies in consumers. Core and templates consume the published runtime package
|
||||
from the Gitea package registry.
|
||||
|
||||
For local external MCP agents, multi-workspace config is
|
||||
`MOLECULE_WORKSPACES=[{"id":"...","token":"...","platform_url":"..."}]`.
|
||||
`platform_url` selects the tenant; `org_id` is not part of this config.
|
||||
Workspace IDs can differ across orgs.
|
||||
|
||||
## Local Testing
|
||||
|
||||
### review-check.sh
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# use this Makefile; CI calls docker compose / go test directly so the
|
||||
# Makefile can evolve without breaking the build.
|
||||
|
||||
.PHONY: help dev up down logs build test e2e-peer-visibility openapi-spec openapi-spec-check
|
||||
.PHONY: help dev up down logs build test e2e-peer-visibility
|
||||
|
||||
help: ## Show this help.
|
||||
@grep -E '^[a-zA-Z0-9_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}'
|
||||
@@ -36,23 +36,3 @@ test: ## Run Go unit tests in workspace-server/.
|
||||
# env contract (CLAUDE_CODE_OAUTH_TOKEN / E2E_MINIMAX_API_KEY / etc).
|
||||
e2e-peer-visibility: ## Run the LOCAL peer-visibility MCP gate vs the running stack (needs `make up` first).
|
||||
bash tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
|
||||
# ─── OpenAPI spec generation (RFC #1706, Phase 1) ─────────────────────
|
||||
# Regenerate workspace-server/docs/openapi/swagger.{yaml,json} from
|
||||
# swaggo annotations on the gin handlers. Commit the output. CI runs
|
||||
# `make openapi-spec-check` to assert no drift between annotations and
|
||||
# the committed file — if a PR changes a handler but forgets to
|
||||
# regenerate, CI fails with a diff.
|
||||
openapi-spec: ## Regenerate OpenAPI spec from workspace-server handler annotations.
|
||||
@command -v swag >/dev/null 2>&1 || go install github.com/swaggo/swag/cmd/swag@v1.16.4
|
||||
cd workspace-server && swag init \
|
||||
--generalInfo cmd/server/main.go \
|
||||
--output docs/openapi \
|
||||
--outputTypes yaml,json \
|
||||
--dir . \
|
||||
--parseDependency=false \
|
||||
--parseInternal=true
|
||||
|
||||
openapi-spec-check: openapi-spec ## CI gate — fail if openapi-spec produces a diff vs the committed file.
|
||||
@git diff --exit-code -- workspace-server/docs/openapi/ \
|
||||
|| (echo "openapi-spec is stale — run 'make openapi-spec' and commit the result" && exit 1)
|
||||
|
||||
@@ -163,11 +163,11 @@ Most agent systems stop at "a smart runtime." Molecule AI pushes further: it giv
|
||||
|
||||
| Core mechanism | Molecule AI module(s) | Why it matters |
|
||||
|---|---|---|
|
||||
| **Durable memory that survives sessions** | `molecule-ai-workspace-runtime/molecule_runtime/builtin_tools/`, `workspace-server/internal/handlers/memories.go` | Memory is not just durable, it is **workspace-scoped** and can route into awareness namespaces tied to the org structure |
|
||||
| **Durable memory that survives sessions** | `workspace/builtin_tools/memory.py`, `workspace/builtin_tools/awareness_client.py`, `workspace-server/internal/handlers/memories.go` | Memory is not just durable, it is **workspace-scoped** and can route into awareness namespaces tied to the org structure |
|
||||
| **Cross-session recall** | `workspace-server/internal/handlers/activity.go` (`/workspaces/:id/session-search`) | Recall spans both activity history and memory rows, so the system can search what happened and what was learned without inventing a separate hidden store |
|
||||
| **Skills built from experience** | `molecule-ai-workspace-runtime/molecule_runtime/builtin_tools/memory.py` (`_maybe_log_skill_promotion`) | Promotion from memory into a skill candidate is surfaced as an explicit platform activity, not a silent internal side effect |
|
||||
| **Skill improvement during use** | `molecule-ai-workspace-runtime/molecule_runtime/skill_loader/`, `molecule-ai-workspace-runtime/molecule_runtime/main.py` | Skills hot-reload into the live runtime, so improvements become available on the next A2A task without restarting the workspace |
|
||||
| **Persistent skill lifecycle** | `workspace-server/cmd/cli/cmd_agent_skill.go`, `molecule-ai-workspace-runtime/molecule_runtime/plugins.py` | Skills are not just generated once; they can be audited, installed, published, shared, mounted by plugins, and governed as reusable operational assets |
|
||||
| **Skills built from experience** | `workspace/builtin_tools/memory.py` (`_maybe_log_skill_promotion`) | Promotion from memory into a skill candidate is surfaced as an explicit platform activity, not a silent internal side effect |
|
||||
| **Skill improvement during use** | `workspace/skill_loader/watcher.py`, `workspace/skill_loader/loader.py`, `workspace/main.py` | Skills hot-reload into the live runtime, so improvements become available on the next A2A task without restarting the workspace |
|
||||
| **Persistent skill lifecycle** | `workspace-server/cmd/cli/cmd_agent_skill.go`, `workspace/plugins.py` | Skills are not just generated once; they can be audited, installed, published, shared, mounted by plugins, and governed as reusable operational assets |
|
||||
|
||||
### Why this matters in Molecule AI
|
||||
|
||||
@@ -208,7 +208,7 @@ The result is not just “an agent that learns.” It is **an organization that
|
||||
|
||||
### Runtime
|
||||
|
||||
- standalone workspace-template images that install `molecule-ai-workspace-runtime` from the Gitea package registry; thin AMI in production (us-east-2)
|
||||
- unified `workspace/` image; thin AMI in production (us-east-2)
|
||||
- adapter-driven execution across **8 runtimes** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw)
|
||||
- Agent Card registration
|
||||
- awareness-backed memory integration; **Memory v2 backed by pgvector** for semantic recall
|
||||
|
||||
@@ -55,7 +55,7 @@ test.describe("Desktop ChatTab", () => {
|
||||
await textarea.fill("What is the weather?");
|
||||
await page.getByRole("button", { name: /Send/ }).first().click();
|
||||
|
||||
await expect(page.getByText("What is the weather?", { exact: true })).toBeVisible({ timeout: 5_000 });
|
||||
await expect(page.getByText("What is the weather?")).toBeVisible({ timeout: 5_000 });
|
||||
await expect(page.getByText("Echo: What is the weather?")).toBeVisible({ timeout: 15_000 });
|
||||
});
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ test.describe("MobileChat", () => {
|
||||
await textarea.fill("Mobile test message");
|
||||
await page.getByRole("button", { name: /Send/ }).first().click();
|
||||
|
||||
await expect(page.getByText("Mobile test message", { exact: true })).toBeVisible({ timeout: 5_000 });
|
||||
await expect(page.getByText("Mobile test message")).toBeVisible({ timeout: 5_000 });
|
||||
await expect(page.getByText("Echo: Mobile test message")).toBeVisible({ timeout: 15_000 });
|
||||
});
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { execFileSync, execSync } from "node:child_process";
|
||||
|
||||
const PLATFORM_URL = process.env.E2E_PLATFORM_URL ?? "http://localhost:8080";
|
||||
|
||||
@@ -24,19 +23,13 @@ export interface SeededWorkspace {
|
||||
* Create an external workspace and wire it to the echo runtime.
|
||||
*/
|
||||
export async function seedWorkspace(echoURL: string): Promise<SeededWorkspace> {
|
||||
// 1. Create external workspace pointing at the in-process echo runtime.
|
||||
// 1. Create external workspace (no URL — platform will mint an auth token).
|
||||
const runId = Math.random().toString(36).slice(2, 8);
|
||||
const wsName = `Chat E2E Agent ${runId}`;
|
||||
const createRes = await fetch(`${PLATFORM_URL}/workspaces`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
name: wsName,
|
||||
tier: 1,
|
||||
external: true,
|
||||
runtime: "external",
|
||||
url: echoURL,
|
||||
}),
|
||||
body: JSON.stringify({ name: wsName, tier: 1, external: true, runtime: "external" }),
|
||||
});
|
||||
if (!createRes.ok) {
|
||||
const text = await createRes.text();
|
||||
@@ -47,10 +40,7 @@ export async function seedWorkspace(echoURL: string): Promise<SeededWorkspace> {
|
||||
name: string;
|
||||
connection?: { auth_token?: string };
|
||||
};
|
||||
let authToken = ws.connection?.auth_token;
|
||||
if (!authToken) {
|
||||
authToken = await mintTestToken(ws.id);
|
||||
}
|
||||
const authToken = ws.connection?.auth_token;
|
||||
if (!authToken) {
|
||||
throw new Error("Workspace created but no auth_token returned");
|
||||
}
|
||||
@@ -83,35 +73,16 @@ export async function seedWorkspace(echoURL: string): Promise<SeededWorkspace> {
|
||||
`-c "UPDATE workspaces SET status = 'online', url = '${echoURL}', platform_inbound_secret = '${inboundSecret}' WHERE id = '${ws.id}'"`,
|
||||
].join(" ");
|
||||
|
||||
const { execSync } = await import("node:child_process");
|
||||
try {
|
||||
execSync(psql, { stdio: "pipe", timeout: 30_000 });
|
||||
} catch (err) {
|
||||
throw new Error(`DB update failed: ${err}`);
|
||||
}
|
||||
|
||||
cacheWorkspaceURL(ws.id, echoURL);
|
||||
|
||||
return { id: ws.id, name: wsName, agentURL: echoURL, authToken };
|
||||
}
|
||||
|
||||
function cacheWorkspaceURL(workspaceId: string, agentURL: string): void {
|
||||
const redisContainer = process.env.REDIS_CONTAINER;
|
||||
if (!redisContainer) return;
|
||||
|
||||
const keys = [`ws:${workspaceId}:url`, `ws:${workspaceId}:internal_url`];
|
||||
for (const key of keys) {
|
||||
try {
|
||||
execFileSync(
|
||||
"docker",
|
||||
["exec", redisContainer, "redis-cli", "SET", key, agentURL],
|
||||
{ stdio: "pipe", timeout: 10_000 },
|
||||
);
|
||||
} catch (err) {
|
||||
throw new Error(`Redis URL cache update failed for ${key}: ${err}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a heartbeat interval that keeps an external workspace alive.
|
||||
* Returns a stop function.
|
||||
@@ -170,6 +141,7 @@ export async function seedChatHistory(
|
||||
|
||||
const sql = `INSERT INTO chat_messages (id, workspace_id, role, content, created_at) VALUES ${values};`;
|
||||
|
||||
const { execSync } = await import("node:child_process");
|
||||
const psql = `PGPASSWORD=${pass} psql -h ${host} -p ${port} -U ${user} -d ${db} -c "${sql}"`;
|
||||
execSync(psql, { stdio: "pipe", timeout: 10_000 });
|
||||
}
|
||||
@@ -191,6 +163,7 @@ export async function cleanupWorkspace(workspaceId: string): Promise<void> {
|
||||
|
||||
const psql = `PGPASSWORD=${pass} psql -h ${host} -p ${port} -U ${user} -d ${db} -c "DELETE FROM workspaces WHERE id = '${workspaceId}'"`;
|
||||
|
||||
const { execSync } = await import("node:child_process");
|
||||
try {
|
||||
execSync(psql, { stdio: "pipe", timeout: 30_000 });
|
||||
} catch {
|
||||
|
||||
@@ -162,10 +162,10 @@ export async function startEchoRuntime(): Promise<EchoRuntime> {
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => server.listen(0, resolve));
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
const port = typeof address === "object" && address ? address.port : 0;
|
||||
const baseURL = `http://localhost:${port}`;
|
||||
const baseURL = `http://127.0.0.1:${port}`;
|
||||
|
||||
return {
|
||||
baseURL,
|
||||
|
||||
@@ -9,8 +9,6 @@ import { DetailsTab } from "./tabs/DetailsTab";
|
||||
import { SkillsTab } from "./tabs/SkillsTab";
|
||||
import { ChatTab } from "./tabs/ChatTab";
|
||||
import { ConfigTab } from "./tabs/ConfigTab";
|
||||
import { ContainerConfigTab } from "./tabs/ContainerConfigTab";
|
||||
import { DisplayTab } from "./tabs/DisplayTab";
|
||||
import { TerminalTab } from "./tabs/TerminalTab";
|
||||
import { FilesTab } from "./tabs/FilesTab";
|
||||
import { MemoryInspectorPanel } from "./MemoryInspectorPanel";
|
||||
@@ -33,8 +31,6 @@ const TABS: { id: PanelTab; label: string; icon: string }[] = [
|
||||
{ id: "details", label: "Details", icon: "◉" },
|
||||
{ id: "skills", label: "Plugins", icon: "✦" },
|
||||
{ id: "terminal", label: "Terminal", icon: "▸" },
|
||||
{ id: "display", label: "Display", icon: "▣" },
|
||||
{ id: "container-config", label: "Container", icon: "▤" },
|
||||
{ id: "config", label: "Config", icon: "⚙" },
|
||||
{ id: "schedule", label: "Schedule", icon: "⏲" },
|
||||
{ id: "channels", label: "Channels", icon: "⇌" },
|
||||
@@ -304,8 +300,6 @@ export function SidePanel() {
|
||||
{panelTab === "activity" && <ActivityTab key={selectedNodeId} workspaceId={selectedNodeId} />}
|
||||
{panelTab === "chat" && <ChatTab key={selectedNodeId} workspaceId={selectedNodeId} data={node.data} />}
|
||||
{panelTab === "terminal" && <TerminalTab key={selectedNodeId} workspaceId={selectedNodeId} data={node.data} />}
|
||||
{panelTab === "display" && <DisplayTab key={selectedNodeId} workspaceId={selectedNodeId} />}
|
||||
{panelTab === "container-config" && <ContainerConfigTab key={selectedNodeId} data={node.data} />}
|
||||
{panelTab === "config" && <ConfigTab key={selectedNodeId} workspaceId={selectedNodeId} />}
|
||||
{panelTab === "schedule" && <ScheduleTab key={selectedNodeId} workspaceId={selectedNodeId} />}
|
||||
{panelTab === "channels" && <ChannelsTab key={selectedNodeId} workspaceId={selectedNodeId} />}
|
||||
|
||||
@@ -68,103 +68,14 @@ export function Toolbar() {
|
||||
return c;
|
||||
}, [nodes]);
|
||||
|
||||
/**
|
||||
* Stop All - task #377 fix.
|
||||
*
|
||||
* BEFORE this PR: directly POSTed `/workspaces/:id/restart`, which tears
|
||||
* the container down and back up. That kills in-flight tool subprocesses
|
||||
* (e.g. `bash -c 'sleep 600'`) but is heavy and discards any in-progress
|
||||
* agent state. It also bypasses the runtime-side fast cancel path (task
|
||||
* #377 PR#40 in template-claude-code) - meaning flipping
|
||||
* `MOLECULE_STOP_PROPAGATE=true` would produce zero canary signal because
|
||||
* nothing ever invokes `executor.cancel()` in production.
|
||||
*
|
||||
* AFTER this PR (two-phase polite cancel):
|
||||
*
|
||||
* 1. POST `tasks/cancel` (A2A JSON-RPC) to each active workspace's
|
||||
* `/workspaces/:id/a2a` proxy. The platform proxies the envelope to
|
||||
* the workspace runtime; the a2a-sdk framework dispatches `tasks/cancel`
|
||||
* to `AgentExecutor.cancel()` (a2a-sdk 1.0.3
|
||||
* `a2a/compat/v0_3/types.py` line 1125 pins the wire literal as
|
||||
* `Literal["tasks/cancel"]`; A2A protocol spec section 9.4.5 maps the
|
||||
* abstract `CancelTask` operation to that wire string). The runtime's
|
||||
* executor cancel path signals the CLI subprocess group with
|
||||
* SIGTERM/grace/SIGKILL (template-claude-code PR#40 `stop_propagate.py`).
|
||||
*
|
||||
* 2. Poll the canvas store (the platform pushes `TASK_UPDATED` over WS
|
||||
* on `active_tasks` changes - `canvas-events.ts` line 400) for up to
|
||||
* `STOP_ALL_DRAIN_TIMEOUT_MS`. A workspace whose `activeTasks` drops
|
||||
* to 0 is considered drained and is NOT restarted.
|
||||
*
|
||||
* 3. For any workspace that DID NOT drain inside the timeout - runtime
|
||||
* is on an old image without the cancel path, or the cancel
|
||||
* propagation is stuck - fall back to the original heavy
|
||||
* `/workspaces/:id/restart`. The original behavior is preserved as a
|
||||
* floor so a stuck workspace still gets stopped; the polite path is
|
||||
* a fast top-up that lets well-behaved workspaces cancel without
|
||||
* losing context.
|
||||
*
|
||||
* The polite-cancel envelope mirrors `ScheduleTab.handleRunNow` (line 168)
|
||||
* which is the only other place in canvas that POSTs `/workspaces/:id/a2a`
|
||||
* directly. Method string `tasks/cancel` and empty `params` match the
|
||||
* a2a-sdk shape verified above. The proxy adds `jsonrpc:"2.0"` and `id`
|
||||
* via `normalizeA2APayload` server-side, so the canvas envelope omits them.
|
||||
*/
|
||||
const stopAll = useCallback(async () => {
|
||||
setStopping(true);
|
||||
const active = nodes.filter((n) => (n.data.activeTasks as number) > 0);
|
||||
const activeIds = active.map((n) => n.id);
|
||||
|
||||
// Phase 1 - polite cancel on every active workspace in parallel.
|
||||
// Errors are swallowed (same shape as the pre-fix /restart
|
||||
// Promise.all): a 4xx/5xx on tasks/cancel just means we fall through
|
||||
// to /restart for that workspace below.
|
||||
await Promise.all(
|
||||
activeIds.map((id) =>
|
||||
api
|
||||
.post(`/workspaces/${id}/a2a`, {
|
||||
method: "tasks/cancel",
|
||||
params: {},
|
||||
})
|
||||
.catch(() => {})
|
||||
active.map((n) =>
|
||||
api.post(`/workspaces/${n.id}/restart`).catch(() => {})
|
||||
)
|
||||
);
|
||||
|
||||
// Phase 2 - poll the store for activeTasks reaching 0, with a hard
|
||||
// timeout. STOP_ALL_DRAIN_TIMEOUT_MS is sized to cover the runtime's
|
||||
// own SIGTERM-grace (5s in template-claude-code stop_propagate.py
|
||||
// `_SIGTERM_GRACE_S`) plus a small WS round-trip buffer for the
|
||||
// TASK_UPDATED push. STOP_ALL_POLL_INTERVAL_MS keeps the poll cheap
|
||||
// (no animation jitter, no busy-wait).
|
||||
const STOP_ALL_DRAIN_TIMEOUT_MS = 8000;
|
||||
const STOP_ALL_POLL_INTERVAL_MS = 250;
|
||||
const deadline = Date.now() + STOP_ALL_DRAIN_TIMEOUT_MS;
|
||||
let undrained = new Set(activeIds);
|
||||
while (undrained.size > 0 && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, STOP_ALL_POLL_INTERVAL_MS));
|
||||
const fresh = useCanvasStore.getState().nodes;
|
||||
const stillActive = new Set<string>();
|
||||
for (const id of undrained) {
|
||||
const n = fresh.find((x) => x.id === id);
|
||||
// Missing node (workspace deleted mid-cancel) is treated as
|
||||
// drained - there's nothing left to restart and reporting it as
|
||||
// "still running" would be a lie.
|
||||
if (n && (n.data.activeTasks as number) > 0) stillActive.add(id);
|
||||
}
|
||||
undrained = stillActive;
|
||||
}
|
||||
|
||||
// Phase 3 - hard-restart anything that did not drain. This is the
|
||||
// same call shape as the pre-fix Stop All, so behavior is strictly a
|
||||
// superset: undrained workspaces still get the heavy stop, drained
|
||||
// ones are spared.
|
||||
if (undrained.size > 0) {
|
||||
await Promise.all(
|
||||
Array.from(undrained).map((id) =>
|
||||
api.post(`/workspaces/${id}/restart`).catch(() => {})
|
||||
)
|
||||
);
|
||||
}
|
||||
setStopping(false);
|
||||
}, [nodes]);
|
||||
|
||||
|
||||
@@ -11,8 +11,6 @@ vi.mock("../tabs/DetailsTab", () => ({ DetailsTab: () => null }));
|
||||
vi.mock("../tabs/SkillsTab", () => ({ SkillsTab: () => null }));
|
||||
vi.mock("../tabs/ChatTab", () => ({ ChatTab: () => null }));
|
||||
vi.mock("../tabs/ConfigTab", () => ({ ConfigTab: () => null }));
|
||||
vi.mock("../tabs/ContainerConfigTab", () => ({ ContainerConfigTab: () => null }));
|
||||
vi.mock("../tabs/DisplayTab", () => ({ DisplayTab: () => null }));
|
||||
vi.mock("../tabs/TerminalTab", () => ({ TerminalTab: () => null }));
|
||||
vi.mock("../tabs/FilesTab", () => ({ FilesTab: () => null }));
|
||||
vi.mock("../MemoryInspectorPanel", () => ({ MemoryInspectorPanel: () => null }));
|
||||
@@ -76,7 +74,7 @@ import { SidePanel } from "../SidePanel";
|
||||
|
||||
const TABS = [
|
||||
"chat", "activity", "details", "skills", "terminal",
|
||||
"display", "container-config", "config", "schedule", "channels", "files", "memory", "traces", "events", "audit",
|
||||
"config", "schedule", "channels", "files", "memory", "traces", "events", "audit",
|
||||
];
|
||||
|
||||
describe("SidePanel — ARIA tablist pattern", () => {
|
||||
@@ -87,20 +85,10 @@ describe("SidePanel — ARIA tablist pattern", () => {
|
||||
expect(tablist.getAttribute("aria-label")).toBe("Workspace panel tabs");
|
||||
});
|
||||
|
||||
it("renders exactly 15 tab buttons", () => {
|
||||
it("renders exactly 13 tab buttons", () => {
|
||||
render(<SidePanel />);
|
||||
const tabs = screen.getAllByRole("tab");
|
||||
expect(tabs.length).toBe(15);
|
||||
});
|
||||
|
||||
it("renders the Display tab", () => {
|
||||
render(<SidePanel />);
|
||||
expect(document.getElementById("tab-display")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders the Container Config tab", () => {
|
||||
render(<SidePanel />);
|
||||
expect(document.getElementById("tab-container-config")).toBeTruthy();
|
||||
expect(tabs.length).toBe(13);
|
||||
});
|
||||
|
||||
it("active tab (chat) has aria-selected='true'", () => {
|
||||
@@ -111,11 +99,11 @@ describe("SidePanel — ARIA tablist pattern", () => {
|
||||
expect(chatTab?.getAttribute("aria-selected")).toBe("true");
|
||||
});
|
||||
|
||||
it("all other 14 tabs have aria-selected='false'", () => {
|
||||
it("all other 12 tabs have aria-selected='false'", () => {
|
||||
render(<SidePanel />);
|
||||
const tabs = screen.getAllByRole("tab");
|
||||
const inactive = tabs.filter((t) => t.id !== "tab-chat");
|
||||
expect(inactive.length).toBe(14);
|
||||
expect(inactive.length).toBe(12);
|
||||
for (const tab of inactive) {
|
||||
expect(tab.getAttribute("aria-selected")).toBe("false");
|
||||
}
|
||||
@@ -128,7 +116,7 @@ describe("SidePanel — ARIA tablist pattern", () => {
|
||||
const minusOnes = tabs.filter((t) => t.getAttribute("tabindex") === "-1");
|
||||
expect(zeros.length).toBe(1);
|
||||
expect(zeros[0].id).toBe("tab-chat");
|
||||
expect(minusOnes.length).toBe(14);
|
||||
expect(minusOnes.length).toBe(12);
|
||||
});
|
||||
|
||||
it("active tab has aria-controls='panel-chat' and id='tab-chat'", () => {
|
||||
|
||||
@@ -131,30 +131,14 @@ const defaultStore = {
|
||||
batchDelete: vi.fn(() => Promise.resolve()),
|
||||
};
|
||||
|
||||
vi.mock("@/store/canvas", () => {
|
||||
// useCanvasStore is used in two shapes:
|
||||
// 1. As a hook: `useCanvasStore((s) => s.x)` — selector path.
|
||||
// 2. As a static accessor: `useCanvasStore.getState().nodes` —
|
||||
// used by stopAll's drain-poll loop (task #377 Toolbar fix) and
|
||||
// restartAll's success-clear loop. Both read the LIVE
|
||||
// defaultStore object so tests that mutate `defaultStore.nodes`
|
||||
// mid-flight (e.g. simulating a TASK_UPDATED that drops
|
||||
// activeTasks to 0) see the update on the next poll tick.
|
||||
const hook = vi.fn((selector: (s: typeof defaultStore) => unknown) =>
|
||||
vi.mock("@/store/canvas", () => ({
|
||||
useCanvasStore: vi.fn((selector: (s: typeof defaultStore) => unknown) =>
|
||||
selector(defaultStore)
|
||||
) as unknown as ((selector: (s: typeof defaultStore) => unknown) => unknown) & {
|
||||
getState: () => typeof defaultStore;
|
||||
};
|
||||
hook.getState = () => defaultStore;
|
||||
return { useCanvasStore: hook };
|
||||
});
|
||||
),
|
||||
}));
|
||||
|
||||
// ── Component under test ───────────────────────────────────────────────────────
|
||||
import { Toolbar } from "../Toolbar";
|
||||
// Imported AFTER vi.mock("@/lib/api", ...) above (hoisted) so this
|
||||
// resolves to the mock module; gives the new task #377 tests a typed
|
||||
// handle on api.post without a CJS require() (Vitest runs ESM).
|
||||
import { api as mockedApi } from "@/lib/api";
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -331,157 +315,3 @@ describe("Toolbar — ? shortcut opens shortcuts dialog", () => {
|
||||
expect(screen.queryByTestId("shortcuts-dialog")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Toolbar — Stop All polite-cancel flow (task #377) ───────────────────────
|
||||
|
||||
describe("Toolbar — Stop All polite cancel before restart (#377)", () => {
|
||||
// `api` resolves to the top-level vi.mock factory's mocked `post`.
|
||||
// We type-cast so TS allows mockReset/mockResolvedValue/mockImplementation
|
||||
// calls without leaking the mock surface into the production type.
|
||||
const api = mockedApi as unknown as { post: ReturnType<typeof vi.fn> };
|
||||
|
||||
/**
|
||||
* Build a working set of two active workspaces so the assertions can
|
||||
* distinguish per-id behavior (drained vs undrained) within one test.
|
||||
*/
|
||||
const seedTwoActive = () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online", "online"], [2, 2]));
|
||||
};
|
||||
|
||||
/**
|
||||
* Drive an async useCallback handler to completion. Vitest's fake
|
||||
* timers don't see microtasks unless we yield between advances; the
|
||||
* helper interleaves `vi.advanceTimersByTimeAsync` with macrotask
|
||||
* yields so pending fetch resolutions and setTimeout callbacks both
|
||||
* settle before the assertion runs.
|
||||
*/
|
||||
const advanceUntilSettled = async (ms: number) => {
|
||||
await vi.advanceTimersByTimeAsync(ms);
|
||||
// One extra tick lets any chained .then() after a setTimeout
|
||||
// resolution fire before the test moves on.
|
||||
await Promise.resolve();
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
api.post.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("phase 1: issues tasks/cancel via /workspaces/:id/a2a BEFORE any /restart", async () => {
|
||||
seedTwoActive();
|
||||
// Hold both tasks/cancel responses open so the click handler is
|
||||
// observably paused at phase 1. We don't actually need to resolve
|
||||
// them for the order assertion — just inspect the call log.
|
||||
let resolveCancels!: () => void;
|
||||
const cancelGate = new Promise<void>((r) => { resolveCancels = r; });
|
||||
api.post.mockImplementation(async (path: string) => {
|
||||
if (path.endsWith("/a2a")) {
|
||||
await cancelGate;
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
|
||||
render(<Toolbar />);
|
||||
const btn = screen.getByRole("button", { name: /stop all running tasks/i });
|
||||
fireEvent.click(btn);
|
||||
|
||||
// Yield once so the click handler enters phase 1 and dispatches the
|
||||
// two /a2a POSTs.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
const a2aCalls = api.post.mock.calls.filter((c) => String(c[0]).endsWith("/a2a"));
|
||||
const restartCalls = api.post.mock.calls.filter((c) => String(c[0]).endsWith("/restart"));
|
||||
expect(a2aCalls.length).toBe(2);
|
||||
expect(restartCalls.length).toBe(0);
|
||||
|
||||
// Each /a2a POST carries the canonical tasks/cancel envelope.
|
||||
for (const call of a2aCalls) {
|
||||
expect(call[1]).toEqual({ method: "tasks/cancel", params: {} });
|
||||
}
|
||||
|
||||
// Release the gate so the test cleanup doesn't dangle.
|
||||
resolveCancels();
|
||||
await advanceUntilSettled(10_000);
|
||||
});
|
||||
|
||||
it("phase 2: when activeTasks drains to 0 during the poll window, /restart is NOT called", async () => {
|
||||
seedTwoActive();
|
||||
api.post.mockResolvedValue(undefined);
|
||||
|
||||
render(<Toolbar />);
|
||||
fireEvent.click(screen.getByRole("button", { name: /stop all running tasks/i }));
|
||||
|
||||
// Let phase 1 fire (the two tasks/cancel calls).
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
// Simulate the platform pushing TASK_UPDATED with active_tasks=0
|
||||
// on both workspaces — emulate by mutating the store directly,
|
||||
// which is what canvas-events.ts does in production.
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online", "online"], [0, 0]));
|
||||
|
||||
// Advance past the first poll interval (250ms) so the loop sees
|
||||
// the drained store and exits early.
|
||||
await advanceUntilSettled(400);
|
||||
// Drain any remaining timers so the handler returns cleanly.
|
||||
await advanceUntilSettled(10_000);
|
||||
|
||||
const restartCalls = api.post.mock.calls.filter((c) => String(c[0]).endsWith("/restart"));
|
||||
expect(restartCalls.length).toBe(0);
|
||||
});
|
||||
|
||||
it("phase 3: when activeTasks does NOT drain inside the timeout, falls through to /restart for each stuck workspace", async () => {
|
||||
seedTwoActive();
|
||||
api.post.mockResolvedValue(undefined);
|
||||
|
||||
render(<Toolbar />);
|
||||
fireEvent.click(screen.getByRole("button", { name: /stop all running tasks/i }));
|
||||
|
||||
// Phase 1 dispatch.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
// Do NOT drain — activeTasks stays at 2 for both. Advance past the
|
||||
// 8000ms drain timeout plus a buffer so phase 3's /restart POSTs fire.
|
||||
await advanceUntilSettled(9_000);
|
||||
await advanceUntilSettled(1_000);
|
||||
|
||||
const a2aCalls = api.post.mock.calls.filter((c) => String(c[0]).endsWith("/a2a"));
|
||||
const restartCalls = api.post.mock.calls.filter((c) => String(c[0]).endsWith("/restart"));
|
||||
expect(a2aCalls.length).toBe(2);
|
||||
expect(restartCalls.length).toBe(2);
|
||||
|
||||
// Order check: every /a2a call comes before every /restart call.
|
||||
const lastA2AIdx = Math.max(
|
||||
...api.post.mock.calls.map((c, i) => (String(c[0]).endsWith("/a2a") ? i : -1))
|
||||
);
|
||||
const firstRestartIdx = Math.min(
|
||||
...api.post.mock.calls.map((c, i) => (String(c[0]).endsWith("/restart") ? i : Infinity))
|
||||
);
|
||||
expect(lastA2AIdx).toBeLessThan(firstRestartIdx);
|
||||
});
|
||||
|
||||
it("phase 3 selective: drains only one of two workspaces — /restart is called only for the stuck one", async () => {
|
||||
seedTwoActive();
|
||||
api.post.mockResolvedValue(undefined);
|
||||
|
||||
render(<Toolbar />);
|
||||
fireEvent.click(screen.getByRole("button", { name: /stop all running tasks/i }));
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
// ws-0 drains immediately, ws-1 stays stuck for the full timeout.
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online", "online"], [0, 2]));
|
||||
await advanceUntilSettled(9_500);
|
||||
|
||||
const restartCalls = api.post.mock.calls.filter((c) => String(c[0]).endsWith("/restart"));
|
||||
expect(restartCalls.length).toBe(1);
|
||||
expect(restartCalls[0][0]).toBe("/workspaces/ws-1/restart");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
'use client';
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { api } from '@/lib/api';
|
||||
import { fetchSession, type Session } from '@/lib/auth';
|
||||
import { getTenantSlug } from '@/lib/tenant';
|
||||
import { Spinner } from '@/components/Spinner';
|
||||
|
||||
/**
|
||||
* Organization-identity surface inside SettingsPanel.
|
||||
*
|
||||
* Closes a chronic UX gap where users (and our own AI agents) had to
|
||||
* call /cp/auth/me or /cp/orgs from browser devtools to read their
|
||||
* org_id UUID. Now: a copy-buttoned view of name + slug + UUID for the
|
||||
* currently-active org, plus a switcher list when the user belongs to
|
||||
* multiple orgs.
|
||||
*
|
||||
* Data path:
|
||||
* 1. fetchSession() → /cp/auth/me → current org_id
|
||||
* 2. api.get('/cp/orgs') → list of all orgs the user belongs to
|
||||
* 3. Match by id === session.org_id; fall back to host-slug match
|
||||
* if the session probe loses the race.
|
||||
*
|
||||
* Read-only — this tab never mutates. Org creation/switching lives at
|
||||
* /orgs (the post-signup landing page).
|
||||
*/
|
||||
|
||||
interface Org {
|
||||
id: string;
|
||||
slug: string;
|
||||
name: string;
|
||||
status?: string;
|
||||
}
|
||||
|
||||
// /cp/orgs may return a bare array or {orgs: []} — see orgs/page.tsx
|
||||
// for the same defensive unwrap.
|
||||
type OrgsResponse = Org[] | { orgs?: Org[] };
|
||||
|
||||
export function OrgInfoTab() {
|
||||
const [orgs, setOrgs] = useState<Org[] | null>(null);
|
||||
const [session, setSession] = useState<Session | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const [sess, body] = await Promise.all([
|
||||
fetchSession().catch(() => null),
|
||||
api.get<OrgsResponse>('/cp/orgs'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
setSession(sess);
|
||||
setOrgs(Array.isArray(body) ? body : body.orgs ?? []);
|
||||
} catch (e) {
|
||||
if (!cancelled) setError(e instanceof Error ? e.message : 'Failed to load org info');
|
||||
} finally {
|
||||
if (!cancelled) setLoading(false);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const tenantSlug = getTenantSlug();
|
||||
const currentOrg =
|
||||
orgs?.find((o) => session && o.id === session.org_id) ??
|
||||
orgs?.find((o) => tenantSlug && o.slug === tenantSlug) ??
|
||||
null;
|
||||
const otherOrgs = orgs?.filter((o) => o.id !== currentOrg?.id) ?? [];
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
className="flex items-center justify-center gap-2 py-6 text-ink-mid text-xs"
|
||||
>
|
||||
<Spinner /> Loading organization…
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (error) {
|
||||
return (
|
||||
<div className="p-4">
|
||||
<div className="px-3 py-2 bg-red-950/40 border border-red-800/50 rounded-lg text-[10px] text-bad">
|
||||
{error}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (!currentOrg) {
|
||||
return (
|
||||
<div className="p-4">
|
||||
<p className="text-xs text-ink-mid">
|
||||
No organization found for this session. If this is unexpected, sign out and back in, or visit{' '}
|
||||
<a href="/orgs" className="underline">/orgs</a>.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="p-4 space-y-4">
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold text-ink mb-1">Current Organization</h3>
|
||||
<p className="text-[10px] text-ink-mid leading-relaxed">
|
||||
IDs you can paste into API calls, support tickets, or CLI arguments. The UUID never changes;
|
||||
the slug is the URL subdomain.
|
||||
</p>
|
||||
</div>
|
||||
<OrgIdentityCard org={currentOrg} highlighted />
|
||||
{otherOrgs.length > 0 && (
|
||||
<div className="space-y-2 pt-2">
|
||||
<h4 className="text-[11px] font-semibold text-ink-mid uppercase tracking-wider">
|
||||
Your other organizations ({otherOrgs.length})
|
||||
</h4>
|
||||
{otherOrgs.map((o) => (
|
||||
<OrgIdentityCard key={o.id} org={o} />
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function OrgIdentityCard({ org, highlighted }: { org: Org; highlighted?: boolean }) {
|
||||
return (
|
||||
<div
|
||||
className={`rounded-lg border p-3 space-y-2 ${
|
||||
highlighted ? 'border-accent/40 bg-accent-strong/5' : 'border-line/40 bg-surface-card/40'
|
||||
}`}
|
||||
data-testid={`org-card-${org.slug}`}
|
||||
>
|
||||
<div className="flex items-baseline justify-between gap-2">
|
||||
<span className="text-[12px] font-medium text-ink truncate">{org.name}</span>
|
||||
{org.status && (
|
||||
<span className="text-[9px] text-ink-mid uppercase tracking-wider shrink-0">{org.status}</span>
|
||||
)}
|
||||
</div>
|
||||
<IdentityRow label="Slug" value={org.slug} />
|
||||
<IdentityRow label="UUID" value={org.id} mono />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function IdentityRow({ label, value, mono }: { label: string; value: string; mono?: boolean }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const onCopy = useCallback(() => {
|
||||
// Best-effort: jsdom + old Safari throw synchronously on writeText.
|
||||
try {
|
||||
navigator.clipboard.writeText(value);
|
||||
} catch {
|
||||
/* user can still triple-click select */
|
||||
}
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
}, [value]);
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-[10px] text-ink-mid w-10 shrink-0">{label}</span>
|
||||
<code
|
||||
className={`flex-1 text-[11px] text-ink bg-surface-sunken/60 px-2 py-1 rounded select-all break-all ${
|
||||
mono ? 'font-mono' : ''
|
||||
}`}
|
||||
>
|
||||
{value}
|
||||
</code>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onCopy}
|
||||
aria-label={`Copy ${label}`}
|
||||
className="shrink-0 px-2 py-1 bg-surface-card/60 hover:bg-surface-card border border-line/40 rounded text-[10px] text-ink-mid hover:text-ink transition-colors focus:outline-none focus-visible:ring-2 focus-visible:ring-accent focus-visible:ring-offset-1"
|
||||
>
|
||||
{copied ? 'Copied' : 'Copy'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import { useKeyboardShortcut } from '@/hooks/use-keyboard-shortcut';
|
||||
import { SecretsTab } from './SecretsTab';
|
||||
import { TokensTab } from './TokensTab';
|
||||
import { OrgTokensTab } from './OrgTokensTab';
|
||||
import { OrgInfoTab } from './OrgInfoTab';
|
||||
import { UnsavedChangesGuard } from './UnsavedChangesGuard';
|
||||
|
||||
/** Module-level ref so TopBar's SettingsButton can receive focus back on close. */
|
||||
@@ -117,9 +116,6 @@ export function SettingsPanel({ workspaceId }: SettingsPanelProps) {
|
||||
<Tabs.Trigger value="org-tokens" className="settings-panel__tab">
|
||||
Org API Keys
|
||||
</Tabs.Trigger>
|
||||
<Tabs.Trigger value="org-info" className="settings-panel__tab">
|
||||
Organization
|
||||
</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
|
||||
<Tabs.Content value="api-keys" className="settings-panel__content">
|
||||
@@ -133,10 +129,6 @@ export function SettingsPanel({ workspaceId }: SettingsPanelProps) {
|
||||
<Tabs.Content value="org-tokens" className="settings-panel__content">
|
||||
<OrgTokensTab />
|
||||
</Tabs.Content>
|
||||
|
||||
<Tabs.Content value="org-info" className="settings-panel__content">
|
||||
<OrgInfoTab />
|
||||
</Tabs.Content>
|
||||
</Tabs.Root>
|
||||
|
||||
<div className="settings-panel__footer">
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
/**
|
||||
* Tests for OrgInfoTab — surfaces current org name/slug/UUID with copy
|
||||
* buttons, plus a list of the user's other orgs when applicable.
|
||||
*
|
||||
* Covers (≥3 cases per the closing-the-UX-gap brief):
|
||||
* - Loading state (spinner + aria-live)
|
||||
* - Renders current org matched by session.org_id, with UUID + slug + name
|
||||
* - Copy button writes the UUID to navigator.clipboard
|
||||
* - Falls back to host-slug match when session lookup fails
|
||||
* - Lists other orgs when user belongs to multiple
|
||||
* - Error banner when /cp/orgs throws
|
||||
* - Empty/no-match state renders the recovery hint, not a crash
|
||||
*/
|
||||
import React from "react";
|
||||
import { render, screen, fireEvent, cleanup, act, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { OrgInfoTab } from "../OrgInfoTab";
|
||||
|
||||
const mockGet = vi.fn();
|
||||
const mockFetchSession = vi.fn();
|
||||
const mockGetTenantSlug = vi.fn();
|
||||
|
||||
vi.mock("@/lib/api", () => ({
|
||||
api: { get: (...args: unknown[]) => mockGet(...args) },
|
||||
}));
|
||||
vi.mock("@/lib/auth", () => ({
|
||||
fetchSession: (...args: unknown[]) => mockFetchSession(...args),
|
||||
}));
|
||||
vi.mock("@/lib/tenant", () => ({
|
||||
getTenantSlug: (...args: unknown[]) => mockGetTenantSlug(...args),
|
||||
}));
|
||||
|
||||
// Stub clipboard
|
||||
vi.stubGlobal("navigator", {
|
||||
clipboard: { writeText: vi.fn().mockResolvedValue(undefined) },
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useRealTimers();
|
||||
mockGet.mockReset();
|
||||
mockFetchSession.mockReset();
|
||||
mockGetTenantSlug.mockReset();
|
||||
mockGetTenantSlug.mockReturnValue("");
|
||||
vi.mocked(navigator.clipboard.writeText).mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
async function flush() {
|
||||
await act(async () => {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
}
|
||||
|
||||
const AGENTS_TEAM = {
|
||||
id: "2355b568-0799-4cc7-9e7f-806747f9958c",
|
||||
slug: "agents-team",
|
||||
name: "Agents Team",
|
||||
status: "running",
|
||||
};
|
||||
const OTHER_ORG = {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
slug: "skunkworks",
|
||||
name: "Skunkworks",
|
||||
status: "running",
|
||||
};
|
||||
|
||||
// ─── Loading ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("OrgInfoTab — loading", () => {
|
||||
it("shows spinner while fetching", () => {
|
||||
mockGet.mockImplementation(() => new Promise(() => {}));
|
||||
mockFetchSession.mockImplementation(() => new Promise(() => {}));
|
||||
render(<OrgInfoTab />);
|
||||
const status = screen.getByRole("status");
|
||||
expect(status).toBeTruthy();
|
||||
expect(status.getAttribute("aria-live")).toBe("polite");
|
||||
expect(status.textContent).toContain("Loading organization");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Current org renders + copy ──────────────────────────────────────────────
|
||||
|
||||
describe("OrgInfoTab — current org", () => {
|
||||
it("renders the org matched by session.org_id with name, slug, UUID", async () => {
|
||||
mockFetchSession.mockResolvedValue({
|
||||
user_id: "u-1",
|
||||
org_id: AGENTS_TEAM.id,
|
||||
email: "hongming@moleculesai.app",
|
||||
});
|
||||
mockGet.mockResolvedValue([AGENTS_TEAM, OTHER_ORG]);
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() => screen.getByText("Current Organization"));
|
||||
|
||||
// Name shown
|
||||
expect(screen.getByText("Agents Team")).toBeTruthy();
|
||||
// Slug shown
|
||||
expect(screen.getByText("agents-team")).toBeTruthy();
|
||||
// UUID shown
|
||||
expect(screen.getByText(AGENTS_TEAM.id)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("copy-UUID button writes the UUID to navigator.clipboard", async () => {
|
||||
mockFetchSession.mockResolvedValue({
|
||||
user_id: "u-1",
|
||||
org_id: AGENTS_TEAM.id,
|
||||
email: "hongming@moleculesai.app",
|
||||
});
|
||||
mockGet.mockResolvedValue([AGENTS_TEAM]);
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() => screen.getByText(AGENTS_TEAM.id));
|
||||
|
||||
const copyUuid = screen.getByRole("button", { name: /Copy UUID/i });
|
||||
fireEvent.click(copyUuid);
|
||||
|
||||
expect(navigator.clipboard.writeText).toHaveBeenCalledWith(AGENTS_TEAM.id);
|
||||
// Optimistic "Copied" label flip
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
screen.getByRole("button", { name: /Copy UUID/i }).textContent,
|
||||
).toContain("Copied"),
|
||||
);
|
||||
});
|
||||
|
||||
it("copy-Slug button writes the slug to navigator.clipboard", async () => {
|
||||
mockFetchSession.mockResolvedValue({
|
||||
user_id: "u-1",
|
||||
org_id: AGENTS_TEAM.id,
|
||||
email: "hongming@moleculesai.app",
|
||||
});
|
||||
mockGet.mockResolvedValue([AGENTS_TEAM]);
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() => screen.getByText(AGENTS_TEAM.slug));
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: /Copy Slug/i }));
|
||||
expect(navigator.clipboard.writeText).toHaveBeenCalledWith(AGENTS_TEAM.slug);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Fallback: host-slug match when session fails ────────────────────────────
|
||||
|
||||
describe("OrgInfoTab — fallbacks", () => {
|
||||
it("falls back to host-slug match when fetchSession rejects", async () => {
|
||||
mockFetchSession.mockRejectedValue(new Error("session probe failed"));
|
||||
mockGetTenantSlug.mockReturnValue("agents-team");
|
||||
mockGet.mockResolvedValue({ orgs: [AGENTS_TEAM, OTHER_ORG] }); // wrapped shape
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() => screen.getByText("Current Organization"));
|
||||
|
||||
expect(screen.getByText("Agents Team")).toBeTruthy();
|
||||
expect(screen.getByText(AGENTS_TEAM.id)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("lists other orgs the user belongs to under a separate header", async () => {
|
||||
mockFetchSession.mockResolvedValue({
|
||||
user_id: "u-1",
|
||||
org_id: AGENTS_TEAM.id,
|
||||
email: "hongming@moleculesai.app",
|
||||
});
|
||||
mockGet.mockResolvedValue([AGENTS_TEAM, OTHER_ORG]);
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() => screen.getByText(/Your other organizations/));
|
||||
|
||||
expect(screen.getByText("Skunkworks")).toBeTruthy();
|
||||
expect(screen.getByText(OTHER_ORG.id)).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Error + empty handling ──────────────────────────────────────────────────
|
||||
|
||||
describe("OrgInfoTab — error + empty", () => {
|
||||
it("renders an error banner when /cp/orgs throws", async () => {
|
||||
mockFetchSession.mockResolvedValue(null);
|
||||
mockGet.mockRejectedValue(new Error("API GET /cp/orgs: 500 boom"));
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() => screen.getByText(/500 boom/));
|
||||
expect(screen.queryByText("Current Organization")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders the recovery hint when no org matches (no crash)", async () => {
|
||||
mockFetchSession.mockResolvedValue(null);
|
||||
mockGetTenantSlug.mockReturnValue("");
|
||||
mockGet.mockResolvedValue([]);
|
||||
|
||||
render(<OrgInfoTab />);
|
||||
await flush();
|
||||
await waitFor(() =>
|
||||
screen.getByText(/No organization found for this session/),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -8,4 +8,3 @@ export { SearchBar } from './SearchBar';
|
||||
export { EmptyState } from './EmptyState';
|
||||
export { DeleteConfirmDialog } from './DeleteConfirmDialog';
|
||||
export { UnsavedChangesGuard } from './UnsavedChangesGuard';
|
||||
export { OrgInfoTab } from './OrgInfoTab';
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { runtimeDisplayName } from "@/lib/runtime-names";
|
||||
import type { WorkspaceNodeData } from "@/store/canvas";
|
||||
|
||||
type Props = {
|
||||
data: Pick<
|
||||
WorkspaceNodeData,
|
||||
"runtime" | "status" | "needsRestart" | "activeTasks" | "deliveryMode"
|
||||
| "workspaceAccess" | "maxConcurrentTasks"
|
||||
>;
|
||||
};
|
||||
|
||||
export function ContainerConfigTab({ data }: Props) {
|
||||
const runtime = data.runtime || "unknown";
|
||||
const workspaceAccess = formatAccess(data.workspaceAccess);
|
||||
const maxConcurrentTasks = data.maxConcurrentTasks ? String(data.maxConcurrentTasks) : "platform-managed";
|
||||
const mountedPath = "/workspace";
|
||||
const privilegeStatus = "standard";
|
||||
const deliveryMode = data.deliveryMode || "push";
|
||||
|
||||
return (
|
||||
<div className="p-4 space-y-4">
|
||||
<section className="rounded-lg border border-line/50 bg-surface-card/40 p-4">
|
||||
<div className="mb-3">
|
||||
<h3 className="text-sm font-semibold text-ink">Container Config</h3>
|
||||
</div>
|
||||
|
||||
<dl className="grid grid-cols-1 gap-2 text-[11px]">
|
||||
<ConfigRow label="Runtime image" value={runtimeDisplayName(runtime)} detail={runtime} />
|
||||
<ConfigRow label="Workspace access" value={workspaceAccess} />
|
||||
<ConfigRow label="Max concurrent tasks" value={maxConcurrentTasks} />
|
||||
<ConfigRow label="Mounted workspace path" value={mountedPath} />
|
||||
<ConfigRow label="Container privileges" value={privilegeStatus} />
|
||||
<ConfigRow label="Delivery mode" value={deliveryMode} />
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<section className="rounded-lg border border-line/50 bg-surface-card/40 p-4">
|
||||
<h3 className="mb-3 text-sm font-semibold text-ink">Session Controls</h3>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<ReadOnlyAction label={data.needsRestart ? "Restart required" : "Restart"} />
|
||||
<ReadOnlyAction label="Reset session" />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="rounded-lg border border-line/50 bg-surface-card/40 p-4">
|
||||
<h3 className="mb-3 text-sm font-semibold text-ink">Status</h3>
|
||||
<dl className="grid grid-cols-1 gap-2 text-[11px]">
|
||||
<ConfigRow label="Container status" value={data.status} />
|
||||
<ConfigRow label="Active tasks" value={String(data.activeTasks ?? 0)} />
|
||||
<ConfigRow label="Mounted path access" value="available" />
|
||||
</dl>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function formatAccess(value: string | null | undefined): string {
|
||||
if (!value) return "none";
|
||||
return value.replace(/_/g, "-");
|
||||
}
|
||||
|
||||
function ConfigRow({
|
||||
label,
|
||||
value,
|
||||
detail,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
detail?: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-start justify-between gap-3 rounded-md bg-surface-sunken/40 px-3 py-2">
|
||||
<dt className="text-ink-mid">{label}</dt>
|
||||
<dd className="min-w-0 text-right">
|
||||
<div className="font-mono text-ink break-words">{value}</div>
|
||||
{detail && detail !== value && (
|
||||
<div className="mt-0.5 font-mono text-[10px] text-ink-mid break-words">{detail}</div>
|
||||
)}
|
||||
</dd>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ReadOnlyAction({ label }: { label: string }) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
disabled
|
||||
className="rounded-md border border-line/50 bg-surface-sunken/40 px-3 py-2 text-[11px] text-ink-mid disabled:cursor-not-allowed disabled:opacity-70"
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
|
||||
interface DisplayStatus {
|
||||
available: boolean;
|
||||
reason?: string;
|
||||
mode?: string;
|
||||
status?: string;
|
||||
protocol?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
}
|
||||
|
||||
interface DisplayControlStatus {
|
||||
controller: "none" | "user" | "agent";
|
||||
controlled_by?: string;
|
||||
expires_at?: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export function DisplayTab({ workspaceId }: Props) {
|
||||
const [status, setStatus] = useState<DisplayStatus | null>(null);
|
||||
const [control, setControl] = useState<DisplayControlStatus | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [controlError, setControlError] = useState<string | null>(null);
|
||||
const [controlBusy, setControlBusy] = useState(false);
|
||||
const requestGeneration = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
const generation = requestGeneration.current + 1;
|
||||
requestGeneration.current = generation;
|
||||
let cancelled = false;
|
||||
setStatus(null);
|
||||
setControl(null);
|
||||
setError(null);
|
||||
setControlError(null);
|
||||
setControlBusy(false);
|
||||
async function load() {
|
||||
try {
|
||||
const displayStatus = await api.get<DisplayStatus>(`/workspaces/${workspaceId}/display`);
|
||||
if (cancelled || requestGeneration.current !== generation) return;
|
||||
setStatus(displayStatus);
|
||||
if (displayStatus.reason === "display_not_enabled") return;
|
||||
try {
|
||||
const displayControl = await api.get<DisplayControlStatus>(`/workspaces/${workspaceId}/display/control`);
|
||||
if (!cancelled && requestGeneration.current === generation) setControl(displayControl);
|
||||
} catch (err) {
|
||||
if (!cancelled && requestGeneration.current === generation) {
|
||||
setControl(null);
|
||||
setControlError("Display control unavailable");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (!cancelled && requestGeneration.current === generation) setError("The display status could not be loaded.");
|
||||
}
|
||||
}
|
||||
load();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [workspaceId]);
|
||||
|
||||
const acquireControl = async () => {
|
||||
const generation = requestGeneration.current;
|
||||
const controlPath = `/workspaces/${workspaceId}/display/control`;
|
||||
setControlBusy(true);
|
||||
setControlError(null);
|
||||
try {
|
||||
const next = await api.post<DisplayControlStatus>(`${controlPath}/acquire`, {
|
||||
controller: "user",
|
||||
ttl_seconds: 300,
|
||||
});
|
||||
if (requestGeneration.current !== generation) return;
|
||||
setControl(next);
|
||||
} catch (err) {
|
||||
if (requestGeneration.current !== generation) return;
|
||||
setControlError("Failed to take control");
|
||||
try {
|
||||
const latest = await api.get<DisplayControlStatus>(controlPath);
|
||||
if (requestGeneration.current !== generation) return;
|
||||
setControl(latest);
|
||||
} catch {
|
||||
if (requestGeneration.current !== generation) return;
|
||||
setControl(null);
|
||||
}
|
||||
} finally {
|
||||
if (requestGeneration.current === generation) setControlBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="p-5">
|
||||
<div className="rounded-lg border border-red-500/20 bg-red-950/20 p-4">
|
||||
<h3 className="text-sm font-medium text-red-200">Display status unavailable</h3>
|
||||
<p className="mt-2 text-[11px] leading-relaxed text-red-200/75">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!status) {
|
||||
return (
|
||||
<div className="p-5">
|
||||
<div className="h-24 rounded-lg border border-line/40 bg-surface-sunken/30 motion-safe:animate-pulse" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!status.available) {
|
||||
const isNotEnabled = status.reason === "display_not_enabled";
|
||||
return (
|
||||
<div className="flex min-h-full flex-col items-center justify-center bg-surface-sunken/30 p-8 text-center">
|
||||
<svg
|
||||
width="72"
|
||||
height="72"
|
||||
viewBox="0 0 72 72"
|
||||
fill="none"
|
||||
aria-hidden="true"
|
||||
className="mb-4 text-ink-mid"
|
||||
>
|
||||
<rect x="12" y="14" width="48" height="36" rx="4" stroke="currentColor" strokeWidth="2.5" opacity="0.65" />
|
||||
<path d="M28 58h16M36 50v8M16 16l40 40" stroke="currentColor" strokeWidth="3" strokeLinecap="round" />
|
||||
</svg>
|
||||
<h3 className="mb-1.5 text-sm font-medium text-ink">
|
||||
{isNotEnabled ? "Display is not enabled for this workspace." : "Display session is not ready."}
|
||||
</h3>
|
||||
<p className="max-w-xs text-[11px] leading-relaxed text-ink-mid">
|
||||
{isNotEnabled
|
||||
? "Recreate this workspace with display enabled to view and take over its desktop."
|
||||
: "This workspace has display configuration, but the desktop session infrastructure is not configured yet."}
|
||||
</p>
|
||||
{!isNotEnabled && (
|
||||
<>
|
||||
<dl className="mt-5 grid grid-cols-2 gap-x-4 gap-y-2 text-left text-[11px]">
|
||||
<dt className="text-ink-mid">Mode</dt>
|
||||
<dd className="font-mono text-ink">{status.mode || "unknown"}</dd>
|
||||
<dt className="text-ink-mid">Status</dt>
|
||||
<dd className="font-mono text-ink">{status.status || "unknown"}</dd>
|
||||
</dl>
|
||||
<div className="mt-5 w-full max-w-xs border-t border-line/50 pt-4">
|
||||
{control ? (
|
||||
<div className="flex items-center justify-between gap-3 text-left">
|
||||
<div className="min-w-0">
|
||||
<p className="text-[11px] font-medium text-ink">
|
||||
{control.controller === "none"
|
||||
? "No active controller"
|
||||
: `Controlled by ${displayControlActorLabel(control)}`}
|
||||
</p>
|
||||
{control.expires_at && (
|
||||
<p className="mt-1 truncate font-mono text-[10px] text-ink-mid">
|
||||
Until {new Date(control.expires_at).toLocaleTimeString()}
|
||||
</p>
|
||||
)}
|
||||
{controlError && <p className="mt-1 text-[10px] leading-snug text-red-200">{controlError}</p>}
|
||||
</div>
|
||||
{control.controller === "none" && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={acquireControl}
|
||||
disabled={controlBusy}
|
||||
className="h-8 shrink-0 rounded border border-line bg-surface px-3 text-[11px] font-medium text-ink hover:bg-surface-elevated disabled:cursor-not-allowed disabled:opacity-60"
|
||||
>
|
||||
Take control
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="text-left">
|
||||
{!controlError && (
|
||||
<div className="h-8 rounded border border-line/40 bg-surface-sunken/30 motion-safe:animate-pulse" />
|
||||
)}
|
||||
{controlError && <p className="mt-2 text-[10px] leading-snug text-red-200">{controlError}</p>}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function displayControlActorLabel(control: DisplayControlStatus): string {
|
||||
if (control.controller === "agent") return "Agent";
|
||||
if (control.controlled_by === "admin-token") return "Admin";
|
||||
if (control.controlled_by?.startsWith("org-token:")) return "Automation";
|
||||
return "User";
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
import { cleanup, render, screen } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/runtime-names", () => ({
|
||||
runtimeDisplayName: (runtime: string) => runtime,
|
||||
}));
|
||||
|
||||
import { ContainerConfigTab } from "../ContainerConfigTab";
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
describe("ContainerConfigTab", () => {
|
||||
it("renders read-only runtime and container settings separate from compute shape", () => {
|
||||
render(
|
||||
<ContainerConfigTab
|
||||
data={{
|
||||
runtime: "claude-code",
|
||||
status: "online",
|
||||
needsRestart: false,
|
||||
activeTasks: 2,
|
||||
maxConcurrentTasks: 3,
|
||||
workspaceAccess: "read_write",
|
||||
deliveryMode: "poll",
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("Runtime image")).toBeTruthy();
|
||||
expect(screen.getByText("claude-code")).toBeTruthy();
|
||||
expect(screen.getByText("Workspace access")).toBeTruthy();
|
||||
expect(screen.getByText("read-write")).toBeTruthy();
|
||||
expect(screen.getByText("Max concurrent tasks")).toBeTruthy();
|
||||
expect(screen.getByText("3")).toBeTruthy();
|
||||
expect(screen.getByText("/workspace")).toBeTruthy();
|
||||
expect(screen.getByText("Container privileges")).toBeTruthy();
|
||||
expect(screen.queryByText("Instance type")).toBeNull();
|
||||
expect(screen.queryByText("Root volume")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,250 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
|
||||
const { mockGet, mockPost } = vi.hoisted(() => ({ mockGet: vi.fn(), mockPost: vi.fn() }));
|
||||
|
||||
vi.mock("@/lib/api", () => ({
|
||||
api: {
|
||||
get: mockGet,
|
||||
post: mockPost,
|
||||
},
|
||||
}));
|
||||
|
||||
import { DisplayTab } from "../DisplayTab";
|
||||
|
||||
describe("DisplayTab", () => {
|
||||
beforeEach(() => {
|
||||
cleanup();
|
||||
mockGet.mockReset();
|
||||
mockPost.mockReset();
|
||||
});
|
||||
|
||||
it("renders unavailable state for non-display workspaces", async () => {
|
||||
mockGet.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_not_enabled",
|
||||
});
|
||||
|
||||
render(<DisplayTab workspaceId="ws-no-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Display is not enabled for this workspace.")).toBeTruthy();
|
||||
});
|
||||
expect(mockGet).toHaveBeenCalledWith("/workspaces/ws-no-display/display");
|
||||
expect(mockGet).not.toHaveBeenCalledWith("/workspaces/ws-no-display/display/control");
|
||||
});
|
||||
|
||||
it("renders control acquisition for display-configured workspaces", async () => {
|
||||
mockGet
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "none",
|
||||
});
|
||||
mockPost.mockResolvedValueOnce({
|
||||
controller: "user",
|
||||
controlled_by: "admin-token",
|
||||
expires_at: "2026-05-23T08:48:27Z",
|
||||
});
|
||||
|
||||
render(<DisplayTab workspaceId="ws-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: "Take control" })).toBeTruthy();
|
||||
});
|
||||
expect(mockGet).toHaveBeenCalledWith("/workspaces/ws-display/display");
|
||||
expect(mockGet).toHaveBeenCalledWith("/workspaces/ws-display/display/control");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Take control" }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Controlled by Admin")).toBeTruthy();
|
||||
});
|
||||
expect(mockPost).toHaveBeenCalledWith("/workspaces/ws-display/display/control/acquire", {
|
||||
controller: "user",
|
||||
ttl_seconds: 300,
|
||||
});
|
||||
});
|
||||
|
||||
it("renders active display control locks as observe-only", async () => {
|
||||
mockGet
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "agent",
|
||||
controlled_by: "sidecar",
|
||||
expires_at: "2026-05-23T08:48:27Z",
|
||||
});
|
||||
|
||||
render(<DisplayTab workspaceId="ws-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Controlled by Agent")).toBeTruthy();
|
||||
});
|
||||
expect(screen.queryByRole("button", { name: "Release" })).toBeNull();
|
||||
expect(screen.queryByRole("button", { name: "Take control" })).toBeNull();
|
||||
expect(mockPost).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("labels org-token display control locks as automation", async () => {
|
||||
mockGet
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "user",
|
||||
controlled_by: "org-token:abc123",
|
||||
expires_at: "2026-05-23T08:48:27Z",
|
||||
});
|
||||
|
||||
render(<DisplayTab workspaceId="ws-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Controlled by Automation")).toBeTruthy();
|
||||
});
|
||||
expect(screen.queryByText("org-token:abc123")).toBeNull();
|
||||
expect(screen.queryByRole("button", { name: "Take control" })).toBeNull();
|
||||
});
|
||||
|
||||
it("refreshes display control state after failed acquisition", async () => {
|
||||
mockGet
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "none",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "agent",
|
||||
controlled_by: "sidecar",
|
||||
expires_at: "2026-05-23T08:48:27Z",
|
||||
});
|
||||
mockPost.mockRejectedValueOnce(new Error("API POST /workspaces/ws-display/display/control/acquire: 409 conflict"));
|
||||
|
||||
render(<DisplayTab workspaceId="ws-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: "Take control" })).toBeTruthy();
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Take control" }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Controlled by Agent")).toBeTruthy();
|
||||
});
|
||||
expect(screen.getByText("Failed to take control")).toBeTruthy();
|
||||
expect(mockGet).toHaveBeenCalledWith("/workspaces/ws-display/display/control");
|
||||
expect(mockGet).toHaveBeenCalledTimes(3);
|
||||
expect(mockPost).toHaveBeenCalledWith("/workspaces/ws-display/display/control/acquire", {
|
||||
controller: "user",
|
||||
ttl_seconds: 300,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps display status visible without takeover actions when control status fails", async () => {
|
||||
mockGet
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockRejectedValueOnce(new Error("API GET /workspaces/ws-display/display/control: 401 unauthorized"));
|
||||
|
||||
render(<DisplayTab workspaceId="ws-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Display session is not ready.")).toBeTruthy();
|
||||
});
|
||||
expect(screen.queryByRole("button", { name: "Take control" })).toBeNull();
|
||||
expect(screen.getByText("Display control unavailable")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("does not render raw display status errors", async () => {
|
||||
mockGet.mockRejectedValueOnce(new Error("API GET /workspaces/ws-display/display: 500 secret backend details"));
|
||||
|
||||
render(<DisplayTab workspaceId="ws-display" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Display status unavailable")).toBeTruthy();
|
||||
});
|
||||
expect(screen.queryByText(/secret backend details/)).toBeNull();
|
||||
});
|
||||
|
||||
it("ignores stale acquire responses after workspace changes", async () => {
|
||||
const acquire = deferred<{ controller: "user"; controlled_by: string; expires_at: string }>();
|
||||
mockGet
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "none",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
available: false,
|
||||
reason: "display_session_unavailable",
|
||||
mode: "desktop-control",
|
||||
status: "not_configured",
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
controller: "none",
|
||||
});
|
||||
mockPost.mockReturnValueOnce(acquire.promise);
|
||||
|
||||
const { rerender } = render(<DisplayTab workspaceId="ws-a" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: "Take control" })).toBeTruthy();
|
||||
});
|
||||
fireEvent.click(screen.getByRole("button", { name: "Take control" }));
|
||||
|
||||
rerender(<DisplayTab workspaceId="ws-b" />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockGet).toHaveBeenCalledWith("/workspaces/ws-b/display/control");
|
||||
});
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: "Take control" })).toBeTruthy();
|
||||
});
|
||||
|
||||
acquire.resolve({
|
||||
controller: "user",
|
||||
controlled_by: "admin-token",
|
||||
expires_at: "2026-05-23T08:48:27Z",
|
||||
});
|
||||
await acquire.promise;
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText("Controlled by Admin")).toBeNull();
|
||||
});
|
||||
expect(screen.getByRole("button", { name: "Take control" })).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
@@ -649,17 +649,7 @@ function WaitingBubbles({ visible }: { visible: CommMessage[] }) {
|
||||
if (!prev || m.timestamp > prev.timestamp) tailByPeer.set(m.peerId, m);
|
||||
}
|
||||
const waitingPeers = Array.from(tailByPeer.values()).filter(
|
||||
// Task #227 — also light the indicator for status="dispatched": that's
|
||||
// the platform's marker for a poll-mode delegation that's been
|
||||
// recorded into the peer's inbox but not yet picked up. Without this
|
||||
// arm, external/MCP peer threads showed an outbound bubble and then
|
||||
// dead silence until the eventual reply landed — no parity with the
|
||||
// native push-path "pending" indicator.
|
||||
(m) =>
|
||||
m.flow === "out" &&
|
||||
(m.status === "pending" ||
|
||||
m.status === "queued" ||
|
||||
m.status === "dispatched"),
|
||||
(m) => m.flow === "out" && (m.status === "pending" || m.status === "queued"),
|
||||
);
|
||||
if (waitingPeers.length === 0) return null;
|
||||
return (
|
||||
@@ -698,9 +688,7 @@ function WaitingBubbles({ visible }: { visible: CommMessage[] }) {
|
||||
<span className="text-[10px]">
|
||||
{m.status === "queued"
|
||||
? `${m.peerName} is busy — reply will arrive when they're free`
|
||||
: m.status === "dispatched"
|
||||
? `Queued — ${m.peerName} will pick up on next poll`
|
||||
: `Waiting for ${m.peerName}…`}
|
||||
: `Waiting for ${m.peerName}…`}
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -41,19 +41,6 @@ describe("inferA2AErrorHint", () => {
|
||||
expect(inferA2AErrorHint("RuntimeException in tool call")).toMatch(/runtime threw an exception/);
|
||||
});
|
||||
|
||||
it("points at the Activity tab (the real in-product logs surface), not 'workspace/container logs' (internal#212)", () => {
|
||||
// Pre-#212 these hints sent users to "workspace logs" / "container
|
||||
// logs" — neither has a UI affordance in the canvas. Activity tab
|
||||
// is the in-product surface where the full row lives. Lock the
|
||||
// copy so a future refactor cannot re-introduce the dangling
|
||||
// pointer.
|
||||
expect(inferA2AErrorHint("Agent error: boom")).toMatch(/Activity tab/);
|
||||
expect(inferA2AErrorHint("some completely novel error nobody has matched yet")).toMatch(/Activity tab/);
|
||||
// And the two strings together must not regress to the old text.
|
||||
expect(inferA2AErrorHint("Agent error: boom")).not.toMatch(/container logs/);
|
||||
expect(inferA2AErrorHint("some novel error")).not.toMatch(/workspace logs/);
|
||||
});
|
||||
|
||||
it("recognises peer-unreachable cases (Activity-tab originals)", () => {
|
||||
expect(inferA2AErrorHint("workspace not found")).toMatch(/can't be reached/);
|
||||
expect(inferA2AErrorHint("not accessible")).toMatch(/can't be reached/);
|
||||
@@ -66,8 +53,7 @@ describe("inferA2AErrorHint", () => {
|
||||
|
||||
it("returns a generic fallback for unrecognised text", () => {
|
||||
const hint = inferA2AErrorHint("some completely novel error nobody has matched yet");
|
||||
// Fallback now sends the user to the Activity tab (post-#212).
|
||||
expect(hint).toMatch(/Activity tab|delivery failure/);
|
||||
expect(hint).toMatch(/Check the workspace logs|delivery failure/);
|
||||
});
|
||||
|
||||
it("Claude SDK wedge wins over the more general timeout pattern", () => {
|
||||
|
||||
@@ -38,11 +38,7 @@ export function inferA2AErrorHint(detail: string): string {
|
||||
return "The connection to the remote agent dropped before a reply arrived. Usually a transient network blip — retry once. If it repeats, the remote container may have crashed mid-request; check its logs.";
|
||||
}
|
||||
if (t.includes("agent error") || t.includes("exception")) {
|
||||
// internal#212 closeout: end users have no "container logs" surface
|
||||
// in the canvas; the Activity tab IS the user-visible logs surface
|
||||
// (full row carries request/response body + error_detail). Point
|
||||
// there so the hint is actionable from inside the product.
|
||||
return "The remote agent's runtime threw an exception. Open the Activity tab for the full row (request body, response, error_detail) — Restart usually clears transient runtime crashes.";
|
||||
return "The remote agent's runtime threw an exception. Check the workspace's container logs for the traceback. Restart usually clears transient runtime crashes.";
|
||||
}
|
||||
if (
|
||||
t.includes("not found") ||
|
||||
@@ -54,9 +50,5 @@ export function inferA2AErrorHint(detail: string): string {
|
||||
if (detail === "") {
|
||||
return "The remote agent returned no error detail (the underlying httpx exception had an empty message — typically a connection-reset or silent timeout). A workspace restart is the safe first move.";
|
||||
}
|
||||
// internal#212 closeout: "workspace logs" pointed at a tab that does
|
||||
// not exist — Activity tab is the in-product logs surface. Keep the
|
||||
// hint generic enough for the unrecognised-detail fallback but point
|
||||
// the user at a real affordance.
|
||||
return "The remote agent reported a delivery failure. Open the Activity tab for the full row, or try restarting the workspace.";
|
||||
return "The remote agent reported a delivery failure. Check the workspace logs or try restarting.";
|
||||
}
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// Task #227 — external/MCP workspace progress UX parity.
|
||||
//
|
||||
// ws-server's `proxyA2ARequest` poll-mode short-circuit
|
||||
// (workspace-server/internal/handlers/a2a_proxy.go:402-432) returns a
|
||||
// synthetic `{status:"queued", delivery_mode:"poll", method:"message/send"}`
|
||||
// HTTP 200 within ~50ms when the target workspace is registered with
|
||||
// `delivery_mode=poll` — i.e. an operator's laptop running
|
||||
// `molecule-mcp-claude-channel`, a hermes/codex MCP bridge, or a Cursor
|
||||
// MCP client. The real agent reply arrives separately via the
|
||||
// AGENT_MESSAGE WebSocket event after the agent's next
|
||||
// `wait_for_message` poll (could be 1s, could be 60s).
|
||||
//
|
||||
// Pre-#227 behaviour: useChatSend treated the queued-200 as a successful
|
||||
// round-trip — extractReplyText returned "", no agent bubble was
|
||||
// created, `releaseSendGuards` flipped `sending` off, and the user saw
|
||||
// dead silence between their user bubble and the eventual reply with
|
||||
// NO progress indicator. That's the user-reported gap this task fixes.
|
||||
//
|
||||
// These tests pin the new behaviour: on a queued-200, the hook MUST NOT
|
||||
// call onAgentMessage (no empty bubble) AND MUST NOT call
|
||||
// releaseSendGuards (spinner persists). The eventual AGENT_MESSAGE WS
|
||||
// event is what clears the spinner — that path is covered by
|
||||
// useChatSocket.test.tsx already.
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { renderHook, act } from "@testing-library/react";
|
||||
|
||||
// Capture the api.post invocations + control responses per-test.
|
||||
const apiPostMock = vi.fn<
|
||||
(url: string, body?: unknown, opts?: unknown) => Promise<unknown>
|
||||
>();
|
||||
vi.mock("@/lib/api", () => ({
|
||||
api: {
|
||||
post: (url: string, body?: unknown, opts?: unknown) =>
|
||||
apiPostMock(url, body, opts),
|
||||
get: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
// uploads — tests don't go through the upload path; stub the helpers
|
||||
// useChatSend imports so the module loads.
|
||||
vi.mock("../../uploads", () => ({
|
||||
uploadChatFiles: vi.fn(),
|
||||
FileTooLargeError: class FileTooLargeError extends Error {},
|
||||
}));
|
||||
|
||||
// types — re-export the createMessage helper unchanged; only the
|
||||
// uploads stub matters above.
|
||||
import { useChatSend } from "../useChatSend";
|
||||
|
||||
beforeEach(() => {
|
||||
apiPostMock.mockReset();
|
||||
});
|
||||
|
||||
describe("useChatSend — poll-mode (external/MCP) queued-200 handling — task #227", () => {
|
||||
it("does NOT call onAgentMessage when the synthetic {status:'queued'} response lands (no empty bubble)", async () => {
|
||||
// Mock the platform's poll-mode short-circuit response shape exactly
|
||||
// as ws-server's `proxyA2ARequest` returns it (a2a_proxy.go:420-431).
|
||||
apiPostMock.mockResolvedValueOnce({
|
||||
status: "queued",
|
||||
delivery_mode: "poll",
|
||||
method: "message/send",
|
||||
});
|
||||
|
||||
const onUserMessage = vi.fn();
|
||||
const onAgentMessage = vi.fn();
|
||||
|
||||
const { result } = renderHook(() =>
|
||||
useChatSend("ws-poll-target", {
|
||||
getHistoryMessages: () => [],
|
||||
onUserMessage,
|
||||
onAgentMessage,
|
||||
}),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.sendMessage("hello external workspace");
|
||||
// Yield one microtask so the .then runs.
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
// User bubble fires — the user typed, that part is unconditional.
|
||||
expect(onUserMessage).toHaveBeenCalledTimes(1);
|
||||
// CRITICAL: no agent bubble. extractReplyText on a queued envelope
|
||||
// returns "" — the pre-#227 code would still have hit the
|
||||
// "releaseSendGuards + no bubble" path, BUT it would have ended
|
||||
// `sending`. The new code returns early BEFORE that release, so the
|
||||
// contract under test is "no synthesised empty bubble".
|
||||
expect(onAgentMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps `sending` true after a queued-200 — the spinner must persist until the real AGENT_MESSAGE arrives", async () => {
|
||||
apiPostMock.mockResolvedValueOnce({
|
||||
status: "queued",
|
||||
delivery_mode: "poll",
|
||||
method: "message/send",
|
||||
});
|
||||
|
||||
const { result } = renderHook(() =>
|
||||
useChatSend("ws-poll-target", {
|
||||
getHistoryMessages: () => [],
|
||||
}),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.sendMessage("waiting for the operator laptop");
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
// The spinner-driving state is `sending`. On a queued-200, it must
|
||||
// remain true — clearing it here is the exact bug task #227
|
||||
// resurfaces (collapsing the spinner before the agent has even seen
|
||||
// the message).
|
||||
expect(result.current.sending).toBe(true);
|
||||
});
|
||||
|
||||
it("ALSO keeps `sending` true even after a follow-up microtask flush — guards against an accidental late release", async () => {
|
||||
// Defense: ensure no chained .then / .finally accidentally calls
|
||||
// releaseSendGuards on the queued path. Run several microtask
|
||||
// ticks and re-assert.
|
||||
apiPostMock.mockResolvedValueOnce({
|
||||
status: "queued",
|
||||
delivery_mode: "poll",
|
||||
});
|
||||
|
||||
const { result } = renderHook(() =>
|
||||
useChatSend("ws-poll-target", {
|
||||
getHistoryMessages: () => [],
|
||||
}),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.sendMessage("late-release-guard");
|
||||
// Flush multiple microtask ticks.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(result.current.sending).toBe(true);
|
||||
});
|
||||
|
||||
it("push-mode (real reply parts) still flips sending=false + creates an agent bubble — non-regression for the default path", async () => {
|
||||
// Sanity-check the push path still works: a real reply must call
|
||||
// onAgentMessage and flip sending=false. Without this assertion an
|
||||
// overzealous "return early on any non-result body" would silently
|
||||
// break the dominant push-mode path.
|
||||
apiPostMock.mockResolvedValueOnce({
|
||||
result: {
|
||||
parts: [{ kind: "text", text: "hi from native workspace" }],
|
||||
},
|
||||
});
|
||||
|
||||
const onAgentMessage = vi.fn();
|
||||
const { result } = renderHook(() =>
|
||||
useChatSend("ws-native-push", {
|
||||
getHistoryMessages: () => [],
|
||||
onAgentMessage,
|
||||
}),
|
||||
);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.sendMessage("native push test");
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(onAgentMessage).toHaveBeenCalledTimes(1);
|
||||
const msg = onAgentMessage.mock.calls[0][0] as {
|
||||
role: string;
|
||||
content: string;
|
||||
};
|
||||
expect(msg.role).toBe("agent");
|
||||
expect(msg.content).toBe("hi from native workspace");
|
||||
expect(result.current.sending).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -116,77 +116,6 @@ describe("useChatSocket — surface error_detail to onSendError (internal#212)",
|
||||
expect(reason.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// Task #227 — external/MCP (poll-mode) workspace progress UX.
|
||||
//
|
||||
// ws-server's `proxyA2ARequest` poll-mode short-circuit fires the
|
||||
// ACTIVITY_LOGGED a2a_receive with status="ok" and NO duration_ms (no
|
||||
// reply yet — the request is queued for the agent's next poll). Before
|
||||
// task #227 the (status==="ok" && durationMs) guard silently dropped
|
||||
// this row, so the chat UI had ZERO progress signal between "user
|
||||
// typed" and "agent eventually polled and replied". Lock the queued
|
||||
// line in so future refactors don't regress to the silent-drop state.
|
||||
it("emits a 'queued — will pick up on next poll' activity line when a2a_receive status=ok has no duration_ms (poll-mode)", () => {
|
||||
const onActivityLog = vi.fn();
|
||||
renderHook(() =>
|
||||
useChatSocket("ws-self", {
|
||||
onActivityLog,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(capturedHandler).not.toBeNull();
|
||||
act(() => {
|
||||
capturedHandler!({
|
||||
event: "ACTIVITY_LOGGED",
|
||||
workspace_id: "ws-self",
|
||||
payload: {
|
||||
activity_type: "a2a_receive",
|
||||
method: "message/send",
|
||||
status: "ok",
|
||||
target_id: "ws-self",
|
||||
// No duration_ms — this is the queued-for-poll signal.
|
||||
},
|
||||
timestamp: "2026-05-20T00:00:00Z",
|
||||
});
|
||||
});
|
||||
|
||||
expect(onActivityLog).toHaveBeenCalledTimes(1);
|
||||
const line = onActivityLog.mock.calls[0][0] as string;
|
||||
// The line MUST be present (not the empty-string silent-drop pattern)
|
||||
// and MUST mention the queued state so the user has actionable signal.
|
||||
expect(line.length).toBeGreaterThan(0);
|
||||
expect(line.toLowerCase()).toMatch(/queued|poll/);
|
||||
});
|
||||
|
||||
// Pair with the above: poll-mode acknowledgement must NOT prematurely
|
||||
// call onSendComplete — the spinner has to stay up until the actual
|
||||
// AGENT_MESSAGE reply lands. (The reply-success path with duration_ms
|
||||
// still calls onSendComplete; that's the push-mode case.)
|
||||
it("does NOT call onSendComplete on a poll-mode queued a2a_receive (spinner must persist)", () => {
|
||||
const onSendComplete = vi.fn();
|
||||
renderHook(() =>
|
||||
useChatSocket("ws-self", {
|
||||
onSendComplete,
|
||||
}),
|
||||
);
|
||||
|
||||
act(() => {
|
||||
capturedHandler!({
|
||||
event: "ACTIVITY_LOGGED",
|
||||
workspace_id: "ws-self",
|
||||
payload: {
|
||||
activity_type: "a2a_receive",
|
||||
method: "message/send",
|
||||
status: "ok",
|
||||
target_id: "ws-self",
|
||||
// No duration_ms.
|
||||
},
|
||||
timestamp: "2026-05-20T00:00:00Z",
|
||||
});
|
||||
});
|
||||
|
||||
expect(onSendComplete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores errors targeted at a different workspace's peer", () => {
|
||||
// Defense against a race where the WS hub fans out to all clients —
|
||||
// each chat panel must only react when target_id matches its own
|
||||
|
||||
@@ -22,28 +22,6 @@ interface A2AResponse {
|
||||
parts?: A2APart[];
|
||||
artifacts?: Array<{ parts: A2APart[] }>;
|
||||
};
|
||||
/** Set by ws-server's poll-mode short-circuit in `proxyA2ARequest`
|
||||
* (a2a_proxy.go:416-431) when the target workspace is registered as
|
||||
* `delivery_mode=poll` — e.g. an operator's laptop running
|
||||
* `molecule-mcp-claude-channel`, a hermes/codex MCP bridge, or a
|
||||
* Cursor MCP client. The HTTP 200 carries the synthetic envelope
|
||||
* `{status:"queued", delivery_mode:"poll", method:"message/send"}`
|
||||
* immediately (~50ms), BEFORE the agent has produced a reply.
|
||||
*
|
||||
* Task #227 routing: when this field is "queued" the caller must NOT
|
||||
* treat the 200 as "agent done" — there are no `result.parts` yet
|
||||
* (the reply will arrive separately via the AGENT_MESSAGE WS event
|
||||
* after the agent's next poll). Keep the spinner up; the eventual
|
||||
* AGENT_MESSAGE flips `sending` off via the existing useChatSocket
|
||||
* `onSendComplete` path. Without this distinction the spinner
|
||||
* disappeared immediately and external/MCP workspaces had no progress
|
||||
* UX between send and reply. */
|
||||
status?: string;
|
||||
/** Companion to `status` — "poll" when the queued short-circuit fired.
|
||||
* Defensive: we key the poll-mode-skip decision on status==="queued"
|
||||
* (the canonical signal) rather than on this field, but it's surfaced
|
||||
* here so future debugging / tests can assert on the full envelope. */
|
||||
delivery_mode?: string;
|
||||
}
|
||||
|
||||
export function extractReplyText(resp: A2AResponse): string {
|
||||
@@ -217,30 +195,6 @@ export function useChatSend(workspaceId: string, options: UseChatSendOptions) {
|
||||
sendInFlightRef.current = false;
|
||||
return;
|
||||
}
|
||||
// Task #227 — poll-mode (external/MCP workspace) queued-200
|
||||
// short-circuit. ws-server's `proxyA2ARequest` returns
|
||||
// `{status:"queued", delivery_mode:"poll", ...}` immediately
|
||||
// when the target has no URL (delivery_mode=poll), BEFORE the
|
||||
// agent has produced any reply. There is no `result.parts`
|
||||
// payload here — the actual reply will arrive separately via
|
||||
// the AGENT_MESSAGE WebSocket event after the agent's next
|
||||
// `wait_for_message` poll.
|
||||
//
|
||||
// Keep the spinner up by deliberately NOT calling
|
||||
// releaseSendGuards: the user-facing "thinking" state must
|
||||
// persist until the AGENT_MESSAGE lands (handled by the
|
||||
// useChatSocket `onAgentMessage`/`onSendComplete` path) or an
|
||||
// explicit error fires (`onSendError` from an ACTIVITY_LOGGED
|
||||
// status="error"). Don't synthesise an empty agent bubble.
|
||||
//
|
||||
// sendInFlightRef stays true intentionally — it's the dedup
|
||||
// guard for the user typing two messages back-to-back; for
|
||||
// poll mode the second message would race the first agent's
|
||||
// reply, so blocking is correct (matches push-mode behaviour
|
||||
// where `sending` blocks the textarea).
|
||||
if (resp?.status === "queued") {
|
||||
return;
|
||||
}
|
||||
const replyText = extractReplyText(resp);
|
||||
const replyFiles = extractFilesFromTask(
|
||||
(resp?.result ?? {}) as Record<string, unknown>,
|
||||
|
||||
@@ -62,25 +62,6 @@ export function useChatSocket(
|
||||
line = `← ${targetName} responded (${sec}s)`;
|
||||
const own = (targetId || msg.workspace_id) === workspaceId;
|
||||
if (own) callbacksRef.current.onSendComplete?.();
|
||||
} else if (status === "ok" && !durationMs) {
|
||||
// Task #227 — poll-mode (external/MCP workspace) queued receipt.
|
||||
// ws-server `logA2AReceiveQueued` writes a "received but no
|
||||
// reply yet" row with status="ok" and NO duration_ms, then
|
||||
// immediately returns the synthetic {status:"queued"} 200 to
|
||||
// the caller. Before this branch the row was silently dropped
|
||||
// by the (status==="ok" && durationMs) guard above — leaving
|
||||
// the chat UI with zero progress signal for the entire window
|
||||
// between "user typed" and "agent eventually polled and
|
||||
// replied". Surface the queued state explicitly so the user
|
||||
// sees acknowledgement (matches the queued-delegation
|
||||
// indicator in AgentCommsPanel.WaitingBubbles).
|
||||
//
|
||||
// We intentionally do NOT call onSendComplete here: the
|
||||
// outbound is not done — only acknowledged. The MyChatPanel
|
||||
// spinner stays up until the actual AGENT_MESSAGE reply lands
|
||||
// (poll path) or an explicit error fires (which still hits
|
||||
// the status==="error" branch below).
|
||||
line = `⧗ ${targetName} queued — agent will pick up on next poll`;
|
||||
} else if (status === "error") {
|
||||
line = `⚠ ${targetName} error`;
|
||||
const own = (targetId || msg.workspace_id) === workspaceId;
|
||||
|
||||
@@ -513,8 +513,6 @@ export function buildNodesAndEdges(
|
||||
parentId: ws.parent_id,
|
||||
currentTask: ws.current_task || "",
|
||||
runtime: ws.runtime || "",
|
||||
workspaceAccess: ws.workspace_access,
|
||||
maxConcurrentTasks: ws.max_concurrent_tasks ?? null,
|
||||
needsRestart: false,
|
||||
budgetLimit: ws.budget_limit ?? null,
|
||||
budgetUsed: ws.budget_used ?? null,
|
||||
@@ -525,9 +523,6 @@ export function buildNodesAndEdges(
|
||||
// that don't yet include these columns in the GET response.
|
||||
broadcastEnabled: ws.broadcast_enabled ?? false,
|
||||
talkToUserEnabled: ws.talk_to_user_enabled ?? true,
|
||||
// A2A delivery mode (task #227). Absent on older ws-server builds
|
||||
// — leave undefined so the chat UI's "?? 'push'" fallback applies.
|
||||
deliveryMode: ws.delivery_mode,
|
||||
},
|
||||
};
|
||||
if (hasParent) {
|
||||
|
||||
@@ -88,8 +88,6 @@ export interface WorkspaceNodeData extends Record<string, unknown> {
|
||||
parentId: string | null;
|
||||
currentTask: string;
|
||||
runtime: string;
|
||||
workspaceAccess?: string | null;
|
||||
maxConcurrentTasks?: number | null;
|
||||
needsRestart: boolean;
|
||||
/** USD spend ceiling set by the user; null = unlimited. Added by issue #541. */
|
||||
budgetLimit: number | null;
|
||||
@@ -108,31 +106,9 @@ export interface WorkspaceNodeData extends Record<string, unknown> {
|
||||
* send_message_to_user / POST /notify return 403 and the canvas
|
||||
* shows a "not enabled" state with a button to re-enable. Default true. */
|
||||
talkToUserEnabled?: boolean;
|
||||
/** A2A inbound delivery mode for this workspace — "push" (default —
|
||||
* synchronous HTTP dispatch by ws-server `proxyA2ARequest`) or "poll"
|
||||
* (workspace has no URL; ws-server logs the request and the agent
|
||||
* consumes it via `wait_for_message` / GET /activity?since_id=).
|
||||
*
|
||||
* Why surfaced to the UI: poll-mode targets (external/MCP workspaces:
|
||||
* `molecule-mcp-claude-channel` on an operator laptop, hermes/codex
|
||||
* bridge clients, Cursor MCP) acknowledge a canvas `message/send` with
|
||||
* a synthetic `{status:"queued"}` 200 within ~50ms. Without this flag
|
||||
* the chat UI cannot tell that gap from a real round-trip — the
|
||||
* spinner disappears immediately and the user sees dead silence until
|
||||
* the agent eventually polls and replies via the AGENT_MESSAGE WS
|
||||
* event (could be seconds, could be minutes). Task #227 — render a
|
||||
* "queued — agent will pick up on next poll" state for poll-mode
|
||||
* sends so external/MCP workspaces have progress UX parity with
|
||||
* native runtimes (claude-code / codex / hermes / openclaw).
|
||||
*
|
||||
* Sourced from the GET /workspaces response (`delivery_mode` snake_case
|
||||
* field, mapped here in canvas-topology.ts). Absent on older platform
|
||||
* builds — that fallthrough is treated as "push" to match
|
||||
* ws-server's `lookupDeliveryMode` default. */
|
||||
deliveryMode?: string;
|
||||
}
|
||||
|
||||
export type PanelTab = "details" | "skills" | "chat" | "terminal" | "display" | "container-config" | "config" | "schedule" | "channels" | "files" | "memory" | "traces" | "events" | "activity" | "audit";
|
||||
export type PanelTab = "details" | "skills" | "chat" | "terminal" | "config" | "schedule" | "channels" | "files" | "memory" | "traces" | "events" | "activity" | "audit";
|
||||
|
||||
export interface ContextMenuState {
|
||||
x: number;
|
||||
|
||||
@@ -320,13 +320,11 @@ export interface WorkspaceData {
|
||||
url: string;
|
||||
parent_id: string | null;
|
||||
active_tasks: number;
|
||||
max_concurrent_tasks?: number | null;
|
||||
last_error_rate: number;
|
||||
last_sample_error: string;
|
||||
uptime_seconds: number;
|
||||
current_task: string;
|
||||
runtime: string;
|
||||
workspace_access?: string | null;
|
||||
x: number;
|
||||
y: number;
|
||||
collapsed: boolean;
|
||||
@@ -344,16 +342,6 @@ export interface WorkspaceData {
|
||||
/** Workspace ability flags (migration 20260514). */
|
||||
broadcast_enabled?: boolean;
|
||||
talk_to_user_enabled?: boolean;
|
||||
/** A2A delivery mode for inbound messages — "push" (default, synchronous
|
||||
* HTTP dispatch to `url`) or "poll" (queued to activity_logs, agent
|
||||
* picks up via `wait_for_message` / GET /activity?since_id=). Surfaced
|
||||
* in the GET /workspaces response since #2339 PR 1; older platform
|
||||
* versions return it absent so the canvas treats absent as "push" (the
|
||||
* documented default in `lookupDeliveryMode`). Used by the chat UI to
|
||||
* render an "agent will pick up on next poll" indicator instead of
|
||||
* collapsing the spinner the moment the synchronous queued-200 returns
|
||||
* (task #227 — external/MCP workspaces had no progress UX). */
|
||||
delivery_mode?: string;
|
||||
}
|
||||
|
||||
let socket: ReconnectingSocket | null = null;
|
||||
|
||||
@@ -17,7 +17,7 @@ Canvas (Next.js :3000) ←WebSocket→ Platform (Go :8080) ←HTTP→ Postgres +
|
||||
|
||||
- **Workspace Server** (`workspace-server/`): Go/Gin control plane — workspace CRUD, registry, discovery, WebSocket hub, liveness monitoring.
|
||||
- **Canvas** (`canvas/`): Next.js 15 + React Flow (@xyflow/react v12) + Zustand + Tailwind — visual workspace graph.
|
||||
- **Workspace Runtime**: Shared runtime published from [`molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime) to the Molecule AI Gitea package registry. Supports LangGraph, Claude Code, OpenClaw, Hermes, Codex, and AutoGen. Each adapter lives in its own standalone template repo (e.g. `molecule-ai-workspace-template-claude-code`). See `docs/workspace-runtime-package.md` for the full picture.
|
||||
- **Workspace Runtime** (`workspace/`): Shared runtime published as [`molecule-ai-workspace-runtime`](https://pypi.org/project/molecule-ai-workspace-runtime/) on PyPI. Supports LangGraph, Claude Code, OpenClaw, DeepAgents, CrewAI, AutoGen. Each adapter lives in its own standalone template repo (e.g. `molecule-ai-workspace-template-claude-code`). See `docs/workspace-runtime-package.md` for the full picture.
|
||||
- **molecli** (`workspace-server/cmd/cli/`): Go TUI dashboard (Bubbletea + Lipgloss) — real-time workspace monitoring, event log, health overview, delete/filter operations.
|
||||
|
||||
## Key Architectural Patterns
|
||||
|
||||
@@ -285,39 +285,6 @@ Canvas requests (no `X-Workspace-ID` header) and system callers
|
||||
|
||||
---
|
||||
|
||||
## Multiple Workspaces From One Local MCP Bridge
|
||||
|
||||
The standalone runtime package includes `molecule-mcp`, a local MCP bridge for
|
||||
external agents such as Claude Code, Codex, Hermes, and other tools that run
|
||||
outside the platform container fleet. One local bridge can serve multiple
|
||||
external workspaces by setting `MOLECULE_WORKSPACES`:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "workspace-id-local-to-hongming-org",
|
||||
"token": "...",
|
||||
"platform_url": "https://hongming.moleculesai.app"
|
||||
},
|
||||
{
|
||||
"id": "different-workspace-id-local-to-agents-team-org",
|
||||
"token": "...",
|
||||
"platform_url": "https://agents-team.moleculesai.app"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
`platform_url` is the tenant routing key. The bridge registers, heartbeats,
|
||||
polls inboxes, and sends outbound A2A calls against the URL attached to the
|
||||
workspace that is doing the work.
|
||||
|
||||
Do not add `org_id` to this config. The tenant already comes from
|
||||
`platform_url`, and the bearer token is issued by that tenant. Workspace IDs
|
||||
also do not need to be shared across orgs; each tenant can return its own
|
||||
workspace ID and token for the same local agent process.
|
||||
|
||||
---
|
||||
|
||||
## Canvas Appearance
|
||||
|
||||
External workspaces appear on the canvas with a purple **REMOTE** badge
|
||||
|
||||
@@ -135,33 +135,6 @@ The `id` field is your workspace ID — remember it.
|
||||
|
||||
---
|
||||
|
||||
## Optional — one local MCP bridge, multiple tenants
|
||||
|
||||
If your local agent runtime uses `molecule-mcp`, one process can serve more
|
||||
than one external workspace:
|
||||
|
||||
```bash
|
||||
export MOLECULE_WORKSPACES='[
|
||||
{
|
||||
"id": "workspace-id-local-to-you-org",
|
||||
"token": "...",
|
||||
"platform_url": "https://you.moleculesai.app"
|
||||
},
|
||||
{
|
||||
"id": "different-workspace-id-local-to-team-org",
|
||||
"token": "...",
|
||||
"platform_url": "https://team.moleculesai.app"
|
||||
}
|
||||
]'
|
||||
molecule-mcp
|
||||
```
|
||||
|
||||
Use the workspace ID and token returned by each tenant. The IDs may differ
|
||||
across orgs. `org_id` is not required here because `platform_url` selects the
|
||||
tenant and the token is tenant-scoped.
|
||||
|
||||
---
|
||||
|
||||
## Step 4 — Chat with it
|
||||
|
||||
1. Open your Molecule canvas at `https://<TENANT>`
|
||||
|
||||
@@ -125,33 +125,6 @@ The agent appears on the canvas with a **purple REMOTE badge** within seconds. F
|
||||
|
||||
---
|
||||
|
||||
## Multi-Tenant Local MCP Bridge
|
||||
|
||||
For local MCP-driven agents, use the standalone runtime's `molecule-mcp`
|
||||
entrypoint. A single local bridge can serve multiple external workspaces by
|
||||
setting `MOLECULE_WORKSPACES`:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "workspace-id-local-to-acme",
|
||||
"token": "...",
|
||||
"platform_url": "https://acme.moleculesai.app"
|
||||
},
|
||||
{
|
||||
"id": "different-workspace-id-local-to-ops",
|
||||
"token": "...",
|
||||
"platform_url": "https://ops.moleculesai.app"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
`platform_url` selects the tenant for registration, heartbeat, inbox polling,
|
||||
and outbound A2A routing. `org_id` is not required in this config, and the
|
||||
workspace IDs do not need to match across tenants.
|
||||
|
||||
---
|
||||
|
||||
## What Phase 30 Covers
|
||||
|
||||
| Phase | What shipped | Endpoint |
|
||||
|
||||
@@ -1,44 +1,304 @@
|
||||
# Workspace Runtime Package
|
||||
# Workspace Runtime PyPI Package
|
||||
|
||||
`molecule-ai-workspace-runtime` is the shared Python runtime consumed by
|
||||
workspace template images and by external MCP integrations.
|
||||
## Requires Python >= 3.11
|
||||
|
||||
## Source Of Truth
|
||||
The wheel pins `requires_python>=3.11`. On Python 3.10 or older, `pip install
|
||||
molecule-ai-workspace-runtime` fails with `Could not find a version that
|
||||
satisfies the requirement (from versions: none)` — the pin filters the only
|
||||
available artifact before pip even attempts install. Upgrade the interpreter
|
||||
(`brew install python@3.12` / `apt install python3.12` / etc.) or use a
|
||||
3.11+ venv.
|
||||
|
||||
The source of truth is the standalone Gitea repo:
|
||||
## Overview
|
||||
|
||||
```text
|
||||
https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime
|
||||
The shared workspace runtime infrastructure has **one editable source** and
|
||||
**one published artifact**:
|
||||
|
||||
1. **Source of truth (monorepo, editable):** `workspace/` — every runtime
|
||||
change lands here. Edit it like any other monorepo code.
|
||||
2. **Published artifact (PyPI, generated):** [`molecule-ai-workspace-runtime`](https://pypi.org/project/molecule-ai-workspace-runtime/)
|
||||
— produced by `.github/workflows/publish-runtime.yml` on every
|
||||
`runtime-vX.Y.Z` tag push. Do NOT edit this independently — it gets
|
||||
overwritten on every publish.
|
||||
|
||||
The legacy sibling repo `molecule-ai-workspace-runtime` (the GitHub repo, as
|
||||
distinct from the PyPI package) is no longer the source-of-truth and should
|
||||
be treated as a publish artifact only. It can be archived or used as a
|
||||
read-only mirror.
|
||||
|
||||
## Where to make changes
|
||||
|
||||
**All runtime edits land in `molecule-monorepo/workspace/`. Period.**
|
||||
|
||||
The GitHub repo `Molecule-AI/molecule-ai-workspace-runtime` is **mirror-only**.
|
||||
It exists so external consumers (template repos, downstream operators) have a
|
||||
git-cloneable artifact that mirrors the PyPI wheel — nothing more.
|
||||
|
||||
- **Direct PRs against `molecule-ai-workspace-runtime` are auto-rejected by
|
||||
the `mirror-guard` CI check.** The check fails any push that did not come
|
||||
from the publish pipeline. There is no opt-out — file the change against
|
||||
`molecule-monorepo/workspace/` instead.
|
||||
- **The mirror + the PyPI wheel both auto-regenerate on every push to
|
||||
`staging`** via `.github/workflows/publish-runtime.yml` (which calls
|
||||
`scripts/build_runtime_package.py`, builds wheel + sdist, smoke-imports,
|
||||
uploads to PyPI via Trusted Publisher, and force-pushes the rewritten tree
|
||||
to the mirror repo). You never touch the mirror by hand.
|
||||
|
||||
If you have an old local clone of the mirror and try to push a fix to it
|
||||
directly, expect a CI failure with a message pointing you here. Re-open the
|
||||
change against `molecule-monorepo/workspace/` and let the publish workflow
|
||||
do the rest.
|
||||
|
||||
## Why this shape
|
||||
|
||||
The 8 workspace template repos (claude-code, langgraph, hermes, etc.) each
|
||||
build their own Docker image and `pip install molecule-ai-workspace-runtime`
|
||||
from PyPI. PyPI is the right distribution channel — semver, reproducible
|
||||
builds, no submodule dance per-repo. But the runtime ALSO needs to evolve
|
||||
in lock-step with the platform's wire protocol (queue shape, A2A metadata,
|
||||
event payloads). Shipping cross-cutting protocol changes as separate
|
||||
runtime + platform PRs in two repos creates ordering pain and broken
|
||||
intermediate states.
|
||||
|
||||
The monorepo + auto-publish split gives both: edit cross-cutting changes
|
||||
in one PR, publish the runtime artifact via a tag.
|
||||
|
||||
## What's in the package
|
||||
|
||||
Everything in `workspace/*.py` plus the `adapters/`, `builtin_tools/`,
|
||||
`plugins_registry/`, `policies/`, `skill_loader/` subpackages. Build
|
||||
artifacts (`Dockerfile`, `*.sh`, `pytest.ini`, `requirements.txt`) are
|
||||
excluded.
|
||||
|
||||
The build script rewrites bare imports so the published package is a
|
||||
proper Python namespace:
|
||||
|
||||
```
|
||||
# In monorepo workspace/:
|
||||
from a2a_client import discover_peer
|
||||
from builtin_tools.memory import store
|
||||
|
||||
# In published molecule_runtime/ (auto-rewritten at publish time):
|
||||
from molecule_runtime.a2a_client import discover_peer
|
||||
from molecule_runtime.builtin_tools.memory import store
|
||||
```
|
||||
|
||||
Do not add runtime source back under `molecule-core/workspace/`. The core repo
|
||||
owns the platform server, canvas, provisioning, and tests around the installed
|
||||
runtime package.
|
||||
The closed allowlist of rewritten module names lives in
|
||||
`scripts/build_runtime_package.py` (`TOP_LEVEL_MODULES` + `SUBPACKAGES`).
|
||||
Add a new top-level module to workspace/? Add it to the allowlist in the
|
||||
same PR.
|
||||
|
||||
## Package Registry
|
||||
## Adapter repos
|
||||
|
||||
The runtime package is published to the Molecule AI Gitea package registry:
|
||||
Each of the 8 adapter template repos contains:
|
||||
- `adapter.py` — runtime-specific `Adapter` class
|
||||
- `requirements.txt` — `molecule-ai-workspace-runtime>=0.1.X` + adapter deps
|
||||
- `Dockerfile` — standalone image with `ENV ADAPTER_MODULE=adapter` and
|
||||
`ENTRYPOINT ["molecule-runtime"]`
|
||||
|
||||
```text
|
||||
https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/
|
||||
| Adapter | Repo |
|
||||
|---------|------|
|
||||
| claude-code | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-claude-code |
|
||||
| langgraph | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-langgraph |
|
||||
| crewai | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-crewai |
|
||||
| autogen | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-autogen |
|
||||
| deepagents | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-deepagents |
|
||||
| hermes | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-hermes |
|
||||
| gemini-cli | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-gemini-cli |
|
||||
| openclaw | https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-openclaw |
|
||||
|
||||
## Adapter discovery (ADAPTER_MODULE)
|
||||
|
||||
Standalone adapter repos set `ENV ADAPTER_MODULE=adapter` in their
|
||||
Dockerfile. The runtime's `get_adapter()` checks this env var first:
|
||||
|
||||
```python
|
||||
# In molecule_runtime/adapters/__init__.py
|
||||
def get_adapter(runtime: str) -> type[BaseAdapter]:
|
||||
adapter_module = os.environ.get("ADAPTER_MODULE")
|
||||
if adapter_module:
|
||||
mod = importlib.import_module(adapter_module)
|
||||
return getattr(mod, "Adapter")
|
||||
raise KeyError(...)
|
||||
```
|
||||
|
||||
PyPI is intentionally not part of the critical path. Template Dockerfiles,
|
||||
external-runtime snippets, and CI install checks should use the Gitea registry.
|
||||
## Publishing a new version
|
||||
|
||||
## Release Flow
|
||||
```bash
|
||||
# From any local checkout of monorepo, after merging your runtime change:
|
||||
git tag runtime-v0.1.6
|
||||
git push origin runtime-v0.1.6
|
||||
```
|
||||
|
||||
1. Land a reviewed PR in `molecule-ai-workspace-runtime`.
|
||||
2. Bump `version =` in that repo's `pyproject.toml`.
|
||||
3. Tag `runtime-vX.Y.Z` on the runtime repo.
|
||||
4. The runtime repo's `publish-runtime` workflow builds the wheel and sdist,
|
||||
publishes to the Gitea registry, verifies install from that registry, then
|
||||
cascades `.runtime-version` pins to workspace template repos.
|
||||
The `publish-runtime` workflow takes over — checks out the tag, runs
|
||||
`scripts/build_runtime_package.py --version 0.1.6`, builds wheel + sdist,
|
||||
runs a smoke import to catch broken rewrites, and uploads to PyPI via
|
||||
the PyPA Trusted Publisher action (OIDC). No static API token is stored
|
||||
in this repo — PyPI verifies the workflow's OIDC claim against the
|
||||
trusted-publisher config registered for `molecule-ai-workspace-runtime`.
|
||||
|
||||
## Core Repo Contract
|
||||
For dev/test releases without tagging, dispatch the workflow manually
|
||||
with an explicit version (e.g. `0.1.6.dev1` — PEP 440 dev/rc/post forms
|
||||
are accepted).
|
||||
|
||||
`molecule-core` must not ship editable runtime code. Its responsibilities are:
|
||||
After publish, the 8 template repos pick up the new version on their
|
||||
next `:latest` rebuild. To force-pull immediately, bump the pin in each
|
||||
template's `requirements.txt`.
|
||||
|
||||
- Test platform behavior against the installed runtime contract.
|
||||
- Keep MCP/registry/TenantGuard behavior compatible with the runtime package.
|
||||
- Fail CI if `workspace/` or legacy build-from-workspace scripts are restored.
|
||||
## End-to-end CD chain
|
||||
|
||||
The full chain from monorepo merge → workspace containers running new code:
|
||||
|
||||
```
|
||||
1. Merge PR with workspace/ changes to main
|
||||
↓
|
||||
2. .github/workflows/auto-tag-runtime.yml fires
|
||||
↓ reads PR labels (release:major/minor) or defaults to patch
|
||||
↓ pushes runtime-vX.Y.Z tag
|
||||
↓
|
||||
3. .github/workflows/publish-runtime.yml fires (on the tag)
|
||||
↓ builds wheel via scripts/build_runtime_package.py
|
||||
↓ smoke-imports the wheel
|
||||
↓ uploads to PyPI
|
||||
↓ cascade job fires repository_dispatch (event-type: runtime-published)
|
||||
↓ to all 8 workspace-template-* repos
|
||||
↓
|
||||
4. Each template's publish-image.yml fires (on repository_dispatch)
|
||||
↓ rebuilds Dockerfile (which pip-installs the new PyPI version)
|
||||
↓ pushes ghcr.io/molecule-ai/workspace-template-<runtime>:latest
|
||||
↓
|
||||
5. Production hosts run scripts/refresh-workspace-images.sh
|
||||
OR an operator hits POST /admin/workspace-images/refresh on the platform
|
||||
↓ docker pull all 8 :latest tags
|
||||
↓ remove + force-recreate any running ws-* containers using a refreshed image
|
||||
↓ canvas re-provisions the workspaces on next interaction
|
||||
```
|
||||
|
||||
Steps 1-4 are fully automated. Step 5 is one-click: a single curl or shell
|
||||
command. SaaS deployments typically wire step 5 into their normal deploy
|
||||
pipeline (every release pulls fresh images on every host); local dev fires
|
||||
it manually after a runtime release lands.
|
||||
|
||||
### Auth
|
||||
|
||||
PyPI publishing uses **Trusted Publisher (OIDC)** — no static token in the
|
||||
monorepo. The trusted-publisher config on PyPI binds the
|
||||
`molecule-ai-workspace-runtime` project to this repo's
|
||||
`publish-runtime.yml` workflow + `pypi-publish` environment. Rotation is
|
||||
moot: there is no shared secret to rotate.
|
||||
|
||||
### Required secrets
|
||||
|
||||
| Secret | Where | Why |
|
||||
|---|---|---|
|
||||
| `TEMPLATE_DISPATCH_TOKEN` | molecule-core repo | Fine-grained PAT with `actions:write` on the 8 template repos. Without it the `cascade` job warns and exits clean — PyPI still publishes; templates just don't auto-rebuild. |
|
||||
|
||||
### Step 5 specifics
|
||||
|
||||
**Local dev (compose stack):**
|
||||
```bash
|
||||
bash scripts/refresh-workspace-images.sh # all runtimes
|
||||
bash scripts/refresh-workspace-images.sh --runtime claude-code
|
||||
bash scripts/refresh-workspace-images.sh --no-recreate # pull only, leave containers
|
||||
```
|
||||
|
||||
**Via platform admin endpoint (any deploy):**
|
||||
```bash
|
||||
curl -X POST "$PLATFORM/admin/workspace-images/refresh"
|
||||
curl -X POST "$PLATFORM/admin/workspace-images/refresh?runtime=claude-code"
|
||||
curl -X POST "$PLATFORM/admin/workspace-images/refresh?recreate=false"
|
||||
```
|
||||
|
||||
The endpoint pulls + recreates from inside the platform container, so it
|
||||
needs Docker socket access (the compose stack mounts
|
||||
`/var/run/docker.sock` already) AND GHCR auth on the host's docker config
|
||||
(`docker login ghcr.io` once per host). On a fresh host without GHCR auth,
|
||||
the pull step warns per runtime and the response surfaces the failures.
|
||||
|
||||
**Fully hands-off (opt-in image auto-refresh):**
|
||||
|
||||
Set `IMAGE_AUTO_REFRESH=true` on the platform process. A watcher polls
|
||||
GHCR every 5 minutes for digest changes on each `workspace-template-*:latest`
|
||||
tag and invokes the same refresh logic the admin endpoint exposes —
|
||||
no operator action required between "runtime PR merged" and
|
||||
"containers running new code". Disabled by default because SaaS deploy
|
||||
pipelines that already pull on every release would do redundant work.
|
||||
|
||||
Optional companion env (same as the admin endpoint):
|
||||
|
||||
- `GHCR_USER` + `GHCR_TOKEN` — required for private template images;
|
||||
unused for the current public set, but harmless if set.
|
||||
|
||||
## Local dev (build the package without publishing)
|
||||
|
||||
```bash
|
||||
python3 scripts/build_runtime_package.py --version 0.1.0-local --out /tmp/runtime-build
|
||||
cd /tmp/runtime-build
|
||||
python -m build # produces dist/*.whl + dist/*.tar.gz
|
||||
pip install dist/*.whl # install into a venv to test locally
|
||||
```
|
||||
|
||||
This is the same pipeline CI runs. Use it to validate import-rewrite
|
||||
correctness before pushing a `runtime-v*` tag.
|
||||
|
||||
## Writing a new adapter
|
||||
|
||||
Use the GitHub template repo
|
||||
[`molecule-ai/molecule-ai-workspace-template-starter`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-starter) (note: the starter repo did not survive the 2026-05-06 GitHub-org-suspension migration; recreation tracked at internal#41)
|
||||
— it ships with the canonical Dockerfile + adapter.py skeleton + config.yaml
|
||||
schema + the `repository_dispatch: [runtime-published]` cascade receiver
|
||||
already wired up. No follow-up setup PR required.
|
||||
|
||||
```bash
|
||||
# Replace <runtime> with your runtime slug (lowercase, hyphenated).
|
||||
gh repo create Molecule-AI/molecule-ai-workspace-template-<runtime> \
|
||||
--template Molecule-AI/molecule-ai-workspace-template-starter \
|
||||
--public \
|
||||
--description "Molecule AI workspace template: <runtime>"
|
||||
|
||||
git clone https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-<runtime>.git
|
||||
cd molecule-ai-workspace-template-<runtime>
|
||||
```
|
||||
|
||||
Then fill in the `TODO` markers in:
|
||||
|
||||
| File | What to fill in |
|
||||
|---|---|
|
||||
| `adapter.py` | Rename class to `<Runtime>Adapter`. Fill in `name()`, `display_name()`, `description()`, `get_config_schema()`. Implement `setup()` and `create_executor()`. |
|
||||
| `requirements.txt` | Add your runtime's pip dependencies (e.g. `langgraph`, `crewai`, `claude-agent-sdk`). |
|
||||
| `Dockerfile` | Add runtime-specific apt deps (most runtimes don't need any). Replace ENTRYPOINT only if you need custom boot logic. |
|
||||
| `config.yaml` | Update top-level `name`/`runtime`/`description`. Add the models your runtime supports to `models[]`. |
|
||||
| `system-prompt.md` | Default agent prompt. |
|
||||
|
||||
After `git push`:
|
||||
|
||||
1. The template's `publish-image.yml` builds + pushes
|
||||
`ghcr.io/molecule-ai/workspace-template-<runtime>:latest` automatically.
|
||||
2. The next `runtime-vX.Y.Z` tag on `molecule-core` cascades a
|
||||
`repository_dispatch` event into your new template, rebuilding the image
|
||||
against the latest runtime — no setup PR required.
|
||||
3. Register the runtime name in the platform's `RuntimeImages` map (in
|
||||
`workspace-server/internal/provisioner/provisioner.go`) so it's
|
||||
selectable in the canvas.
|
||||
|
||||
## When the starter itself needs to evolve
|
||||
|
||||
If the canonical shape changes (e.g. `config.yaml` schema gets a new field,
|
||||
the `BaseAdapter` interface adds a method, the reusable CI workflow
|
||||
signature changes), update the
|
||||
[starter](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-template-starter) (recreation pending — see note above)
|
||||
**first**. Existing templates can either migrate at their own pace or be
|
||||
touched in a coordinated cleanup PR. Either way, future templates pick up
|
||||
the new shape from day one.
|
||||
|
||||
## Migration note
|
||||
|
||||
Prior to this workflow, the runtime was duplicated across monorepo
|
||||
`workspace/` AND a sibling repo `molecule-ai-workspace-runtime`, with no
|
||||
sync mechanism. That caused 30+ files to drift between the two trees and
|
||||
tonight's chat-leak / queued-classification fixes existed only in the
|
||||
monorepo copy until manually ported.
|
||||
|
||||
If you have an old local checkout of `molecule-ai-workspace-runtime`, treat
|
||||
it as outdated. The monorepo `workspace/` is now authoritative; the PyPI
|
||||
artifact is rebuilt from it on every `runtime-v*` tag.
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# local-e2e — session-continuity canary harness
|
||||
|
||||
Self-contained Docker-Compose harness that gates RFC#600-class template
|
||||
changes (session continuity, file-only messages, multimodal prompts,
|
||||
cross-session memory) **before** they reach customer canary.
|
||||
|
||||
Per CTO standing directive "fully tested + separate CI": this is a
|
||||
dedicated, *fast* (target <3 min), *small-surface* harness that uses a
|
||||
Python tenant-CP simulator (not the full `workspace-server` Go service)
|
||||
to exercise the runtime image end-to-end against canonical canary turns.
|
||||
|
||||
See [`feedback_no_single_source_of_truth`] — the harness IS the canonical
|
||||
session-continuity validator. Per-runtime unit tests still cover their
|
||||
own guard logic; the harness covers the live conversational behaviour
|
||||
that those unit tests cannot prove.
|
||||
|
||||
See [`feedback_image_promote_is_not_user_live`] — every assertion reads
|
||||
state back from the *running container*, never from a publish-pipeline
|
||||
ack.
|
||||
|
||||
## What it tests (the 4 canaries)
|
||||
|
||||
| # | Scenario | Asserts |
|
||||
|---|----------|---------|
|
||||
| 1 | 2-turn name canary | turn 2 reply contains "Hongming" → SessionStore continuity |
|
||||
| 2 | File-only message (no caption) | NOT "(empty prompt — nothing to do)" + reply references filename or asks for clarification |
|
||||
| 3 | File + caption ("summarize this") | reply addresses attachment + caption |
|
||||
| 4 | Cross-session memory recall | new session pulls "blue" via memory tool |
|
||||
|
||||
Each scenario re-uses the same A2A wire-shape that the production
|
||||
`workspace-server` POSTs to runtime `:8000` (canvas-thread-id semantics
|
||||
via `context_id`).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
local-e2e/
|
||||
docker-compose.yml # runtime under test + cp_sim
|
||||
cp_sim/ # ≈300 LoC Python A2A poster + file uploader
|
||||
cp_sim.py
|
||||
Dockerfile
|
||||
requirements.txt
|
||||
canary/
|
||||
conftest.py
|
||||
test_session_continuity.py # 4 canary scenarios
|
||||
test_layer_diagnostics.py # SessionStore state probe + key derivation
|
||||
scripts/
|
||||
run-canary.sh # one-shot orchestration entrypoint
|
||||
```
|
||||
|
||||
The CP simulator emits the **exact** JSON-RPC `message/send` envelope
|
||||
that `workspace-server` produces (verified against
|
||||
`tests/e2e/test_chat_attachments_e2e.sh`). No Go service is in the loop —
|
||||
this keeps the harness lean per the CTO directive.
|
||||
|
||||
## Run locally
|
||||
|
||||
```bash
|
||||
# from molecule-core repo root:
|
||||
export TEMPLATE_IMAGE=ghcr.io/molecule-ai/workspace-template-hermes:latest
|
||||
./local-e2e/scripts/run-canary.sh
|
||||
```
|
||||
|
||||
Exit code 0 = all 4 canaries pass. Non-zero = at least one canary failed
|
||||
and the harness dumped SessionStore state + last 200 log lines from the
|
||||
runtime container into `./local-e2e/artifacts/`.
|
||||
|
||||
## How it integrates into CI
|
||||
|
||||
Each template repo's `.gitea/workflows/session-continuity-e2e.yml` calls
|
||||
`run-canary.sh` with its own freshly-built `TEMPLATE_IMAGE`. The
|
||||
template repo's Gitea branch-protection lists
|
||||
`session-continuity-e2e (pull_request)` as a required context.
|
||||
|
||||
Rollout order (deliberate — per `feedback_image_promote_is_not_user_live`
|
||||
we bake before we cascade):
|
||||
|
||||
1. `molecule-ai-workspace-template-hermes` — highest-traffic + most
|
||||
recent RFC#600-class fixes — REQUIRED gate
|
||||
2. Bake for 5 business days
|
||||
3. Cascade to claude-code, langgraph, autogen, openclaw, smolagents,
|
||||
google-adk (one PR per template — see `scripts/onboard-template.sh`)
|
||||
|
||||
## Future extensions (out of scope for the initial PR)
|
||||
|
||||
- Multi-session memory consistency (3+ sessions deep)
|
||||
- Tool-use canary (workspace seeded with skills/, agent must invoke)
|
||||
- Streaming-cancellation canary (mid-stream client disconnect)
|
||||
- Cross-runtime A2A peer call (currently covered by `e2e-peer-visibility`)
|
||||
|
||||
## Why a thin Python simulator and not the real `workspace-server`?
|
||||
|
||||
`workspace-server` is a 60+ MB Go binary that requires Postgres, Redis,
|
||||
admin-token wiring, registry plumbing, and a 30+ second cold-boot. None
|
||||
of that touches session-continuity behaviour, which is fully owned by
|
||||
the runtime container's `a2a_executor.py`. Per CTO directive "separate
|
||||
CI as possible" + the <3 min target, we excise the platform-tenant Go
|
||||
service from the loop and emit identical wire-shape envelopes from a
|
||||
single Python file.
|
||||
|
||||
If the simulator diverges from `workspace-server` wire shape, the gate
|
||||
goes red — fix the simulator to match production. The wire shape is
|
||||
asserted in `tests/e2e/test_chat_attachments_e2e.sh` and the runtime's
|
||||
`workspace/a2a_executor.py:_core_execute`.
|
||||
@@ -0,0 +1,19 @@
|
||||
# Python tenant-CP simulator + canary test driver.
|
||||
# Single image — pytest + httpx + the canary tests baked in.
|
||||
FROM python:3.11-slim@sha256:e78299e55776ca065dcb769f80161f48465ad352014240eb5fe4712e22505e9b
|
||||
|
||||
WORKDIR /harness
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Test files are bind-mounted by docker-compose at run time so a `pytest -x`
|
||||
# rerun loop doesn't require a rebuild. The COPY here is for the
|
||||
# self-contained image used by Gitea Actions (where bind mounts are awkward).
|
||||
COPY cp_sim.py /harness/cp_sim.py
|
||||
COPY canary /harness/canary
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Default: run the 4 canaries with verbose output + JUnit XML for CI.
|
||||
CMD ["pytest", "-v", "--tb=short", "--junitxml=/harness/artifacts/junit.xml", "canary/"]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Shared pytest fixtures for the canary suite."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
# cp_sim.py lives one dir up — make it importable without packaging.
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import pytest # noqa: E402
|
||||
|
||||
from cp_sim import CPSim, CPSimConfig # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sim() -> CPSim:
|
||||
"""Fresh CPSim per test — cheap, isolates connection state."""
|
||||
return CPSim(
|
||||
cfg=CPSimConfig(
|
||||
runtime_url=os.environ.get("RUNTIME_URL", "http://localhost:18000"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def context_id() -> str:
|
||||
"""A unique canvas-thread-id per test — guarantees SessionStore isolation
|
||||
between scenarios so a failing canary doesn't poison the next one."""
|
||||
return f"canary-ctx-{uuid.uuid4().hex[:12]}"
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Layer-isolation diagnostics — runs alongside the 4 canaries.
|
||||
|
||||
These probes are not strict pass/fail gates by themselves; they exist so
|
||||
when a canary fails, the artifacts include enough state to tell whether
|
||||
the regression is in the wire-shape layer, the SessionStore layer, or
|
||||
the memory layer. Each test always passes (returns early) when the
|
||||
underlying surface is unavailable on the runtime under test — different
|
||||
templates expose different debug endpoints.
|
||||
|
||||
Cross-refs:
|
||||
- feedback_verify_actual_endstate_not_ack_follow_sop — we read state
|
||||
back, not the side-effect ack.
|
||||
- feedback_image_promote_is_not_user_live — the verification is at
|
||||
the running-container layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
|
||||
from cp_sim import CPSim
|
||||
|
||||
|
||||
def test_diag_agent_card_advertises_a2a(sim: CPSim) -> None:
|
||||
"""The runtime's /agent-card must advertise A2A capabilities.
|
||||
|
||||
If this fails, the canaries' transport assumption (POST /a2a) is
|
||||
already broken — diagnose the runtime image, not the canary.
|
||||
"""
|
||||
url = f"{sim.cfg.runtime_url}/agent-card"
|
||||
r = httpx.get(url, timeout=10.0)
|
||||
assert r.status_code == 200, (
|
||||
f"/agent-card returned {r.status_code}: {r.text[:300]!r}"
|
||||
)
|
||||
body = r.json()
|
||||
# AgentCard spec: capabilities object must exist, even if empty.
|
||||
assert isinstance(body, dict), f"/agent-card body not an object: {body!r}"
|
||||
# We don't require any specific capability flag — different templates
|
||||
# advertise different sets. The point of this diag is "is the card
|
||||
# there at all", which signals the runtime booted past entrypoint.
|
||||
|
||||
|
||||
def test_diag_context_id_required_for_continuity(sim: CPSim) -> None:
|
||||
"""Same context_id in two turns must not crash the runtime.
|
||||
|
||||
Pure smoke probe — proves the executor accepts a continuation
|
||||
message without 5xx-ing. The substantive assertion is canary 1; this
|
||||
one just guarantees the path is reachable.
|
||||
"""
|
||||
ctx = f"diag-{uuid.uuid4().hex[:8]}"
|
||||
r1 = sim.send_text("ping", context_id=ctx)
|
||||
r2 = sim.send_text("ping again", context_id=ctx, task_id=r1.get("result", {}).get("id"))
|
||||
# Both replies must parse — non-empty envelope, no JSON-RPC error.
|
||||
for label, env in (("turn1", r1), ("turn2", r2)):
|
||||
assert "error" not in env, f"{label} returned JSON-RPC error: {env['error']}"
|
||||
|
||||
|
||||
def test_diag_memory_root_writable_in_canary_mode(sim: CPSim) -> None:
|
||||
"""When MOLECULE_CANARY_MODE=1, the memory root must accept writes.
|
||||
|
||||
Probes via the recall_memory MCP tool — if /mcp is not exposed,
|
||||
returns early (skip-style; we still pass because some templates
|
||||
proxy MCP elsewhere).
|
||||
"""
|
||||
# We can't write directly here — only confirm the read path doesn't
|
||||
# 500 on a missing key. A real write happens in canary 4.
|
||||
key = f"canary-probe-{uuid.uuid4().hex[:8]}"
|
||||
try:
|
||||
val = sim.probe_memory(key)
|
||||
except Exception as e:
|
||||
# /mcp may not be exposed on this template — canary 4 will
|
||||
# surface the real defect if memory is actually broken.
|
||||
if os.environ.get("CANARY_STRICT_MCP") == "1":
|
||||
raise
|
||||
return
|
||||
# Unknown key → None is fine. The point is the call didn't crash.
|
||||
assert val is None or isinstance(val, str)
|
||||
@@ -0,0 +1,204 @@
|
||||
"""The 4 canonical session-continuity canaries (task #342, RFC#600 class).
|
||||
|
||||
These tests speak A2A directly to the runtime under test. They are the
|
||||
authoritative gate that the runtime preserves conversation continuity,
|
||||
handles file-only messages without dropping to the empty-prompt error,
|
||||
addresses multimodal prompts, and persists memory across sessions.
|
||||
|
||||
Wire-shape source of truth: see ../cp_sim.py docstring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from cp_sim import CPSim
|
||||
|
||||
|
||||
# ---------- canary 1: 2-turn name continuity -------------------------------
|
||||
|
||||
|
||||
def test_canary_1_two_turn_name_continuity(sim: CPSim, context_id: str) -> None:
|
||||
"""SessionStore continuity — turn 2 must recall the name from turn 1.
|
||||
|
||||
Empirically tests:
|
||||
- ``a2a_executor._core_execute`` injects prior-turn history via
|
||||
``_extract_history(context)`` (workspace/a2a_executor.py:313).
|
||||
- The runtime's session store is keyed on ``context_id`` (canvas
|
||||
thread id) NOT ``task_id`` — task_id is per-turn, context_id is
|
||||
per-conversation. Regressions to that key derivation were the
|
||||
root cause of the 2026-05 multi-turn-amnesia incidents
|
||||
(#a60623344 diagnosis).
|
||||
"""
|
||||
# Turn 1 — establish the fact.
|
||||
r1 = sim.send_text(
|
||||
"Hi, my name is Hongming.",
|
||||
context_id=context_id,
|
||||
)
|
||||
reply1 = sim.extract_text_parts(r1)
|
||||
assert reply1, f"Turn 1 produced empty reply. envelope={r1!r}"
|
||||
|
||||
# Turn 2 — ask back. Same context_id → same SessionStore key.
|
||||
r2 = sim.send_text(
|
||||
"What's my name?",
|
||||
context_id=context_id,
|
||||
)
|
||||
reply2 = sim.extract_text_parts(r2)
|
||||
assert reply2, f"Turn 2 produced empty reply. envelope={r2!r}"
|
||||
|
||||
# Substring match, case-insensitive — agents may reply
|
||||
# "Your name is Hongming." or "It's Hongming!" or similar.
|
||||
assert re.search(r"\bhongming\b", reply2, flags=re.IGNORECASE), (
|
||||
f"Turn 2 reply does not contain 'Hongming' — SessionStore "
|
||||
f"continuity regression suspected. context_id={context_id} "
|
||||
f"turn1_reply={reply1[:200]!r} turn2_reply={reply2[:400]!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------- canary 2: file-only message (no caption) -----------------------
|
||||
|
||||
|
||||
_DROPPED_TURN_MARKERS = (
|
||||
"(empty prompt — nothing to do)",
|
||||
"empty prompt",
|
||||
"message contained no text content",
|
||||
"no text content",
|
||||
)
|
||||
|
||||
|
||||
def test_canary_2_file_only_message(sim: CPSim, context_id: str) -> None:
|
||||
"""File-attached A2A message with NO text part must not be dropped.
|
||||
|
||||
Root cause this guards against: a long-standing executor bug where
|
||||
``extract_message_text`` returned "" for file-only messages and the
|
||||
executor short-circuited with the "Error: message contained no text
|
||||
content." reply, even though the attached file was the entire point
|
||||
of the turn.
|
||||
|
||||
Hard assertions:
|
||||
- Reply is non-empty AND not the dropped-turn marker.
|
||||
- Reply references the file by name OR asks an actionable
|
||||
clarifying question (NOT a flat error).
|
||||
"""
|
||||
file_name = f"canary-{uuid.uuid4().hex[:8]}.txt"
|
||||
file_body = b"Project status: nominal. Lighthouse score 98."
|
||||
|
||||
r = sim.send_with_file(
|
||||
context_id=context_id,
|
||||
text=None, # ← THE CANARY: no caption.
|
||||
file_name=file_name,
|
||||
file_bytes=file_body,
|
||||
mime_type="text/plain",
|
||||
)
|
||||
reply = sim.extract_text_parts(r)
|
||||
assert reply, f"File-only message produced empty reply. envelope={r!r}"
|
||||
|
||||
low = reply.lower()
|
||||
for marker in _DROPPED_TURN_MARKERS:
|
||||
assert marker.lower() not in low, (
|
||||
f"File-only message was dropped — reply contains "
|
||||
f"{marker!r}. Full reply: {reply[:500]!r}"
|
||||
)
|
||||
|
||||
# Soft assertion: reply must engage with the file (reference its
|
||||
# name) OR ask an actionable clarification. We require ONE of those —
|
||||
# a generic "Hello! How can I help?" reply is also a drop.
|
||||
name_referenced = file_name.lower() in low or "file" in low or "attach" in low
|
||||
asks_clarification = (
|
||||
"what" in low or "would you like" in low or "?" in reply
|
||||
)
|
||||
assert name_referenced or asks_clarification, (
|
||||
f"File-only reply neither references the file nor asks a "
|
||||
f"clarifying question. Reply: {reply[:500]!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------- canary 3: file + prompt (multimodal) ---------------------------
|
||||
|
||||
|
||||
def test_canary_3_file_with_prompt(sim: CPSim, context_id: str) -> None:
|
||||
"""File-attached A2A message WITH a caption — multimodal happy path.
|
||||
|
||||
Lower bar than canary 2: assert the agent acknowledges the file was
|
||||
received and tries to address the caption. We deliberately don't
|
||||
require a perfect summary because canary mode replies are canned —
|
||||
the goal is to prove the executor's multimodal code path doesn't
|
||||
drop EITHER the file OR the caption.
|
||||
"""
|
||||
file_name = f"canary-doc-{uuid.uuid4().hex[:8]}.txt"
|
||||
file_body = (
|
||||
b"Quarterly review. Revenue up 14%. Churn down 3%. "
|
||||
b"Team headcount steady. Action: ship RFC#600 by end of week."
|
||||
)
|
||||
r = sim.send_with_file(
|
||||
context_id=context_id,
|
||||
text="summarize this",
|
||||
file_name=file_name,
|
||||
file_bytes=file_body,
|
||||
mime_type="text/plain",
|
||||
)
|
||||
reply = sim.extract_text_parts(r)
|
||||
assert reply, f"File+prompt produced empty reply. envelope={r!r}"
|
||||
|
||||
low = reply.lower()
|
||||
for marker in _DROPPED_TURN_MARKERS:
|
||||
assert marker.lower() not in low, (
|
||||
f"File+prompt was dropped — reply contains {marker!r}. "
|
||||
f"Full reply: {reply[:500]!r}"
|
||||
)
|
||||
|
||||
# At minimum: the reply must mention file/attach/summary semantics,
|
||||
# demonstrating the executor accepted both parts.
|
||||
engaged = any(
|
||||
kw in low for kw in ("file", "attach", "summary", "summarize", "content", file_name.lower())
|
||||
)
|
||||
assert engaged, (
|
||||
f"Multimodal reply doesn't engage with attached file or caption. "
|
||||
f"Reply: {reply[:500]!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------- canary 4: cross-session memory recall --------------------------
|
||||
|
||||
|
||||
def test_canary_4_cross_session_memory_recall(sim: CPSim) -> None:
|
||||
"""Memory persists across distinct context_ids → memory layer (NOT
|
||||
SessionStore) is the storage.
|
||||
|
||||
Two distinct context_ids in this test — SessionStore CANNOT bridge
|
||||
them. The bridge is the runtime's persistent memory (MOLECULE_MEMORY_ROOT
|
||||
in canary mode). If the recall returns "blue" in session 2, the
|
||||
memory layer is wired correctly.
|
||||
|
||||
Note: we ask the agent to commit the memory explicitly in session 1
|
||||
so that the canary doesn't depend on memory auto-extraction
|
||||
heuristics (which vary by runtime). The commit goes through the
|
||||
same MCP tool the canvas would invoke.
|
||||
"""
|
||||
ctx_a = f"canary-ctx-{uuid.uuid4().hex[:12]}"
|
||||
ctx_b = f"canary-ctx-{uuid.uuid4().hex[:12]}"
|
||||
|
||||
# Session 1 — commit a fact via the memory tool. Use the explicit
|
||||
# "remember" verb so canary-mode agents (which short-circuit to a
|
||||
# deterministic tool-call) reliably invoke `commit_memory`.
|
||||
r1 = sim.send_text(
|
||||
"Please use the memory tool to remember: my favorite color is blue.",
|
||||
context_id=ctx_a,
|
||||
)
|
||||
reply1 = sim.extract_text_parts(r1)
|
||||
assert reply1, f"Session 1 produced empty reply. envelope={r1!r}"
|
||||
|
||||
# Session 2 — different context_id. Same workspace, same memory.
|
||||
r2 = sim.send_text(
|
||||
"Use the memory tool to recall my favorite color, then tell me what it is.",
|
||||
context_id=ctx_b,
|
||||
)
|
||||
reply2 = sim.extract_text_parts(r2)
|
||||
assert reply2, f"Session 2 produced empty reply. envelope={r2!r}"
|
||||
|
||||
assert re.search(r"\bblue\b", reply2, flags=re.IGNORECASE), (
|
||||
f"Session 2 reply does not contain 'blue' — cross-session memory "
|
||||
f"recall regression suspected. ctx_a={ctx_a} ctx_b={ctx_b} "
|
||||
f"session1_reply={reply1[:200]!r} session2_reply={reply2[:400]!r}"
|
||||
)
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Tenant control-plane simulator.
|
||||
|
||||
Emits the byte-identical JSON-RPC `message/send` wire shape that the
|
||||
production `workspace-server` POSTs to the runtime's :8000 — see
|
||||
``workspace-server/internal/handlers/a2a.go`` and the canonical sample
|
||||
in ``tests/e2e/test_chat_attachments_e2e.sh``.
|
||||
|
||||
This file is purposefully small (~250 LoC). It is NOT a re-implementation
|
||||
of `workspace-server`; it is just the minimum surface required to drive
|
||||
the 4 session-continuity canaries.
|
||||
|
||||
If the runtime asserts on a header / envelope field that the production
|
||||
platform sets but this simulator omits, FIX THE SIMULATOR — never weaken
|
||||
the runtime to accept divergent wire shapes. The simulator is the
|
||||
canonical contract emitter for canary purposes
|
||||
(``feedback_no_single_source_of_truth``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class CPSimConfig:
|
||||
runtime_url: str
|
||||
"""Base URL of the runtime under test (e.g. http://runtime:8000)."""
|
||||
request_timeout_s: float = 60.0
|
||||
"""Per-A2A-call timeout. Generous — canary mode replies are fast,
|
||||
but a real Provider-backed runtime under cold cache can take 30+s."""
|
||||
|
||||
|
||||
class CPSim:
|
||||
"""Thin client matching workspace-server's wire shape."""
|
||||
|
||||
def __init__(self, cfg: CPSimConfig | None = None) -> None:
|
||||
self.cfg = cfg or CPSimConfig(
|
||||
runtime_url=os.environ.get("RUNTIME_URL", "http://localhost:18000"),
|
||||
)
|
||||
self._client = httpx.Client(timeout=self.cfg.request_timeout_s)
|
||||
|
||||
# ------------------------------------------------------------------ A2A
|
||||
|
||||
def send_text(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
context_id: str,
|
||||
task_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""POST a text-only A2A message. Returns the JSON-RPC envelope."""
|
||||
msg_id = f"canary-{uuid.uuid4().hex[:12]}"
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"method": "message/send",
|
||||
"params": {
|
||||
"message": {
|
||||
"role": "user",
|
||||
"messageId": msg_id,
|
||||
"kind": "message",
|
||||
"contextId": context_id,
|
||||
"taskId": task_id,
|
||||
"parts": [{"kind": "text", "text": text}],
|
||||
},
|
||||
"configuration": {
|
||||
"acceptedOutputModes": ["text/plain"],
|
||||
"blocking": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
return self._post(payload)
|
||||
|
||||
def send_with_file(
|
||||
self,
|
||||
*,
|
||||
context_id: str,
|
||||
text: str | None,
|
||||
file_name: str,
|
||||
file_bytes: bytes,
|
||||
mime_type: str = "text/plain",
|
||||
task_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""POST an A2A message with an inline file part.
|
||||
|
||||
Uses the inline `bytes` form of A2A file parts (RFC#600 — the
|
||||
no-URI variant added precisely so canary tests don't need a
|
||||
`/chat/uploads` round-trip). Each runtime's executor calls
|
||||
``extract_attached_files`` which handles both forms — verified
|
||||
in ``workspace/executor_helpers.py:903``.
|
||||
"""
|
||||
msg_id = f"canary-{uuid.uuid4().hex[:12]}"
|
||||
parts: list[dict[str, Any]] = []
|
||||
if text:
|
||||
parts.append({"kind": "text", "text": text})
|
||||
parts.append(
|
||||
{
|
||||
"kind": "file",
|
||||
"file": {
|
||||
"name": file_name,
|
||||
"mimeType": mime_type,
|
||||
"bytes": base64.b64encode(file_bytes).decode("ascii"),
|
||||
},
|
||||
}
|
||||
)
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"method": "message/send",
|
||||
"params": {
|
||||
"message": {
|
||||
"role": "user",
|
||||
"messageId": msg_id,
|
||||
"kind": "message",
|
||||
"contextId": context_id,
|
||||
"taskId": task_id,
|
||||
"parts": parts,
|
||||
},
|
||||
"configuration": {
|
||||
"acceptedOutputModes": ["text/plain"],
|
||||
"blocking": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
return self._post(payload)
|
||||
|
||||
# ------------------------------------------------------------ helpers
|
||||
|
||||
def _post(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
url = f"{self.cfg.runtime_url}/a2a"
|
||||
try:
|
||||
r = self._client.post(url, json=payload)
|
||||
except httpx.HTTPError as e:
|
||||
raise CPSimError(f"A2A POST failed: {e}") from e
|
||||
if r.status_code != 200:
|
||||
raise CPSimError(
|
||||
f"A2A non-200: status={r.status_code} body={r.text[:500]}"
|
||||
)
|
||||
try:
|
||||
return r.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise CPSimError(f"A2A body not JSON: {r.text[:500]}") from e
|
||||
|
||||
@staticmethod
|
||||
def extract_text_parts(envelope: dict[str, Any]) -> str:
|
||||
"""Return concatenated text from all text parts of a reply.
|
||||
|
||||
Handles both top-level `result.parts` (the canonical shape) and
|
||||
`result.artifacts[*].parts` (which some runtimes emit when the
|
||||
reply was streamed as artifact chunks). Matches the extractor in
|
||||
``tests/e2e/test_chat_attachments_e2e.sh``.
|
||||
"""
|
||||
result = envelope.get("result") or {}
|
||||
chunks: list[str] = []
|
||||
for p in result.get("parts", []) or []:
|
||||
if p.get("kind") == "text":
|
||||
chunks.append(p.get("text", ""))
|
||||
for art in result.get("artifacts", []) or []:
|
||||
for p in art.get("parts", []) or []:
|
||||
if p.get("kind") == "text":
|
||||
chunks.append(p.get("text", ""))
|
||||
# Some runtimes return a status.message instead of/in addition to parts.
|
||||
status = result.get("status") or {}
|
||||
status_msg = status.get("message") or {}
|
||||
for p in status_msg.get("parts", []) or []:
|
||||
if p.get("kind") == "text":
|
||||
chunks.append(p.get("text", ""))
|
||||
return "\n".join(chunks).strip()
|
||||
|
||||
# ----------------------------------------------------- memory probe
|
||||
|
||||
def probe_memory(self, key: str) -> str | None:
|
||||
"""Read a memory value via the runtime's MCP memory tool.
|
||||
|
||||
Uses the same MCP transport the canvas uses
|
||||
(``POST /workspaces/:id/mcp``-shaped JSON-RPC over /mcp). Returns
|
||||
the recalled string or None if the key is missing.
|
||||
"""
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": f"canary-mem-{uuid.uuid4().hex[:8]}",
|
||||
"method": "tools/call",
|
||||
"params": {"name": "recall_memory", "arguments": {"key": key}},
|
||||
}
|
||||
try:
|
||||
r = self._client.post(f"{self.cfg.runtime_url}/mcp", json=payload)
|
||||
except httpx.HTTPError as e:
|
||||
raise CPSimError(f"MCP POST failed: {e}") from e
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
body = r.json()
|
||||
result = body.get("result") or {}
|
||||
# MCP responses wrap the tool output in result.content[*].text per
|
||||
# the JSON-RPC tools/call contract.
|
||||
for c in result.get("content", []) or []:
|
||||
if c.get("type") == "text":
|
||||
return c.get("text")
|
||||
return None
|
||||
|
||||
|
||||
class CPSimError(RuntimeError):
|
||||
"""Raised on transport / envelope failures (NOT canary assertion failures).
|
||||
|
||||
Distinct from AssertionError so pytest reports them as ERROR not
|
||||
FAILED — a transport-layer fault should be debugged differently from
|
||||
a real session-continuity regression.
|
||||
"""
|
||||
@@ -0,0 +1,5 @@
|
||||
# Pinned (not floating) so the harness is reproducible across CI runs.
|
||||
# These versions match what tests/e2e/_lib.sh and tests/e2e/conftest.py use.
|
||||
httpx==0.27.2
|
||||
pytest==8.3.3
|
||||
pytest-asyncio==0.24.0
|
||||
@@ -0,0 +1,58 @@
|
||||
# local-e2e/docker-compose.yml — minimal harness stack.
|
||||
#
|
||||
# Two services:
|
||||
# runtime — the template image under test (TEMPLATE_IMAGE env var).
|
||||
# Exposes :8000 for A2A traffic. The simulator POSTs to it.
|
||||
# cp_sim — thin Python tenant-CP simulator. Drives the canary turns.
|
||||
#
|
||||
# Deliberately NO postgres, NO redis, NO platform Go service. SessionStore
|
||||
# continuity is a runtime-internal concern (a2a_executor + executor_helpers);
|
||||
# we test it without dragging the platform-tenant Go binary into the loop.
|
||||
# See README.md "Why a thin Python simulator" for rationale.
|
||||
|
||||
services:
|
||||
runtime:
|
||||
image: ${TEMPLATE_IMAGE:?TEMPLATE_IMAGE env required, e.g. ghcr.io/molecule-ai/workspace-template-hermes:latest}
|
||||
# The runtime entrypoint (workspace/entrypoint.sh) refuses to start when
|
||||
# any operator-scope env var is present. We deliberately set no creds —
|
||||
# the canary doesn't invoke a real LLM provider (see TEST_NO_PROVIDER below).
|
||||
environment:
|
||||
# Disable provider calls during canary — the runtime returns canned
|
||||
# echo-style replies so the harness can assert continuity / file-handling
|
||||
# behaviour without burning provider quota. The template image must
|
||||
# honour MOLECULE_CANARY_MODE=1 (added in molecule-ai-workspace-runtime
|
||||
# PR #46 — see molecule_runtime/a2a_executor.py canary short-circuit).
|
||||
MOLECULE_CANARY_MODE: "1"
|
||||
# Anonymous workspace identity so RBAC paths exercise the same code
|
||||
# they would in tenant production.
|
||||
WORKSPACE_ID: "canary-${CANARY_RUN_ID:-local}"
|
||||
# Memory tool requires a writable scope; point at /tmp inside the
|
||||
# container so cross-session canary (#4) works without bind mounts.
|
||||
MOLECULE_MEMORY_ROOT: "/tmp/canary-memory"
|
||||
# The provisioner's forbidden-env guard exits non-zero when any
|
||||
# operator-scope literal is present; the canary intentionally sets
|
||||
# zero of them. Leave guard ON (do NOT set MOLECULE_TENANT_GUARD_DISABLE)
|
||||
# so we exercise the prod entrypoint code path verbatim.
|
||||
ports:
|
||||
- "${RUNTIME_PORT:-18000}:8000"
|
||||
healthcheck:
|
||||
# /agent-card is the universal A2A discovery endpoint — every template
|
||||
# exposes it. /health varies per template.
|
||||
test: ["CMD-SHELL", "wget -qO /dev/null --tries=1 http://localhost:8000/agent-card || exit 1"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 30s
|
||||
|
||||
cp_sim:
|
||||
build:
|
||||
context: ./cp_sim
|
||||
depends_on:
|
||||
runtime:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
RUNTIME_URL: "http://runtime:8000"
|
||||
CANARY_RUN_ID: "${CANARY_RUN_ID:-local}"
|
||||
# cp_sim doesn't expose a port — it's a one-shot driver invoked by
|
||||
# run-canary.sh via `docker compose run cp_sim pytest ...`.
|
||||
profiles: ["driver"]
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# onboard-template.sh — gitops helper to wire local-e2e into a new template.
|
||||
#
|
||||
# Drops .gitea/workflows/session-continuity-e2e.yml into the target template
|
||||
# repo (a thin shim that clones molecule-core's local-e2e harness, then runs
|
||||
# run-canary.sh against the locally-built template image). Opens a PR.
|
||||
#
|
||||
# Usage:
|
||||
# ./local-e2e/scripts/onboard-template.sh molecule-ai-workspace-template-claude-code
|
||||
#
|
||||
# Per task #342 sequencing: do NOT run this for every template at once.
|
||||
# Bake the gate on hermes for ≥5 business days first; expand only after
|
||||
# the canary is empirically stable.
|
||||
#
|
||||
# Cross-refs:
|
||||
# feedback_no_single_source_of_truth — the workflow content is identical
|
||||
# across templates; this helper guarantees it.
|
||||
# feedback_image_promote_is_not_user_live — we wire the gate at the
|
||||
# CI layer; flipping it to REQUIRED in branch_protection is a
|
||||
# separate step (see README.md).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO="${1:?usage: onboard-template.sh <template-repo-name>}"
|
||||
HARNESS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
|
||||
|
||||
# Sanity: ensure the template-side workflow file exists in this repo.
|
||||
TEMPLATE_WORKFLOW="$HARNESS_ROOT/templates/session-continuity-e2e.yml"
|
||||
[ -f "$TEMPLATE_WORKFLOW" ] || {
|
||||
echo "ERROR: $TEMPLATE_WORKFLOW not found in this harness checkout"
|
||||
exit 1
|
||||
}
|
||||
|
||||
WORK_DIR=$(mktemp -d -t e2e-onboard-XXXXXX)
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
|
||||
cd "$WORK_DIR"
|
||||
|
||||
# Use mol_clone — preserves the persona credential model.
|
||||
# shellcheck disable=SC1090
|
||||
source "$HOME/.molecule-ai/ops.sh"
|
||||
mol_clone "$REPO"
|
||||
cd "$REPO"
|
||||
|
||||
git checkout -b "task342/session-continuity-e2e-gate"
|
||||
|
||||
mkdir -p .gitea/workflows
|
||||
cp "$TEMPLATE_WORKFLOW" .gitea/workflows/session-continuity-e2e.yml
|
||||
|
||||
git add .gitea/workflows/session-continuity-e2e.yml
|
||||
git commit -m "ci: add local-e2e session-continuity canary gate (task #342)
|
||||
|
||||
Wires this template into the cross-template session-continuity harness
|
||||
in molecule-ai/molecule-core/local-e2e/. The gate boots THIS repo's
|
||||
locally-built image, drives 4 canonical canaries (2-turn name continuity,
|
||||
file-only message, file+prompt, cross-session memory recall), and fails
|
||||
PRs that regress any of them.
|
||||
|
||||
Per CTO directive: required-context flip in branch_protection is a
|
||||
SEPARATE step after 5 business days of bake."
|
||||
|
||||
# Push branch; do not auto-open PR — leave that to the operator so the
|
||||
# review-relay routing follows the same rules as a normal change.
|
||||
git push -u origin "task342/session-continuity-e2e-gate"
|
||||
|
||||
echo
|
||||
echo "DONE. Branch pushed to $REPO. Open PR manually:"
|
||||
echo " https://git.moleculesai.app/molecule-ai/$REPO/compare/main...task342/session-continuity-e2e-gate"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user