Compare commits
45 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b57cebf8d4 | |||
| 15e2d93989 | |||
| 3eb06e40e6 | |||
| 9d05335b1a | |||
| f470f589c0 | |||
| 0a2e1e9a97 | |||
| d7e163d2a8 | |||
| 05e6443e2c | |||
| b62b18b523 | |||
| e70955298b | |||
| db647de1cd | |||
| 94b08ef0de | |||
| 1a2cfb9417 | |||
| 3d572d97a3 | |||
| beea0e9b88 | |||
| 2747246519 | |||
| 67762ca422 | |||
| 71cfb70a6f | |||
| c2d27d2b3f | |||
| ce06b8cd59 | |||
| e0bbba801e | |||
| 5c10ee0d73 | |||
| 8f1d24f33f | |||
| ae30cdef87 | |||
| dd992fcc9b | |||
| 00f0a1066f | |||
| 65f34711bc | |||
| df2e69b32f | |||
| 4a7e1bd988 | |||
| 0911ee1a89 | |||
| cebd9ab916 | |||
| d0ed03edc6 | |||
| 5a67b1dc5e | |||
| 26a04c2a99 | |||
| cc2c810637 | |||
| deda8ddccf | |||
| eeef790afa | |||
| 20c72cfb62 | |||
| 32f32cafca | |||
| f91d34c9e4 | |||
| 4ed3dbdfb7 | |||
| ff5186dbc3 | |||
| 2d096aa7ae | |||
| eda6b987a2 | |||
| c7e1642ffb |
Executable
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
# sop-tier-refire — re-evaluate sop-tier-check and POST status to PR head SHA.
|
||||
#
|
||||
# Invoked from `.gitea/workflows/sop-tier-refire.yml` when a repo
|
||||
# MEMBER/OWNER/COLLABORATOR comments `/refire-tier-check` on a PR.
|
||||
#
|
||||
# Behavior:
|
||||
#
|
||||
# 1. Resolve PR head SHA + author from PR_NUMBER.
|
||||
# 2. Rate-limit: if the sop-tier-check context has been POSTed in the
|
||||
# last 30 seconds, skip (prevents comment-spam status thrash).
|
||||
# 3. Invoke `.gitea/scripts/sop-tier-check.sh` with the same env the
|
||||
# canonical workflow provides. This is DRY: we re-use the exact AND-
|
||||
# composition gate logic, not a watered-down approving-count check.
|
||||
# 4. POST the resulting status (success on exit 0, failure on non-zero)
|
||||
# to `/repos/.../statuses/{HEAD_SHA}` with context
|
||||
# "sop-tier-check / tier-check (pull_request)" — the same context name
|
||||
# branch protection requires.
|
||||
#
|
||||
# Required env (set by sop-tier-refire.yml):
|
||||
# GITEA_TOKEN — org-level SOP_TIER_CHECK_TOKEN (read:org/user/issue/repo)
|
||||
# GITEA_HOST — e.g. git.moleculesai.app
|
||||
# REPO — owner/name
|
||||
# PR_NUMBER — PR number from issue_comment payload
|
||||
# COMMENT_AUTHOR — login of the commenter (logged for audit)
|
||||
#
|
||||
# Optional:
|
||||
# SOP_DEBUG=1 — verbose per-API-call diagnostics
|
||||
# SOP_REFIRE_RATE_LIMIT_SEC — override the 30s rate-limit (default 30)
|
||||
# SOP_REFIRE_DISABLE_RATE_LIMIT=1 — for tests; skips the rate-limit check
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
debug() {
|
||||
if [ "${SOP_DEBUG:-}" = "1" ]; then
|
||||
echo " [debug] $*" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN required}"
|
||||
: "${GITEA_HOST:?GITEA_HOST required}"
|
||||
: "${REPO:?REPO required (owner/name)}"
|
||||
: "${PR_NUMBER:?PR_NUMBER required}"
|
||||
: "${COMMENT_AUTHOR:=unknown}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
API="https://${GITEA_HOST}/api/v1"
|
||||
AUTH="Authorization: token ${GITEA_TOKEN}"
|
||||
CONTEXT="sop-tier-check / tier-check (pull_request)"
|
||||
RATE_LIMIT_SEC="${SOP_REFIRE_RATE_LIMIT_SEC:-30}"
|
||||
|
||||
echo "::notice::sop-tier-refire start: repo=$OWNER/$NAME pr=$PR_NUMBER commenter=$COMMENT_AUTHOR"
|
||||
|
||||
# 1. Fetch PR details — need head.sha and user.login.
|
||||
PR_FILE=$(mktemp)
|
||||
trap 'rm -f "$PR_FILE"' EXIT
|
||||
PR_HTTP=$(curl -sS -o "$PR_FILE" -w '%{http_code}' -H "$AUTH" \
|
||||
"${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}")
|
||||
if [ "$PR_HTTP" != "200" ]; then
|
||||
echo "::error::GET /pulls/$PR_NUMBER returned HTTP $PR_HTTP (body $(head -c 200 "$PR_FILE"))"
|
||||
exit 1
|
||||
fi
|
||||
HEAD_SHA=$(jq -r '.head.sha' <"$PR_FILE")
|
||||
PR_AUTHOR=$(jq -r '.user.login' <"$PR_FILE")
|
||||
PR_STATE=$(jq -r '.state' <"$PR_FILE")
|
||||
if [ -z "$HEAD_SHA" ] || [ "$HEAD_SHA" = "null" ]; then
|
||||
echo "::error::Could not resolve head.sha from PR #$PR_NUMBER response"
|
||||
exit 1
|
||||
fi
|
||||
debug "head_sha=$HEAD_SHA pr_author=$PR_AUTHOR state=$PR_STATE"
|
||||
|
||||
if [ "$PR_STATE" != "open" ]; then
|
||||
echo "::notice::PR #$PR_NUMBER state is $PR_STATE; refire is a no-op on closed PRs."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 2. Rate-limit: skip if our context was updated in the last $RATE_LIMIT_SEC.
|
||||
# Gitea statuses endpoint returns latest first; we check the most recent
|
||||
# entry for our context name.
|
||||
if [ "${SOP_REFIRE_DISABLE_RATE_LIMIT:-}" != "1" ]; then
|
||||
STATUSES_FILE=$(mktemp)
|
||||
trap 'rm -f "$PR_FILE" "$STATUSES_FILE"' EXIT
|
||||
ST_HTTP=$(curl -sS -o "$STATUSES_FILE" -w '%{http_code}' -H "$AUTH" \
|
||||
"${API}/repos/${OWNER}/${NAME}/statuses/${HEAD_SHA}?limit=50&sort=newest")
|
||||
debug "statuses-list HTTP=$ST_HTTP"
|
||||
if [ "$ST_HTTP" = "200" ]; then
|
||||
LAST_UPDATED=$(jq -r --arg c "$CONTEXT" \
|
||||
'[.[] | select(.context == $c)] | first | .updated_at // ""' \
|
||||
<"$STATUSES_FILE")
|
||||
if [ -n "$LAST_UPDATED" ] && [ "$LAST_UPDATED" != "null" ]; then
|
||||
# Parse RFC3339 → epoch. Use python -c for portability (date(1) -d
|
||||
# differs between BSD/GNU; the Gitea runner is Ubuntu so GNU date
|
||||
# works, but we keep python for future container variance).
|
||||
LAST_EPOCH=$(python3 -c "import sys,datetime;print(int(datetime.datetime.fromisoformat(sys.argv[1].replace('Z','+00:00')).timestamp()))" "$LAST_UPDATED" 2>/dev/null || echo "0")
|
||||
NOW_EPOCH=$(date -u +%s)
|
||||
AGE=$((NOW_EPOCH - LAST_EPOCH))
|
||||
debug "last status update: $LAST_UPDATED ($AGE seconds ago)"
|
||||
if [ "$AGE" -lt "$RATE_LIMIT_SEC" ] && [ "$AGE" -ge 0 ]; then
|
||||
echo "::notice::sop-tier-refire rate-limited — last status update was ${AGE}s ago (<${RATE_LIMIT_SEC}s window). Try again shortly."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects. Capture exit code.
|
||||
# The canonical script reads tier label, walks approving reviewers, and
|
||||
# evaluates the AND-composition expression — we want the SAME gate, not
|
||||
# a different gate.
|
||||
#
|
||||
# SOP_REFIRE_TIER_CHECK_SCRIPT env var lets tests substitute a mock —
|
||||
# sop-tier-check.sh uses bash 4+ associative arrays which trigger a known
|
||||
# bash 3.2 parser bug (`tier: unbound variable` from declare -A with
|
||||
# `set -u`). Linux Gitea runners ship bash 4/5 so production is fine;
|
||||
# the override exists so the bash 3.2 dev box can still exercise the
|
||||
# refire glue logic end-to-end.
|
||||
SCRIPT="${SOP_REFIRE_TIER_CHECK_SCRIPT:-$(dirname "$0")/sop-tier-check.sh}"
|
||||
if [ ! -f "$SCRIPT" ]; then
|
||||
echo "::error::sop-tier-check.sh not found at $SCRIPT — refire requires the canonical script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Re-invoke. Pipe stdout/stderr through so the runner log shows the
|
||||
# tier-check decision inline.
|
||||
set +e
|
||||
GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
GITEA_HOST="$GITEA_HOST" \
|
||||
REPO="$REPO" \
|
||||
PR_NUMBER="$PR_NUMBER" \
|
||||
PR_AUTHOR="$PR_AUTHOR" \
|
||||
SOP_DEBUG="${SOP_DEBUG:-0}" \
|
||||
SOP_LEGACY_CHECK="${SOP_LEGACY_CHECK:-0}" \
|
||||
bash "$SCRIPT"
|
||||
TIER_EXIT=$?
|
||||
set -e
|
||||
debug "sop-tier-check.sh exit=$TIER_EXIT"
|
||||
|
||||
# 4. POST the resulting status.
|
||||
if [ "$TIER_EXIT" -eq 0 ]; then
|
||||
STATE="success"
|
||||
DESCRIPTION="Refired via /refire-tier-check by $COMMENT_AUTHOR"
|
||||
else
|
||||
STATE="failure"
|
||||
DESCRIPTION="Refired via /refire-tier-check; tier-check failed (see workflow log)"
|
||||
fi
|
||||
|
||||
# Status target_url points at the runner log so a curious reviewer can
|
||||
# follow it back. SERVER_URL + RUN_ID + JOB_ID isn't trivially constructible
|
||||
# from the bash env on Gitea 1.22.6, so we point at the PR itself.
|
||||
TARGET_URL="https://${GITEA_HOST}/${OWNER}/${NAME}/pulls/${PR_NUMBER}"
|
||||
|
||||
POST_BODY=$(jq -nc \
|
||||
--arg state "$STATE" \
|
||||
--arg context "$CONTEXT" \
|
||||
--arg description "$DESCRIPTION" \
|
||||
--arg target_url "$TARGET_URL" \
|
||||
'{state:$state, context:$context, description:$description, target_url:$target_url}')
|
||||
|
||||
POST_FILE=$(mktemp)
|
||||
trap 'rm -f "$PR_FILE" "${STATUSES_FILE:-}" "$POST_FILE"' EXIT
|
||||
POST_HTTP=$(curl -sS -o "$POST_FILE" -w '%{http_code}' \
|
||||
-X POST -H "$AUTH" -H "Content-Type: application/json" \
|
||||
-d "$POST_BODY" \
|
||||
"${API}/repos/${OWNER}/${NAME}/statuses/${HEAD_SHA}")
|
||||
if [ "$POST_HTTP" != "200" ] && [ "$POST_HTTP" != "201" ]; then
|
||||
echo "::error::POST /statuses/$HEAD_SHA returned HTTP $POST_HTTP (body $(head -c 200 "$POST_FILE"))"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::sop-tier-refire posted state=$STATE for context=\"$CONTEXT\" on sha=$HEAD_SHA"
|
||||
exit "$TIER_EXIT"
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mock sop-tier-check.sh for sop-tier-refire tests.
|
||||
#
|
||||
# Exits 0 ("PASS") if $MOCK_TIER_RESULT == "pass", else exits 1.
|
||||
# This lets the refire tests cover the success + failure status-POST
|
||||
# paths without invoking the real sop-tier-check.sh (which uses bash 4+
|
||||
# associative arrays — known parser bug on macOS bash 3.2 dev box).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
case "${MOCK_TIER_RESULT:-pass}" in
|
||||
pass)
|
||||
echo "::notice::mock tier-check: PASS"
|
||||
exit 0
|
||||
;;
|
||||
fail_no_label)
|
||||
echo "::error::mock tier-check: no tier label"
|
||||
exit 1
|
||||
;;
|
||||
fail_no_approvals)
|
||||
echo "::error::mock tier-check: no approving reviews"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "::error::mock tier-check: unknown MOCK_TIER_RESULT=${MOCK_TIER_RESULT:-}"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stub Gitea API for sop-tier-refire test scenarios.
|
||||
|
||||
Reads $FIXTURE_STATE_DIR/scenario to decide what to return for each
|
||||
endpoint the sop-tier-refire.sh + sop-tier-check.sh scripts call.
|
||||
Captures every POST to /statuses/{sha} into posted_statuses.jsonl so
|
||||
the test can assert what the script tried to write.
|
||||
|
||||
Scenarios:
|
||||
T1_success — tier:low + APPROVED by engineer → tier-check passes
|
||||
T2_no_tier_label — no tier label → tier-check exits 1 before POST
|
||||
T3_no_approvals — tier:low but zero approving reviews → exits 1
|
||||
T4_closed — PR state=closed → refire is a no-op
|
||||
T5_rate_limited — last status update 5 seconds ago → skip
|
||||
|
||||
Usage:
|
||||
FIXTURE_STATE_DIR=/tmp/x python3 _refire_fixture.py 8080
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
|
||||
STATE_DIR = os.environ["FIXTURE_STATE_DIR"]
|
||||
|
||||
|
||||
def scenario() -> str:
|
||||
p = os.path.join(STATE_DIR, "scenario")
|
||||
if not os.path.isfile(p):
|
||||
return "T1_success"
|
||||
with open(p) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
|
||||
|
||||
def append_post(body: dict) -> None:
|
||||
with open(os.path.join(STATE_DIR, "posted_statuses.jsonl"), "a") as f:
|
||||
f.write(json.dumps(body) + "\n")
|
||||
|
||||
|
||||
def pr_payload() -> dict:
|
||||
sc = scenario()
|
||||
state = "closed" if sc == "T4_closed" else "open"
|
||||
return {
|
||||
"number": 999,
|
||||
"state": state,
|
||||
"head": {"sha": "deadbeef0000111122223333444455556666"},
|
||||
"user": {"login": "feature-author"},
|
||||
}
|
||||
|
||||
|
||||
def labels_payload() -> list:
|
||||
sc = scenario()
|
||||
if sc == "T2_no_tier_label":
|
||||
return [{"name": "bug"}]
|
||||
# All other scenarios use tier:low
|
||||
return [{"name": "tier:low"}, {"name": "ci"}]
|
||||
|
||||
|
||||
def reviews_payload() -> list:
|
||||
sc = scenario()
|
||||
if sc == "T3_no_approvals":
|
||||
return []
|
||||
# All other scenarios have one APPROVED review by an engineer
|
||||
return [
|
||||
{
|
||||
"state": "APPROVED",
|
||||
"user": {"login": "reviewer-engineer"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def teams_payload() -> list:
|
||||
# Mirror the real molecule-ai org teams referenced in TIER_EXPR
|
||||
return [
|
||||
{"id": 5, "name": "ceo"},
|
||||
{"id": 2, "name": "engineers"},
|
||||
{"id": 6, "name": "managers"},
|
||||
]
|
||||
|
||||
|
||||
def statuses_payload() -> list:
|
||||
sc = scenario()
|
||||
if sc == "T5_rate_limited":
|
||||
recent = (
|
||||
datetime.datetime.now(datetime.timezone.utc)
|
||||
- datetime.timedelta(seconds=5)
|
||||
).isoformat()
|
||||
return [
|
||||
{
|
||||
"context": "sop-tier-check / tier-check (pull_request)",
|
||||
"state": "failure",
|
||||
"updated_at": recent,
|
||||
}
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def user_payload() -> dict:
|
||||
# Mirrors the WHOAMI probe in sop-tier-check.sh
|
||||
return {"login": "sop-tier-bot-fixture"}
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
# Quiet — keep stdout for explicit logs only.
|
||||
def log_message(self, *args, **kwargs): # noqa: D401
|
||||
pass
|
||||
|
||||
def _json(self, code: int, body) -> None:
|
||||
payload = json.dumps(body).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def _empty(self, code: int) -> None:
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def do_GET(self): # noqa: N802
|
||||
u = urllib.parse.urlparse(self.path)
|
||||
path = u.path
|
||||
|
||||
if path == "/_ping":
|
||||
return self._json(200, {"ok": True})
|
||||
if path == "/api/v1/user":
|
||||
return self._json(200, user_payload())
|
||||
|
||||
# /api/v1/repos/{owner}/{name}/pulls/{n}
|
||||
m = re.match(r"^/api/v1/repos/[^/]+/[^/]+/pulls/(\d+)$", path)
|
||||
if m:
|
||||
return self._json(200, pr_payload())
|
||||
|
||||
# /api/v1/repos/{owner}/{name}/issues/{n}/labels
|
||||
if re.match(r"^/api/v1/repos/[^/]+/[^/]+/issues/\d+/labels$", path):
|
||||
return self._json(200, labels_payload())
|
||||
|
||||
# /api/v1/repos/{owner}/{name}/pulls/{n}/reviews
|
||||
if re.match(r"^/api/v1/repos/[^/]+/[^/]+/pulls/\d+/reviews$", path):
|
||||
return self._json(200, reviews_payload())
|
||||
|
||||
# /api/v1/orgs/{owner}/teams
|
||||
if re.match(r"^/api/v1/orgs/[^/]+/teams$", path):
|
||||
return self._json(200, teams_payload())
|
||||
|
||||
# /api/v1/teams/{id}/members/{login} → 204 if user is an engineer
|
||||
m = re.match(r"^/api/v1/teams/(\d+)/members/([^/]+)$", path)
|
||||
if m:
|
||||
team_id, login = m.group(1), m.group(2)
|
||||
# In our fixture reviewer-engineer ∈ engineers (id=2)
|
||||
if team_id == "2" and login == "reviewer-engineer":
|
||||
return self._empty(204)
|
||||
return self._empty(404)
|
||||
|
||||
# /api/v1/orgs/{owner}/members/{login} — fallback path used when
|
||||
# team-member probes all 403. We don't need it for these tests.
|
||||
if re.match(r"^/api/v1/orgs/[^/]+/members/[^/]+$", path):
|
||||
return self._empty(404)
|
||||
|
||||
# /api/v1/repos/{owner}/{name}/statuses/{sha}
|
||||
if re.match(r"^/api/v1/repos/[^/]+/[^/]+/statuses/[^/]+$", path):
|
||||
return self._json(200, statuses_payload())
|
||||
|
||||
return self._json(404, {"path": path, "msg": "fixture: no route"})
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
u = urllib.parse.urlparse(self.path)
|
||||
path = u.path
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
raw = self.rfile.read(length) if length else b""
|
||||
try:
|
||||
body = json.loads(raw) if raw else {}
|
||||
except Exception:
|
||||
body = {"_raw": raw.decode(errors="replace")}
|
||||
|
||||
if re.match(r"^/api/v1/repos/[^/]+/[^/]+/statuses/[^/]+$", path):
|
||||
append_post(body)
|
||||
# Echo back something status-shaped — script only checks HTTP code.
|
||||
return self._json(
|
||||
201,
|
||||
{
|
||||
"context": body.get("context"),
|
||||
"state": body.get("state"),
|
||||
"created_at": now_iso(),
|
||||
},
|
||||
)
|
||||
|
||||
return self._json(404, {"path": path, "msg": "fixture: no route"})
|
||||
|
||||
|
||||
def main():
|
||||
port = int(sys.argv[1])
|
||||
srv = http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler)
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+297
@@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tests for sop-tier-refire.{yml,sh} — internal#292.
|
||||
#
|
||||
# Behavior matrix:
|
||||
#
|
||||
# T1: PR open + APPROVED via tier:low → script invokes sop-tier-check
|
||||
# and POSTs status=success.
|
||||
# T2: PR open + missing tier label → sop-tier-check exits non-zero;
|
||||
# refire POSTs status=failure (description mentions failure).
|
||||
# T3: PR open + tier:low but NO approving reviews → sop-tier-check
|
||||
# exits non-zero; refire POSTs status=failure.
|
||||
# T4: PR CLOSED → refire exits 0 with no status POST (no-op on closed).
|
||||
# T5: Rate-limit — recent status update within 30s → refire skips,
|
||||
# no new POST.
|
||||
# T6 (yaml-lint): workflow `if:` expression contains author_association
|
||||
# gate + slash-command-trigger gate + PR-not-issue gate.
|
||||
# T7 (yaml-lint): workflow file is parseable YAML.
|
||||
#
|
||||
# Tests T1-T5 run the real script against a local-fixture HTTP server
|
||||
# (python http.server with a stub handler — `tests/_refire_fixture.py`)
|
||||
# so the script's Gitea API calls hit the fixture, not the real Gitea.
|
||||
#
|
||||
# Tests T6/T7 are pure YAML checks against the workflow file.
|
||||
#
|
||||
# Hostile-self-review (per feedback_assert_exact_not_substring):
|
||||
# this test MUST FAIL if the workflow or script is absent. Verified by
|
||||
# running the test before the files exist (covered in the PR body).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
|
||||
WORKFLOW_DIR="$(cd "$THIS_DIR/../../workflows" && pwd)"
|
||||
WORKFLOW="$WORKFLOW_DIR/sop-tier-refire.yml"
|
||||
SCRIPT="$SCRIPT_DIR/sop-tier-refire.sh"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAILED_TESTS=""
|
||||
|
||||
assert_eq() {
|
||||
local label="$1"
|
||||
local expected="$2"
|
||||
local got="$3"
|
||||
if [ "$expected" = "$got" ]; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label"
|
||||
echo " expected: <$expected>"
|
||||
echo " got: <$got>"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local label="$1"
|
||||
local needle="$2"
|
||||
local haystack="$3"
|
||||
if printf '%s' "$haystack" | grep -qF "$needle"; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label"
|
||||
echo " needle: <$needle>"
|
||||
echo " haystack: <$(printf '%s' "$haystack" | head -c 400)>"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_file_exists() {
|
||||
local label="$1"
|
||||
local path="$2"
|
||||
if [ -f "$path" ]; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label (not found: $path)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Existence (foundation — every other test depends on these)
|
||||
echo
|
||||
echo "== existence =="
|
||||
assert_file_exists "workflow file exists" "$WORKFLOW"
|
||||
assert_file_exists "script file exists" "$SCRIPT"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo
|
||||
echo "------"
|
||||
echo "PASS=$PASS FAIL=$FAIL (existence)"
|
||||
echo "Cannot proceed without these files."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# T6 / T7 — workflow YAML structure
|
||||
echo
|
||||
echo "== T6/T7 workflow yaml =="
|
||||
|
||||
# YAML parseability
|
||||
PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T7 workflow parses as YAML" "ok" "$PARSE_OUT"
|
||||
|
||||
# Three required gates in the `if:` expression
|
||||
WORKFLOW_CONTENT=$(cat "$WORKFLOW")
|
||||
assert_contains "T6a workflow if: contains author_association gate" \
|
||||
"github.event.comment.author_association" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6b workflow if: gates on MEMBER/OWNER/COLLABORATOR" \
|
||||
'["MEMBER","OWNER","COLLABORATOR"]' "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6c workflow if: contains slash-command trigger" \
|
||||
"/refire-tier-check" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6d workflow if: gates on PR-not-issue" \
|
||||
"github.event.issue.pull_request" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6e workflow listens on issue_comment" \
|
||||
"issue_comment" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6f workflow requests statuses:write permission" \
|
||||
"statuses: write" "$WORKFLOW_CONTENT"
|
||||
# Does NOT check out PR HEAD (security)
|
||||
if grep -q 'ref: \${{ github.event.pull_request.head' "$WORKFLOW"; then
|
||||
echo " FAIL T6g workflow MUST NOT check out PR head (security)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T6g"
|
||||
else
|
||||
echo " PASS T6g workflow does not check out PR head"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
|
||||
# T1-T5 — script behavior against a local Gitea-fixture
|
||||
echo
|
||||
echo "== T1-T5 script behavior (vs local fixture) =="
|
||||
|
||||
# Spin up the fixture HTTP server.
|
||||
FIXTURE_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$FIXTURE_DIR"; [ -n "${FIX_PID:-}" ] && kill "$FIX_PID" 2>/dev/null || true' EXIT
|
||||
FIXTURE_PY="$THIS_DIR/_refire_fixture.py"
|
||||
if [ ! -f "$FIXTURE_PY" ]; then
|
||||
echo "::error::fixture server $FIXTURE_PY missing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FIX_LOG="$FIXTURE_DIR/fixture.log"
|
||||
FIX_STATE_DIR="$FIXTURE_DIR/state"
|
||||
mkdir -p "$FIX_STATE_DIR"
|
||||
|
||||
# Find an unused port.
|
||||
FIX_PORT=$(python3 -c 'import socket;s=socket.socket();s.bind(("127.0.0.1",0));print(s.getsockname()[1]);s.close()')
|
||||
|
||||
FIXTURE_STATE_DIR="$FIX_STATE_DIR" python3 "$FIXTURE_PY" "$FIX_PORT" \
|
||||
>"$FIX_LOG" 2>&1 &
|
||||
FIX_PID=$!
|
||||
|
||||
# Wait for fixture readiness.
|
||||
for _ in $(seq 1 50); do
|
||||
if curl -fsS "http://127.0.0.1:${FIX_PORT}/_ping" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
if ! curl -fsS "http://127.0.0.1:${FIX_PORT}/_ping" >/dev/null 2>&1; then
|
||||
echo "::error::fixture server failed to start. Log:"
|
||||
cat "$FIX_LOG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Helper: set fixture state for a scenario, then run the script.
|
||||
# tier_result is one of: pass | fail_no_label | fail_no_approvals.
|
||||
# The refire script's tier-check invocation is mocked because the real
|
||||
# sop-tier-check.sh uses bash 4+ associative arrays — incompatible with
|
||||
# the macOS bash 3.2 dev shell. Linux Gitea runners use bash 4/5 so
|
||||
# production runs the real script. The mock exercises the success +
|
||||
# failure branches of refire's status-POST glue.
|
||||
run_scenario() {
|
||||
local scenario="$1"
|
||||
local tier_result="${2:-pass}"
|
||||
echo "$scenario" >"$FIX_STATE_DIR/scenario"
|
||||
: >"$FIX_STATE_DIR/posted_statuses.jsonl" # clear status log
|
||||
|
||||
local out
|
||||
set +e
|
||||
out=$(
|
||||
PATH="$FIXTURE_DIR/bin:$PATH" \
|
||||
GITEA_TOKEN="fixture-token" \
|
||||
GITEA_HOST="fixture.local" \
|
||||
REPO="molecule-ai/molecule-core" \
|
||||
PR_NUMBER="999" \
|
||||
COMMENT_AUTHOR="test-runner" \
|
||||
SOP_REFIRE_DISABLE_RATE_LIMIT="1" \
|
||||
SOP_REFIRE_TIER_CHECK_SCRIPT="$THIS_DIR/_mock_tier_check.sh" \
|
||||
MOCK_TIER_RESULT="$tier_result" \
|
||||
FIXTURE_PORT="$FIX_PORT" \
|
||||
bash "$SCRIPT" 2>&1
|
||||
)
|
||||
local rc=$?
|
||||
set -e
|
||||
echo "$out" >"$FIX_STATE_DIR/last_run.log"
|
||||
echo "$rc" >"$FIX_STATE_DIR/last_rc"
|
||||
}
|
||||
|
||||
# Install a curl shim that rewrites https://fixture.local → http://127.0.0.1:$PORT
|
||||
# Use bash prefix-strip (${var#prefix}) — it sidesteps the `/` delimiter
|
||||
# confusion of ${var/pattern/replacement}.
|
||||
mkdir -p "$FIXTURE_DIR/bin"
|
||||
cat >"$FIXTURE_DIR/bin/curl" <<SHIM
|
||||
#!/usr/bin/env bash
|
||||
# Test shim: rewrite https://fixture.local/* -> http://127.0.0.1:${FIX_PORT}/*
|
||||
# The fixture doesn't authenticate; -H Authorization passes through harmlessly.
|
||||
new_args=()
|
||||
for a in "\$@"; do
|
||||
if [[ "\$a" == https://fixture.local/* ]]; then
|
||||
rest="\${a#https://fixture.local}"
|
||||
a="http://127.0.0.1:${FIX_PORT}\${rest}"
|
||||
fi
|
||||
new_args+=("\$a")
|
||||
done
|
||||
exec /usr/bin/curl "\${new_args[@]}"
|
||||
SHIM
|
||||
chmod +x "$FIXTURE_DIR/bin/curl"
|
||||
|
||||
# T1: tier:low + 1 APPROVED + author is in engineers team → success
|
||||
run_scenario "T1_success" "pass"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T1 exit code 0 (success)" "0" "$RC"
|
||||
assert_contains "T1 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
assert_contains "T1 POST context is sop-tier-check / tier-check" \
|
||||
'"context": "sop-tier-check / tier-check (pull_request)"' "$POSTED"
|
||||
assert_contains "T1 description names commenter" "test-runner" "$POSTED"
|
||||
|
||||
# T2: missing tier label → tier-check fails → failure status POSTed
|
||||
run_scenario "T2_no_tier_label" "fail_no_label"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
# tier-check.sh exits 1; refire script forwards that exit, so RC != 0
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T2 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T2 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T2_rc"
|
||||
fi
|
||||
assert_contains "T2 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T3: tier:low present but ZERO approving reviews → failure
|
||||
run_scenario "T3_no_approvals" "fail_no_approvals"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T3 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T3 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T3_rc"
|
||||
fi
|
||||
assert_contains "T3 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T4: closed PR — refire is a no-op (no POST, exit 0)
|
||||
run_scenario "T4_closed" "pass"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T4 closed PR exits 0" "0" "$RC"
|
||||
assert_eq "T4 closed PR posts no status" "" "$POSTED"
|
||||
|
||||
# T5: rate-limit — disable the env override and let scenario set a
|
||||
# recent statuses entry. Re-enable rate-limit for this scenario by NOT
|
||||
# passing SOP_REFIRE_DISABLE_RATE_LIMIT.
|
||||
echo "T5_rate_limited" >"$FIX_STATE_DIR/scenario"
|
||||
: >"$FIX_STATE_DIR/posted_statuses.jsonl"
|
||||
set +e
|
||||
T5_OUT=$(
|
||||
PATH="$FIXTURE_DIR/bin:$PATH" \
|
||||
GITEA_TOKEN="fixture-token" \
|
||||
GITEA_HOST="fixture.local" \
|
||||
REPO="molecule-ai/molecule-core" \
|
||||
PR_NUMBER="999" \
|
||||
COMMENT_AUTHOR="test-runner" \
|
||||
FIXTURE_PORT="$FIX_PORT" \
|
||||
bash "$SCRIPT" 2>&1
|
||||
)
|
||||
T5_RC=$?
|
||||
set -e
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T5 rate-limited exits 0" "0" "$T5_RC"
|
||||
assert_contains "T5 rate-limited log says skipped" "rate-limited" "$T5_OUT"
|
||||
assert_eq "T5 rate-limited posts no status" "" "$POSTED"
|
||||
|
||||
echo
|
||||
echo "------"
|
||||
echo "PASS=$PASS FAIL=$FAIL"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo "Failed:$FAILED_TESTS"
|
||||
fi
|
||||
[ "$FAIL" -eq 0 ]
|
||||
@@ -56,7 +56,7 @@ on:
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# 3. Avoid the :30 heavy slot (staging-smoke /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
|
||||
@@ -124,7 +124,10 @@ jobs:
|
||||
env:
|
||||
CANVAS_E2E_STAGING: '1'
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# 2026-05-11: secret canonicalised from MOLECULE_STAGING_ADMIN_TOKEN
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@@ -145,7 +148,7 @@ jobs:
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
|
||||
echo "::error::Missing CP_STAGING_ADMIN_API_TOKEN"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -207,7 +210,7 @@ jobs:
|
||||
- name: Teardown safety net
|
||||
if: always() && needs.detect-changes.outputs.canvas == 'true'
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
STATE_FILE=".playwright-staging-state.json"
|
||||
|
||||
@@ -89,7 +89,10 @@ jobs:
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# 2026-05-11: secret canonicalised from MOLECULE_STAGING_ADMIN_TOKEN
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
|
||||
@@ -104,7 +107,7 @@ jobs:
|
||||
# missing — silent skip would mask infra rot. Manual dispatch
|
||||
# gets the same hard-fail; an operator running this on a fork
|
||||
# without secrets configured needs to know up-front.
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
@@ -129,7 +132,7 @@ jobs:
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
|
||||
@@ -86,7 +86,10 @@ jobs:
|
||||
# Single admin-bearer secret drives provision + tenant-token
|
||||
# retrieval + teardown. Configure in
|
||||
# Settings → Secrets and variables → Actions → Repository secrets.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# 2026-05-11: secret canonicalised from MOLECULE_STAGING_ADMIN_TOKEN
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -95,7 +98,7 @@ jobs:
|
||||
# ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
|
||||
# MINIMAX_API_KEY at boot — separate billing account so an
|
||||
# OpenAI quota collapse no longer wedges the gate. Mirrors the
|
||||
# canary-staging.yml + continuous-synth-e2e.yml migrations.
|
||||
# staging-smoke.yml + continuous-synth-e2e.yml migrations.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
@@ -122,7 +125,7 @@ jobs:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
@@ -189,7 +192,7 @@ jobs:
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
# Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
|
||||
# nuke them. Catches the case where the script died before
|
||||
|
||||
@@ -11,11 +11,11 @@ name: E2E Staging Sanity (leak-detection self-check)
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Periodic assertion that the teardown safety nets in e2e-staging-saas
|
||||
# and canary-staging actually work. Runs the E2E harness with
|
||||
# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
|
||||
# the org is provisioned. The workspace-provision step then fails, the
|
||||
# script exits non-zero, and the EXIT trap + workflow always()-step
|
||||
# must still tear down cleanly.
|
||||
# and staging-smoke (formerly canary-staging) actually work. Runs the
|
||||
# E2E harness with E2E_INTENTIONAL_FAILURE=1, which poisons the tenant
|
||||
# admin token after the org is provisioned. The workspace-provision
|
||||
# step then fails, the script exits non-zero, and the EXIT trap +
|
||||
# workflow always()-step must still tear down cleanly.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -42,8 +42,11 @@ jobs:
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_MODE: canary
|
||||
# 2026-05-11: secret canonicalised from MOLECULE_STAGING_ADMIN_TOKEN
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
E2E_MODE: smoke
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
E2E_INTENTIONAL_FAILURE: "1"
|
||||
@@ -54,7 +57,7 @@ jobs:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -118,7 +121,7 @@ jobs:
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
@@ -127,8 +130,14 @@ jobs:
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
today = __import__('datetime').date.today().strftime('%Y%m%d')
|
||||
# Match both the new e2e-smoke- prefix (post-2026-05-11 rename)
|
||||
# and the legacy e2e-canary- prefix for one rollout cycle so
|
||||
# any in-flight org provisioned under the old prefix on an
|
||||
# older runner checkout still gets cleaned up. Remove the
|
||||
# canary fallback after one week of no-old-prefix observations.
|
||||
prefixes = (f'e2e-smoke-{today}-sanity-', f'e2e-canary-{today}-sanity-')
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# gate-check-v3 — automated PR gate detector
|
||||
#
|
||||
# Runs on every open PR (push/synchronize) and hourly via cron.
|
||||
# Posts a structured [gate-check-v3] STATUS: comment on the PR.
|
||||
#
|
||||
# Inputs:
|
||||
# PR_NUMBER — set via ${{ github.event.pull_request.number }} from the trigger
|
||||
# POST_COMMENT — "true" to post/update comment on PR
|
||||
#
|
||||
# Gating logic (MVP signals 1,2,3,6):
|
||||
# 1. Author-aware agent-tag comment scan
|
||||
# 2. REQUEST_CHANGES reviews state machine
|
||||
# 3. Staleness detection (SOP-12: review.commit_id != PR.head_sha + >1 working day)
|
||||
# 6. CI required-checks awareness
|
||||
#
|
||||
# Exit code: 0=CLEAR, 1=BLOCKED, 2=ERROR
|
||||
|
||||
name: gate-check-v3
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, synchronize, reopened]
|
||||
schedule:
|
||||
# Hourly: refresh all open PRs
|
||||
- cron: '8 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: 'PR number to check (omit for all open PRs)'
|
||||
required: false
|
||||
type: string
|
||||
post_comment:
|
||||
description: 'Post comment on PR'
|
||||
required: false
|
||||
type: string
|
||||
default: 'true'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
gate-check:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true # Never block on our own detector failing
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha || github.ref_name }}
|
||||
|
||||
- name: Run gate-check-v3 (single PR mode)
|
||||
if: github.event_name == 'pull_request_target' || github.event.inputs.pr_number != ''
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
|
||||
POST_COMMENT: ${{ github.event.inputs.post_comment || 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 tools/gate-check-v3/gate_check.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "$PR_NUMBER" \
|
||||
$([ "$POST_COMMENT" = "true" ] && echo "--post-comment")
|
||||
echo "verdict=$?" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run gate-check-v3 (all open PRs — cron mode)
|
||||
if: github.event_name == 'schedule'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch all open PRs and run gate-check on each
|
||||
pr_numbers=$(python3 -c "
|
||||
import urllib.request, json, os
|
||||
token = os.environ['GITEA_TOKEN']
|
||||
req = urllib.request.Request(
|
||||
'https://git.moleculesai.app/api/v1/repos/${{ github.repository }}/pulls?state=open&limit=100',
|
||||
headers={'Authorization': f'token {token}', 'Accept': 'application/json'}
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
prs = json.loads(r.read())
|
||||
for pr in prs:
|
||||
print(pr['number'])
|
||||
")
|
||||
for pr in $pr_numbers; do
|
||||
echo "Checking PR #$pr..."
|
||||
python3 tools/gate-check-v3/gate_check.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "$pr" \
|
||||
--post-comment \
|
||||
|| true
|
||||
done
|
||||
@@ -68,7 +68,35 @@ jobs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Fetch base branch tip for diff
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# With the default fetch-depth: 1, actions/checkout only fetches the
|
||||
# PR head commit. The base commit is NOT in the local history, so
|
||||
# `git diff "$BASE" "$GITHUB_SHA"` fails. Fetch the base branch at
|
||||
# depth 1 — the base commit is the immediate parent of the PR head
|
||||
# on the base branch, so depth=1 is sufficient.
|
||||
#
|
||||
# Network: Gitea Actions runner (5.78.80.188) cannot reach the git
|
||||
# remote over HTTPS (confirmed: git fetch times out at ~15s). The runner
|
||||
# is on the same host as Gitea, but the container network namespace
|
||||
# cannot reach the Gitea HTTPS endpoint.
|
||||
#
|
||||
# Fallback: if the base commit does not exist locally, skip the diff
|
||||
# and set run=true (always run harness). This is safe: PRs where the
|
||||
# base is unavailable still run the harness (correct), PRs where the
|
||||
# base IS available get the correct path-based diff.
|
||||
#
|
||||
# Timeout: 20s. If the fetch completes, great. If it times out, the
|
||||
# step exits non-zero and we fall through to run=true.
|
||||
if timeout 20 git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1; then
|
||||
echo "::notice::base branch fetched successfully"
|
||||
else
|
||||
echo "::warning::git fetch origin ${{ github.event.pull_request.base.ref }} --depth=1 timed out"
|
||||
echo "::warning::Skipping diff — detect-changes will run the harness unconditionally."
|
||||
fi
|
||||
- id: decide
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# workflow_dispatch: always run (manual trigger)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
@@ -95,7 +123,13 @@ jobs:
|
||||
fi
|
||||
|
||||
# GitHub Actions and Gitea Actions both expose github.sha for HEAD.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
|
||||
# git diff exits 1 when BASE is not in local history (e.g. shallow
|
||||
# checkout where the base commit was never fetched). Capture and
|
||||
# swallow that exit code — the empty diff means "run everything".
|
||||
# The runner network cannot reach the git remote (confirmed: git fetch
|
||||
# times out at ~15s), so a failed fetch is expected and we always fall
|
||||
# through to the unconditional run=true below.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null) || true
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
|
||||
|
||||
@@ -11,7 +11,7 @@ name: publish-canvas-image
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Open question for review**: this workflow pushes the canvas
|
||||
# image to `ghcr.io`. GHCR was retired during the 2026-05-06
|
||||
# Gitea migration in favor of ECR (per canary-verify.yml header
|
||||
# Gitea migration in favor of ECR (per staging-verify.yml header
|
||||
# notes). The image may not be consumable post-migration. Two
|
||||
# options for follow-up: (a) retarget to
|
||||
# `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas`,
|
||||
|
||||
@@ -43,10 +43,18 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Fetch full tag list so the bump logic can sanity-check against
|
||||
# what's already in this repo (catches collision with prior
|
||||
# manual tag pushes).
|
||||
fetch-depth: 0
|
||||
# Shallow clone — depth 1 is enough for the workspace-diff check.
|
||||
# Tags needed for the collision check below are fetched explicitly
|
||||
# in the next step, bypassing the runner-network timeout that
|
||||
# full-history fetch triggers on Gitea Actions runners
|
||||
# (runbooks/gitea-operational-quirks.md §runner-network-isolation).
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Fetch tags for collision check
|
||||
# fetch-depth: 1 gets only the most recent commit's refs, not the
|
||||
# tag that points at it. Do a targeted tag fetch so git tag --list
|
||||
# below can detect collision with prior manual pushes.
|
||||
run: git fetch origin --tags --depth=1
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
|
||||
@@ -32,7 +32,7 @@ name: redeploy-tenants-on-main
|
||||
#
|
||||
# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
|
||||
# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
|
||||
# Gitea suspension migration. The canary-verify.yml promote step now
|
||||
# Gitea suspension migration. The staging-verify.yml promote step now
|
||||
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
|
||||
#
|
||||
# Runtime ordering:
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
# `staging-<sha>` to roll back to a known-good build.
|
||||
# 2. Default → `staging-<short_head_sha>`. The just-published
|
||||
# digest. Bypasses the `:latest` retag path that's currently
|
||||
# dead (canary-verify soft-skips without canary fleet, so
|
||||
# dead (staging-verify soft-skips without canary fleet, so
|
||||
# the only thing retagging `:latest` today is the manual
|
||||
# promote-latest.yml — last run 2026-04-28). Auto-trigger
|
||||
# from workflow_run uses workflow_run.head_sha; manual
|
||||
@@ -359,7 +359,7 @@ jobs:
|
||||
|
||||
# Belt-and-suspenders sanity floor: same logic as the staging
|
||||
# variant — see that file's comment for the full rationale.
|
||||
# Floor only applies when fleet >= 4; below that, canary-verify
|
||||
# Floor only applies when fleet >= 4; below that, staging-verify
|
||||
# is the actual gate.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
|
||||
@@ -21,7 +21,7 @@ name: redeploy-tenants-on-staging
|
||||
#
|
||||
# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
|
||||
# the :staging-latest tag. Sister workflow exists for prod (rolls
|
||||
# :latest after canary-verify). Both share the same shape — just
|
||||
# :latest after staging-verify). Both share the same shape — just
|
||||
# different CP_URL + target_tag + admin token secret.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image now builds
|
||||
@@ -336,7 +336,7 @@ jobs:
|
||||
# crashes on startup), not a teardown race. Hard-fail.
|
||||
#
|
||||
# Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
|
||||
# canary-verify step is the actual gate for "all tenants down"
|
||||
# staging-verify step is the actual gate for "all tenants down"
|
||||
# detection (it runs against the canary first and aborts the
|
||||
# rollout if the canary fails to come up). Without the >=4 gate,
|
||||
# a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# sop-tier-refire — issue_comment-triggered refire of sop-tier-check.
|
||||
#
|
||||
# Closes internal#292. Gitea 1.22.6 doesn't refire workflows on the
|
||||
# `pull_request_review` event (go-gitea/gitea#33700); the `sop-tier-check`
|
||||
# workflow's review-event subscription is silently dead. The result:
|
||||
# PRs that get their approving review AFTER the tier-check ran on open/
|
||||
# synchronize keep their failing status check forever, and the only way
|
||||
# to merge is the admin force-merge path (audited via `audit-force-merge`
|
||||
# but the audit trail keeps growing; see `feedback_never_admin_merge_bypass`).
|
||||
#
|
||||
# Workaround pattern from `feedback_pull_request_review_no_refire`:
|
||||
# `issue_comment` events DO fire reliably on 1.22.6. When a repo
|
||||
# MEMBER/OWNER/COLLABORATOR comments `/refire-tier-check` on a PR, this
|
||||
# workflow re-runs the sop-tier-check logic and POSTs the resulting
|
||||
# status to the PR head SHA directly. No empty commit, no git history
|
||||
# bloat, no cascade re-fire of every other workflow on the PR.
|
||||
#
|
||||
# SECURITY MODEL:
|
||||
#
|
||||
# 1. `pull_request` exists on the issue (issue_comment fires on issues
|
||||
# AND PRs; we only want PRs).
|
||||
# 2. `comment.author_association` must be MEMBER/OWNER/COLLABORATOR.
|
||||
# Per the internal#292 core-security review (review#1066 ask): anyone
|
||||
# can comment, but only repo collaborators+ can flip the status.
|
||||
# Without this gate, a drive-by commenter on a public-issue-tracker
|
||||
# surface could trigger a status flip.
|
||||
# 3. Comment body must contain `/refire-tier-check` — a slash-command-
|
||||
# shaped trigger (not just any comment word). Prevents accidental
|
||||
# triggering from prose like "we should refire tests" in a review.
|
||||
# 4. This workflow does NOT check out PR HEAD code. Like sop-tier-check,
|
||||
# it only HTTP-calls the Gitea API. Trust boundary preserved.
|
||||
#
|
||||
# Note: `issue_comment` fires from the BASE branch's workflow file. There
|
||||
# is no `pull_request_target` equivalent to set; the trigger inherently
|
||||
# loads the workflow from the default branch.
|
||||
#
|
||||
# Rate-limit: a 1s pre-sleep + a "skip if status posted in last 30s"
|
||||
# guard prevents comment-spam from thrashing the status. See the script.
|
||||
|
||||
name: sop-tier-check refire (issue_comment)
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
refire:
|
||||
# Three gates, all required:
|
||||
# - comment is on a PR (not a plain issue)
|
||||
# - commenter is MEMBER, OWNER, or COLLABORATOR
|
||||
# - comment body contains the slash-command trigger
|
||||
if: |
|
||||
github.event.issue.pull_request != null &&
|
||||
contains(fromJson('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) &&
|
||||
contains(github.event.comment.body, '/refire-tier-check')
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Load the script from the default branch (main), matching the
|
||||
# sop-tier-check.yml security model.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
- name: Re-evaluate sop-tier-check and POST status
|
||||
env:
|
||||
# Same org-level secret sop-tier-check.yml + audit-force-merge.yml use.
|
||||
# Fallback to GITHUB_TOKEN with a clear error if missing.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
|
||||
# Set to '1' for diagnostic per-API-call output. Off by default.
|
||||
SOP_DEBUG: '0'
|
||||
run: bash .gitea/scripts/sop-tier-refire.sh
|
||||
@@ -1,6 +1,8 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
name: Staging SaaS smoke (every 30 min)
|
||||
|
||||
# Ported from .github/workflows/canary-staging.yml on 2026-05-11 per RFC
|
||||
# Renamed from canary-staging.yml on 2026-05-11 per Hongming directive
|
||||
# ("canary naming changed to staging for all"). Originally ported from
|
||||
# .github/workflows/canary-staging.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
@@ -21,21 +23,21 @@ name: Canary — staging SaaS smoke (every 30 min)
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# Lean mode: E2E_MODE=smoke skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
# a few minutes under load — that's fine for a smoke check.
|
||||
- cron: '*/30 * * * *'
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
# e2e-staging-saas since we don't mind queueing smoke runs behind one
|
||||
# full run, but two smoke runs SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
group: staging-smoke
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
@@ -47,32 +49,47 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
smoke:
|
||||
name: Staging SaaS smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# NOTE: Phase 3 (RFC #219 §1) `continue-on-error: true` removed
|
||||
# 2026-05-11. The "surface broken workflows without blocking"
|
||||
# rationale was correctly applied to advisory/lint workflows but
|
||||
# wrong for this smoke — it is the 30-min canary cadence for the
|
||||
# entire staging SaaS stack, and silent failure here masks the
|
||||
# exact regressions the smoke exists to surface (AMI rot, CF cert
|
||||
# drift, WorkOS session breakage, secret rotations). Same class of
|
||||
# failure as PR#461 (`sweep-stale-e2e-orgs`) where Phase-3 silent
|
||||
# failure leaked EC2. The four other `e2e-staging-*` workflows
|
||||
# KEEP `continue-on-error: true` per RFC #219 §1 — they are
|
||||
# advisory and matrix-style; this one is the canary. A follow-up
|
||||
# `notify-failure` step below also surfaces breakage to ops even
|
||||
# if branch-protection wiring is adjusted to keep this off the
|
||||
# required-checks list.
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping the
|
||||
# smoke tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# 2026-05-11: secret canonicalised from MOLECULE_STAGING_ADMIN_TOKEN
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax is the smoke's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# the smoke red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# smoke. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
@@ -86,16 +103,16 @@ jobs:
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_MODE: smoke
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# Pin the smoke to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
E2E_RUN_ID: "smoke-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# the smoke script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
@@ -109,7 +126,7 @@ jobs:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
@@ -119,7 +136,7 @@ jobs:
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# SECRETS_JSON branch and the smoke fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
@@ -155,8 +172,8 @@ jobs:
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
- name: Smoke run
|
||||
id: smoke
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
@@ -184,6 +201,9 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
# Title kept stable across the canary-staging.yml → staging-smoke.yml
|
||||
# rename (2026-05-11) so any open alert issue from the old name
|
||||
# still title-matches and auto-closes on the next green run.
|
||||
TITLE="Canary failing: staging SaaS smoke"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
@@ -194,18 +214,18 @@ jobs:
|
||||
if [ -n "$EXISTING" ]; then
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Canary still failing. " + $run)}')" >/dev/null
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Smoke still failing. " + $run)}')" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
BODY=$(jq -nc --arg t "$TITLE" --arg now "$NOW" --arg run "$RUN_URL" \
|
||||
'{title: $t, body: ("Canary run failed at " + $now + ".\n\nRun: " + $run + "\n\nThis issue auto-closes on the next green canary run. Consecutive failures add a comment here rather than a new issue.")}')
|
||||
'{title: $t, body: ("Smoke run failed at " + $now + ".\n\nRun: " + $run + "\n\nThis issue auto-closes on the next green smoke run. Consecutive failures add a comment here rather than a new issue.")}')
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$BODY" >/dev/null
|
||||
echo "Opened canary failure issue (first red)"
|
||||
echo "Opened smoke failure issue (first red)"
|
||||
fi
|
||||
|
||||
- name: Auto-close canary issue on success (Gitea API)
|
||||
- name: Auto-close smoke issue on success (Gitea API)
|
||||
if: success()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -215,6 +235,8 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
# Title kept stable across the canary-staging.yml → staging-smoke.yml
|
||||
# rename so open alert issues from the old name still match.
|
||||
TITLE="Canary failing: staging SaaS smoke"
|
||||
|
||||
NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
@@ -225,37 +247,36 @@ jobs:
|
||||
for N in $NUMS; do
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}/comments" \
|
||||
-d "$(jq -nc --arg now "$NOW" '{body: ("Canary recovered at " + $now + ". Closing.")}')" >/dev/null
|
||||
-d "$(jq -nc --arg now "$NOW" '{body: ("Smoke recovered at " + $now + ". Closing.")}')" >/dev/null
|
||||
curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
|
||||
echo "Closed recovered canary issue #${N}"
|
||||
echo "Closed recovered smoke issue #${N}"
|
||||
done
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
# in smoke mode:
|
||||
# SLUG="e2e-smoke-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier (pre-2026-05-11 canary→staging rename) the prefix was
|
||||
# `e2e-canary-`; both prefixes are matched here for one
|
||||
# release cycle so cleanup still catches any in-flight org
|
||||
# provisioned under the old prefix on an older runner that
|
||||
# hasn't picked up the renamed script. Remove the canary
|
||||
# fallback after one week of no-old-prefix observations.
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# Scope to slugs from THIS smoke run when GITHUB_RUN_ID is
|
||||
# available; the smoke workflow sets E2E_RUN_ID='smoke-\${run_id}'
|
||||
# so the slug suffix is '-smoke-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
@@ -265,9 +286,11 @@ jobs:
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
prefixes = tuple(f'e2e-smoke-{d}-smoke-{run_id}' for d in dates) \
|
||||
+ tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
prefixes = tuple(f'e2e-smoke-{d}-' for d in dates) \
|
||||
+ tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
@@ -280,8 +303,8 @@ jobs:
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# on cleanup failure — a single-smoke cleanup miss shouldn't
|
||||
# fail-flag the smoke itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
@@ -290,21 +313,34 @@ jobs:
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
curl -sS -o /tmp/smoke-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/smoke-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
code=$(cat /tmp/smoke-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
echo "::warning::smoke teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/smoke-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
echo "::warning::smoke teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
|
||||
- name: Notify on smoke failure
|
||||
# Fail-loud companion to dropping `continue-on-error: true`.
|
||||
# The Open-issue-on-failure step above handles the human-facing
|
||||
# alert; this step emits a clearly-tagged ::error:: line that
|
||||
# log-tail consumers (Loki SOPRefireRule, orchestrator triage
|
||||
# loop) can grep on. Mirrors PR#461's sweep-stale-e2e-orgs
|
||||
# pattern. Runs AFTER the teardown safety net (which is
|
||||
# if: always()) so failures don't suppress cleanup.
|
||||
if: failure()
|
||||
run: |
|
||||
echo "::error::staging-smoke FAILED — staging SaaS canary is red. See prior step logs + the auto-filed alert issue. Common causes: (a) CP_STAGING_ADMIN_API_TOKEN secret missing/rotated, (b) staging-api.moleculesai.app 5xx, (c) MiniMax/Anthropic LLM key dead, (d) AMI/CF/WorkOS drift. The 30-min cron will retry, but a chronic red here indicates the staging SaaS stack is broken end-to-end."
|
||||
exit 1
|
||||
@@ -1,6 +1,8 @@
|
||||
name: canary-verify
|
||||
name: Staging verify
|
||||
|
||||
# Ported from .github/workflows/canary-verify.yml on 2026-05-11 per RFC
|
||||
# Renamed from canary-verify.yml on 2026-05-11 per Hongming directive
|
||||
# ("canary naming changed to staging for all"). Originally ported from
|
||||
# .github/workflows/canary-verify.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
@@ -23,13 +25,22 @@ name: canary-verify
|
||||
# digest. On red, :latest stays on the prior known-good digest and
|
||||
# prod is untouched.
|
||||
#
|
||||
# Terminology note (2026-05-11): The deployment STRATEGY here is still
|
||||
# called "canary release" (a small subset of tenants gets the new image
|
||||
# first, the rest follow on green). The "canary" word stays for the
|
||||
# pre-fan-out cohort concept (see docs/architecture/canary-release.md
|
||||
# and CANARY_SLUG in redeploy-tenants-on-*.yml). What changed is the
|
||||
# FILE NAME and the SECRETS feeding this workflow — both are renamed
|
||||
# to drop the redundant "canary-" prefix that conflated workflow
|
||||
# identity with deployment strategy.
|
||||
#
|
||||
# Registry note (2026-05-10): This workflow previously used GHCR
|
||||
# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
|
||||
# during the 2026-05-06 Gitea suspension migration when publish-
|
||||
# workspace-server-image.yml switched to the operator's ECR org
|
||||
# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
|
||||
# platform-tenant). The GHCR → ECR migration was never applied to
|
||||
# this file, so canary-verify was silently smoke-testing the stale
|
||||
# this file, so this workflow was silently smoke-testing the stale
|
||||
# GHCR image while the actual staging/prod tenants ran the ECR image.
|
||||
# Result: smoke tests could not catch a broken ECR build. Fix:
|
||||
# - Wait step: reads SHA from running canary /health (tenant-
|
||||
@@ -43,8 +54,9 @@ name: canary-verify
|
||||
# to ECR on staging and main merges.
|
||||
# - Canary tenants are configured to pull :staging-<sha> from ECR
|
||||
# (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
|
||||
# - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
|
||||
# CANARY_CP_SHARED_SECRET are populated.
|
||||
# - Repo secrets MOLECULE_STAGING_TENANT_URLS /
|
||||
# MOLECULE_STAGING_ADMIN_TOKENS / MOLECULE_STAGING_CP_SHARED_SECRET
|
||||
# are populated.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
@@ -65,7 +77,7 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
canary-smoke:
|
||||
staging-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
# workflow_dispatch trigger dropped in this Gitea port; only the
|
||||
# workflow_run path remains.
|
||||
@@ -97,15 +109,15 @@ jobs:
|
||||
# other registry — the canary is telling us what it's actually
|
||||
# running, which is the ground truth for smoke testing.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
MOLECULE_STAGING_TENANT_URLS: ${{ secrets.MOLECULE_STAGING_TENANT_URLS }}
|
||||
EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
|
||||
run: |
|
||||
if [ -z "$CANARY_TENANT_URLS" ]; then
|
||||
if [ -z "$MOLECULE_STAGING_TENANT_URLS" ]; then
|
||||
echo "No canary URLs configured — falling back to 60s wait"
|
||||
sleep 60
|
||||
exit 0
|
||||
fi
|
||||
IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
|
||||
IFS=',' read -ra URLS <<< "$MOLECULE_STAGING_TENANT_URLS"
|
||||
MAX_WAIT=420 # 7 minutes
|
||||
INTERVAL=30
|
||||
ELAPSED=0
|
||||
@@ -129,7 +141,7 @@ jobs:
|
||||
done
|
||||
echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
|
||||
|
||||
- name: Run canary smoke suite
|
||||
- name: Run staging smoke suite
|
||||
id: smoke
|
||||
# Graceful-skip when no canary fleet is configured (Phase 2 not yet
|
||||
# stood up — see molecule-controlplane/docs/canary-tenants.md).
|
||||
@@ -138,29 +150,29 @@ jobs:
|
||||
# promote-latest.yml is the release gate while canary is absent.
|
||||
# Once the fleet is real: delete the early-exit branch.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
|
||||
CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
|
||||
MOLECULE_STAGING_TENANT_URLS: ${{ secrets.MOLECULE_STAGING_TENANT_URLS }}
|
||||
MOLECULE_STAGING_ADMIN_TOKENS: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKENS }}
|
||||
MOLECULE_STAGING_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_STAGING_CP_SHARED_SECRET: ${{ secrets.MOLECULE_STAGING_CP_SHARED_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CANARY_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
|
||||
if [ -z "${MOLECULE_STAGING_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${MOLECULE_STAGING_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${MOLECULE_STAGING_CP_SHARED_SECRET:-}" ]; then
|
||||
{
|
||||
echo "## ⚠️ canary-verify skipped"
|
||||
echo "## ⚠️ staging-verify skipped"
|
||||
echo
|
||||
echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
|
||||
echo "One or more canary secrets are unset (\`MOLECULE_STAGING_TENANT_URLS\`, \`MOLECULE_STAGING_ADMIN_TOKENS\`, \`MOLECULE_STAGING_CP_SHARED_SECRET\`)."
|
||||
echo "Phase 2 canary fleet has not been stood up yet —"
|
||||
echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
|
||||
echo
|
||||
echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ran=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::canary-verify: skipped — no canary fleet configured"
|
||||
echo "::notice::staging-verify: skipped — no canary fleet configured"
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/canary-smoke.sh
|
||||
bash scripts/staging-smoke.sh
|
||||
echo "ran=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Summary on failure
|
||||
@@ -173,7 +185,7 @@ jobs:
|
||||
echo ":latest stays pinned to the prior good digest — prod is untouched."
|
||||
echo
|
||||
echo "Fix forward and merge again, or investigate the specific failed"
|
||||
echo "assertions in the canary-smoke step log above."
|
||||
echo "assertions in the staging-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
promote-to-latest:
|
||||
@@ -188,13 +200,13 @@ jobs:
|
||||
# silently promoting a stale GHCR image while actual prod tenants
|
||||
# pulled from ECR. Canary smoke tests were GHCR-targeted and could
|
||||
# not catch a broken ECR build.
|
||||
needs: canary-smoke
|
||||
if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
|
||||
needs: staging-smoke
|
||||
if: ${{ needs.staging-smoke.result == 'success' && needs.staging-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
SHA: ${{ needs.canary-smoke.outputs.sha }}
|
||||
SHA: ${{ needs.staging-smoke.outputs.sha }}
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
# CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
|
||||
# Stored at the repo level so all workflows pick it up automatically.
|
||||
@@ -264,9 +276,9 @@ jobs:
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Canary verified — :latest promoted via CP redeploy-fleet"
|
||||
echo "## Staging verified — :latest promoted via CP redeploy-fleet"
|
||||
echo ""
|
||||
echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
|
||||
echo "- **Target tag:** \`staging-${{ needs.staging-smoke.outputs.sha }}\`"
|
||||
echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
|
||||
echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "- **Batch size:** ${BATCH_SIZE:-3}"
|
||||
@@ -63,12 +63,21 @@ jobs:
|
||||
sweep:
|
||||
name: Sweep e2e orgs
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# NOTE: Phase 3 (RFC #219 §1) `continue-on-error: true` removed
|
||||
# 2026-05-11. The "surface broken workflows without blocking"
|
||||
# rationale was correctly applied to advisory/lint workflows but
|
||||
# wrong for this janitor — silent failure here masks real-money
|
||||
# tenant leaks. Hongming observed 15 leaked EC2 in molecule-canary
|
||||
# (004947743811) us-east-2 at 11:05Z 2026-05-11 because the sweep
|
||||
# had been exiting 2 every tick and the failure was swallowed.
|
||||
# See `feedback_strict_root_only_after_class_a` — critical janitors
|
||||
# must fail loud. A follow-up `notify-failure` step below also
|
||||
# surfaces breakage to ops even if branch-protection wiring is
|
||||
# adjusted to keep this off the required-checks list.
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
# Refuse to delete more than this many orgs in one tick. If the
|
||||
@@ -81,7 +90,7 @@ jobs:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
@@ -99,7 +108,8 @@ jobs:
|
||||
|
||||
# Filter:
|
||||
# 1. slug starts with one of the ephemeral test prefixes:
|
||||
# - 'e2e-' — covers e2e-canary-, e2e-canvas-*, etc.
|
||||
# - 'e2e-' — covers e2e-smoke- (formerly e2e-canary-),
|
||||
# e2e-canvas-*, etc.
|
||||
# - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
|
||||
# missing this prefix left two such tenants
|
||||
# orphaned 8h on staging (2026-05-03), then
|
||||
@@ -241,3 +251,17 @@ jobs:
|
||||
if: env.DRY_RUN == 'true'
|
||||
run: |
|
||||
echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."
|
||||
|
||||
- name: Notify on sweep failure
|
||||
# Fail-loud companion to dropping `continue-on-error: true`.
|
||||
# If any prior step failed (missing token, CP 5xx, safety-cap
|
||||
# tripped, etc.) emit a clearly-tagged ::error:: line so the
|
||||
# Gitea runs UI + any log-tail consumer (Loki SOPRefireRule)
|
||||
# flags this. Without this step, an early `exit 2` shows as a
|
||||
# red run but the message can scroll past in busy log windows;
|
||||
# the explicit tag here is greppable from the orchestrator
|
||||
# triage loop.
|
||||
if: failure()
|
||||
run: |
|
||||
echo "::error::sweep-stale-e2e-orgs FAILED — staging tenants are LEAKING. See prior step logs. Common causes: (a) CP_STAGING_ADMIN_API_TOKEN secret missing/rotated, (b) staging-api.moleculesai.app 5xx, (c) safety-cap tripped (CP admin API returning malformed orgs). Manual cleanup of leaked EC2 + DNS may be required while this is broken."
|
||||
exit 1
|
||||
|
||||
@@ -96,6 +96,7 @@ export function WorkspaceNode({ id, data }: NodeProps<Node<WorkspaceNodeData>>)
|
||||
<div
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
data-testid="workspace-node"
|
||||
aria-label={
|
||||
isMisconfigured && configurationError
|
||||
? `${data.name} workspace — agent not configured: ${configurationError}`
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
* Covers: renders nothing when no approvals, polls /approvals/pending,
|
||||
* shows approval cards, approve/deny decisions, toast notifications.
|
||||
*
|
||||
* Note: does NOT mock @/lib/api — uses vi.spyOn on the real module.
|
||||
* vi.restoreAllMocks() is omitted from afterEach so queued mock values
|
||||
* (set up via mockResolvedValueOnce in beforeEach) are preserved for the
|
||||
* component's useEffect to consume.
|
||||
* All blocks use vi.useFakeTimers() consistently in beforeEach/afterEach to
|
||||
* avoid polluting the fake-timer state for subsequent test files. The
|
||||
* vi.spyOn mocks are reset per-spy via mockReset() in afterEach so each
|
||||
* test gets a clean mock state without touching the module-level api mock.
|
||||
*/
|
||||
import React from "react";
|
||||
import { render, screen, fireEvent, cleanup, act } from "@testing-library/react";
|
||||
@@ -41,9 +41,10 @@ const pendingApproval = (id = "a1", workspaceId = "ws-1"): {
|
||||
created_at: "2026-05-10T10:00:00Z",
|
||||
});
|
||||
|
||||
// Shared spy reference so individual tests can call mockGet.mockRestore()
|
||||
// without needing to pass it through beforeEach → it scope chain.
|
||||
// Shared spy references so individual tests can reset or reject the POST mock
|
||||
// without needing to call spyOn again (which would create a duplicate spy).
|
||||
let mockGet: ReturnType<typeof vi.spyOn>;
|
||||
let mockPost: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -55,7 +56,7 @@ describe("ApprovalBanner — empty state", () => {
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
it("renders nothing when there are no pending approvals", async () => {
|
||||
@@ -83,7 +84,8 @@ describe("ApprovalBanner — renders approval cards", () => {
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
mockGet?.mockReset();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
it("renders an alert card for each pending approval", async () => {
|
||||
@@ -91,7 +93,6 @@ describe("ApprovalBanner — renders approval cards", () => {
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
const alerts = screen.getAllByRole("alert");
|
||||
expect(alerts).toHaveLength(2);
|
||||
mockGet.mockRestore();
|
||||
});
|
||||
|
||||
it("displays the workspace name and action text", async () => {
|
||||
@@ -139,13 +140,15 @@ describe("ApprovalBanner — renders approval cards", () => {
|
||||
describe("ApprovalBanner — decisions", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.spyOn(api, "get").mockResolvedValueOnce([pendingApproval("a1")]);
|
||||
vi.spyOn(api, "post").mockResolvedValue({});
|
||||
mockGet = vi.spyOn(api, "get").mockResolvedValueOnce([pendingApproval("a1")]);
|
||||
mockPost = vi.spyOn(api, "post").mockResolvedValue({});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
mockGet?.mockReset();
|
||||
mockPost?.mockReset();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
it("calls POST /workspaces/:id/approvals/:id/decide on Approve click", async () => {
|
||||
@@ -196,7 +199,7 @@ describe("ApprovalBanner — decisions", () => {
|
||||
});
|
||||
|
||||
it("shows an error toast when POST fails", async () => {
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error("Network error"));
|
||||
mockPost.mockReset().mockRejectedValue(new Error("Network error"));
|
||||
render(<ApprovalBanner />);
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
fireEvent.click(screen.getAllByRole("button", { name: /approve/i })[0]);
|
||||
@@ -208,8 +211,9 @@ describe("ApprovalBanner — decisions", () => {
|
||||
});
|
||||
|
||||
it("keeps the card visible when the POST fails", async () => {
|
||||
// Use mockRejectedValueOnce on the same spy as beforeEach (don't call spyOn again)
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error("Network error"));
|
||||
// Reset the post mock before rejecting so the beforeEach's resolved value
|
||||
// is gone and we get a clean rejection instead of a resolved→rejected queue.
|
||||
mockPost.mockReset().mockRejectedValue(new Error("Network error"));
|
||||
render(<ApprovalBanner />);
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
fireEvent.click(screen.getAllByRole("button", { name: /approve/i })[0]);
|
||||
@@ -226,7 +230,7 @@ describe("ApprovalBanner — handles empty list from server", () => {
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
it("shows nothing when the API returns an empty array on first poll", async () => {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
* window.location.search in the jsdom environment.
|
||||
*/
|
||||
import React from "react";
|
||||
import { render, screen, fireEvent, cleanup, act } from "@testing-library/react";
|
||||
import { render, screen, fireEvent, cleanup, act, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { PurchaseSuccessModal } from "../PurchaseSuccessModal";
|
||||
|
||||
@@ -30,9 +30,13 @@ function clearSearch() {
|
||||
setSearch("");
|
||||
}
|
||||
|
||||
// Helper: wait for dialog to appear (real timers)
|
||||
// Helper: wait for the dialog to appear after React useEffect batch.
|
||||
// Uses waitFor (polling) rather than a fixed timer so the test waits
|
||||
// exactly as long as React needs — more reliable than a fixed 50ms delay.
|
||||
async function waitForDialog() {
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 50)); });
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByRole("dialog")).toBeTruthy();
|
||||
}, { timeout: 2000 });
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
@@ -104,6 +108,7 @@ describe("PurchaseSuccessModal — render conditions", () => {
|
||||
describe("PurchaseSuccessModal — dismiss", () => {
|
||||
beforeEach(() => {
|
||||
setSearch("?purchase_success=1&item=TestItem");
|
||||
vi.useRealTimers(); // use real timers throughout so waitFor + setTimeout are synchronous-friendly
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -116,52 +121,45 @@ describe("PurchaseSuccessModal — dismiss", () => {
|
||||
it("closes the dialog when the close button is clicked", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Close" }));
|
||||
await waitForDialog();
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 100)); });
|
||||
expect(screen.queryByRole("dialog")).toBeNull();
|
||||
});
|
||||
|
||||
it("closes the dialog when the backdrop is clicked", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
const backdrop = document.body.querySelector('[aria-hidden="true"]');
|
||||
if (backdrop) fireEvent.click(backdrop);
|
||||
await waitForDialog();
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 100)); });
|
||||
expect(screen.queryByRole("dialog")).toBeNull();
|
||||
});
|
||||
|
||||
it("closes on Escape key", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
fireEvent.keyDown(window, { key: "Escape" });
|
||||
await waitForDialog();
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 100)); });
|
||||
expect(screen.queryByRole("dialog")).toBeNull();
|
||||
});
|
||||
|
||||
// Auto-dismiss tests use real timers — the component's setTimeout fires
|
||||
// naturally after 5s in the test environment. vi.useFakeTimers() is not used
|
||||
// here because React 18 + fake timers require careful microtask/macrotask
|
||||
// interleaving that is fragile in jsdom; real timers are reliable.
|
||||
// naturally after 5s in the test environment.
|
||||
it("auto-dismisses after 5 seconds", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
// The component's AUTO_DISMISS_MS = 5000ms. In jsdom, setTimeout fires
|
||||
// reliably. Wait long enough for 2 dismiss cycles to ensure the first fires.
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 11000)); });
|
||||
// AUTO_DISMISS_MS = 5000ms. Wait 6s to ensure dismiss has fired + React updated.
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 6000)); });
|
||||
expect(screen.queryByRole("dialog")).toBeNull();
|
||||
}, 15000); // extended timeout for real-timer wait
|
||||
}, 10000);
|
||||
|
||||
it("does not auto-dismiss before 5 seconds", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
const dialog = screen.getByRole("dialog");
|
||||
// Wait 4s — just under the 5s auto-dismiss threshold
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 4000)); });
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
expect(screen.queryByRole("dialog")).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -210,27 +208,28 @@ describe("PurchaseSuccessModal — accessibility", () => {
|
||||
|
||||
it("has aria-modal=true on the dialog", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
const dialog = screen.getByRole("dialog");
|
||||
expect(dialog.getAttribute("aria-modal")).toBe("true");
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("dialog").getAttribute("aria-modal")).toBe("true");
|
||||
});
|
||||
});
|
||||
|
||||
it("has aria-labelledby pointing to the title", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await waitForDialog();
|
||||
const dialog = screen.getByRole("dialog");
|
||||
const labelledby = dialog.getAttribute("aria-labelledby");
|
||||
expect(labelledby).toBeTruthy();
|
||||
expect(document.getElementById(labelledby!)).toBeTruthy();
|
||||
expect(document.getElementById(labelledby!)?.textContent).toMatch(/purchase successful/i);
|
||||
await waitFor(() => {
|
||||
const dialog = screen.getByRole("dialog");
|
||||
const labelledby = dialog.getAttribute("aria-labelledby");
|
||||
expect(labelledby).toBeTruthy();
|
||||
expect(document.getElementById(labelledby!)).toBeTruthy();
|
||||
expect(document.getElementById(labelledby!)?.textContent).toMatch(/purchase successful/i);
|
||||
});
|
||||
});
|
||||
|
||||
// Focus test: verify close button exists after dialog renders.
|
||||
// We test presence (not focus) since rAF focus is tricky in jsdom.
|
||||
it("moves focus to the close button on open", async () => {
|
||||
render(<PurchaseSuccessModal />);
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 100)); });
|
||||
// Use getByRole which is more reliable than querySelector
|
||||
expect(screen.getByRole("button", { name: "Close" })).toBeTruthy();
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("button", { name: "Close" })).toBeTruthy();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
// @vitest-environment jsdom
|
||||
/**
|
||||
* Toolbar tests.
|
||||
*
|
||||
* Covers:
|
||||
* - Renders with 0 workspaces
|
||||
* - Shows online/offline/failed/provisioning status pills when nodes exist
|
||||
* - WebSocket status pill: connected → "Live"
|
||||
* - WebSocket status pill: connecting → "Reconnecting"
|
||||
* - WebSocket status pill: disconnected → "Offline"
|
||||
* - Stop All button visible when activeTasks > 0
|
||||
* - Restart Pending button visible when needsRestart nodes exist
|
||||
* - Help button opens the help popover
|
||||
* - Help popover closes on Escape or pointer-outside
|
||||
* - KeyboardShortcutsDialog opens via ? shortcut (when not in input)
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { render, screen, fireEvent, cleanup } from "@testing-library/react";
|
||||
import React from "react";
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
// Reset store state between tests so mutations don't leak.
|
||||
beforeEach(() => {
|
||||
defaultStore.nodes = [];
|
||||
defaultStore.wsStatus = "connected";
|
||||
defaultStore.showA2AEdges = false;
|
||||
defaultStore.selectedNodeId = null;
|
||||
mockSetShowA2AEdges.mockClear();
|
||||
mockSetPanelTab.mockClear();
|
||||
mockSetSearchOpen.mockClear();
|
||||
mockUpdateNodeData.mockClear();
|
||||
});
|
||||
|
||||
// ── Mock targets ───────────────────────────────────────────────────────────────
|
||||
|
||||
vi.mock("@/components/Toaster", () => ({
|
||||
showToast: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/ConfirmDialog", () => ({
|
||||
ConfirmDialog: () => null,
|
||||
}));
|
||||
|
||||
vi.mock("@/components/settings/SettingsButton", () => ({
|
||||
SettingsButton: () => null,
|
||||
}));
|
||||
|
||||
vi.mock("@/components/settings/SettingsPanel", () => ({
|
||||
settingsGearRef: { current: null },
|
||||
}));
|
||||
|
||||
vi.mock("@/components/ThemeToggle", () => ({
|
||||
ThemeToggle: () => null,
|
||||
}));
|
||||
|
||||
vi.mock("@/components/KeyboardShortcutsDialog", () => ({
|
||||
KeyboardShortcutsDialog: ({ open }: { open: boolean; onClose: () => void }) =>
|
||||
open ? <div role="dialog" data-testid="shortcuts-dialog">Shortcuts</div> : null,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/design-tokens", () => ({
|
||||
statusDotClass: (status: string) => {
|
||||
const map: Record<string, string> = {
|
||||
online: "bg-emerald-400",
|
||||
offline: "bg-zinc-500",
|
||||
paused: "bg-indigo-400",
|
||||
degraded: "bg-amber-400",
|
||||
failed: "bg-red-400",
|
||||
provisioning: "bg-sky-400",
|
||||
};
|
||||
return map[status] ?? "bg-zinc-500";
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/api", () => ({
|
||||
api: {
|
||||
post: vi.fn(() => Promise.resolve()),
|
||||
},
|
||||
}));
|
||||
|
||||
// ── Store mocks ────────────────────────────────────────────────────────────────
|
||||
|
||||
const mockSetShowA2AEdges = vi.fn();
|
||||
const mockSetPanelTab = vi.fn();
|
||||
const mockSetSearchOpen = vi.fn();
|
||||
const mockUpdateNodeData = vi.fn();
|
||||
|
||||
const makeNodes = (
|
||||
statuses: Array<"online" | "offline" | "failed" | "provisioning">,
|
||||
activeTasks: number[] = [],
|
||||
needsRestart: boolean[] = [],
|
||||
parentIds: (string | null)[] = []
|
||||
) => {
|
||||
return statuses.map((status, i) => ({
|
||||
id: `ws-${i}`,
|
||||
data: {
|
||||
name: `Workspace ${i}`,
|
||||
role: "agent",
|
||||
tier: 1,
|
||||
status,
|
||||
parentId: parentIds[i] ?? null,
|
||||
activeTasks: activeTasks[i] ?? 0,
|
||||
needsRestart: needsRestart[i] ?? false,
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
// Nodes must be React Flow nodes (id + data), but Toolbar only reads data fields.
|
||||
// makeNodes returns { id, data: { activeTasks, needsRestart, ... } }.
|
||||
const toStoreNodes = (nodes: ReturnType<typeof makeNodes>) =>
|
||||
nodes.map((n) => ({ id: n.id, data: n.data }));
|
||||
|
||||
const defaultStore = {
|
||||
nodes: [] as ReturnType<typeof makeNodes>,
|
||||
wsStatus: "connected" as "connected" | "connecting" | "disconnected",
|
||||
showA2AEdges: false,
|
||||
selectedNodeId: null as string | null,
|
||||
sidePanelWidth: 480,
|
||||
setShowA2AEdges: mockSetShowA2AEdges,
|
||||
setPanelTab: mockSetPanelTab,
|
||||
setSearchOpen: mockSetSearchOpen,
|
||||
updateNodeData: mockUpdateNodeData,
|
||||
selectedNodeIds: new Set<string>(),
|
||||
clearSelection: vi.fn(),
|
||||
batchRestart: vi.fn(() => Promise.resolve()),
|
||||
batchPause: vi.fn(() => Promise.resolve()),
|
||||
batchDelete: vi.fn(() => Promise.resolve()),
|
||||
};
|
||||
|
||||
vi.mock("@/store/canvas", () => ({
|
||||
useCanvasStore: vi.fn((selector: (s: typeof defaultStore) => unknown) =>
|
||||
selector(defaultStore)
|
||||
),
|
||||
}));
|
||||
|
||||
// ── Component under test ───────────────────────────────────────────────────────
|
||||
import { Toolbar } from "../Toolbar";
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("Toolbar — workspace count display", () => {
|
||||
it("shows '0 workspaces' when the canvas is empty", () => {
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByText(/0 workspaces?/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows 'N workspaces' when nodes exist", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online", "online"]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByText(/2 workspaces?/)).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — status pills", () => {
|
||||
it("shows the online pill when nodes are online", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online"]));
|
||||
render(<Toolbar />);
|
||||
// StatusPill uses aria-label
|
||||
expect(screen.getByLabelText(/1 online/i)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows the offline pill only when offline nodes exist", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["offline"]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByLabelText(/1 offline/i)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows the failed pill when failed nodes exist", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["failed"]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByLabelText(/1 failed/i)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows the provisioning pill when provisioning nodes exist", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["provisioning"]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByLabelText(/1 starting/i)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses offline pill when no offline nodes", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online", "online"]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.queryByLabelText(/offline/i)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — WebSocket status pill", () => {
|
||||
it('shows "Live" when connected', () => {
|
||||
defaultStore.wsStatus = "connected";
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByText("Live")).toBeTruthy();
|
||||
});
|
||||
|
||||
it('shows "Reconnecting" when connecting', () => {
|
||||
defaultStore.wsStatus = "connecting";
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByText("Reconnecting")).toBeTruthy();
|
||||
});
|
||||
|
||||
it('shows "Offline" when disconnected', () => {
|
||||
defaultStore.wsStatus = "disconnected";
|
||||
render(<Toolbar />);
|
||||
expect(screen.getByText("Offline")).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — Stop All", () => {
|
||||
it("is hidden when no active tasks", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online"], [0]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.queryByRole("button", { name: /Stop All/i })).toBeNull();
|
||||
});
|
||||
|
||||
it("is visible when active tasks > 0", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online", "online"], [2, 2]));
|
||||
render(<Toolbar />);
|
||||
// aria-label: "Stop all running tasks (2)"
|
||||
expect(screen.getByRole("button", { name: /stop all running tasks/i })).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — Restart Pending", () => {
|
||||
it("is hidden when no nodes need restart", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online"], [], [false]));
|
||||
render(<Toolbar />);
|
||||
expect(screen.queryByRole("button", { name: /Restart Pending/i })).toBeNull();
|
||||
});
|
||||
|
||||
it("is visible when nodes need restart", () => {
|
||||
defaultStore.nodes = toStoreNodes(makeNodes(["online"], [], [true]));
|
||||
render(<Toolbar />);
|
||||
// aria-label: "Restart 1 workspaces pending config or secret changes"
|
||||
expect(screen.getByRole("button", { name: /restart 1 workspace/i })).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — Help popover", () => {
|
||||
it("opens when help button is clicked", () => {
|
||||
render(<Toolbar />);
|
||||
const helpBtn = screen.getByRole("button", { name: /open shortcuts and tips/i });
|
||||
fireEvent.click(helpBtn);
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("closes when close button is clicked", () => {
|
||||
render(<Toolbar />);
|
||||
const helpBtn = screen.getByRole("button", { name: /open shortcuts and tips/i });
|
||||
fireEvent.click(helpBtn);
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
const closeBtn = screen.getByRole("button", { name: /close help dialog/i });
|
||||
fireEvent.click(closeBtn);
|
||||
expect(screen.queryByRole("dialog")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — A2A edges toggle", () => {
|
||||
it("calls setShowA2AEdges on click", () => {
|
||||
defaultStore.showA2AEdges = false;
|
||||
render(<Toolbar />);
|
||||
const toggle = screen.getByRole("button", { name: /show a2a edges/i });
|
||||
fireEvent.click(toggle);
|
||||
expect(mockSetShowA2AEdges).toHaveBeenCalledWith(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Toolbar — ? shortcut opens shortcuts dialog", () => {
|
||||
it("opens KeyboardShortcutsDialog when ? is pressed outside an input", () => {
|
||||
render(<Toolbar />);
|
||||
expect(screen.queryByTestId("shortcuts-dialog")).toBeNull();
|
||||
fireEvent.keyDown(window, { key: "?" });
|
||||
expect(screen.getByTestId("shortcuts-dialog")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("does not fire ? shortcut when focus is in an input", () => {
|
||||
render(
|
||||
<div>
|
||||
<input data-testid="test-input" type="text" />
|
||||
<Toolbar />
|
||||
</div>
|
||||
);
|
||||
const input = screen.getByTestId("test-input");
|
||||
fireEvent.focus(input);
|
||||
// Fire on the input element so e.target.tagName === "INPUT" is true
|
||||
fireEvent.keyDown(input, { key: "?" });
|
||||
expect(screen.queryByTestId("shortcuts-dialog")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,592 @@
|
||||
// @vitest-environment jsdom
|
||||
/**
|
||||
* WorkspaceNode tests.
|
||||
*
|
||||
* Covers:
|
||||
* - Renders name, status dot, tier badge, role, skills
|
||||
* - Status gradient bar colored by STATUS_CONFIG
|
||||
* - Online/offline/failed/degraded/provisioning states
|
||||
* - Misconfigured state (online + not_configured)
|
||||
* - Click → select, Shift+click → batch select
|
||||
* - Keyboard Enter/Space → select/deselect
|
||||
* - Context menu on right-click
|
||||
* - Double-click collapsed parent → expands
|
||||
* - Double-click expanded parent → zoom to team
|
||||
* - Needs restart button visible when needsRestart=true
|
||||
* - Current task banner when activeTasks > 0
|
||||
* - Descendant count badge when node has children
|
||||
* - Drag-target highlight class when dragOverNodeId matches
|
||||
* - Batch-selected highlight class
|
||||
* - OrgCancelButton renders on deploying root
|
||||
* - Degraded error preview
|
||||
* - Configuration error preview for misconfigured nodes
|
||||
* - TeamMemberChip: name, status, skills, extract button, recursive
|
||||
* - Handle anchors: top = extract, bottom = nest (keyboard accessible)
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { render, screen, fireEvent, cleanup } from "@testing-library/react";
|
||||
import React from "react";
|
||||
|
||||
// ── Mock @xyflow/react ────────────────────────────────────────────────────────
|
||||
vi.mock("@xyflow/react", () => {
|
||||
const Handle = ({
|
||||
type,
|
||||
position,
|
||||
"aria-label": ariaLabel,
|
||||
onKeyDown,
|
||||
...rest
|
||||
}: {
|
||||
type: string;
|
||||
position: string;
|
||||
"aria-label"?: string;
|
||||
onKeyDown?: (e: React.KeyboardEvent) => void;
|
||||
[key: string]: unknown;
|
||||
}) => (
|
||||
<div
|
||||
role="button"
|
||||
aria-label={ariaLabel}
|
||||
data-handle-type={type}
|
||||
data-handle-position={position}
|
||||
tabIndex={0}
|
||||
onKeyDown={onKeyDown}
|
||||
{...rest}
|
||||
>
|
||||
handle
|
||||
</div>
|
||||
);
|
||||
return {
|
||||
__esModule: true,
|
||||
default: ({ children }: { children?: React.ReactNode }) => (
|
||||
<div data-testid="react-flow-root">{children}</div>
|
||||
),
|
||||
NodeResizer: () => null,
|
||||
Handle,
|
||||
Position: { Top: "top", Bottom: "bottom", Left: "left", Right: "right" },
|
||||
useReactFlow: () => ({ fitView: vi.fn(), setViewport: vi.fn() }),
|
||||
applyNodeChanges: vi.fn((_: unknown, n: unknown) => n),
|
||||
ReactFlowProvider: ({ children }: { children?: React.ReactNode }) => <>{children}</>,
|
||||
};
|
||||
});
|
||||
|
||||
// ── Mock dependencies ─────────────────────────────────────────────────────────
|
||||
const mockGetConfigurationStatus = vi.fn(() => "configured");
|
||||
const mockGetConfigurationError = vi.fn(() => null);
|
||||
|
||||
vi.mock("@/store/canvas-topology", () => ({
|
||||
getConfigurationStatus: (...args: unknown[]) => mockGetConfigurationStatus(...args),
|
||||
getConfigurationError: (...args: unknown[]) => mockGetConfigurationError(...args),
|
||||
}));
|
||||
|
||||
// Expose for per-test override
|
||||
const useConfigStatus = mockGetConfigurationStatus;
|
||||
const useConfigError = mockGetConfigurationError;
|
||||
|
||||
vi.mock("@/components/Toaster", () => ({
|
||||
showToast: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/Tooltip", () => ({
|
||||
Tooltip: ({ text, children }: { text: string; children: React.ReactNode }) => (
|
||||
<div title={text} data-testid="tooltip-wrapper">{children}</div>
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/canvas/useOrgDeployState", () => ({
|
||||
useOrgDeployState: vi.fn(() => ({
|
||||
isActivelyProvisioning: false,
|
||||
isDeployingRoot: false,
|
||||
isLockedChild: false,
|
||||
descendantProvisioningCount: 0,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/design-tokens", () => ({
|
||||
STATUS_CONFIG: {
|
||||
online: { dot: "bg-emerald-400", glow: "shadow-emerald-400/50", bar: "to-emerald-500/30", label: "ONLINE" },
|
||||
offline: { dot: "bg-zinc-500", glow: "", bar: "to-zinc-600/30", label: "OFFLINE" },
|
||||
failed: { dot: "bg-red-400", glow: "", bar: "to-red-600/30", label: "FAILED" },
|
||||
degraded: { dot: "bg-amber-400", glow: "", bar: "to-amber-600/30", label: "DEGRADED" },
|
||||
provisioning: { dot: "bg-sky-400", glow: "", bar: "to-sky-600/30", label: "STARTING" },
|
||||
not_configured: { dot: "bg-amber-400", glow: "", bar: "to-amber-600/30", label: "NOT CONFIGURED" },
|
||||
},
|
||||
TIER_CONFIG: {
|
||||
1: { label: "T1", color: "text-zinc-400 bg-zinc-800" },
|
||||
2: { label: "T2", color: "text-blue-400 bg-blue-900/50" },
|
||||
3: { label: "T3", color: "text-purple-400 bg-purple-900/50" },
|
||||
4: { label: "T4", color: "text-amber-400 bg-amber-900/50" },
|
||||
},
|
||||
}));
|
||||
|
||||
// ── Store mock ────────────────────────────────────────────────────────────────
|
||||
// Uses a global object to share mock state between the factory (which runs
|
||||
// when the module is imported) and the test body (beforeEach/afterEach).
|
||||
declare global {
|
||||
// eslint-disable-next-line no-var
|
||||
var __workspaceNodeMocks: {
|
||||
selectNode: ReturnType<typeof vi.fn>;
|
||||
openContextMenu: ReturnType<typeof vi.fn>;
|
||||
toggleNodeSelection: ReturnType<typeof vi.fn>;
|
||||
nestNode: ReturnType<typeof vi.fn>;
|
||||
restartWorkspace: ReturnType<typeof vi.fn>;
|
||||
store: {
|
||||
nodes: Array<{ id: string; data: Record<string, unknown> }>;
|
||||
selectedNodeId: string | null;
|
||||
dragOverNodeId: string | null;
|
||||
selectedNodeIds: Set<string>;
|
||||
};
|
||||
} | undefined;
|
||||
}
|
||||
|
||||
vi.mock("@/store/canvas", () => {
|
||||
const mockSelectNode = vi.fn();
|
||||
const mockOpenContextMenu = vi.fn();
|
||||
const mockToggleNodeSelection = vi.fn();
|
||||
const mockNestNode = vi.fn();
|
||||
const mockRestartWorkspace = vi.fn(() => Promise.resolve());
|
||||
|
||||
const store = {
|
||||
nodes: [] as Array<{ id: string; data: Record<string, unknown> }>,
|
||||
selectedNodeId: null as string | null,
|
||||
dragOverNodeId: null as string | null,
|
||||
selectedNodeIds: new Set<string>(),
|
||||
selectNode: mockSelectNode,
|
||||
openContextMenu: mockOpenContextMenu,
|
||||
toggleNodeSelection: mockToggleNodeSelection,
|
||||
nestNode: mockNestNode,
|
||||
restartWorkspace: mockRestartWorkspace,
|
||||
};
|
||||
|
||||
const mockFn = (selector: (s: typeof store) => unknown) => selector(store);
|
||||
Object.defineProperty(mockFn, "getState", { value: () => store });
|
||||
|
||||
// Expose via global for test body access
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(globalThis as any).__workspaceNodeMocks = {
|
||||
selectNode: mockSelectNode,
|
||||
openContextMenu: mockOpenContextMenu,
|
||||
toggleNodeSelection: mockToggleNodeSelection,
|
||||
nestNode: mockNestNode,
|
||||
restartWorkspace: mockRestartWorkspace,
|
||||
store,
|
||||
};
|
||||
|
||||
return { useCanvasStore: mockFn, __esModule: true };
|
||||
});
|
||||
|
||||
// ── Component ────────────────────────────────────────────────────────────────
|
||||
import { WorkspaceNode } from "../WorkspaceNode";
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// Main node card uses data-testid to distinguish from handle anchors (also role=button)
|
||||
const getNode = () => screen.getByTestId("workspace-node");
|
||||
|
||||
// Typed access to the shared mock state (set by the vi.mock factory)
|
||||
const mocks = () => globalThis.__workspaceNodeMocks!;
|
||||
const store = () => mocks().store;
|
||||
|
||||
const makeNode = (overrides: Record<string, unknown> = {}) => ({
|
||||
id: "ws-1",
|
||||
data: {
|
||||
name: "Test Workspace",
|
||||
role: "Test Agent",
|
||||
tier: 1,
|
||||
status: "online" as const,
|
||||
parentId: null,
|
||||
activeTasks: 0,
|
||||
needsRestart: false,
|
||||
currentTask: null as string | null,
|
||||
lastSampleError: null as string | null,
|
||||
collapsed: false,
|
||||
agentCard: null,
|
||||
runtime: null as string | null,
|
||||
...overrides,
|
||||
},
|
||||
});
|
||||
|
||||
const renderNode = (nodeOverrides: Record<string, unknown> = {}) => {
|
||||
const node = makeNode(nodeOverrides);
|
||||
// WorkspaceNode expects NodeProps — it receives { id, data } as props
|
||||
return render(<WorkspaceNode id={node.id as string} data={node.data as never} />);
|
||||
};
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
beforeEach(() => {
|
||||
const m = globalThis.__workspaceNodeMocks!;
|
||||
m.store.nodes = [];
|
||||
m.store.selectedNodeId = null;
|
||||
m.store.dragOverNodeId = null;
|
||||
m.store.selectedNodeIds = new Set();
|
||||
m.selectNode.mockClear();
|
||||
m.openContextMenu.mockClear();
|
||||
m.toggleNodeSelection.mockClear();
|
||||
m.nestNode.mockClear();
|
||||
m.restartWorkspace.mockClear();
|
||||
mockGetConfigurationStatus.mockClear().mockReturnValue("configured");
|
||||
mockGetConfigurationError.mockClear().mockReturnValue(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — basic rendering", () => {
|
||||
it("renders the workspace name", () => {
|
||||
renderNode({ name: "My Workspace" });
|
||||
expect(screen.getByText("My Workspace")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders the role text", () => {
|
||||
renderNode({ role: "Frontend Engineer" });
|
||||
expect(screen.getByText("Frontend Engineer")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders the tier badge", () => {
|
||||
renderNode({ tier: 2 });
|
||||
expect(screen.getByText("T2")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders status dot with online class", () => {
|
||||
renderNode({ status: "online" });
|
||||
const dot = getNode().querySelector(".bg-emerald-400");
|
||||
expect(dot).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders role text clamped to 2 lines", () => {
|
||||
renderNode({ role: "A very long role description that might overflow" });
|
||||
expect(screen.getByText(/A very long role description/i)).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — status states", () => {
|
||||
it("shows status label for failed node", () => {
|
||||
renderNode({ status: "failed" });
|
||||
expect(screen.getByText("FAILED")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows status label for degraded node", () => {
|
||||
renderNode({ status: "degraded" });
|
||||
expect(screen.getByText("DEGRADED")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows status label for provisioning node", () => {
|
||||
renderNode({ status: "provisioning" });
|
||||
expect(screen.getByText("STARTING")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses status label for online node", () => {
|
||||
renderNode({ status: "online" });
|
||||
expect(screen.queryByText("ONLINE")).toBeNull();
|
||||
});
|
||||
|
||||
it("shows degraded error preview when status is degraded and lastSampleError is set", () => {
|
||||
renderNode({ status: "degraded", lastSampleError: "Connection timeout" });
|
||||
expect(screen.getByText("Connection timeout")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses degraded error preview when no error", () => {
|
||||
renderNode({ status: "degraded", lastSampleError: null });
|
||||
expect(screen.queryByText(/timeout/i)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — misconfigured state", () => {
|
||||
it("shows 'NOT CONFIGURED' label when agent is online but not_configured", () => {
|
||||
vi.mocked(useConfigStatus).mockReturnValueOnce("not_configured");
|
||||
vi.mocked(useConfigError).mockReturnValueOnce("ANTHROPIC_API_KEY is missing");
|
||||
renderNode({ status: "online" });
|
||||
expect(screen.getByText("NOT CONFIGURED")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows configuration error preview when misconfigured", () => {
|
||||
vi.mocked(useConfigStatus).mockReturnValueOnce("not_configured");
|
||||
vi.mocked(useConfigError).mockReturnValueOnce("OPENAI_API_KEY missing");
|
||||
renderNode({ status: "online" });
|
||||
expect(screen.getByText("OPENAI_API_KEY missing")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("aria-label includes name and status by default", () => {
|
||||
// Mock set to default "configured" — no misconfigured label
|
||||
renderNode({ status: "online" });
|
||||
const btn = getNode();
|
||||
expect(btn.getAttribute("aria-label")).toMatch(/Test Workspace/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — click interactions", () => {
|
||||
it("calls selectNode(id) on click", () => {
|
||||
renderNode();
|
||||
fireEvent.click(getNode());
|
||||
expect(mocks().selectNode).toHaveBeenCalledWith("ws-1");
|
||||
});
|
||||
|
||||
it("calls selectNode(null) on click when already selected", () => {
|
||||
store().selectedNodeId = "ws-1";
|
||||
renderNode();
|
||||
fireEvent.click(getNode());
|
||||
expect(mocks().selectNode).toHaveBeenCalledWith(null);
|
||||
});
|
||||
|
||||
it("calls toggleNodeSelection on Shift+click", () => {
|
||||
renderNode();
|
||||
fireEvent.click(getNode(), { shiftKey: true });
|
||||
expect(mocks().toggleNodeSelection).toHaveBeenCalledWith("ws-1");
|
||||
});
|
||||
|
||||
it("opens context menu on right-click", () => {
|
||||
renderNode();
|
||||
fireEvent.contextMenu(getNode(), {
|
||||
clientX: 100,
|
||||
clientY: 200,
|
||||
});
|
||||
expect(mocks().openContextMenu).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ nodeId: "ws-1", x: 100, y: 200 })
|
||||
);
|
||||
});
|
||||
|
||||
it("stops propagation to prevent canvas background click from firing", () => {
|
||||
renderNode();
|
||||
const btn = getNode();
|
||||
// React synthetic events fire regardless of native bubbles. We just verify
|
||||
// selectNode was called — the stopPropagation() call inside the handler
|
||||
// prevents the event from reaching canvas background listeners.
|
||||
expect(mocks().selectNode).not.toHaveBeenCalled(); // no click yet
|
||||
fireEvent.click(btn, { bubbles: true });
|
||||
expect(mocks().selectNode).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — keyboard interactions", () => {
|
||||
it("selects node on Enter key", () => {
|
||||
renderNode();
|
||||
fireEvent.keyDown(getNode(), { key: "Enter" });
|
||||
expect(mocks().selectNode).toHaveBeenCalledWith("ws-1");
|
||||
});
|
||||
|
||||
it("deselects node on Enter key when already selected", () => {
|
||||
store().selectedNodeId = "ws-1";
|
||||
renderNode();
|
||||
fireEvent.keyDown(getNode(), { key: "Enter" });
|
||||
expect(mocks().selectNode).toHaveBeenCalledWith(null);
|
||||
});
|
||||
|
||||
it("toggles batch selection on Shift+Enter", () => {
|
||||
renderNode();
|
||||
fireEvent.keyDown(getNode(), { key: "Enter", shiftKey: true });
|
||||
expect(mocks().toggleNodeSelection).toHaveBeenCalledWith("ws-1");
|
||||
});
|
||||
|
||||
it("opens context menu on ContextMenu key", () => {
|
||||
renderNode();
|
||||
fireEvent.keyDown(getNode(), { key: "ContextMenu" });
|
||||
expect(mocks().openContextMenu).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ nodeId: "ws-1" })
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — double-click interactions", () => {
|
||||
it("does nothing on double-click when node has no children", () => {
|
||||
renderNode({ collapsed: false });
|
||||
fireEvent.doubleClick(getNode());
|
||||
// No exception thrown = fine. The actual zoom-to-team event is dispatched
|
||||
// on the window, which jsdom handles silently.
|
||||
expect(mocks().selectNode).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("sets collapsed=false on double-click of collapsed parent (no children in store)", () => {
|
||||
renderNode({ collapsed: true });
|
||||
fireEvent.doubleClick(getNode());
|
||||
// When hasChildren is false (no child nodes in store), the handler returns early.
|
||||
expect(mocks().selectNode).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — active tasks", () => {
|
||||
it("shows active tasks badge when activeTasks > 0", () => {
|
||||
renderNode({ activeTasks: 3 });
|
||||
expect(screen.getByText("3 tasks")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows singular 'task' when activeTasks is 1", () => {
|
||||
renderNode({ activeTasks: 1 });
|
||||
expect(screen.getByText("1 task")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses badge when no active tasks", () => {
|
||||
renderNode({ activeTasks: 0 });
|
||||
expect(screen.queryByText(/task/)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — current task banner", () => {
|
||||
it("shows current task banner when currentTask is set", () => {
|
||||
renderNode({ currentTask: "Writing unit tests" });
|
||||
expect(screen.getByText("Writing unit tests")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses current task banner when null", () => {
|
||||
renderNode({ currentTask: null });
|
||||
expect(screen.queryByText(/Writing unit tests/)).toBeNull();
|
||||
});
|
||||
|
||||
it("shows both currentTask and needsRestart — currentTask takes visual priority", () => {
|
||||
renderNode({ currentTask: "Active work", needsRestart: true });
|
||||
// Current task banner renders; needs restart button is conditionally hidden
|
||||
// behind `!data.currentTask` in the component
|
||||
expect(screen.getByText("Active work")).toBeTruthy();
|
||||
expect(screen.queryByRole("button", { name: /restart/i })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — needs restart", () => {
|
||||
it("shows restart button when needsRestart=true and no currentTask", () => {
|
||||
renderNode({ needsRestart: true, currentTask: null });
|
||||
expect(screen.getByRole("button", { name: /restart to apply changes/i })).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses restart button when currentTask is active", () => {
|
||||
renderNode({ needsRestart: true, currentTask: "Working" });
|
||||
expect(screen.queryByRole("button", { name: /restart/i })).toBeNull();
|
||||
});
|
||||
|
||||
it("suppresses restart button when needsRestart=false", () => {
|
||||
renderNode({ needsRestart: false });
|
||||
expect(screen.queryByRole("button", { name: /restart/i })).toBeNull();
|
||||
});
|
||||
|
||||
it("restart button calls restartWorkspace on click", () => {
|
||||
renderNode({ needsRestart: true, currentTask: null });
|
||||
fireEvent.click(screen.getByRole("button", { name: /restart to apply changes/i }));
|
||||
expect(mocks().restartWorkspace).toHaveBeenCalledWith("ws-1");
|
||||
});
|
||||
|
||||
it("restart button stops propagation", () => {
|
||||
renderNode({ needsRestart: true, currentTask: null });
|
||||
fireEvent.click(screen.getByRole("button", { name: /restart/i }));
|
||||
// If propagation wasn't stopped, selectNode would also be called
|
||||
expect(mocks().selectNode).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — descendant badge", () => {
|
||||
it("shows descendant count badge when node has children in store", () => {
|
||||
store().nodes = [
|
||||
makeNode({ id: "ws-1" }),
|
||||
{ id: "child-1", data: { ...makeNode({ id: "ws-1" }).data, parentId: "ws-1" } },
|
||||
];
|
||||
renderNode();
|
||||
expect(screen.getByText("1 sub")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses badge when node has no children", () => {
|
||||
store().nodes = [makeNode({ id: "ws-1" })];
|
||||
renderNode();
|
||||
expect(screen.queryByText(/sub/)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — skills pills", () => {
|
||||
it("renders up to 4 skill pills", () => {
|
||||
renderNode({
|
||||
agentCard: {
|
||||
skills: [
|
||||
{ name: "code-review" },
|
||||
{ name: "tdd" },
|
||||
{ name: "debugging" },
|
||||
{ name: "refactoring" },
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(screen.getByText("code-review")).toBeTruthy();
|
||||
expect(screen.getByText("refactoring")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows +N overflow when more than 4 skills", () => {
|
||||
renderNode({
|
||||
agentCard: {
|
||||
skills: [
|
||||
{ name: "s1" }, { name: "s2" }, { name: "s3" }, { name: "s4" }, { name: "s5" },
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(screen.getByText("+1")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses skills section when no skills", () => {
|
||||
renderNode({ agentCard: null });
|
||||
// No skill text rendered
|
||||
expect(screen.queryByText(/code-review/i)).toBeNull();
|
||||
});
|
||||
|
||||
it("handles agentCard with no skills array", () => {
|
||||
renderNode({ agentCard: { name: "Test Agent" } });
|
||||
expect(screen.queryByText(/code-review/i)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — runtime badge", () => {
|
||||
it("shows runtime badge when runtime is set", () => {
|
||||
renderNode({ runtime: "hermes" });
|
||||
expect(screen.getByText("hermes")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows REMOTE badge for external runtime", () => {
|
||||
renderNode({ runtime: "external" });
|
||||
expect(screen.getByText("★ REMOTE")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("suppresses runtime badge when runtime is null", () => {
|
||||
renderNode({ runtime: null });
|
||||
expect(screen.queryByText("hermes")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — selection aria", () => {
|
||||
it('has aria-pressed="false" when not selected', () => {
|
||||
store().selectedNodeId = null;
|
||||
renderNode();
|
||||
expect(getNode().getAttribute("aria-pressed")).toBe("false");
|
||||
});
|
||||
|
||||
it('has aria-pressed="true" when selected', () => {
|
||||
store().selectedNodeId = "ws-1";
|
||||
renderNode();
|
||||
expect(getNode().getAttribute("aria-pressed")).toBe("true");
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — aria-label", () => {
|
||||
it("includes name and status in aria-label", () => {
|
||||
renderNode({ name: "MyAgent", status: "online" });
|
||||
const label = getNode().getAttribute("aria-label");
|
||||
expect(label).toContain("MyAgent");
|
||||
expect(label).toContain("online");
|
||||
});
|
||||
});
|
||||
|
||||
describe("WorkspaceNode — handle anchors accessibility", () => {
|
||||
it("top handle has aria-label for extract", () => {
|
||||
renderNode({ parentId: "parent-1" });
|
||||
const handles = screen.getAllByRole("button");
|
||||
const topHandle = handles.find((h) => h.getAttribute("data-handle-type") === "target");
|
||||
expect(topHandle?.getAttribute("aria-label")).toMatch(/extract/i);
|
||||
});
|
||||
|
||||
it("bottom handle has aria-label for nest", () => {
|
||||
renderNode();
|
||||
const handles = screen.getAllByRole("button");
|
||||
const bottomHandle = handles.find((h) => h.getAttribute("data-handle-type") === "source");
|
||||
expect(bottomHandle?.getAttribute("aria-label")).toMatch(/nest/i);
|
||||
});
|
||||
|
||||
it("top handle extract is no-op when node has no parent", () => {
|
||||
renderNode({ parentId: null });
|
||||
const handles = screen.getAllByRole("button");
|
||||
const topHandle = handles.find((h) => h.getAttribute("data-handle-type") === "target");
|
||||
fireEvent.keyDown(topHandle!, { key: "Enter" });
|
||||
// Should be a no-op — no exception
|
||||
expect(mocks().nestNode).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -55,10 +55,10 @@ describe("statusDotClass", () => {
|
||||
|
||||
describe("TIER_CONFIG", () => {
|
||||
it("has entries for all four tier levels", () => {
|
||||
expect(TIER_CONFIG).toHaveProperty(1);
|
||||
expect(TIER_CONFIG).toHaveProperty(2);
|
||||
expect(TIER_CONFIG).toHaveProperty(3);
|
||||
expect(TIER_CONFIG).toHaveProperty(4);
|
||||
expect(TIER_CONFIG).toHaveProperty("1");
|
||||
expect(TIER_CONFIG).toHaveProperty("2");
|
||||
expect(TIER_CONFIG).toHaveProperty("3");
|
||||
expect(TIER_CONFIG).toHaveProperty("4");
|
||||
});
|
||||
|
||||
it("each tier has label, color, and border fields", () => {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
How a workspace-server code change reaches the prod tenant fleet — and how to stop it if something's wrong.
|
||||
|
||||
> **⚠️ State note (2026-04-22):** this doc describes the **intended design**. As of this write, the canary fleet described below is **not actually running** — no canary tenants are provisioned, `CANARY_TENANT_URLS` / `CANARY_ADMIN_TOKENS` / `CANARY_CP_SHARED_SECRET` are empty in repo secrets, and `canary-verify.yml` fails every run.
|
||||
> **⚠️ State note (2026-04-22, secret names refreshed 2026-05-11):** this doc describes the **intended design**. As of this write, the canary fleet described below is **not actually running** — no canary tenants are provisioned, `MOLECULE_STAGING_TENANT_URLS` / `MOLECULE_STAGING_ADMIN_TOKENS` / `MOLECULE_STAGING_CP_SHARED_SECRET` are empty in repo secrets, and `staging-verify.yml` (formerly `canary-verify.yml`) fails every run.
|
||||
>
|
||||
> Current merges gate on manual `promote-latest.yml` dispatches, not canary. See [molecule-controlplane/docs/canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/src/branch/main/docs/canary-tenants.md) for the Phase 1 code work that's already shipped + the Phase 2 plan for actually standing up the fleet + a "should we even do this now?" decision framework.
|
||||
>
|
||||
@@ -22,7 +22,7 @@ publish-workspace-server-image.yml ← pushes :staging-<sha> ONLY
|
||||
Canary tenants auto-update to :staging-<sha>
|
||||
│ (5-min auto-updater cycle on each canary EC2)
|
||||
▼
|
||||
canary-verify.yml waits 6 min, runs scripts/canary-smoke.sh
|
||||
staging-verify.yml waits 6 min, runs scripts/staging-smoke.sh
|
||||
│
|
||||
├─► GREEN → crane tag :staging-<sha> → :latest
|
||||
│ │
|
||||
@@ -42,7 +42,7 @@ Canary tenants are configured to pull `:staging-<sha>` (not `:latest`) via `TENA
|
||||
|
||||
## Smoke suite
|
||||
|
||||
`scripts/canary-smoke.sh` hits each canary tenant (URL + ADMIN_TOKEN pair) and asserts:
|
||||
`scripts/staging-smoke.sh` hits each canary tenant (URL + ADMIN_TOKEN pair) and asserts:
|
||||
|
||||
- `/admin/liveness` returns a subsystems map (tenant booted, AdminAuth reachable)
|
||||
- `/workspaces` returns a JSON array (wsAuth + DB healthy)
|
||||
@@ -59,8 +59,8 @@ Expand by editing the script — each `check "name" "expected" "$response"` call
|
||||
3. Re-trigger provision (or delete + recreate if the org was already provisioned into staging) — the fresh EC2 lands in the canary AWS account (see internal runbook for the specific ID)
|
||||
|
||||
Then set repo secrets:
|
||||
- `CANARY_TENANT_URLS` — append the new tenant's URL
|
||||
- `CANARY_ADMIN_TOKENS` — append its ADMIN_TOKEN in the same position
|
||||
- `MOLECULE_STAGING_TENANT_URLS` — append the new tenant's URL
|
||||
- `MOLECULE_STAGING_ADMIN_TOKENS` — append its ADMIN_TOKEN in the same position
|
||||
|
||||
## Rolling back `:latest`
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ pipeline.
|
||||
| `check-merge-group-trigger.yml` | The workflow's own header (lines 18-23) documents that it's vacuously satisfied on Gitea — Gitea has no merge queue, no `merge_group:` event type, no `gh-readonly-queue/...` refs. Nothing to lint. |
|
||||
| `codeql.yml` | The workflow's own header (lines 3-67) documents that `github/codeql-action/init@v4` hits api.github.com bundle endpoints not implemented by Gitea (observed: `::error::404 page not found` in Initialize CodeQL step). Per Hongming decision 2026-05-07 (task #156): CodeQL is ADVISORY/non-blocking until a Gitea-compatible SAST pipeline lands. Replacement options (Semgrep self-host, Sonatype, GitHub-mirror-for-SAST) tracked in #156. |
|
||||
| `pr-guards.yml` | The workflow's own header documents that Gitea has no `gh pr merge --auto` primitive — the guard is a structural no-op on Gitea. Branch protection on `main` does NOT reference any `pr-guards` check name; deletion is safe. |
|
||||
| `promote-latest.yml` | Uses `imjasonh/setup-crane` against `ghcr.io/molecule-ai/platform` — the GHCR registry was retired during the 2026-05-06 Gitea migration (per `canary-verify.yml` header notes, the canonical tenant image moved to ECR `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant`). The workflow can no longer find any image to retag. Follow-up issue suggested if an ECR-based retag promote is desired. |
|
||||
| `promote-latest.yml` | Uses `imjasonh/setup-crane` against `ghcr.io/molecule-ai/platform` — the GHCR registry was retired during the 2026-05-06 Gitea migration (per `staging-verify.yml` header notes — file was renamed from `canary-verify.yml` on 2026-05-11; the canonical tenant image moved to ECR `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant`). The workflow can no longer find any image to retag. Follow-up issue suggested if an ECR-based retag promote is desired. |
|
||||
|
||||
## Category C — ported to .gitea/
|
||||
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
# Gitea Actions operational quirks (molecule-core)
|
||||
|
||||
Documents persistent operational findings about Gitea Actions runner behaviour
|
||||
that differ from GitHub Actions and require workarounds in workflow YAML or
|
||||
runbooks.
|
||||
|
||||
> Last updated: 2026-05-11 (core-devops-agent)
|
||||
|
||||
---
|
||||
|
||||
## Gitea 1.22.6 runner network isolation
|
||||
|
||||
### Finding
|
||||
|
||||
The Gitea Actions runner (container on host `5.78.80.188`) cannot reach the
|
||||
git remote (`https://git.moleculesai.app`) over HTTPS from inside the runner
|
||||
container. Any `git fetch`, `git clone`, or `git push` command that contacts
|
||||
the remote times out at 12–15 s.
|
||||
|
||||
This is **not a Gitea Actions bug** — it is an operator-level network policy
|
||||
where the runner container's network namespace is restricted from reaching the
|
||||
Gitea host HTTPS endpoint. The runner can reach external hosts (GitHub,
|
||||
Docker Hub, PyPI) normally.
|
||||
|
||||
### Impact
|
||||
|
||||
Workflows that rely on `git fetch origin <ref>` or `actions/checkout` with
|
||||
`fetch-depth: 0` (full history) will hang or time out.
|
||||
|
||||
Specifically:
|
||||
- `actions/checkout@v*` with `fetch-depth: 0` hangs (fetching full repo
|
||||
history takes >30 s before hitting the timeout).
|
||||
- `git fetch origin main --depth=1` times out at ~15 s.
|
||||
- `git clone <url>` times out at ~15 s.
|
||||
|
||||
### Affected workflows
|
||||
|
||||
| Workflow | Issue | Workaround |
|
||||
|---|---|---|
|
||||
| `harness-replays.yml` detect-changes job | `git fetch origin main --depth=1` times out | Added `timeout 20` + graceful fallback to `run=true` (always run harness) per PR #441 |
|
||||
| `publish-workspace-server-image.yml` | In-image `git clone` of workspace templates | Pre-clone manifest deps before compose build (Task #173 pattern) |
|
||||
| Any workflow using `fetch-depth: 0` | Full history fetch times out | Use `fetch-depth: 1` + explicit `git fetch` for needed refs |
|
||||
|
||||
### How to diagnose
|
||||
|
||||
```bash
|
||||
# From inside the runner (add as a debug step):
|
||||
timeout 20 git fetch origin main --depth=1
|
||||
# If this times out: runner cannot reach git remote
|
||||
```
|
||||
|
||||
### Verification
|
||||
|
||||
Confirmed 2026-05-11 by running `timeout 20 git fetch origin main --depth=1`
|
||||
in the `detect-changes` job of `harness-replays.yml` — consistently times
|
||||
out at 15 s. Runner can reach `https://api.github.com` and `https://pypi.org`
|
||||
without issue.
|
||||
|
||||
### References
|
||||
|
||||
- PR #441: fix for `harness-replays.yml` detect-changes
|
||||
- Task #173: pre-clone manifest deps pattern for compose build
|
||||
- internal#102: tracking customer-private + marketplace third-party repos
|
||||
- `feedback_oss_first_repo_visibility_default`: 5 workspace-template repos
|
||||
flipped public to allow pre-clone without auth
|
||||
|
||||
---
|
||||
|
||||
## `continue-on-error` only works at step level, not job level
|
||||
|
||||
### Finding
|
||||
|
||||
Gitea Actions (1.22.6) does not honour `continue-on-error: true` at the **job**
|
||||
level the way GitHub Actions does. A job with `continue-on-error: true` that
|
||||
fails still reports `status: failure` in the commit status API.
|
||||
|
||||
Only `continue-on-error: true` at the **step** level works as expected.
|
||||
|
||||
### Impact
|
||||
|
||||
If you want a job to always "pass" in the status API (so dependent jobs can
|
||||
run and the overall CI does not show `failure`), you must add
|
||||
`continue-on-error: true` to every step that can fail, AND ensure each step
|
||||
exits with code 0 (e.g., append `|| true` to commands that might fail).
|
||||
|
||||
### Affected workflows
|
||||
|
||||
| Workflow | Fix |
|
||||
|---|---|
|
||||
| `harness-replays.yml` detect-changes | Added `continue-on-error: true` to fetch step + decide step; added `|| true` to `DIFF=$(git diff ...)` per PR #441 |
|
||||
|
||||
### How to diagnose
|
||||
|
||||
```yaml
|
||||
# WRONG — job reports as failure despite flag
|
||||
jobs:
|
||||
my-job:
|
||||
continue-on-error: true # ← ignored by Gitea
|
||||
steps:
|
||||
- run: git diff ... # ← if this fails, job = failure
|
||||
# job-level flag does not help
|
||||
|
||||
# RIGHT — step-level flag prevents step from failing
|
||||
jobs:
|
||||
my-job:
|
||||
steps:
|
||||
- run: git diff ... || true # ← step exits 0
|
||||
continue-on-error: true # ← belt and suspenders
|
||||
```
|
||||
|
||||
### References
|
||||
|
||||
- Gitea Actions quirk #10 (from migration checklist)
|
||||
- PR #441: fix applied to `harness-replays.yml`
|
||||
|
||||
---
|
||||
|
||||
## `workflow_dispatch.inputs` not supported
|
||||
|
||||
Gitea 1.22.6 parser rejects `workflow_dispatch.inputs`. Drop from all workflow
|
||||
YAML files ported from GitHub Actions. Manual triggers should use
|
||||
`workflow_dispatch` without `inputs:`.
|
||||
|
||||
**Reference**: `feedback_gitea_workflow_dispatch_inputs_unsupported`
|
||||
|
||||
---
|
||||
|
||||
## `merge_group` not supported
|
||||
|
||||
Gitea has no merge queue concept. Drop `merge_group:` triggers from all
|
||||
workflow YAML files.
|
||||
|
||||
---
|
||||
|
||||
## `environment:` blocks not supported
|
||||
|
||||
Gitea has no environments concept. Drop `environment:` from all workflow YAML
|
||||
files. Secrets and variables are repo-level.
|
||||
|
||||
---
|
||||
|
||||
## `fetch-depth: 0` on `actions/checkout` times out
|
||||
|
||||
`actions/checkout` with `fetch-depth: 0` triggers a full repo history fetch
|
||||
which exceeds the runner's network timeout to the git remote (~15 s).
|
||||
|
||||
**Workaround**: Use `fetch-depth: 1` (default) and add explicit
|
||||
`git fetch origin <ref> --depth=1` for any additional refs needed.
|
||||
|
||||
**Reference**: PR #441 detect-changes fetch step.
|
||||
+1
-1
@@ -43,7 +43,7 @@ endpoint handler for the supported range.
|
||||
- `cleanup-rogue-workspaces.sh` — emergency teardown for leaked
|
||||
workspaces. Prompts for confirmation. Pair with the harnesses if a
|
||||
cleanup trap fails (see `cleanup_*_failed` events).
|
||||
- `canary-smoke.sh` — quick smoke test for canary releases.
|
||||
- `staging-smoke.sh` — quick smoke test for the staging canary fleet (formerly `canary-smoke.sh`).
|
||||
- `dev-start.sh` — local-dev platform bring-up.
|
||||
|
||||
The rest are self-documenting in their header comments.
|
||||
|
||||
@@ -1,29 +1,40 @@
|
||||
#!/bin/bash
|
||||
# canary-smoke.sh — runs the post-deploy smoke suite against the
|
||||
# staging canary tenant fleet. Called by the canary-verify.yml GitHub
|
||||
# staging-smoke.sh — runs the post-deploy smoke suite against the
|
||||
# staging canary tenant fleet. Called by the staging-verify.yml Gitea
|
||||
# Actions workflow after a new workspace-server image lands in ECR;
|
||||
# exits non-zero on any failure so the workflow can block the
|
||||
# redeploy-fleet promotion that would otherwise release broken code
|
||||
# to the prod tenant fleet.
|
||||
#
|
||||
# Naming note (2026-05-11): The script (and its input env vars) were
|
||||
# renamed from canary-smoke.sh / CANARY_* to staging-smoke.sh /
|
||||
# MOLECULE_STAGING_* per Hongming directive. The tested COHORT is still
|
||||
# called the "canary fleet" (a small subset of staging tenants that
|
||||
# ingest :staging-<sha> before the rest of the fleet); that strategy
|
||||
# concept is unchanged.
|
||||
#
|
||||
# Registry note: GHCR was retired 2026-05-06. Images are now pushed
|
||||
# to the operator's ECR org (153263036946.dkr.ecr.us-east-2.amazonaws.com/
|
||||
# molecule-ai/platform-tenant). The registry URL is a runtime concern for
|
||||
# the CI push step; this script tests the running tenant directly.
|
||||
#
|
||||
# Environment:
|
||||
# CANARY_TENANT_URLS space-sep list of canary tenant base URLs
|
||||
# (e.g. "https://canary-pm.staging.moleculesai.app
|
||||
# https://canary-mcp.staging.moleculesai.app")
|
||||
# CANARY_ADMIN_TOKENS space-sep list of ADMIN_TOKENs, positionally
|
||||
# matched to CANARY_TENANT_URLS. Canary tenants
|
||||
# are provisioned with known ADMIN_TOKENs so CI
|
||||
# can hit their admin-gated endpoints.
|
||||
# CANARY_CP_BASE_URL CP base URL the canaries call back to
|
||||
# (https://staging-api.moleculesai.app)
|
||||
# CANARY_CP_SHARED_SECRET matches CP's PROVISION_SHARED_SECRET so this
|
||||
# script can also exercise /cp/workspaces/* via
|
||||
# the canary's own CPProvisioner identity.
|
||||
# MOLECULE_STAGING_TENANT_URLS space-sep list of canary tenant base
|
||||
# URLs (e.g. "https://canary-pm.staging.
|
||||
# moleculesai.app https://canary-mcp.
|
||||
# staging.moleculesai.app")
|
||||
# MOLECULE_STAGING_ADMIN_TOKENS space-sep list of ADMIN_TOKENs,
|
||||
# positionally matched to
|
||||
# MOLECULE_STAGING_TENANT_URLS.
|
||||
# Canary tenants are provisioned with
|
||||
# known ADMIN_TOKENs so CI can hit
|
||||
# their admin-gated endpoints.
|
||||
# MOLECULE_STAGING_CP_BASE_URL CP base URL the canaries call back to
|
||||
# (https://staging-api.moleculesai.app)
|
||||
# MOLECULE_STAGING_CP_SHARED_SECRET matches CP's PROVISION_SHARED_SECRET
|
||||
# so this script can also exercise
|
||||
# /cp/workspaces/* via the canary's
|
||||
# own CPProvisioner identity.
|
||||
#
|
||||
# Exit codes: 0 = all green, 1 = assertion failure, 2 = setup/env problem.
|
||||
|
||||
@@ -31,12 +42,12 @@ set -euo pipefail
|
||||
|
||||
# ── Setup ────────────────────────────────────────────────────────────────
|
||||
|
||||
: "${CANARY_TENANT_URLS:?space-sep list of canary base URLs required}"
|
||||
: "${CANARY_ADMIN_TOKENS:?space-sep list of ADMIN_TOKENs required, same order as URLs}"
|
||||
: "${CANARY_CP_BASE_URL:?CP base URL required}"
|
||||
: "${MOLECULE_STAGING_TENANT_URLS:?space-sep list of canary base URLs required}"
|
||||
: "${MOLECULE_STAGING_ADMIN_TOKENS:?space-sep list of ADMIN_TOKENs required, same order as URLs}"
|
||||
: "${MOLECULE_STAGING_CP_BASE_URL:?CP base URL required}"
|
||||
|
||||
read -r -a URLS <<< "$CANARY_TENANT_URLS"
|
||||
read -r -a TOKENS <<< "$CANARY_ADMIN_TOKENS"
|
||||
read -r -a URLS <<< "$MOLECULE_STAGING_TENANT_URLS"
|
||||
read -r -a TOKENS <<< "$MOLECULE_STAGING_ADMIN_TOKENS"
|
||||
|
||||
if [ "${#URLS[@]}" -ne "${#TOKENS[@]}" ]; then
|
||||
echo "ERROR: URLS(${#URLS[@]}) and TOKENS(${#TOKENS[@]}) length mismatch" >&2
|
||||
@@ -69,7 +80,7 @@ check() {
|
||||
# tenant never gets the wrong token.
|
||||
acurl() {
|
||||
local base="$1" token="$2"; shift 2
|
||||
curl -sS --max-time 20 -H "Authorization: Bearer $token" "$@" -- "$base${CANARY_ACURL_PATH:-}"
|
||||
curl -sS --max-time 20 -H "Authorization: Bearer $token" "$@" -- "$base${ACURL_PATH:-}"
|
||||
}
|
||||
|
||||
# ── Checks (run per canary tenant) ───────────────────────────────────────
|
||||
@@ -80,7 +91,7 @@ for i in "${!URLS[@]}"; do
|
||||
printf "\n── %s ──\n" "$base"
|
||||
|
||||
# 1. Liveness — the tenant is up and responding to admin auth.
|
||||
CANARY_ACURL_PATH="/admin/liveness" resp=$(acurl "$base" "$token" || true)
|
||||
ACURL_PATH="/admin/liveness" resp=$(acurl "$base" "$token" || true)
|
||||
check "liveness returns a subsystems map" '"subsystems"' "$resp"
|
||||
|
||||
# 2. CP env refresh — the workspace-server fetched MOLECULE_CP_SHARED_SECRET
|
||||
@@ -89,25 +100,25 @@ for i in "${!URLS[@]}"; do
|
||||
# booted without crashing on the refresh call. A startup failure in
|
||||
# refreshEnvFromCP logs but still boots (best-effort semantics), so
|
||||
# this is a sanity check, not a proof.
|
||||
CANARY_ACURL_PATH="/workspaces" resp=$(acurl "$base" "$token" || true)
|
||||
ACURL_PATH="/workspaces" resp=$(acurl "$base" "$token" || true)
|
||||
check "workspace list is JSON array" "[" "$resp"
|
||||
|
||||
# 3. Memory commit round-trip — scope=LOCAL so test data stays on this
|
||||
# tenant. Verifies encryption + scrubber + retrieval end-to-end.
|
||||
probe_id="canary-smoke-$(date +%s)-$i"
|
||||
body=$(printf '{"scope":"LOCAL","namespace":"canary-smoke","content":"probe-%s"}' "$probe_id")
|
||||
CANARY_ACURL_PATH="/memories/commit" resp=$(curl -sS --max-time 20 \
|
||||
ACURL_PATH="/memories/commit" resp=$(curl -sS --max-time 20 \
|
||||
-X POST -H "Content-Type: application/json" -H "Authorization: Bearer $token" \
|
||||
--data "$body" "$base/memories/commit" || true)
|
||||
check "memory commit accepted" '"id"' "$resp"
|
||||
|
||||
CANARY_ACURL_PATH="/memories/search?query=probe-${probe_id}" \
|
||||
ACURL_PATH="/memories/search?query=probe-${probe_id}" \
|
||||
resp=$(curl -sS --max-time 20 -H "Authorization: Bearer $token" \
|
||||
"$base/memories/search?query=probe-${probe_id}" || true)
|
||||
check "memory search finds the probe" "probe-${probe_id}" "$resp"
|
||||
|
||||
# 4. Events admin read — AdminAuth path (C4 fail-closed proof on SaaS).
|
||||
CANARY_ACURL_PATH="/events" resp=$(acurl "$base" "$token" || true)
|
||||
ACURL_PATH="/events" resp=$(acurl "$base" "$token" || true)
|
||||
check "events endpoint returns JSON" "[" "$resp"
|
||||
|
||||
# 5. Negative: unauth'd admin call must 401 (C4 regression gate).
|
||||
@@ -117,7 +128,7 @@ for i in "${!URLS[@]}"; do
|
||||
# 6. POST /org/import unauth → 401. Proves the route is compiled in
|
||||
# and AdminAuth is enforced. A missing route returns 404 (the failure
|
||||
# mode caught by issue #213). Regression guard for the silent-GHCR-
|
||||
# migration gap: canary-verify was testing a stale GHCR image while
|
||||
# migration gap: staging-verify (formerly canary-verify) was testing a stale GHCR image while
|
||||
# actual tenants ran ECR — this test would have caught a missing-route
|
||||
# binary before it reached prod.
|
||||
unauth_code=$(curl -sS -o /dev/null -w '%{http_code}' \
|
||||
@@ -7,11 +7,11 @@ Four workflows + a shared bash harness that together cover the SaaS stack end to
|
||||
| Workflow | Cadence | Wall time | Scope |
|
||||
|---|---|---|---|
|
||||
| `e2e-staging-saas.yml` | push + nightly 07:00 UTC | ~20 min | Full API: org → tenant → 2 workspaces → A2A → HMA → delegation → leak check |
|
||||
| `canary-staging.yml` | every 30 min | ~8 min | Minimum smoke + self-managed alert issue |
|
||||
| `staging-smoke.yml` | every 30 min | ~8 min | Minimum smoke + self-managed alert issue |
|
||||
| `e2e-staging-canvas.yml` | push + weekly Sunday 08:00 | ~25 min | All 13 canvas workspace-panel tabs via Playwright |
|
||||
| `e2e-staging-sanity.yml` | weekly Monday 06:00 | ~10 min | Intentional-failure: teardown safety-net self-check |
|
||||
|
||||
`tests/e2e/test_staging_full_saas.sh` is the shared harness all workflows invoke (with `E2E_MODE={full|canary}` and `E2E_INTENTIONAL_FAILURE={0|1}` toggles).
|
||||
`tests/e2e/test_staging_full_saas.sh` is the shared harness all workflows invoke (with `E2E_MODE={full|smoke}` and `E2E_INTENTIONAL_FAILURE={0|1}` toggles).
|
||||
|
||||
### Full-SaaS checklist (sections)
|
||||
|
||||
@@ -49,7 +49,15 @@ Runs the harness with `E2E_INTENTIONAL_FAILURE=1`, which poisons the tenant admi
|
||||
|
||||
Set in **Settings → Secrets and variables → Actions → Repository secrets**:
|
||||
|
||||
### `MOLECULE_STAGING_ADMIN_TOKEN`
|
||||
### `CP_STAGING_ADMIN_API_TOKEN`
|
||||
|
||||
> **Historical-rename note (2026-05-11):** previously named
|
||||
> `MOLECULE_STAGING_ADMIN_TOKEN`. Canonicalised to
|
||||
> `CP_STAGING_ADMIN_API_TOKEN` per internal#322 (the Railway staging
|
||||
> service exposes it as `CP_ADMIN_API_TOKEN`; the `CP_*` repo-secret
|
||||
> prefix matches the upstream env name + makes the service it talks
|
||||
> to obvious in workflow YAMLs). See the original PR for the
|
||||
> cross-workflow sweep.
|
||||
|
||||
The `CP_ADMIN_API_TOKEN` env currently set on the Railway staging molecule-platform → controlplane service.
|
||||
|
||||
@@ -82,7 +90,7 @@ bash tests/e2e/test_staging_full_saas.sh
|
||||
## Cost
|
||||
|
||||
- Full run: ~20 min, ~$0.007
|
||||
- Canary (48/day): ~$0.06/day
|
||||
- Smoke (48/day): ~$0.06/day
|
||||
- Canvas (few/week): ~$0.01/day
|
||||
- Sanity (weekly): ~$0.002/week
|
||||
- **Total staging burn: < $0.15/day** at expected CI load
|
||||
|
||||
@@ -27,7 +27,11 @@
|
||||
# E2E_PROVISION_TIMEOUT_SECS default 900 (15 min cold EC2 budget)
|
||||
# E2E_KEEP_ORG 1 → skip teardown (debugging only)
|
||||
# E2E_RUN_ID Slug suffix; CI: ${GITHUB_RUN_ID}
|
||||
# E2E_MODE full (default) | canary
|
||||
# E2E_MODE full (default) | smoke
|
||||
# (legacy alias `canary` still accepted —
|
||||
# mapped to `smoke` for back-compat with
|
||||
# any in-flight runner picking up an older
|
||||
# workflow checkout)
|
||||
# E2E_INTENTIONAL_FAILURE 1 → poison tenant token mid-run so the
|
||||
# script fails; the EXIT trap MUST still
|
||||
# tear down cleanly (and exit 4 on leak).
|
||||
@@ -49,15 +53,23 @@ RUNTIME="${E2E_RUNTIME:-hermes}"
|
||||
PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-900}"
|
||||
RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
|
||||
MODE="${E2E_MODE:-full}"
|
||||
# `canary` is a legacy alias for `smoke` retained for back-compat with
|
||||
# any in-flight runner picking up an older workflow checkout during the
|
||||
# 2026-05-11 canary→staging rename rollout. Both map to the same slug
|
||||
# prefix below. Remove the `canary` alias after one week of no-old-mode
|
||||
# observations.
|
||||
if [ "$MODE" = "canary" ]; then
|
||||
MODE="smoke"
|
||||
fi
|
||||
case "$MODE" in
|
||||
full|canary) ;;
|
||||
*) echo "E2E_MODE must be 'full' or 'canary' (got: $MODE)" >&2; exit 2 ;;
|
||||
full|smoke) ;;
|
||||
*) echo "E2E_MODE must be 'full' or 'smoke' (got: $MODE)" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# Canary runs get a distinct prefix so their safety-net sweeper only
|
||||
# Smoke runs get a distinct slug prefix so their safety-net sweeper only
|
||||
# touches their own runs, not in-flight full runs.
|
||||
if [ "$MODE" = "canary" ]; then
|
||||
SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
if [ "$MODE" = "smoke" ]; then
|
||||
SLUG="e2e-smoke-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
else
|
||||
SLUG="e2e-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
gate-check-v3 — SOP-6 + CI gate detector for Gitea PRs.
|
||||
|
||||
Emits structured verdict + human-readable summary. Designed to run as:
|
||||
1. CLI: python gate_check.py --repo org/repo --pr N
|
||||
2. Gitea Actions step: runs this script, captures stdout JSON
|
||||
|
||||
Signals (MVP — signals 1,2,3,6):
|
||||
1. Author-aware agent-tag comment scan
|
||||
2. REQUEST_CHANGES reviews state machine
|
||||
3. Staleness detection (review.commit_id != PR.head_sha)
|
||||
6. CI required-checks awareness
|
||||
|
||||
Exit codes:
|
||||
0 — all gates pass (verdict=CLEAR)
|
||||
1 — one or more gates blocking (verdict=BLOCKED)
|
||||
2 — API error / usage error (verdict=ERROR)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
# ── Gitea API client ────────────────────────────────────────────────────────
|
||||
|
||||
GITEA_HOST = os.environ.get("GITEA_HOST", "git.moleculesai.app")
|
||||
GITEA_TOKEN = os.environ.get("GITEA_TOKEN", os.environ.get("GITHUB_TOKEN", ""))
|
||||
API_BASE = f"https://{GITEA_HOST}/api/v1"
|
||||
|
||||
|
||||
def api_get(path: str) -> dict | list:
|
||||
url = f"{API_BASE}{path}"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode(errors="replace")
|
||||
raise GiteaError(f"GET {url} → {e.code}: {body[:300]}")
|
||||
|
||||
|
||||
def api_list(path: str, per_page: int = 100) -> list:
|
||||
"""Paginate a list endpoint using Link headers (Gitea/GitHub convention)."""
|
||||
results = []
|
||||
page = 1
|
||||
while True:
|
||||
paged_path = f"{path}?per_page={per_page}&page={page}"
|
||||
result = api_get(paged_path)
|
||||
if isinstance(result, list):
|
||||
results.extend(result)
|
||||
if len(result) < per_page:
|
||||
break
|
||||
page += 1
|
||||
else:
|
||||
# Some endpoints return an object with a data/items key
|
||||
data = result.get("data", result.get("items", result))
|
||||
if isinstance(data, list):
|
||||
results.extend(data)
|
||||
break
|
||||
# Safety cap to avoid runaway pagination
|
||||
if page > 20:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
class GiteaError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# ── Signal 1: Author-aware agent-tag comment scan ─────────────────────────────
|
||||
# Matches: [core-{role}-agent] VERDICT in comment body.
|
||||
# Must be authored by the agent whose role is tagged.
|
||||
# Scans BOTH issue comments (/issues/{N}/comments) and PR comments
|
||||
# (/pulls/{N}/comments) since agents post on both.
|
||||
|
||||
# Matches [core-{role}-agent] VERDICT anywhere in the comment body.
|
||||
AGENT_TAG_RE = re.compile(
|
||||
r"\[core-([a-z]+)-agent\]\s+(APPROVED|N/?A|CHANGES_REQUESTED|COMMENT|BLOCKED|ACK)\b",
|
||||
)
|
||||
|
||||
# Map agent role → canonical login (from workspace registry)
|
||||
AGENT_LOGIN_MAP = {
|
||||
"qa": "core-qa",
|
||||
"security": "core-security",
|
||||
"uiux": "core-uiux",
|
||||
"lead": "core-lead",
|
||||
"devops": "core-devops",
|
||||
"be": "core-be",
|
||||
"fe": "core-fe",
|
||||
"offsec": "core-offsec",
|
||||
}
|
||||
|
||||
# SOP-6 tier → required agent groups
|
||||
# tier:low → engineers,managers,ceo (OR: any one suffices)
|
||||
# tier:medium → managers AND engineers AND qa,security (AND)
|
||||
# tier:high → ceo (OR, but single)
|
||||
# "?" = teams not yet created; treated as optional for MVP
|
||||
TIER_AGENTS = {
|
||||
"tier:low": {"managers": "core-lead", "engineers": "core-devops", "ceo": "ceo"},
|
||||
"tier:medium": {"managers": "core-lead", "engineers": "core-devops", "qa": "core-qa", "security": "core-security"},
|
||||
"tier:high": {"ceo": "ceo"},
|
||||
}
|
||||
|
||||
POSITIVE_VERDICTS = {"APPROVED", "N/A", "ACK"}
|
||||
|
||||
|
||||
def _get_pr_tier(pr_number: int, repo: str) -> str:
|
||||
"""Get the PR's tier label."""
|
||||
owner, name = repo.split("/", 1)
|
||||
try:
|
||||
pr = api_get(f"/repos/{owner}/{name}/pulls/{pr_number}")
|
||||
for label in pr.get("labels", []):
|
||||
name_l = label.get("name", "")
|
||||
if name_l in TIER_AGENTS:
|
||||
return name_l
|
||||
except GiteaError:
|
||||
pass
|
||||
return "tier:low" # Default for untagged PRs
|
||||
|
||||
|
||||
def signal_1_comment_scan(pr_number: int, repo: str) -> dict:
|
||||
"""
|
||||
Scan issue + PR comments AND reviews for agent-tag policy gates.
|
||||
Matches tag AND author. Filters to tier-relevant agents.
|
||||
Returns: {signal, results, verdict}
|
||||
"""
|
||||
owner, name = repo.split("/", 1)
|
||||
|
||||
# Get tier label to determine relevant agents
|
||||
tier = _get_pr_tier(pr_number, repo)
|
||||
relevant_roles = TIER_AGENTS.get(tier, TIER_AGENTS["tier:low"])
|
||||
|
||||
# Build reverse map: login -> (group, agent_key)
|
||||
login_to_group = {}
|
||||
for group, login in relevant_roles.items():
|
||||
for role, l in AGENT_LOGIN_MAP.items():
|
||||
if l == login:
|
||||
login_to_group[l] = (group, f"core-{role}")
|
||||
|
||||
# Collect all agent-tag matches from comments
|
||||
comments = []
|
||||
try:
|
||||
comments.extend(api_list(f"/repos/{owner}/{name}/issues/{pr_number}/comments"))
|
||||
except GiteaError:
|
||||
pass
|
||||
try:
|
||||
comments.extend(api_list(f"/repos/{owner}/{name}/pulls/{pr_number}/comments"))
|
||||
except GiteaError:
|
||||
pass
|
||||
|
||||
# Collect APPROVED reviews from agent logins
|
||||
try:
|
||||
reviews = api_list(f"/repos/{owner}/{name}/pulls/{pr_number}/reviews")
|
||||
for r in reviews:
|
||||
login = r.get("user", {}).get("login", "")
|
||||
if login in login_to_group and r.get("state") == "APPROVED":
|
||||
comments.append(
|
||||
{
|
||||
"id": f"review-{r['id']}",
|
||||
"user": {"login": login},
|
||||
"body": f"[{login}-agent] APPROVED",
|
||||
"created_at": r.get("submitted_at") or r.get("created_at", ""),
|
||||
"source": "review",
|
||||
}
|
||||
)
|
||||
except GiteaError:
|
||||
pass
|
||||
|
||||
# Find latest verdict per agent login
|
||||
findings = {}
|
||||
for login, (group, agent_key) in login_to_group.items():
|
||||
matches = []
|
||||
for c in comments:
|
||||
body = c.get("body", "") or ""
|
||||
user_login = c.get("user", {}).get("login", "")
|
||||
if user_login != login:
|
||||
continue
|
||||
for m in AGENT_TAG_RE.finditer(body):
|
||||
tag_role, verdict = m.group(1), m.group(2)
|
||||
# Match the role part of the login (e.g. "core-devops" → "devops")
|
||||
login_role = login.replace("core-", "")
|
||||
if tag_role == login_role:
|
||||
matches.append(
|
||||
{
|
||||
"comment_id": c["id"],
|
||||
"verdict": verdict,
|
||||
"user": user_login,
|
||||
"created_at": c["created_at"],
|
||||
"source": c.get("source", "comment"),
|
||||
}
|
||||
)
|
||||
latest = max(matches, key=lambda x: x["created_at"], default=None) if matches else None
|
||||
findings[agent_key] = {
|
||||
"group": group,
|
||||
"tier": tier,
|
||||
"found": latest,
|
||||
"verdict": latest["verdict"] if latest else "MISSING",
|
||||
}
|
||||
|
||||
# Compute gate verdict using tier-specific logic:
|
||||
# - tier:low / tier:high (OR gate): ANY positive = CLEAR, ANY negative = BLOCKED
|
||||
# - tier:medium (AND gate): ALL must be positive = CLEAR, ANY negative = BLOCKED
|
||||
verdicts = [f["verdict"] for f in findings.values()]
|
||||
if not verdicts:
|
||||
gate_verdict = "N/A"
|
||||
elif tier in ("tier:low", "tier:high"):
|
||||
# OR gate: one positive is enough
|
||||
if any(v in POSITIVE_VERDICTS for v in verdicts):
|
||||
gate_verdict = "CLEAR"
|
||||
elif any(v in ("BLOCKED", "CHANGES_REQUESTED", "COMMENT") for v in verdicts):
|
||||
gate_verdict = "BLOCKED"
|
||||
else:
|
||||
gate_verdict = "INCOMPLETE"
|
||||
else:
|
||||
# AND gate (tier:medium): all must be positive
|
||||
if all(v in POSITIVE_VERDICTS for v in verdicts):
|
||||
gate_verdict = "CLEAR"
|
||||
elif any(v in ("BLOCKED", "CHANGES_REQUESTED", "COMMENT") for v in verdicts):
|
||||
gate_verdict = "BLOCKED"
|
||||
else:
|
||||
gate_verdict = "INCOMPLETE"
|
||||
|
||||
return {"signal": "agent_tag_comments", "results": findings, "verdict": gate_verdict, "tier": tier}
|
||||
|
||||
|
||||
# ── Signal 2: REQUEST_CHANGES reviews state machine ────────────────────────────
|
||||
|
||||
def signal_2_reviews(pr_number: int, repo: str) -> dict:
|
||||
"""
|
||||
Check /pulls/{N}/reviews for active REQUEST_CHANGES with dismissed=false.
|
||||
This is the layer that empirically blocks Gitea merges.
|
||||
Returns: {blocking_reviews: [...], verdict}
|
||||
"""
|
||||
owner, name = repo.split("/", 1)
|
||||
reviews = api_list(f"/repos/{owner}/{name}/pulls/{pr_number}/reviews")
|
||||
|
||||
blocking = []
|
||||
for r in reviews:
|
||||
if r.get("state") == "REQUEST_CHANGES" and not r.get("dismissed", False):
|
||||
blocking.append(
|
||||
{
|
||||
"review_id": r["id"],
|
||||
"user": r["user"]["login"],
|
||||
"commit_id": r.get("commit_id", ""),
|
||||
"created_at": r.get("submitted_at") or r.get("created_at", ""),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"signal": "request_changes_reviews",
|
||||
"blocking_reviews": blocking,
|
||||
"verdict": "BLOCKED" if blocking else "CLEAR",
|
||||
}
|
||||
|
||||
|
||||
# ── Signal 3: Staleness detection ────────────────────────────────────────────
|
||||
|
||||
WORKING_DAY_SECONDS = 9 * 3600 # SOP-12: 1 working day threshold
|
||||
|
||||
|
||||
def signal_3_staleness(pr_number: int, repo: str) -> dict:
|
||||
"""
|
||||
Flag reviews where review.commit_id != PR.head_sha AND
|
||||
time_since_review > 1 working day. Per SOP-12 (internal#282).
|
||||
Returns: {stale_reviews: [...], verdict}
|
||||
"""
|
||||
owner, name = repo.split("/", 1)
|
||||
|
||||
# Get PR head sha
|
||||
pr = api_get(f"/repos/{owner}/{name}/pulls/{pr_number}")
|
||||
head_sha = pr["head"]["sha"]
|
||||
|
||||
reviews = api_list(f"/repos/{owner}/{name}/pulls/{pr_number}/reviews")
|
||||
|
||||
stale = []
|
||||
now = datetime.now(timezone.utc)
|
||||
for r in reviews:
|
||||
review_commit = r.get("commit_id", "")
|
||||
if review_commit and review_commit != head_sha:
|
||||
# Review predates current head
|
||||
try:
|
||||
created = datetime.fromisoformat(r["created_at"].replace("Z", "+00:00"))
|
||||
except (KeyError, ValueError):
|
||||
continue
|
||||
age_seconds = (now - created).total_seconds()
|
||||
if age_seconds > WORKING_DAY_SECONDS:
|
||||
stale.append(
|
||||
{
|
||||
"review_id": r["id"],
|
||||
"user": r["user"]["login"],
|
||||
"review_commit": review_commit,
|
||||
"pr_head": head_sha,
|
||||
"age_hours": round(age_seconds / 3600, 1),
|
||||
"created_at": r.get("submitted_at") or r.get("created_at", ""),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"signal": "stale_reviews",
|
||||
"stale_reviews": stale,
|
||||
"verdict": "STALE-RC" if stale else "CLEAR",
|
||||
}
|
||||
|
||||
|
||||
# ── Signal 6: CI required-checks awareness ───────────────────────────────────
|
||||
|
||||
def signal_6_ci(pr_number: int, repo: str, branch: str = "main") -> dict:
|
||||
"""
|
||||
Query combined CI status for PR head commit.
|
||||
Find required status checks on target branch.
|
||||
Surface any failing required check as primary blocker.
|
||||
"""
|
||||
owner, name = repo.split("/", 1)
|
||||
|
||||
pr = api_get(f"/repos/{owner}/{name}/pulls/{pr_number}")
|
||||
head_sha = pr["head"]["sha"]
|
||||
|
||||
# Combined status of PR head
|
||||
combined = api_get(f"/repos/{owner}/{name}/commits/{head_sha}/status")
|
||||
ci_state = combined.get("state", "null")
|
||||
|
||||
# Individual check statuses
|
||||
# Gitea Actions uses "status" (pending/success/failure) not "state" for
|
||||
# individual check entries. "state" is null for pending runs.
|
||||
check_statuses = {}
|
||||
for s in combined.get("statuses") or []:
|
||||
check_statuses[s["context"]] = s.get("status", "pending")
|
||||
|
||||
# Try to get branch protection for required checks
|
||||
required_checks = []
|
||||
try:
|
||||
protection = api_get(f"/repos/{owner}/{name}/branches/{branch}/protection")
|
||||
for check in protection.get("required_status_checks", {}).get("checks", []):
|
||||
required_checks.append(check["context"])
|
||||
except GiteaError:
|
||||
pass # No protection or no read access
|
||||
|
||||
failing_required = []
|
||||
passing_required = []
|
||||
for ctx in required_checks:
|
||||
state = check_statuses.get(ctx, "null")
|
||||
if state == "failure":
|
||||
failing_required.append(ctx)
|
||||
elif state in ("success", "neutral"):
|
||||
passing_required.append(ctx)
|
||||
else:
|
||||
passing_required.append(f"{ctx} (pending)")
|
||||
|
||||
if failing_required:
|
||||
verdict = "CI_FAIL"
|
||||
elif ci_state == "failure":
|
||||
verdict = "CI_FAIL"
|
||||
elif ci_state == "pending":
|
||||
verdict = "CI_PENDING"
|
||||
else:
|
||||
verdict = "CLEAR"
|
||||
|
||||
return {
|
||||
"signal": "ci_checks",
|
||||
"combined_state": ci_state,
|
||||
"required_checks": required_checks,
|
||||
"failing_required": failing_required,
|
||||
"passing_required": passing_required,
|
||||
"all_check_statuses": check_statuses,
|
||||
"verdict": verdict,
|
||||
}
|
||||
|
||||
|
||||
# ── Gate evaluation ───────────────────────────────────────────────────────────
|
||||
|
||||
VERDICT_ORDER = {"ERROR": 0, "CI_FAIL": 1, "BLOCKED": 2, "STALE-RC": 3, "CI_PENDING": 4, "N/A": 5, "CLEAR": 6}
|
||||
|
||||
|
||||
def compute_verdict(gates: list[dict]) -> tuple[str, list[dict]]:
|
||||
"""Compute overall verdict from gate results. Worst gate wins."""
|
||||
worst = "CLEAR"
|
||||
blockers = []
|
||||
for g in gates:
|
||||
v = g.get("verdict", "N/A")
|
||||
if VERDICT_ORDER.get(v, 99) < VERDICT_ORDER.get(worst, 0):
|
||||
worst = v
|
||||
if v in ("BLOCKED", "CI_FAIL", "STALE-RC", "ERROR"):
|
||||
blockers.append(g)
|
||||
return worst, blockers
|
||||
|
||||
|
||||
def format_gate_verdict(v: str) -> tuple[str, str]:
|
||||
"""Return (icon, label) for a gate verdict."""
|
||||
if v in ("APPROVED", "CLEAR"):
|
||||
return "✅", v
|
||||
if v in ("BLOCKED", "CI_FAIL", "ERROR"):
|
||||
return "❌", v
|
||||
return "⚠️", v
|
||||
|
||||
|
||||
def format_comment(repo: str, pr_number: int, verdict: str, gates: list[dict], blockers: list[dict]) -> str:
|
||||
"""Format human-readable Gitea PR comment."""
|
||||
gate_labels = {
|
||||
"agent_tag_comments": "Agent-tag gates",
|
||||
"request_changes_reviews": "REQUEST_CHANGES reviews",
|
||||
"stale_reviews": "Staleness check",
|
||||
"ci_checks": "CI required checks",
|
||||
}
|
||||
|
||||
lines = [f"[gate-check-v3] STATUS: **{verdict}**", ""]
|
||||
|
||||
# Per-gate summary
|
||||
for g in gates:
|
||||
sig = g.get("signal", "?")
|
||||
label = gate_labels.get(sig, sig)
|
||||
v = g.get("verdict", "N/A")
|
||||
icon, _ = format_gate_verdict(v)
|
||||
lines.append(f"{icon} **{label}**: {v}")
|
||||
|
||||
# Gate-specific detail
|
||||
if blockers:
|
||||
lines.append("")
|
||||
lines.append("### Blockers")
|
||||
for b in blockers:
|
||||
sig = b.get("signal", "?")
|
||||
if sig == "request_changes_reviews":
|
||||
for r in b.get("blocking_reviews", []):
|
||||
lines.append(f" - @{r['user']} requested changes (review id={r['review_id']})")
|
||||
elif sig == "ci_checks":
|
||||
combined = b.get("combined_state", "?")
|
||||
lines.append(f" - CI combined state: **{combined}**")
|
||||
for c in b.get("failing_required", []):
|
||||
lines.append(f" - required check failing: **{c}**")
|
||||
for c in b.get("all_check_statuses", {}).items():
|
||||
ctx, state = c
|
||||
lines.append(f" - {ctx}: {state}")
|
||||
elif sig == "stale_reviews":
|
||||
for r in b.get("stale_reviews", []):
|
||||
lines.append(
|
||||
f" - @{r['user']} stale (commit={r.get('review_commit','?')[:7]}, age={r.get('age_hours','?')}h)"
|
||||
)
|
||||
elif sig == "agent_tag_comments":
|
||||
for agent, res in b.get("results", {}).items():
|
||||
v = res.get("verdict", "MISSING")
|
||||
icon, _ = format_gate_verdict(v)
|
||||
if v == "MISSING":
|
||||
lines.append(f" {icon} {agent}: no agent-tag comment found")
|
||||
else:
|
||||
lines.append(f" {icon} {agent}: {v}")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"_gate-check-v3 · repo={repo} · pr={pr_number}_")
|
||||
return "\n".join(lines)
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"_gate-check-v3 · repo={repo} · pr={pr_number}_")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run(repo: str, pr_number: int, post_comment: bool = False) -> dict:
|
||||
try:
|
||||
gates = [
|
||||
signal_1_comment_scan(pr_number, repo),
|
||||
signal_2_reviews(pr_number, repo),
|
||||
signal_3_staleness(pr_number, repo),
|
||||
signal_6_ci(pr_number, repo),
|
||||
]
|
||||
verdict, blockers = compute_verdict(gates)
|
||||
|
||||
result = {
|
||||
"verdict": verdict,
|
||||
"repo": repo,
|
||||
"pr": pr_number,
|
||||
"gates": gates,
|
||||
"blockers": blockers,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
# Print human-readable to stdout for Gitea Actions log
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
# Optionally post comment
|
||||
if post_comment:
|
||||
owner, name = repo.split("/", 1)
|
||||
comment_body = format_comment(repo, pr_number, verdict, gates, blockers)
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
# Check if a gate-check comment already exists to avoid spamming
|
||||
existing = api_list(f"/repos/{owner}/{name}/issues/{pr_number}/comments")
|
||||
our_comments = [c for c in existing if "[gate-check-v3]" in (c.get("body") or "")]
|
||||
if our_comments:
|
||||
# Update latest
|
||||
comment_id = our_comments[-1]["id"]
|
||||
url = f"{API_BASE}/repos/{owner}/{name}/issues/comments/{comment_id}"
|
||||
req = urllib.request.Request(url, data=json.dumps({"body": comment_body}).encode(), headers=headers, method="PATCH")
|
||||
with urllib.request.urlopen(req) as r:
|
||||
r.read()
|
||||
else:
|
||||
url = f"{API_BASE}/repos/{owner}/{name}/issues/{pr_number}/comments"
|
||||
req = urllib.request.Request(url, data=json.dumps({"body": comment_body}).encode(), headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req) as r:
|
||||
r.read()
|
||||
|
||||
return result
|
||||
|
||||
except GiteaError as e:
|
||||
result = {"verdict": "ERROR", "error": str(e), "repo": repo, "pr": pr_number}
|
||||
print(json.dumps(result, indent=2), file=sys.stderr)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="gate-check-v3 — PR gate detector")
|
||||
parser.add_argument("--repo", required=True, help="org/repo (e.g. molecule-ai/molecule-core)")
|
||||
parser.add_argument("--pr", type=int, required=True, help="PR number")
|
||||
parser.add_argument("--post-comment", action="store_true", help="Post/update comment on PR")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = run(args.repo, args.pr, post_comment=args.post_comment)
|
||||
verdict = result.get("verdict", "ERROR")
|
||||
|
||||
if verdict == "ERROR":
|
||||
return 2
|
||||
elif verdict in ("BLOCKED", "CI_FAIL", "STALE-RC", "ERROR"):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+30
-14
@@ -48,6 +48,27 @@ def get_machine_ip() -> str: # pragma: no cover
|
||||
return "127.0.0.1"
|
||||
|
||||
|
||||
def _check_delegation_results_pending() -> bool:
|
||||
"""Check if there are unconsumed delegation results waiting.
|
||||
|
||||
Reads ``DELEGATION_RESULTS_FILE``. Returns ``True`` if the file
|
||||
exists and contains non-whitespace content (after stripping) — meaning
|
||||
the idle loop should skip this tick. Returns ``False`` if the file is
|
||||
absent, empty, or contains only whitespace.
|
||||
|
||||
The extracted form lets unit tests call this directly rather than mirroring
|
||||
the logic (anti-pattern flagged as #401).
|
||||
"""
|
||||
from heartbeat import DELEGATION_RESULTS_FILE
|
||||
|
||||
try:
|
||||
with open(DELEGATION_RESULTS_FILE) as rf:
|
||||
rf.seek(0)
|
||||
return bool(rf.read().strip())
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
# Re-exported from transcript_auth for the inline /transcript handler.
|
||||
# Separate module keeps the security-critical gate import-light + unit-testable.
|
||||
from transcript_auth import transcript_authorized as _transcript_authorized
|
||||
@@ -678,20 +699,15 @@ async def main(): # pragma: no cover
|
||||
# heartbeat's own self-message wake the agent after results are
|
||||
# written. The agent then sees the results in _prepare_prompt()
|
||||
# and processes them before composing.
|
||||
from heartbeat import DELEGATION_RESULTS_FILE as _DRF
|
||||
try:
|
||||
with open(_DRF) as _rf:
|
||||
_rf.seek(0)
|
||||
_content = _rf.read().strip()
|
||||
if _content:
|
||||
print(
|
||||
f"Idle loop: skipping — {len(_content)} bytes of unconsumed "
|
||||
f"delegation results pending (heartbeat will notify agent)",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
pass # No results file — normal, proceed with idle prompt
|
||||
# Guard logic extracted to _check_delegation_results_pending() for
|
||||
# direct unit-testing (#401 follow-up).
|
||||
if _check_delegation_results_pending():
|
||||
print(
|
||||
"Idle loop: skipping — unconsumed delegation results pending "
|
||||
"(heartbeat will notify agent)",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
|
||||
# Self-post the idle prompt via the platform A2A proxy (same
|
||||
# path as initial_prompt). The agent's own concurrency control
|
||||
|
||||
@@ -4,77 +4,82 @@ The idle loop skips sending the idle prompt when DELEGATION_RESULTS_FILE
|
||||
contains unconsumed results, preventing the agent from composing a stale tick
|
||||
before processing pending delegation notifications from the heartbeat.
|
||||
|
||||
Source: workspace/main.py:_run_idle_loop() pending-results guard.
|
||||
Source: ``workspace/main.py:_check_delegation_results_pending()`` (extracted from
|
||||
``_run_idle_loop()`` guard; see PR #432 follow-up).
|
||||
|
||||
The guard is extracted into a module-level function so unit tests call the
|
||||
real production logic directly — not a mirror copy. This avoids the
|
||||
test-mirror anti-pattern (issue #401) where a copied implementation
|
||||
drifts from the production code it is supposed to test.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def check_results_pending(file_path: str) -> bool:
|
||||
"""Mirror the guard logic from workspace/main.py:_run_idle_loop().
|
||||
|
||||
Returns True if the results file exists and is non-empty,
|
||||
meaning the idle loop should skip this tick.
|
||||
"""
|
||||
try:
|
||||
with open(file_path) as rf:
|
||||
rf.seek(0)
|
||||
content = rf.read().strip()
|
||||
return bool(content)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
from main import _check_delegation_results_pending
|
||||
|
||||
|
||||
class TestIdleLoopPendingCheck:
|
||||
"""Tests for the idle-loop pending-delegation-results guard."""
|
||||
"""Tests for the idle-loop pending-delegation-results guard.
|
||||
|
||||
def test_no_file_means_proceed(self, tmp_path):
|
||||
Each test patches ``builtins.open`` so ``_check_delegation_results_pending``
|
||||
reads the controlled payload instead of the real DELEGATION_RESULTS_FILE.
|
||||
No filesystem side-effects.
|
||||
"""
|
||||
|
||||
def _patch_open(self, payload: str | None):
|
||||
"""Patch builtins.open for _check_delegation_results_pending.
|
||||
|
||||
Args:
|
||||
payload: file contents to return. None → FileNotFoundError.
|
||||
"""
|
||||
if payload is None:
|
||||
return patch("builtins.open", side_effect=FileNotFoundError)
|
||||
else:
|
||||
fake_file = io.StringIO(payload)
|
||||
return patch("builtins.open", return_value=fake_file)
|
||||
|
||||
def test_no_file_means_proceed(self):
|
||||
"""No delegation results file → idle loop fires normally."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
assert not check_results_pending(str(results_file))
|
||||
with self._patch_open(None):
|
||||
assert _check_delegation_results_pending() is False
|
||||
|
||||
def test_empty_file_means_proceed(self, tmp_path):
|
||||
def test_empty_file_means_proceed(self):
|
||||
"""Empty file → no pending results → idle loop fires."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text("", encoding="utf-8")
|
||||
assert not check_results_pending(str(results_file))
|
||||
with self._patch_open(""):
|
||||
assert _check_delegation_results_pending() is False
|
||||
|
||||
def test_whitespace_only_file_means_proceed(self, tmp_path):
|
||||
def test_whitespace_only_file_means_proceed(self):
|
||||
"""File with only whitespace → treated as empty → idle loop fires."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text(" \n ", encoding="utf-8")
|
||||
assert not check_results_pending(str(results_file))
|
||||
with self._patch_open(" \n "):
|
||||
assert _check_delegation_results_pending() is False
|
||||
|
||||
def test_single_result_means_skip(self, tmp_path):
|
||||
def test_single_result_means_skip(self):
|
||||
"""File with one delegation result → skip idle tick."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text(
|
||||
payload = (
|
||||
json.dumps({
|
||||
"status": "completed",
|
||||
"delegation_id": "del-abc",
|
||||
"summary": "Done",
|
||||
}) + "\n",
|
||||
encoding="utf-8",
|
||||
}) + "\n"
|
||||
)
|
||||
assert check_results_pending(str(results_file))
|
||||
with self._patch_open(payload):
|
||||
assert _check_delegation_results_pending() is True
|
||||
|
||||
def test_multiple_results_means_skip(self, tmp_path):
|
||||
def test_multiple_results_means_skip(self):
|
||||
"""File with multiple delegation results → skip idle tick."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text(
|
||||
payload = (
|
||||
json.dumps({"status": "completed", "delegation_id": "del-1", "summary": "A"})
|
||||
+ "\n"
|
||||
+ json.dumps({"status": "failed", "delegation_id": "del-2", "summary": "B"})
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
+ "\n"
|
||||
)
|
||||
assert check_results_pending(str(results_file))
|
||||
with self._patch_open(payload):
|
||||
assert _check_delegation_results_pending() is True
|
||||
|
||||
def test_file_with_only_newline_means_proceed(self, tmp_path):
|
||||
def test_file_with_only_newline_means_proceed(self):
|
||||
"""File with only a newline character → stripped to empty → fires."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text("\n", encoding="utf-8")
|
||||
assert not check_results_pending(str(results_file))
|
||||
with self._patch_open("\n"):
|
||||
assert _check_delegation_results_pending() is False
|
||||
|
||||
Reference in New Issue
Block a user