Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 16a1210abd |
@@ -1 +0,0 @@
|
||||
refire:1778784369
|
||||
@@ -49,11 +49,6 @@ if [ "$MERGED" != "true" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# NOTE: no || true — with set -euo pipefail, jq parse failures (e.g. field
|
||||
# missing from API response) propagate as hard errors. Use jq's // operator
|
||||
# for graceful defaults instead of bash || true guards. This was re-added by
|
||||
# 8c343e3a ("fix(gitea): add || true guards to jq pipelines") — reverted
|
||||
# here because the guards mask silent failures that hide malformed API responses.
|
||||
MERGE_SHA=$(echo "$PR" | jq -r '.merge_commit_sha // empty')
|
||||
MERGED_BY=$(echo "$PR" | jq -r '.merged_by.login // "unknown"')
|
||||
TITLE=$(echo "$PR" | jq -r '.title // ""')
|
||||
@@ -102,9 +97,6 @@ fi
|
||||
|
||||
# 5. Emit structured audit event.
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
# jq -R (raw input) converts each line to a JSON string; jq -s wraps into array.
|
||||
# If FAILED_CHECKS is unexpectedly empty (shouldn't happen — we exit above),
|
||||
# this produces []. No || true needed.
|
||||
FAILED_JSON=$(printf '%s\n' "${FAILED_CHECKS[@]}" | jq -R . | jq -s .)
|
||||
|
||||
# Print as a single-line JSON so Vector's parse_json transform can pick
|
||||
|
||||
@@ -203,17 +203,12 @@ def ci_jobs_all(ci_doc: dict) -> set[str]:
|
||||
|
||||
def ci_job_names(ci_doc: dict) -> set[str]:
|
||||
"""Set of job keys in ci.yml MINUS the sentinel itself MINUS jobs
|
||||
whose `if:` gates on `github.event_name` or `github.ref` (those are
|
||||
event-scoped and can legitimately be `skipped` for a given trigger;
|
||||
if we required them under the sentinel `needs:`, every PR-only job
|
||||
whose `if:` gates on `github.event_name` (those are event-scoped
|
||||
and can legitimately be `skipped` for a given trigger; if we
|
||||
required them under the sentinel `needs:`, every PR-only job
|
||||
would be `skipped` on push and the sentinel would interpret
|
||||
`skipped != success` as failure). RFC §4 spec.
|
||||
|
||||
`github.ref` is the companion gate for jobs that run only on direct
|
||||
pushes to specific branches (e.g. `github.ref == 'refs/heads/main'`).
|
||||
These never execute in a PR context, so flagging them as missing
|
||||
from `all-required.needs:` is a false positive (mc#958 / mc#959).
|
||||
|
||||
Used for F1 (jobs missing from sentinel needs). NOT used for F1b
|
||||
(typos in needs) — see `ci_jobs_all` for that."""
|
||||
jobs = ci_doc.get("jobs")
|
||||
@@ -226,9 +221,7 @@ def ci_job_names(ci_doc: dict) -> set[str]:
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
gate = v.get("if")
|
||||
if isinstance(gate, str) and (
|
||||
"github.event_name" in gate or "github.ref" in gate
|
||||
):
|
||||
if isinstance(gate, str) and "github.event_name" in gate:
|
||||
continue
|
||||
names.add(k)
|
||||
return names
|
||||
@@ -308,19 +301,7 @@ def expected_context(job_key: str, workflow_name: str = "ci") -> str:
|
||||
# Drift detection
|
||||
# --------------------------------------------------------------------------
|
||||
def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
"""Returns (findings, debug). Empty findings == no drift.
|
||||
|
||||
Raises:
|
||||
ApiError: propagated from the protection fetch only when the
|
||||
failure is likely a transient Gitea outage (5xx).
|
||||
403/404 from the protection endpoint is treated as
|
||||
"cannot determine drift for this branch" — a token-
|
||||
scope issue (missing repo-admin on DRIFT_BOT_TOKEN) or
|
||||
a repo with no protection set should not turn the
|
||||
hourly cron red. The workflow continues to the next
|
||||
branch; no [ci-drift] issue is filed for a branch
|
||||
whose protection cannot be read.
|
||||
"""
|
||||
"""Returns (findings, debug). Empty findings == no drift."""
|
||||
findings: list[str] = []
|
||||
|
||||
ci_doc = load_yaml(CI_WORKFLOW_PATH)
|
||||
@@ -332,50 +313,9 @@ def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
env_set = required_checks_env(audit_doc)
|
||||
|
||||
# Protection
|
||||
# api() raises ApiError on non-2xx. Transient 5xx should fail loud.
|
||||
# 403/404 means the token lacks repo-admin scope (Gitea 1.22.6's
|
||||
# branch_protections endpoint requires it — see DRIFT_BOT_TOKEN
|
||||
# provisioning trail in ci-required-drift.yml). Treat as
|
||||
# "cannot determine drift for this branch" — skip without turning
|
||||
# the workflow red. Surface a clear diagnostic so the operator
|
||||
# knows what to fix.
|
||||
contexts: set[str] = set()
|
||||
protection_path = f"/repos/{OWNER}/{NAME}/branch_protections/{branch}"
|
||||
try:
|
||||
_, protection = api("GET", protection_path)
|
||||
except ApiError as e:
|
||||
# Isolate the HTTP status from the error message.
|
||||
http_status: int | None = None
|
||||
msg = str(e)
|
||||
# ApiError message format: "{method} {path} → HTTP {status}: {body}"
|
||||
import re as _re
|
||||
|
||||
m = _re.search(r"HTTP (\d{3})", msg)
|
||||
if m:
|
||||
http_status = int(m.group(1))
|
||||
if http_status in (403, 404):
|
||||
# Token lacks scope OR branch has no protection. Cannot
|
||||
# determine drift — skip this branch. Do NOT exit non-zero;
|
||||
# the issue IS the alarm, not a red workflow.
|
||||
sys.stderr.write(
|
||||
f"::error::GET {protection_path} returned HTTP {http_status} — "
|
||||
f"DRIFT_BOT_TOKEN lacks repo-admin scope (Gitea 1.22.6 "
|
||||
f"requires it for this endpoint) OR branch has no protection "
|
||||
f"configured. Cannot determine drift for {branch}; "
|
||||
f"skipping. Fix: grant repo-admin to mc-drift-bot or "
|
||||
f"configure protection on {branch}.\n"
|
||||
)
|
||||
debug = {
|
||||
"branch": branch,
|
||||
"ci_jobs": sorted(jobs),
|
||||
"sentinel_needs": sorted(needs),
|
||||
"protection_contexts_skipped": True,
|
||||
"protection_http_status": http_status,
|
||||
"audit_env_checks": sorted(env_set),
|
||||
}
|
||||
return [], debug
|
||||
# 5xx — propagate (transient outage, fail loud per design).
|
||||
raise
|
||||
# api() raises ApiError on non-2xx; let it propagate so a transient
|
||||
# 500 fails the run loudly rather than producing a "no drift" lie.
|
||||
_, protection = api("GET", f"/repos/{OWNER}/{NAME}/branch_protections/{branch}")
|
||||
if not isinstance(protection, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::protection response for {branch} not a JSON object\n"
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract changed-file list from Gitea Compare API JSON response.
|
||||
|
||||
Gitea Compare API returns changed files nested inside commits, not at the
|
||||
top level:
|
||||
{"commits": [{"files": [{"filename": "path/to/file"}]}]}
|
||||
|
||||
Usage:
|
||||
compare-api-diff-files.py < API_RESPONSE.json
|
||||
|
||||
Exits 0 with filenames on stdout, one per line.
|
||||
Exits 1 on malformed input (caller should handle as "no files").
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
|
||||
filenames: list[str] = []
|
||||
for commit in data.get("commits", []):
|
||||
for f in commit.get("files", []):
|
||||
fn = f.get("filename", "")
|
||||
if fn:
|
||||
filenames.append(fn)
|
||||
|
||||
if filenames:
|
||||
sys.stdout.write("\n".join(filenames))
|
||||
sys.stdout.write("\n")
|
||||
# else: empty stdout = no files, caller treats as empty list
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,174 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared path-filter helper for Gitea Actions workflows.
|
||||
|
||||
Computes changed files against the PR base SHA or push-before SHA and writes
|
||||
boolean outputs to GITHUB_OUTPUT. If the diff base is missing or untrusted, the
|
||||
helper fails open by setting every output in the selected profile to true.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROFILES: dict[str, dict[str, str]] = {
|
||||
"ci": {
|
||||
"platform": r"^workspace-server/",
|
||||
"canvas": r"^canvas/",
|
||||
"python": r"^workspace/",
|
||||
"scripts": r"^tests/e2e/|^scripts/|^infra/scripts/",
|
||||
},
|
||||
"handlers-postgres": {
|
||||
"handlers": (
|
||||
r"^workspace-server/internal/handlers/"
|
||||
r"|^workspace-server/internal/wsauth/"
|
||||
r"|^workspace-server/migrations/"
|
||||
r"|^\.gitea/workflows/handlers-postgres-integration\.yml$"
|
||||
),
|
||||
},
|
||||
"e2e-api": {
|
||||
"api": r"^workspace-server/|^tests/e2e/|^\.gitea/workflows/e2e-api\.yml$",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def classify(profile: str, paths: list[str]) -> dict[str, bool]:
|
||||
patterns = PROFILES[profile]
|
||||
return {
|
||||
name: any(re.search(pattern, path) for path in paths)
|
||||
for name, pattern in patterns.items()
|
||||
}
|
||||
|
||||
|
||||
def all_true(profile: str) -> dict[str, bool]:
|
||||
return {name: True for name in PROFILES[profile]}
|
||||
|
||||
|
||||
def resolve_base(event_name: str, pr_base_sha: str, push_before: str) -> str:
|
||||
if event_name == "pull_request" and pr_base_sha:
|
||||
return pr_base_sha
|
||||
return push_before
|
||||
|
||||
|
||||
def is_zero_sha(value: str) -> bool:
|
||||
return not value or bool(re.fullmatch(r"0+", value))
|
||||
|
||||
|
||||
def run_git(args: list[str], *, timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["git", *args],
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def base_exists(base: str) -> bool:
|
||||
return run_git(["cat-file", "-e", base]).returncode == 0
|
||||
|
||||
|
||||
def fetch_base(base: str, base_ref: str) -> None:
|
||||
# Gitea may reject fetching an arbitrary unadvertised SHA from a shallow
|
||||
# PR checkout. Fetch the advertised base branch first, then fall back to
|
||||
# the SHA for hosts that allow it.
|
||||
if base_ref:
|
||||
run_git(["fetch", "--depth=1", "origin", base_ref])
|
||||
if not base_exists(base):
|
||||
run_git(["fetch", "--depth=1", "origin", base])
|
||||
|
||||
|
||||
def deepen_base_ref(base_ref: str) -> None:
|
||||
if base_ref:
|
||||
run_git(["fetch", "--deepen=200", "origin", base_ref], timeout=60)
|
||||
|
||||
|
||||
def merge_base(base: str) -> str | None:
|
||||
proc = run_git(["merge-base", base, "HEAD"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
value = proc.stdout.strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def changed_paths(base: str, *, use_merge_base: bool) -> list[str] | None:
|
||||
compare_base = base
|
||||
if use_merge_base:
|
||||
compare_base = merge_base(base) or ""
|
||||
if not compare_base:
|
||||
return None
|
||||
|
||||
proc = run_git(["diff", "--name-only", compare_base, "HEAD"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return [line for line in proc.stdout.splitlines() if line]
|
||||
|
||||
|
||||
def write_outputs(values: dict[str, bool], output_path: str | None) -> None:
|
||||
lines = [f"{name}={'true' if value else 'false'}" for name, value in values.items()]
|
||||
if output_path:
|
||||
with Path(output_path).open("a", encoding="utf-8") as fh:
|
||||
for line in lines:
|
||||
fh.write(line + "\n")
|
||||
else:
|
||||
for line in lines:
|
||||
print(line)
|
||||
|
||||
|
||||
def detect(
|
||||
profile: str,
|
||||
event_name: str,
|
||||
pr_base_sha: str,
|
||||
push_before: str,
|
||||
base_ref: str = "",
|
||||
) -> dict[str, bool]:
|
||||
base = resolve_base(event_name, pr_base_sha, push_before)
|
||||
if is_zero_sha(base):
|
||||
return all_true(profile)
|
||||
|
||||
if not base_exists(base):
|
||||
fetch_base(base, base_ref)
|
||||
if not base_exists(base):
|
||||
return all_true(profile)
|
||||
|
||||
use_merge_base = event_name == "pull_request"
|
||||
if use_merge_base and base_ref and merge_base(base) is None:
|
||||
deepen_base_ref(base_ref)
|
||||
|
||||
paths = changed_paths(base, use_merge_base=use_merge_base)
|
||||
if paths is None:
|
||||
return all_true(profile)
|
||||
return classify(profile, paths)
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--profile", required=True, choices=sorted(PROFILES))
|
||||
parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME", ""))
|
||||
parser.add_argument("--pr-base-sha", default="")
|
||||
parser.add_argument("--base-ref", default="")
|
||||
parser.add_argument("--push-before", default=os.environ.get("GITHUB_EVENT_BEFORE", ""))
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
values = detect(
|
||||
args.profile,
|
||||
args.event_name,
|
||||
args.pr_base_sha,
|
||||
args.push_before,
|
||||
args.base_ref,
|
||||
)
|
||||
write_outputs(values, os.environ.get("GITHUB_OUTPUT"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -1,501 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""gitea-merge-queue — conservative serialized merge bot for Gitea.
|
||||
|
||||
Gitea 1.22.6 has auto-merge (`pull_auto_merge`) but no GitHub-style merge
|
||||
queue. This script provides the missing serialized policy in user space:
|
||||
|
||||
1. Pick the oldest open PR carrying QUEUE_LABEL.
|
||||
2. Refuse to act unless main is green.
|
||||
3. Refuse fork PRs; the queue may only mutate same-repo branches.
|
||||
4. If the PR branch does not contain current main, call Gitea's
|
||||
/pulls/{n}/update endpoint and stop. CI must rerun on the updated head.
|
||||
5. If the updated PR head has all required contexts green, merge with the
|
||||
non-bypass merge actor token.
|
||||
|
||||
The script is intentionally one-PR-per-run. Workflow/cron concurrency should
|
||||
serialize invocations so two green PRs cannot merge against the same main.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _env(key: str, *, default: str = "") -> str:
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN")
|
||||
GITEA_HOST = _env("GITEA_HOST")
|
||||
REPO = _env("REPO")
|
||||
WATCH_BRANCH = _env("WATCH_BRANCH", default="main")
|
||||
QUEUE_LABEL = _env("QUEUE_LABEL", default="merge-queue")
|
||||
HOLD_LABEL = _env("HOLD_LABEL", default="merge-queue-hold")
|
||||
UPDATE_STYLE = _env("UPDATE_STYLE", default="merge")
|
||||
REQUIRED_CONTEXTS_RAW = _env(
|
||||
"REQUIRED_CONTEXTS",
|
||||
default=(
|
||||
"CI / all-required (pull_request),"
|
||||
"sop-checklist / all-items-acked (pull_request)"
|
||||
),
|
||||
)
|
||||
# Required contexts for push (main/staging) runs. The push CI uses the same
|
||||
# aggregator names with " (push)" suffix. Checking these explicitly instead of
|
||||
# the combined state avoids false-pause when non-blocking jobs (e.g. Platform
|
||||
# Go with continue-on-error: true due to mc#774) have failed — their failures
|
||||
# pollute the combined state but do not block merges.
|
||||
PUSH_REQUIRED_CONTEXTS_RAW = _env(
|
||||
"PUSH_REQUIRED_CONTEXTS",
|
||||
default="CI / all-required (push)",
|
||||
)
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
|
||||
class ApiError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class MergePermissionError(ApiError):
|
||||
"""Merge failed with a permanent permission error (403/404/405).
|
||||
The queue should skip this PR and move to the next one."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class MergeDecision:
|
||||
ready: bool
|
||||
action: str
|
||||
reason: str
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "WATCH_BRANCH", "QUEUE_LABEL"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
if UPDATE_STYLE not in {"merge", "rebase"}:
|
||||
sys.stderr.write("::error::UPDATE_STYLE must be merge or rebase\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read()
|
||||
status = exc.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {path} -> HTTP {status}: {snippet}")
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
if expect_json:
|
||||
raise ApiError(f"{method} {path} -> HTTP {status} non-JSON: {exc}") from exc
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
def required_contexts(raw: str) -> list[str]:
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def push_required_contexts() -> list[str]:
|
||||
"""Required contexts for push (branch) CI runs. See PUSH_REQUIRED_CONTEXTS_RAW."""
|
||||
return required_contexts(PUSH_REQUIRED_CONTEXTS_RAW)
|
||||
|
||||
|
||||
def status_state(status: dict) -> str:
|
||||
return str(status.get("status") or status.get("state") or "").lower()
|
||||
|
||||
|
||||
def latest_statuses_by_context(statuses: list[dict]) -> dict[str, dict]:
|
||||
# Gitea /statuses endpoint returns entries in ascending id order (oldest
|
||||
# first). We need the LAST occurrence of each context, so iterate in
|
||||
# reverse to prefer newer entries.
|
||||
latest: dict[str, dict] = {}
|
||||
for status in reversed(statuses):
|
||||
context = status.get("context")
|
||||
if isinstance(context, str):
|
||||
latest[context] = status # overwrite: reverse order → newest wins
|
||||
return latest
|
||||
|
||||
|
||||
def _is_tier_low_pending_ok(
|
||||
latest_statuses: dict[str, dict],
|
||||
context: str,
|
||||
pr_labels: set[str],
|
||||
) -> bool:
|
||||
"""Return True if tier:low PR can tolerate sop-checklist pending state.
|
||||
|
||||
Per sop-checklist-config.yaml tier_failure_mode, tier:low uses soft-fail:
|
||||
sop-checklist posts state=pending when acks are satisfied (missing
|
||||
manager/ceo acks are informational only). The queue should accept
|
||||
pending instead of waiting for success.
|
||||
"""
|
||||
if "tier:low" not in pr_labels:
|
||||
return False
|
||||
if "sop-checklist" not in context:
|
||||
return False
|
||||
status = latest_statuses.get(context) or {}
|
||||
return status_state(status) == "pending"
|
||||
|
||||
|
||||
def required_contexts_green(
|
||||
latest_statuses: dict[str, dict],
|
||||
contexts: list[str],
|
||||
pr_labels: set[str] | None = None,
|
||||
) -> tuple[bool, list[str]]:
|
||||
missing_or_bad: list[str] = []
|
||||
for context in contexts:
|
||||
status = latest_statuses.get(context)
|
||||
state = status_state(status or {})
|
||||
if state != "success":
|
||||
if pr_labels and _is_tier_low_pending_ok(latest_statuses, context, pr_labels):
|
||||
continue # tier:low soft-fail: accept pending sop-checklist
|
||||
missing_or_bad.append(f"{context}={state or 'missing'}")
|
||||
return not missing_or_bad, missing_or_bad
|
||||
|
||||
|
||||
def label_names(issue: dict) -> set[str]:
|
||||
return {
|
||||
label["name"]
|
||||
for label in issue.get("labels", [])
|
||||
if isinstance(label, dict) and isinstance(label.get("name"), str)
|
||||
}
|
||||
|
||||
|
||||
def choose_next_queued_issue(
|
||||
issues: list[dict],
|
||||
*,
|
||||
queue_label: str,
|
||||
hold_label: str = "",
|
||||
) -> dict | None:
|
||||
candidates = []
|
||||
for issue in issues:
|
||||
labels = label_names(issue)
|
||||
if queue_label not in labels:
|
||||
continue
|
||||
if hold_label and hold_label in labels:
|
||||
continue
|
||||
if "pull_request" not in issue:
|
||||
continue
|
||||
candidates.append(issue)
|
||||
candidates.sort(key=lambda issue: (issue.get("created_at") or "", int(issue["number"])))
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def pr_contains_base_sha(commits: list[dict], base_sha: str) -> bool:
|
||||
for commit in commits:
|
||||
sha = commit.get("sha") or commit.get("id")
|
||||
if sha == base_sha:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def pr_has_current_base(pr: dict, commits: list[dict], main_sha: str) -> bool:
|
||||
if pr.get("merge_base") == main_sha:
|
||||
return True
|
||||
return pr_contains_base_sha(commits, main_sha)
|
||||
|
||||
|
||||
def evaluate_merge_readiness(
|
||||
*,
|
||||
main_status: dict,
|
||||
pr_status: dict,
|
||||
required_contexts: list[str],
|
||||
pr_has_current_base: bool,
|
||||
pr_labels: set[str] | None = None,
|
||||
) -> MergeDecision:
|
||||
# Check push-required contexts explicitly instead of combined state.
|
||||
# Combined state can be "failure" due to non-blocking jobs
|
||||
# (continue-on-error: true) that don't actually gate merges.
|
||||
# CI / all-required (push) is the authoritative gate — it respects
|
||||
# continue-on-error and correctly aggregates all blocking failures.
|
||||
main_latest = latest_statuses_by_context(main_status.get("statuses") or [])
|
||||
main_ok, main_bad = required_contexts_green(main_latest, push_required_contexts())
|
||||
if not main_ok:
|
||||
return MergeDecision(False, "pause", "main required contexts not green: " + ", ".join(main_bad))
|
||||
if not pr_has_current_base:
|
||||
return MergeDecision(False, "update", "PR head does not contain current main")
|
||||
|
||||
# Check explicit required contexts instead of combined state. Combined state
|
||||
# can be "failure" due to non-blocking jobs with continue-on-error: true
|
||||
# (e.g. publish-runtime-autobump/pr-validate, qa-review on stale tokens).
|
||||
# The required_contexts list is the authoritative gate — it includes only
|
||||
# the checks that actually block merges.
|
||||
latest = latest_statuses_by_context(pr_status.get("statuses") or [])
|
||||
ok, missing_or_bad = required_contexts_green(latest, required_contexts, pr_labels)
|
||||
if not ok:
|
||||
return MergeDecision(False, "wait", "required contexts not green: " + ", ".join(missing_or_bad))
|
||||
return MergeDecision(True, "merge", "ready")
|
||||
|
||||
|
||||
def get_branch_head(branch: str) -> str:
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/branches/{branch}")
|
||||
commit = body.get("commit") if isinstance(body, dict) else None
|
||||
sha = commit.get("id") if isinstance(commit, dict) else None
|
||||
if not isinstance(sha, str) or len(sha) < 7:
|
||||
raise ApiError(f"branch {branch} response missing commit id")
|
||||
return sha
|
||||
|
||||
|
||||
def get_combined_status(sha: str) -> dict:
|
||||
"""Combined status + all individual statuses for `sha`.
|
||||
|
||||
The /status endpoint caps the `statuses` array at 30 entries (Gitea
|
||||
default page size), so we fetch the full list via /statuses with a
|
||||
higher limit. The combined `state` still comes from /status.
|
||||
"""
|
||||
_, combined = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
|
||||
if not isinstance(combined, dict):
|
||||
raise ApiError(f"status for {sha} response not object")
|
||||
combined_statuses: list[dict] = combined.get("statuses") or []
|
||||
try:
|
||||
_, all_statuses_raw = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/commits/{sha}/statuses",
|
||||
query={"limit": "50"},
|
||||
)
|
||||
if isinstance(all_statuses_raw, list):
|
||||
all_statuses: list[dict] = list(all_statuses_raw)
|
||||
else:
|
||||
all_statuses = []
|
||||
except (ApiError, urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
sys.stderr.write(f"::warning::could not fetch full statuses list for {sha[:8]}: {exc}\n")
|
||||
all_statuses = []
|
||||
# Build latest per context: process combined (ascending→reverse=newest
|
||||
# first), then fill gaps from all_statuses (already newest-first).
|
||||
latest: dict[str, dict] = {}
|
||||
for status in reversed(sorted(combined_statuses, key=lambda s: s.get("id") or 0)):
|
||||
ctx = status.get("context")
|
||||
if isinstance(ctx, str) and ctx not in latest:
|
||||
latest[ctx] = status
|
||||
for status in all_statuses:
|
||||
ctx = status.get("context")
|
||||
if isinstance(ctx, str) and ctx not in latest:
|
||||
latest[ctx] = status
|
||||
combined["statuses"] = list(latest.values())
|
||||
return combined
|
||||
|
||||
|
||||
def list_queued_issues() -> list[dict]:
|
||||
_, body = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={
|
||||
"state": "open",
|
||||
"type": "pulls",
|
||||
"labels": QUEUE_LABEL,
|
||||
"limit": "50",
|
||||
},
|
||||
)
|
||||
if not isinstance(body, list):
|
||||
raise ApiError("queued issues response not list")
|
||||
return body
|
||||
|
||||
|
||||
def get_pull(pr_number: int) -> dict:
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"PR #{pr_number} response not object")
|
||||
return body
|
||||
|
||||
|
||||
def get_pull_commits(pr_number: int) -> list[dict]:
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/commits")
|
||||
if not isinstance(body, list):
|
||||
raise ApiError(f"PR #{pr_number} commits response not list")
|
||||
return body
|
||||
|
||||
|
||||
def post_comment(pr_number: int, body: str, *, dry_run: bool) -> None:
|
||||
print(f"::notice::comment PR #{pr_number}: {body.splitlines()[0][:160]}")
|
||||
if dry_run:
|
||||
return
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/issues/{pr_number}/comments", body={"body": body})
|
||||
|
||||
|
||||
def update_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
print(f"::notice::updating PR #{pr_number} with base branch via style={UPDATE_STYLE}")
|
||||
if dry_run:
|
||||
return
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/update",
|
||||
query={"style": UPDATE_STYLE},
|
||||
expect_json=False,
|
||||
)
|
||||
|
||||
|
||||
def merge_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
payload = {
|
||||
"Do": "merge",
|
||||
"MergeTitleField": f"Merge PR #{pr_number} via Gitea merge queue",
|
||||
"MergeMessageField": (
|
||||
"Serialized merge by gitea-merge-queue after current-main, "
|
||||
"SOP, and required CI checks were green."
|
||||
),
|
||||
}
|
||||
print(f"::notice::merging PR #{pr_number}")
|
||||
if dry_run:
|
||||
return
|
||||
try:
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
|
||||
except ApiError as exc:
|
||||
# Re-raise permission-like errors so process_once can skip this PR.
|
||||
# 403 = no push access, 404 = repo/pr not found, 405 = not allowed.
|
||||
msg = str(exc)
|
||||
for code in ("403", "404", "405"):
|
||||
if code in msg:
|
||||
raise MergePermissionError(msg) from exc
|
||||
raise # re-raise other ApiErrors unchanged
|
||||
|
||||
|
||||
def process_once(*, dry_run: bool = False) -> int:
|
||||
contexts = required_contexts(REQUIRED_CONTEXTS_RAW)
|
||||
main_sha = get_branch_head(WATCH_BRANCH)
|
||||
main_status = get_combined_status(main_sha)
|
||||
# Check push-required contexts explicitly instead of combined state.
|
||||
# See evaluate_merge_readiness for rationale.
|
||||
main_latest = latest_statuses_by_context(main_status.get("statuses") or [])
|
||||
main_ok, main_bad = required_contexts_green(main_latest, push_required_contexts())
|
||||
if not main_ok:
|
||||
print(f"::notice::queue paused: {WATCH_BRANCH}@{main_sha[:8]} required contexts not green: {', '.join(main_bad)}")
|
||||
return 0
|
||||
|
||||
issue = choose_next_queued_issue(
|
||||
list_queued_issues(),
|
||||
queue_label=QUEUE_LABEL,
|
||||
hold_label=HOLD_LABEL,
|
||||
)
|
||||
if not issue:
|
||||
print("::notice::merge queue empty")
|
||||
return 0
|
||||
|
||||
pr_number = int(issue["number"])
|
||||
pr = get_pull(pr_number)
|
||||
if pr.get("state") != "open":
|
||||
print(f"::notice::PR #{pr_number} is not open; skipping")
|
||||
return 0
|
||||
if pr.get("base", {}).get("ref") != WATCH_BRANCH:
|
||||
post_comment(pr_number, f"merge-queue: skipped; base branch is not `{WATCH_BRANCH}`.", dry_run=dry_run)
|
||||
return 0
|
||||
if pr.get("head", {}).get("repo_id") != pr.get("base", {}).get("repo_id"):
|
||||
post_comment(pr_number, "merge-queue: skipped; fork PRs are not supported by the serialized queue.", dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
head_sha = pr.get("head", {}).get("sha")
|
||||
if not isinstance(head_sha, str) or len(head_sha) < 7:
|
||||
raise ApiError(f"PR #{pr_number} missing head sha")
|
||||
commits = get_pull_commits(pr_number)
|
||||
current_base = pr_has_current_base(pr, commits, main_sha)
|
||||
pr_status = get_combined_status(head_sha)
|
||||
pr_labels = label_names(pr)
|
||||
decision = evaluate_merge_readiness(
|
||||
main_status=main_status,
|
||||
pr_status=pr_status,
|
||||
required_contexts=contexts,
|
||||
pr_has_current_base=current_base,
|
||||
pr_labels=pr_labels,
|
||||
)
|
||||
|
||||
print(f"::notice::PR #{pr_number} decision={decision.action}: {decision.reason}")
|
||||
if decision.action == "update":
|
||||
update_pull(pr_number, dry_run=dry_run)
|
||||
post_comment(
|
||||
pr_number,
|
||||
(
|
||||
f"merge-queue: updated this branch with `{WATCH_BRANCH}` at "
|
||||
f"`{main_sha[:12]}`. Waiting for CI on the refreshed head."
|
||||
),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
return 0
|
||||
if decision.ready:
|
||||
latest_main_sha = get_branch_head(WATCH_BRANCH)
|
||||
if latest_main_sha != main_sha:
|
||||
print(
|
||||
f"::notice::main moved {main_sha[:8]} -> {latest_main_sha[:8]}; "
|
||||
"deferring to next tick"
|
||||
)
|
||||
return 0
|
||||
try:
|
||||
merge_pull(pr_number, dry_run=dry_run)
|
||||
except MergePermissionError as exc:
|
||||
# Permanent merge failure (HTTP 403/404/405). Post a comment so
|
||||
# maintainers know why, then return 0 so this tick is done.
|
||||
# The PR stays in the queue; future ticks can retry after the
|
||||
# permission issue is resolved.
|
||||
sys.stderr.write(f"::error::merge permission error for PR #{pr_number}: {exc}\n")
|
||||
post_comment(
|
||||
pr_number,
|
||||
(
|
||||
"merge-queue: merge failed with HTTP 405 'User not allowed to merge PR'. "
|
||||
"No available token has Can-merge permission on this repo. "
|
||||
"Fix: grant Can-merge to a token, or add a maintain/admin collaborator. "
|
||||
"Skipping to next queued PR on next tick."
|
||||
),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
return 0
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
_require_runtime_env()
|
||||
try:
|
||||
return process_once(dry_run=args.dry_run)
|
||||
except ApiError as exc:
|
||||
# API errors (401/403/404/500) are transient for a queue tick —
|
||||
# log and exit 0 so the workflow is not marked failed and the next
|
||||
# tick can retry. Returning non-zero would permanently fail the
|
||||
# workflow run, blocking future ticks.
|
||||
sys.stderr.write(f"::error::queue API error: {exc}\n")
|
||||
return 0
|
||||
except urllib.error.URLError as exc:
|
||||
sys.stderr.write(f"::error::queue network error: {exc}\n")
|
||||
return 0
|
||||
except TimeoutError as exc:
|
||||
sys.stderr.write(f"::error::queue timeout: {exc}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lint workflow bash for curl status-code capture pollution.
|
||||
|
||||
The bad shape is:
|
||||
|
||||
HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
|
||||
|
||||
`curl -w` writes the HTTP code to stdout before returning non-zero, so
|
||||
fallback output inside the same command substitution appends another code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
|
||||
SELF = ".gitea/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
|
||||
class Finding(NamedTuple):
|
||||
path: str
|
||||
snippet: str
|
||||
|
||||
|
||||
BAD_STATUS_CAPTURE = re.compile(
|
||||
r"""
|
||||
\$\(\s*
|
||||
curl\b
|
||||
[^)]*
|
||||
-w\s*['"]%\{http_code\}['"]
|
||||
[^)]*
|
||||
\|\|\s*
|
||||
(?:
|
||||
echo\s+['"]?000['"]?
|
||||
|
|
||||
printf\s+['"]000['"]
|
||||
)
|
||||
\s*\)
|
||||
""",
|
||||
re.DOTALL | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _logical_shell(content: str) -> str:
|
||||
"""Collapse bash line continuations so one curl command is one string."""
|
||||
return re.sub(r"\\\s*\n\s*", " ", content)
|
||||
|
||||
|
||||
def scan_content(path: str, content: str) -> list[Finding]:
|
||||
flat = _logical_shell(content)
|
||||
return [
|
||||
Finding(path=path, snippet=re.sub(r"\s+", " ", match.group(0)).strip()[:160])
|
||||
for match in BAD_STATUS_CAPTURE.finditer(flat)
|
||||
]
|
||||
|
||||
|
||||
def scan_paths(paths: list[str]) -> list[Finding]:
|
||||
findings: list[Finding] = []
|
||||
for path in paths:
|
||||
if path == SELF:
|
||||
continue
|
||||
content = Path(path).read_text(encoding="utf-8")
|
||||
findings.extend(scan_content(path, content))
|
||||
return findings
|
||||
|
||||
|
||||
def default_paths() -> list[str]:
|
||||
return sorted(glob.glob(".gitea/workflows/*.yml"))
|
||||
|
||||
|
||||
def print_report(findings: list[Finding]) -> None:
|
||||
if not findings:
|
||||
print("OK No curl-status-capture pollution patterns detected")
|
||||
return
|
||||
|
||||
print(f"::error::Found {len(findings)} curl-status-capture pollution site(s):")
|
||||
for finding in findings:
|
||||
print(
|
||||
f"::error file={finding.path}::Curl status-capture pollution: "
|
||||
"'|| echo/printf 000' inside a $(curl ... -w '%{http_code}' ...) "
|
||||
"subshell. On non-2xx or connection failure, curl's -w writes a "
|
||||
"status, then exits non-zero, then the fallback appends another "
|
||||
"status. Fix: route -w into a tempfile so the exit code cannot "
|
||||
"pollute stdout."
|
||||
)
|
||||
print(f" matched: {finding.snippet}...")
|
||||
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(" set +e")
|
||||
print(" curl ... -w '%{http_code}' >code.txt 2>/dev/null")
|
||||
print(" set -e")
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("paths", nargs="*", help="workflow files to scan")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
paths = args.paths or default_paths()
|
||||
findings = scan_paths(paths)
|
||||
print_report(findings)
|
||||
return 1 if findings else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,404 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint-required-no-paths — structural enforcement of
|
||||
`feedback_path_filtered_workflow_cant_be_required`.
|
||||
|
||||
For every workflow whose status-check context appears in
|
||||
`branch_protections/<branch>.status_check_contexts`, assert that the
|
||||
workflow's `on:` block has NO `paths:` and NO `paths-ignore:` filter.
|
||||
|
||||
A required-check workflow with a paths filter silently degrades the
|
||||
merge gate:
|
||||
|
||||
- If the PR's diff doesn't match the `paths:` glob, the workflow
|
||||
never fires.
|
||||
- Gitea (1.22.6) reports the required context as `pending` (never as
|
||||
`skipped == success`), so the PR cannot merge.
|
||||
- For a docs-only PR against `paths: ['**.go']`, the PR is
|
||||
blocked forever — no human action can produce a green.
|
||||
|
||||
The class was previously prevented only by reviewer vigilance + the
|
||||
saved memory `feedback_path_filtered_workflow_cant_be_required`. This
|
||||
script makes it a hard CI gate so a future PR adding `paths:` to a
|
||||
required workflow fails fast at PR time, not after merge when the next
|
||||
docs PR wedges main.
|
||||
|
||||
The lint runs as `.gitea/workflows/lint-required-no-paths.yml` on every
|
||||
PR. The lint workflow ITSELF must not have a paths-filter (otherwise it
|
||||
could be circumvented by a paths-non-matching PR) — that's enforced by
|
||||
self-reference and by the workflow's own `on:` block deliberately
|
||||
omitting filters.
|
||||
|
||||
Sources of truth:
|
||||
- `branch_protections/<branch>` `status_check_contexts` (the merge gate)
|
||||
- `.gitea/workflows/*.yml` `name:` + `on:` (the workflow set)
|
||||
|
||||
Context-format note (Gitea 1.22.6):
|
||||
Status-check contexts are formatted `{workflow_name} / {job_name_or_key} ({event})`.
|
||||
We parse the workflow_name prefix and walk `.gitea/workflows/*.yml` for
|
||||
a file whose `name:` attr matches. (The filename is NOT the source of
|
||||
truth; `name:` is, because Gitea formats the context from `name:`.)
|
||||
|
||||
Exit codes:
|
||||
0 — no required workflow has a paths/paths-ignore filter (clean) OR
|
||||
branch_protections endpoint returned 403/404 (token-scope issue;
|
||||
surfaced via ::error:: but non-fatal so a missing scope doesn't
|
||||
red-X every PR — fix the token, not the lint).
|
||||
1 — at least one required workflow has a paths/paths-ignore filter
|
||||
(the gate-degrading defect class).
|
||||
2 — env contract violation (missing GITEA_TOKEN/HOST/REPO/BRANCH).
|
||||
3 — workflows directory missing or workflow YAML unparseable.
|
||||
4 — protection response shape unexpected (non-dict body on 2xx).
|
||||
|
||||
Auth note: `GET /repos/.../branch_protections/{branch}` requires
|
||||
repo-admin role in Gitea 1.22.6. The workflow-default `GITHUB_TOKEN`
|
||||
is non-admin; we re-use `DRIFT_BOT_TOKEN` (same persona that powers
|
||||
ci-required-drift.yml). If `DRIFT_BOT_TOKEN` is unavailable in a future
|
||||
context, the script falls through gracefully (exit 0 + ::error::).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml # PyYAML 6.0.2 — installed by the workflow before this runs.
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment
|
||||
# --------------------------------------------------------------------------
|
||||
def _env(key: str, *, required: bool = True, default: str | None = None) -> str:
|
||||
val = os.environ.get(key, default)
|
||||
if required and not val:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return val or ""
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN", required=False)
|
||||
GITEA_HOST = _env("GITEA_HOST", required=False)
|
||||
REPO = _env("REPO", required=False)
|
||||
BRANCH = _env("BRANCH", required=False, default="main")
|
||||
WORKFLOWS_DIR = _env(
|
||||
"WORKFLOWS_DIR", required=False, default=".gitea/workflows"
|
||||
)
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `run()` only. Tests import
|
||||
individual functions without setting the full env contract."""
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "BRANCH"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper (mirrors ci-required-drift.py contract:
|
||||
# raise on non-2xx and on JSON-decode-fail when JSON expected, per
|
||||
# `feedback_api_helper_must_raise_not_return_dict`).
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API call cannot be trusted to have succeeded."""
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {path} → HTTP {status}: {snippet}")
|
||||
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
if expect_json:
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status} but body is not JSON: {e}"
|
||||
) from e
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Status-check context parser
|
||||
# --------------------------------------------------------------------------
|
||||
# Format: "<workflow_name> / <job_name_or_key> (<event>)"
|
||||
# Examples observed on molecule-core/main:
|
||||
# "Secret scan / Scan diff for credential-shaped strings (pull_request)"
|
||||
# "sop-tier-check / tier-check (pull_request)"
|
||||
#
|
||||
# Split strategy: peel off the trailing ` (<event>)` first, then split
|
||||
# the leading `<workflow> / <rest>` on the FIRST ` / ` (workflow names
|
||||
# come from `name:` attrs which conventionally don't embed ' / '; job
|
||||
# names CAN, so we keep the rest of the slash-divided text as the job
|
||||
# name). This matches Gitea's `name: ` semantics.
|
||||
_CONTEXT_RE = re.compile(r"^(?P<workflow>.+?) / (?P<job>.+) \((?P<event>[^)]+)\)$")
|
||||
|
||||
|
||||
def parse_context(ctx: str) -> tuple[str, str, str] | None:
|
||||
"""Parse `<workflow> / <job> (<event>)` → (workflow, job, event) or None."""
|
||||
if not ctx:
|
||||
return None
|
||||
m = _CONTEXT_RE.match(ctx)
|
||||
if not m:
|
||||
return None
|
||||
return m.group("workflow"), m.group("job"), m.group("event")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# workflow-name → file resolution
|
||||
# --------------------------------------------------------------------------
|
||||
def _iter_workflow_files() -> list[Path]:
|
||||
d = Path(WORKFLOWS_DIR)
|
||||
if not d.is_dir():
|
||||
sys.stderr.write(f"::error::workflows directory not found: {d}\n")
|
||||
sys.exit(3)
|
||||
# `.yml` and `.yaml` — Gitea accepts both (rarely used `.yaml`, but
|
||||
# don't silently miss it if a future port uses it).
|
||||
return sorted(list(d.glob("*.yml")) + list(d.glob("*.yaml")))
|
||||
|
||||
|
||||
def resolve_workflow_file(workflow_name: str) -> Path | None:
|
||||
"""Find the YAML file whose `name:` attr matches `workflow_name`.
|
||||
|
||||
Returns None if no match. Filename is NOT used as a fallback —
|
||||
Gitea's context format uses `name:`, so a `name:`-less workflow
|
||||
won't even appear in the protection list. (A YAML with no `name:`
|
||||
would default the context to the file basename, but our protection
|
||||
contexts on molecule-core are all `name:`-derived; we trust the
|
||||
same.)
|
||||
"""
|
||||
for f in _iter_workflow_files():
|
||||
try:
|
||||
doc = yaml.safe_load(f.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(f"::error::YAML parse error in {f}: {e}\n")
|
||||
sys.exit(3)
|
||||
if isinstance(doc, dict) and doc.get("name") == workflow_name:
|
||||
return f
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# paths-filter detection
|
||||
# --------------------------------------------------------------------------
|
||||
# Triggers that accept `paths:` / `paths-ignore:` (per GitHub Actions /
|
||||
# Gitea Actions docs): pull_request, pull_request_target, push.
|
||||
# We don't enumerate — any sub-key named `paths` or `paths-ignore`
|
||||
# inside an event mapping is flagged.
|
||||
_PATHS_KEYS = ("paths", "paths-ignore")
|
||||
|
||||
|
||||
def detect_paths_filters(workflow_path: Path) -> list[str]:
|
||||
"""Walk the workflow's `on:` block and return a list of findings, one
|
||||
per offending `paths`/`paths-ignore` key.
|
||||
|
||||
Returns:
|
||||
Empty list if the workflow has no paths/paths-ignore filter
|
||||
anywhere in its `on:` block. Otherwise, a list of human-readable
|
||||
strings naming the event and filter key + the filter contents.
|
||||
"""
|
||||
try:
|
||||
doc = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(f"::error::YAML parse error in {workflow_path}: {e}\n")
|
||||
sys.exit(3)
|
||||
if not isinstance(doc, dict):
|
||||
return []
|
||||
|
||||
on_block = doc.get("on") or doc.get(True) # PyYAML 6 quirk: `on:`
|
||||
# under default constructor sometimes becomes the bool key `True`
|
||||
# because YAML 1.1 treats `on` as a boolean. Tolerate both.
|
||||
if on_block is None:
|
||||
return []
|
||||
|
||||
findings: list[str] = []
|
||||
|
||||
# Shape A: `on: pull_request` (string shorthand) — cannot carry filters.
|
||||
if isinstance(on_block, str):
|
||||
return []
|
||||
# Shape B: `on: [pull_request, push]` (list shorthand) — cannot carry filters.
|
||||
if isinstance(on_block, list):
|
||||
return []
|
||||
# Shape C: `on: { event: { ... } }` — the standard mapping case.
|
||||
if isinstance(on_block, dict):
|
||||
# Defensive: top-level malformed `on.paths` (someone wrote
|
||||
# `on: { paths: ['x'] }` thinking it's a workflow-level filter).
|
||||
# This is invalid syntax, but if present, flag it — it might
|
||||
# not block the workflow from registering (Gitea may ignore the
|
||||
# unknown key) and would create a false sense of "filter exists"
|
||||
# the lint should still surface.
|
||||
for k in _PATHS_KEYS:
|
||||
if k in on_block:
|
||||
v = on_block[k]
|
||||
findings.append(
|
||||
f"top-level `on.{k}` filter (malformed but present): {v!r}"
|
||||
)
|
||||
for event, event_body in on_block.items():
|
||||
if event in _PATHS_KEYS:
|
||||
continue # already handled above
|
||||
if not isinstance(event_body, dict):
|
||||
# `pull_request: null` / `pull_request: [opened]` shapes —
|
||||
# no place for a paths filter to live; skip.
|
||||
continue
|
||||
for k in _PATHS_KEYS:
|
||||
if k in event_body:
|
||||
v = event_body[k]
|
||||
findings.append(
|
||||
f"`on.{event}.{k}` filter present: {v!r}"
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Driver
|
||||
# --------------------------------------------------------------------------
|
||||
def run() -> int:
|
||||
"""Main lint entrypoint. Returns the process exit code.
|
||||
|
||||
Exit semantics (see module docstring for full table):
|
||||
0 — clean (no offending paths-filter on any required workflow),
|
||||
OR protection unreadable (403/404) — surfaced as ::error::
|
||||
but treated as non-fatal so token-scope issues don't red-X
|
||||
every PR.
|
||||
1 — at least one required workflow carries a paths/paths-ignore
|
||||
filter — the regression class this lint exists to prevent.
|
||||
"""
|
||||
_require_runtime_env()
|
||||
|
||||
protection_path = f"/repos/{OWNER}/{NAME}/branch_protections/{BRANCH}"
|
||||
try:
|
||||
_, protection = api("GET", protection_path)
|
||||
except ApiError as e:
|
||||
msg = str(e)
|
||||
m = re.search(r"HTTP (\d{3})", msg)
|
||||
http_status = int(m.group(1)) if m else None
|
||||
if http_status in (403, 404):
|
||||
sys.stderr.write(
|
||||
f"::error::GET {protection_path} returned HTTP {http_status} — "
|
||||
f"DRIFT_BOT_TOKEN lacks repo-admin scope (Gitea 1.22.6 "
|
||||
f"requires it for this endpoint) OR branch '{BRANCH}' has "
|
||||
f"no protection configured. Cannot enumerate required "
|
||||
f"checks; skipping lint with exit 0 to avoid red-X on "
|
||||
f"every PR. Fix: grant repo-admin to mc-drift-bot.\n"
|
||||
)
|
||||
return 0
|
||||
raise
|
||||
|
||||
if not isinstance(protection, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::protection response for {BRANCH} not a JSON object\n"
|
||||
)
|
||||
return 4
|
||||
|
||||
contexts: list[str] = list(protection.get("status_check_contexts") or [])
|
||||
if not contexts:
|
||||
print(
|
||||
f"::notice::branch_protections/{BRANCH} has 0 required "
|
||||
f"status_check_contexts; nothing to lint. (no required contexts)"
|
||||
)
|
||||
return 0
|
||||
|
||||
print(f"::notice::Linting {len(contexts)} required context(s) for paths-filter regressions:")
|
||||
for c in contexts:
|
||||
print(f" - {c}")
|
||||
|
||||
offenders: list[tuple[str, Path, list[str]]] = []
|
||||
unresolved: list[str] = []
|
||||
|
||||
for ctx in contexts:
|
||||
parsed = parse_context(ctx)
|
||||
if parsed is None:
|
||||
print(
|
||||
f"::warning::could not parse context '{ctx}' "
|
||||
f"(expected `<workflow> / <job> (<event>)`); skipping"
|
||||
)
|
||||
unresolved.append(ctx)
|
||||
continue
|
||||
workflow_name, _job, _event = parsed
|
||||
wf_path = resolve_workflow_file(workflow_name)
|
||||
if wf_path is None:
|
||||
print(
|
||||
f"::warning::no workflow file in {WORKFLOWS_DIR} has "
|
||||
f"`name: {workflow_name}` (required context '{ctx}'); "
|
||||
f"skipping paths-filter check. "
|
||||
f"(orphaned-context detection is ci-required-drift's job.)"
|
||||
)
|
||||
unresolved.append(ctx)
|
||||
continue
|
||||
findings = detect_paths_filters(wf_path)
|
||||
if findings:
|
||||
offenders.append((workflow_name, wf_path, findings))
|
||||
else:
|
||||
print(f"::notice::OK {wf_path.name} ({workflow_name}) — no paths filter")
|
||||
|
||||
if offenders:
|
||||
print("")
|
||||
print(f"::error::Found {len(offenders)} required workflow(s) with paths/paths-ignore filters:")
|
||||
for workflow_name, wf_path, findings in offenders:
|
||||
for finding in findings:
|
||||
# ::error file=... lets Gitea Actions surface a per-file
|
||||
# annotation in the PR UI (when annotations are wired).
|
||||
print(
|
||||
f"::error file={wf_path}::Required workflow "
|
||||
f"'{workflow_name}' ({wf_path.name}) has a paths "
|
||||
f"filter that would degrade the merge gate to a "
|
||||
f"silent indefinite pending: {finding}. "
|
||||
f"See feedback_path_filtered_workflow_cant_be_required. "
|
||||
f"Fix: remove the filter and instead gate per-step "
|
||||
f"inside the job with `if: contains(steps.changed.outputs.files, ...)` "
|
||||
f"or refactor to a single-job-with-per-step-if shape."
|
||||
)
|
||||
return 1
|
||||
|
||||
print("")
|
||||
print(
|
||||
f"::notice::OK — all {len(contexts) - len(unresolved)} resolvable "
|
||||
f"required workflow(s) clean (no paths/paths-ignore filters)."
|
||||
)
|
||||
if unresolved:
|
||||
print(
|
||||
f"::notice::{len(unresolved)} required context(s) were not "
|
||||
f"resolved to a workflow file (warn-not-fail); see warnings above."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -1,519 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint-workflow-yaml — catch Gitea-1.22.6-hostile workflow YAML shapes.
|
||||
|
||||
This script enforces six structural rules that have historically caused
|
||||
silent CI failures on Gitea Actions (1.22.6) — workflows that the server's
|
||||
YAML parser rejects with `[W] ignore invalid workflow ...` and registers
|
||||
for zero events, or shape conventions that produce ambiguous status
|
||||
contexts. Each rule maps to a documented incident in saved memory.
|
||||
|
||||
Rules (4 fatal + 1 fatal cross-file + 1 heuristic-warn):
|
||||
1. `workflow_dispatch.inputs:` block — Gitea 1.22.6 mis-parses the
|
||||
`inputs` keys as sibling event types and rejects the whole file.
|
||||
Memory: feedback_gitea_workflow_dispatch_inputs_unsupported.
|
||||
Origin: 2026-05-11 PyPI freeze (publish-runtime).
|
||||
2. `on: workflow_run:` event — not enumerated in Gitea 1.22.6's
|
||||
supported event list (verified via modules/actions/workflows.go
|
||||
enumeration; task #81). Workflow registers, fires for 0 events.
|
||||
3. `name:` containing `/` — breaks the
|
||||
`<workflow> / <job> (<event>)` commit-status context convention;
|
||||
downstream parsers (sop-tier-check, status-reaper) tokenize on `/`.
|
||||
4. `name:` collision across files — Gitea routes commit-status updates
|
||||
by `name` and behavior on collision is undefined (status-reaper
|
||||
rev1 fail-loud).
|
||||
5. Cross-repo `uses: org/repo/path@ref` — blocked while
|
||||
`[actions].DEFAULT_ACTIONS_URL=github` is the server default;
|
||||
resolves to github.com/<org-suspended>/... and 404s.
|
||||
Memory: feedback_gitea_cross_repo_uses_blocked. Cross-link: task #109.
|
||||
6. (HEURISTIC, warn-not-fail) Steps reference `https://api.github.com`
|
||||
or `https://github.com/.../releases/download` without a
|
||||
workflow-level `env.GITHUB_SERVER_URL` set to the Gitea instance.
|
||||
Memory: feedback_act_runner_github_server_url.
|
||||
7. Production deploy/redeploy workflows may not rely on Gitea
|
||||
`concurrency.cancel-in-progress: false` for serialization. Gitea
|
||||
1.22.6 can cancel queued runs despite that setting.
|
||||
8. Production deploy/redeploy workflows may not dump raw CP responses or
|
||||
raw `.error` fields into CI logs/summaries.
|
||||
9. Production deploy/redeploy workflows must expose an operational control:
|
||||
kill switch for auto deploys or rollback tag for manual deploys.
|
||||
10. Docker health checks must not run `docker info | head` under pipefail.
|
||||
`head` closes the pipe early, `docker info` can exit nonzero from
|
||||
SIGPIPE, and the step can falsely report Docker daemon failure.
|
||||
|
||||
Per `feedback_smoke_test_vendor_truth_not_shape_match`: fixtures used to
|
||||
validate this lint must mirror real Gitea 1.22.6 YAML semantics, not
|
||||
Python yaml-parser quirks. The test suite at tests/test_lint_workflow_yaml.py
|
||||
includes a vendor-truth fixture (the exact publish-runtime regression).
|
||||
|
||||
Usage:
|
||||
python3 .gitea/scripts/lint-workflow-yaml.py
|
||||
Lint every `*.yml` in `.gitea/workflows/`.
|
||||
|
||||
python3 .gitea/scripts/lint-workflow-yaml.py --workflow-dir <path>
|
||||
Lint a custom directory (used by tests/test_lint_workflow_yaml.py).
|
||||
|
||||
Exit codes:
|
||||
0 — clean OR only heuristic-warnings emitted.
|
||||
1 — at least one fatal rule (1-5) violated.
|
||||
2 — YAML parse error or argv usage error.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("::error::PyYAML is required. Install with: pip install PyYAML", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# YAML quirk: bare `on:` at the top level parses to the Python `True`
|
||||
# (because `on` is a YAML 1.1 boolean alias). Handle both keys.
|
||||
def _get_on(d: dict) -> Any:
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
if "on" in d:
|
||||
return d["on"]
|
||||
if True in d:
|
||||
return d[True]
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 1 — workflow_dispatch.inputs block (Gitea 1.22.6 parser rejects)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_workflow_dispatch_inputs(filename: str, doc: Any) -> list[str]:
|
||||
"""Return per-violation error lines if `workflow_dispatch.inputs` is set."""
|
||||
errors: list[str] = []
|
||||
on = _get_on(doc)
|
||||
if not isinstance(on, dict):
|
||||
return errors
|
||||
wd = on.get("workflow_dispatch")
|
||||
if isinstance(wd, dict) and wd.get("inputs"):
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 1 (FATAL): "
|
||||
f"`on.workflow_dispatch.inputs:` block detected. Gitea 1.22.6 "
|
||||
f"silently rejects the entire workflow with `[W] ignore invalid "
|
||||
f"workflow: unknown on type: map[...]`. Drop the `inputs:` block "
|
||||
f"and derive parameters from tag name / env / external query. "
|
||||
f"Memory: feedback_gitea_workflow_dispatch_inputs_unsupported."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 2 — on: workflow_run (not supported on Gitea 1.22.6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_workflow_run_event(filename: str, doc: Any) -> list[str]:
|
||||
"""Return per-violation error lines if `on: workflow_run:` is used."""
|
||||
errors: list[str] = []
|
||||
on = _get_on(doc)
|
||||
if isinstance(on, dict) and "workflow_run" in on:
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 2 (FATAL): `on: workflow_run:` "
|
||||
f"event used. Gitea 1.22.6 does NOT support `workflow_run` "
|
||||
f"(verified via modules/actions/workflows.go enumeration; "
|
||||
f"task #81). Workflow will fire for zero events. Use a "
|
||||
f"`schedule:` cron OR a `push:` trigger with `paths:` filter "
|
||||
f"on the upstream workflow file as the cross-workflow gate."
|
||||
)
|
||||
elif isinstance(on, list) and "workflow_run" in on:
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 2 (FATAL): `on: workflow_run` "
|
||||
f"in event list. Not supported on Gitea 1.22.6 — task #81."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 3 — name: contains "/" (breaks status-context tokenization)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_name_with_slash(filename: str, doc: Any) -> list[str]:
|
||||
"""Return per-violation error lines if workflow `name:` contains a slash."""
|
||||
errors: list[str] = []
|
||||
if not isinstance(doc, dict):
|
||||
return errors
|
||||
name = doc.get("name")
|
||||
if isinstance(name, str) and "/" in name:
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 3 (FATAL): workflow `name: "
|
||||
f"{name!r}` contains `/`. The commit-status context convention "
|
||||
f"is `<workflow> / <job> (<event>)`; embedding `/` in the "
|
||||
f"workflow name makes downstream parsers (sop-tier-check, "
|
||||
f"status-reaper) tokenize ambiguously. Rename to use `-` or "
|
||||
f"` ` instead."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 4 — cross-file name collision
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_name_collision_across_files(
|
||||
docs_by_file: dict[str, Any],
|
||||
) -> list[str]:
|
||||
"""Return per-collision error lines if two files share the same `name:`."""
|
||||
errors: list[str] = []
|
||||
by_name: dict[str, list[str]] = collections.defaultdict(list)
|
||||
for filename, doc in docs_by_file.items():
|
||||
if isinstance(doc, dict):
|
||||
n = doc.get("name")
|
||||
if isinstance(n, str) and n:
|
||||
by_name[n].append(filename)
|
||||
for n, files in sorted(by_name.items()):
|
||||
if len(files) > 1:
|
||||
errors.append(
|
||||
f"::error::Rule 4 (FATAL): workflow `name: {n!r}` collision "
|
||||
f"across {len(files)} files: {files}. Gitea routes "
|
||||
f"commit-status updates by `name`; collision yields "
|
||||
f"undefined behavior. Give each workflow a unique `name:`."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 5 — cross-repo `uses: org/repo/path@ref`
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# `uses: <foo>@<ref>` — match the value form Gitea/act actually parse.
|
||||
# We need to distinguish:
|
||||
# - `actions/checkout@<sha>` OK (bare org/repo@ref, no subpath)
|
||||
# - `./.gitea/actions/foo` OK (local path)
|
||||
# - `docker://image:tag` OK (docker-image form)
|
||||
# - `molecule-ai/molecule-ci/.gitea/actions/audit-force-merge@main` BAD
|
||||
USES_CROSS_REPO_RE = re.compile(
|
||||
r"""^
|
||||
(?P<owner>[A-Za-z0-9_.\-]+)
|
||||
/
|
||||
(?P<repo>[A-Za-z0-9_.\-]+)
|
||||
/ # mandatory subpath separator => cross-repo composite/reusable
|
||||
(?P<path>[^@\s]+)
|
||||
@
|
||||
(?P<ref>\S+)
|
||||
$""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _iter_uses(doc: Any) -> Iterable[str]:
|
||||
"""Yield every `uses:` string from job steps in a workflow document."""
|
||||
if not isinstance(doc, dict):
|
||||
return
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return
|
||||
for job in jobs.values():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
# reusable workflow: `uses:` at the job level
|
||||
if isinstance(job.get("uses"), str):
|
||||
yield job["uses"]
|
||||
steps = job.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
continue
|
||||
for step in steps:
|
||||
if isinstance(step, dict) and isinstance(step.get("uses"), str):
|
||||
yield step["uses"]
|
||||
|
||||
|
||||
def _iter_run_blocks(doc: Any) -> Iterable[str]:
|
||||
"""Yield every shell `run:` block from job steps in a workflow document."""
|
||||
if not isinstance(doc, dict):
|
||||
return
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return
|
||||
for job in jobs.values():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
steps = job.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
continue
|
||||
for step in steps:
|
||||
if isinstance(step, dict) and isinstance(step.get("run"), str):
|
||||
yield step["run"]
|
||||
|
||||
|
||||
def check_cross_repo_uses(filename: str, doc: Any) -> list[str]:
|
||||
"""Return per-violation error lines for cross-repo `uses:` references."""
|
||||
errors: list[str] = []
|
||||
for uses in _iter_uses(doc):
|
||||
# Skip docker:// and local ./
|
||||
if uses.startswith(("docker://", "./", "../")):
|
||||
continue
|
||||
m = USES_CROSS_REPO_RE.match(uses.strip())
|
||||
if m:
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 5 (FATAL): cross-repo "
|
||||
f"`uses: {uses}` detected. Gitea 1.22.6 with "
|
||||
f"`[actions].DEFAULT_ACTIONS_URL=github` resolves this to "
|
||||
f"github.com/{m.group('owner')}/{m.group('repo')} which "
|
||||
f"404s (org suspended 2026-05-06). Inline the shared bash "
|
||||
f"into `.gitea/scripts/` until task #109 (actions mirror) "
|
||||
f"ships. Memory: feedback_gitea_cross_repo_uses_blocked."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 6 — heuristic: github.com/api refs without workflow-level
|
||||
# GITHUB_SERVER_URL (WARN-not-FAIL per halt-condition 3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Match `https://api.github.com/...` (API call) — that's the actionable
|
||||
# pattern. We intentionally do NOT match `https://github.com/.../releases/
|
||||
# download/...` (jq-release pin) nor `https://github.com/${{ github.repository
|
||||
# }}` (OCI label) because those are documented benign references on current
|
||||
# main and would 100% false-positive (3 hits, per Phase 1 audit).
|
||||
GITHUB_API_REF_RE = re.compile(
|
||||
r"https://api\.github\.com\b|https://github\.com/api/",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
PROD_CP_URL_RE = re.compile(r"https://api\.moleculesai\.app\b")
|
||||
REDEPLOY_FLEET_RE = re.compile(r"\b/cp/admin/tenants/redeploy-fleet\b")
|
||||
RUN_SETS_PIPEFAIL_RE = re.compile(r"(?m)^\s*set\s+-[^\n]*o\s+pipefail\b")
|
||||
DOCKER_INFO_HEAD_PIPE_RE = re.compile(
|
||||
r"(?m)^\s*docker\s+info\b[^\n|]*\|\s*head\b"
|
||||
)
|
||||
RAW_CP_RESPONSE_RE = re.compile(
|
||||
r"""(?x)
|
||||
(?:\bjq\s+\.\s+["']?\$HTTP_RESPONSE["']?)
|
||||
|
|
||||
(?:\bcat\s+["']?\$HTTP_RESPONSE["']?)
|
||||
|
|
||||
(?:\|\s*\.error\b)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _has_workflow_level_server_url(doc: Any) -> bool:
|
||||
if not isinstance(doc, dict):
|
||||
return False
|
||||
env = doc.get("env")
|
||||
if isinstance(env, dict) and "GITHUB_SERVER_URL" in env:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_github_server_url_missing(filename: str, doc: Any, raw: str) -> list[str]:
|
||||
"""Return warn-lines (NOT errors) if api.github.com is referenced without
|
||||
workflow-level GITHUB_SERVER_URL. Heuristic — false-positives possible.
|
||||
"""
|
||||
warns: list[str] = []
|
||||
if not GITHUB_API_REF_RE.search(raw):
|
||||
return warns
|
||||
if _has_workflow_level_server_url(doc):
|
||||
return warns
|
||||
warns.append(
|
||||
f"::warning file={filename}::Rule 6 (WARN, heuristic): file "
|
||||
f"references `https://api.github.com` without a workflow-level "
|
||||
f"`env.GITHUB_SERVER_URL: https://git.moleculesai.app`. The "
|
||||
f"act_runner default for `${{{{ github.server_url }}}}` is "
|
||||
f"github.com, which can break actions that auth-condition on "
|
||||
f"server_url (e.g. actions/setup-go). If this curl is "
|
||||
f"intentionally hitting GitHub (e.g. public release pin), ignore. "
|
||||
f"Memory: feedback_act_runner_github_server_url."
|
||||
)
|
||||
return warns
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 7-9 — production CI/CD hardening rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _is_production_redeploy_workflow(raw: str) -> bool:
|
||||
"""Heuristic production-side-effect detector.
|
||||
|
||||
We intentionally key on the production CP host plus the redeploy-fleet
|
||||
endpoint. Staging workflows call the same endpoint on staging-api and are
|
||||
governed by looser staging verification policy.
|
||||
"""
|
||||
|
||||
return bool(PROD_CP_URL_RE.search(raw) and REDEPLOY_FLEET_RE.search(raw))
|
||||
|
||||
|
||||
def _iter_concurrency_blocks(doc: Any) -> Iterable[dict[str, Any]]:
|
||||
if not isinstance(doc, dict):
|
||||
return
|
||||
top = doc.get("concurrency")
|
||||
if isinstance(top, dict):
|
||||
yield top
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return
|
||||
for job in jobs.values():
|
||||
if isinstance(job, dict) and isinstance(job.get("concurrency"), dict):
|
||||
yield job["concurrency"]
|
||||
|
||||
|
||||
def check_production_concurrency(filename: str, doc: Any, raw: str) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _is_production_redeploy_workflow(raw):
|
||||
return errors
|
||||
for block in _iter_concurrency_blocks(doc):
|
||||
if block.get("cancel-in-progress") is False:
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 7 (FATAL): production deploy "
|
||||
f"workflow uses `concurrency.cancel-in-progress: false`. "
|
||||
f"Gitea 1.22.6 can cancel queued runs despite that setting, "
|
||||
f"so this is not a safe production serialization primitive. "
|
||||
f"Use an external queue/lock or make the deploy idempotent."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def check_production_raw_response_logging(filename: str, raw: str) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _is_production_redeploy_workflow(raw):
|
||||
return errors
|
||||
if RAW_CP_RESPONSE_RE.search(raw):
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 8 (FATAL): production deploy "
|
||||
f"workflow appears to print a raw production CP response or raw "
|
||||
f"`.error` field. CI logs are persistent and broad-read. Redact "
|
||||
f"runtime/SSM error details; print counts, booleans, status "
|
||||
f"codes, and links to restricted observability instead."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def check_production_operational_control(filename: str, raw: str) -> list[str]:
|
||||
errors: list[str] = []
|
||||
if not _is_production_redeploy_workflow(raw):
|
||||
return errors
|
||||
has_kill_switch = "PROD_AUTO_DEPLOY_DISABLED" in raw
|
||||
has_rollback = "PROD_MANUAL_REDEPLOY_TARGET_TAG" in raw
|
||||
if not (has_kill_switch or has_rollback):
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 9 (FATAL): production deploy "
|
||||
f"workflow calls redeploy-fleet without an operational control. "
|
||||
f"Auto deploys need a `PROD_AUTO_DEPLOY_DISABLED` kill switch; "
|
||||
f"manual deploys need a `PROD_MANUAL_REDEPLOY_TARGET_TAG` "
|
||||
f"rollback/pin path."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rule 10 — docker info piped to head under pipefail
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_docker_info_head_pipefail(filename: str, doc: Any) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for run_block in _iter_run_blocks(doc):
|
||||
if not (
|
||||
RUN_SETS_PIPEFAIL_RE.search(run_block)
|
||||
and DOCKER_INFO_HEAD_PIPE_RE.search(run_block)
|
||||
):
|
||||
continue
|
||||
errors.append(
|
||||
f"::error file={filename}::Rule 10 (FATAL): workflow runs "
|
||||
f"`docker info | head` after enabling `pipefail`. `head` can "
|
||||
f"close the pipe early, making `docker info` exit nonzero and "
|
||||
f"falsely fail the Docker daemon health check. Capture "
|
||||
f"`docker_info=\"$(docker info 2>&1)\"` first, then print a "
|
||||
f"bounded preview with `printf ... | sed -n '1,5p'`."
|
||||
)
|
||||
break
|
||||
return errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Lint Gitea Actions workflow YAML for 1.22.6-hostile shapes."
|
||||
)
|
||||
p.add_argument(
|
||||
"--workflow-dir",
|
||||
default=".gitea/workflows",
|
||||
help="Directory of workflow *.yml files (default: .gitea/workflows).",
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
wf_dir = Path(args.workflow_dir)
|
||||
if not wf_dir.exists():
|
||||
# Empty / missing dir = nothing to lint, not a failure.
|
||||
print(f"::notice::No workflow directory at {wf_dir}; skipping.")
|
||||
return 0
|
||||
|
||||
yml_paths = sorted(
|
||||
glob.glob(str(wf_dir / "*.yml")) + glob.glob(str(wf_dir / "*.yaml"))
|
||||
)
|
||||
if not yml_paths:
|
||||
print(f"::notice::No workflow files in {wf_dir}; nothing to lint.")
|
||||
return 0
|
||||
|
||||
fatal_errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
docs_by_file: dict[str, Any] = {}
|
||||
|
||||
for path in yml_paths:
|
||||
rel = os.path.relpath(path)
|
||||
try:
|
||||
raw = Path(path).read_text()
|
||||
doc = yaml.safe_load(raw)
|
||||
except yaml.YAMLError as e:
|
||||
fatal_errors.append(
|
||||
f"::error file={rel}::YAML parse error: {e}. Cannot lint "
|
||||
f"a file the parser rejects."
|
||||
)
|
||||
continue
|
||||
docs_by_file[rel] = doc
|
||||
|
||||
# Per-file checks
|
||||
fatal_errors.extend(check_workflow_dispatch_inputs(rel, doc))
|
||||
fatal_errors.extend(check_workflow_run_event(rel, doc))
|
||||
fatal_errors.extend(check_name_with_slash(rel, doc))
|
||||
fatal_errors.extend(check_cross_repo_uses(rel, doc))
|
||||
fatal_errors.extend(check_production_concurrency(rel, doc, raw))
|
||||
fatal_errors.extend(check_production_raw_response_logging(rel, raw))
|
||||
fatal_errors.extend(check_production_operational_control(rel, raw))
|
||||
fatal_errors.extend(check_docker_info_head_pipefail(rel, doc))
|
||||
warnings.extend(check_github_server_url_missing(rel, doc, raw))
|
||||
|
||||
# Cross-file checks
|
||||
fatal_errors.extend(check_name_collision_across_files(docs_by_file))
|
||||
|
||||
# Emit warnings first (non-blocking)
|
||||
for w in warnings:
|
||||
print(w)
|
||||
|
||||
if not fatal_errors:
|
||||
n = len(yml_paths)
|
||||
print(
|
||||
f"::notice::lint-workflow-yaml: {n} workflow file(s) checked, "
|
||||
f"no fatal Gitea-1.22.6-hostile shapes. "
|
||||
f"({len(warnings)} heuristic warning(s) emitted.)"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Emit fatal errors
|
||||
print(
|
||||
f"::error::lint-workflow-yaml: {len(fatal_errors)} fatal violation(s) "
|
||||
f"across {len(yml_paths)} workflow file(s). See rule documentation "
|
||||
f"in .gitea/scripts/lint-workflow-yaml.py docstring."
|
||||
)
|
||||
for e in fatal_errors:
|
||||
print(e)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,509 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_bp_context_emit_match — Tier 2f per internal#350.
|
||||
|
||||
Rule
|
||||
----
|
||||
For a given protected branch, every context in
|
||||
`branch_protections/<branch>.status_check_contexts` MUST be emitted
|
||||
by at least one workflow in `.gitea/workflows/*.yml`. Two contexts
|
||||
match when:
|
||||
|
||||
1. The workflow's `name:` equals the context's workflow-part (the
|
||||
prefix before ` / `).
|
||||
2. Some job in that workflow has a `name:` (or default-fallback
|
||||
job-key) equal to the context's job-part (between ` / ` and
|
||||
` (`).
|
||||
3. The workflow's `on:` block includes the context's event-part
|
||||
(in parens at the end), with Gitea's event-name mapping:
|
||||
- `pull_request` and `pull_request_target` BOTH emit
|
||||
`(pull_request)` contexts (verified empirically on
|
||||
molecule-core/main).
|
||||
- `push` emits `(push)`.
|
||||
|
||||
A BP context with no emitter blocks merges forever — Gitea treats
|
||||
absent-as-`pending`, NOT absent-as-`skipped`-as-`success`. This is
|
||||
the phantom-required-check class
|
||||
(`feedback_phantom_required_check_after_gitea_migration`).
|
||||
|
||||
The inverse direction (emitter without BP context) is INFORMATIONAL
|
||||
only — Tier 2g handles that direction at PR-time. Flagging it here
|
||||
on a daily schedule would falsely surface every transitional state
|
||||
during a BP rollout.
|
||||
|
||||
How the gate works
|
||||
------------------
|
||||
Daily scheduled run + workflow_dispatch:
|
||||
|
||||
1. GET `branch_protections/{BRANCH}` (needs DRIFT_BOT_TOKEN with
|
||||
repo-admin scope; same persona as ci-required-drift.yml).
|
||||
Graceful-degrade on 403/404 per Tier 2a contract.
|
||||
|
||||
2. Walk `.gitea/workflows/*.yml` via PyYAML AST. For each workflow,
|
||||
enumerate its emitted contexts: `{workflow.name} / {job.name or
|
||||
job-key} ({event})` for each event in `on:` that emits a status.
|
||||
|
||||
3. For each BP context, look for an emitter match. Aggregate
|
||||
orphans.
|
||||
|
||||
4. If orphans exist:
|
||||
- File or PATCH a `[ci-bp-drift]` issue (idempotency contract:
|
||||
search for exact title prefix, edit existing if open).
|
||||
- Apply labels `tier:high` + `ci-bp-drift` (lookup IDs per
|
||||
repo; per `feedback_tier_label_ids_are_per_repo`).
|
||||
- Exit 1.
|
||||
|
||||
5. If no orphans:
|
||||
- Close any existing `[ci-bp-drift]` issue with a clean-state
|
||||
comment.
|
||||
- Exit 0.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
0 — clean OR API 403/404 (graceful-degrade, surfaces ::error::).
|
||||
1 — at least one BP context has no emitter.
|
||||
2 — env contract violation, workflows-dir missing, or YAML parse
|
||||
error.
|
||||
|
||||
Env
|
||||
---
|
||||
GITEA_TOKEN — DRIFT_BOT_TOKEN (repo-admin for branch_protections)
|
||||
GITEA_HOST — e.g. git.moleculesai.app
|
||||
REPO — owner/name
|
||||
BRANCH — defaults to `main`
|
||||
WORKFLOWS_DIR — defaults to `.gitea/workflows`
|
||||
DRIFT_LABEL — defaults to `ci-bp-drift`
|
||||
|
||||
Memory cross-links
|
||||
------------------
|
||||
- internal#350 (the RFC that specs this lint)
|
||||
- feedback_phantom_required_check_after_gitea_migration
|
||||
- feedback_tier_label_ids_are_per_repo
|
||||
- reference_post_suspension_pipeline
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write(
|
||||
"::error::PyYAML is required. Install with: pip install PyYAML\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# Status-check context regex (mirrors lint-required-no-paths.py).
|
||||
_CONTEXT_RE = re.compile(
|
||||
r"^(?P<workflow>.+?) / (?P<job>.+) \((?P<event>[^)]+)\)$"
|
||||
)
|
||||
|
||||
# Map a workflow `on:` event-key to the context's event-part. Gitea's
|
||||
# emitter convention (verified on molecule-core):
|
||||
# - pull_request → `(pull_request)`
|
||||
# - pull_request_target → `(pull_request)` (same surface)
|
||||
# - push → `(push)`
|
||||
# - schedule → no PR status; scheduled runs don't post
|
||||
# commit-statuses unless the workflow itself does so explicitly.
|
||||
# - workflow_dispatch → manually dispatched runs may or may not
|
||||
# emit; safest to treat as "no PR status" (informational notice
|
||||
# only).
|
||||
_EVENT_MAP = {
|
||||
"pull_request": "pull_request",
|
||||
"pull_request_target": "pull_request",
|
||||
"push": "push",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env
|
||||
# ---------------------------------------------------------------------------
|
||||
def _env(key: str, default: str | None = None) -> str:
|
||||
v = os.environ.get(key, default)
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def _require_env(key: str) -> str:
|
||||
v = os.environ.get(key)
|
||||
if not v:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API helper. Mirrors lint-required-no-paths.py's contract: returns
|
||||
# (status, payload) tuple with status ∈ {"ok", "not_found", "forbidden",
|
||||
# "error"}.
|
||||
# ---------------------------------------------------------------------------
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
) -> tuple[str, Any]:
|
||||
host = _env("GITEA_HOST")
|
||||
token = _env("GITEA_TOKEN")
|
||||
url = f"https://{host}/api/v1{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(
|
||||
url, method=method, data=data, headers=headers
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
if not raw:
|
||||
return ("ok", None)
|
||||
return ("ok", json.loads(raw))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return ("not_found", None)
|
||||
if e.code in (401, 403):
|
||||
return ("forbidden", None)
|
||||
return ("error", None)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return ("error", None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _get_on(d: Any) -> Any:
|
||||
"""YAML 1.1 boolean quirk: bare `on:` may parse to True. Handle both."""
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
if "on" in d:
|
||||
return d["on"]
|
||||
if True in d:
|
||||
return d[True]
|
||||
return None
|
||||
|
||||
|
||||
def _on_events(doc: Any) -> set[str]:
|
||||
"""Return the set of event keys in a workflow's `on:` block.
|
||||
|
||||
Accepts all three shapes (string / list / mapping). String/list
|
||||
shapes can't carry filters but they DO emit. Returns the
|
||||
Gitea-mapped event names per `_EVENT_MAP`.
|
||||
"""
|
||||
on = _get_on(doc)
|
||||
raw_events: set[str] = set()
|
||||
if on is None:
|
||||
return raw_events
|
||||
if isinstance(on, str):
|
||||
raw_events.add(on)
|
||||
elif isinstance(on, list):
|
||||
for e in on:
|
||||
if isinstance(e, str):
|
||||
raw_events.add(e)
|
||||
elif isinstance(on, dict):
|
||||
for k in on:
|
||||
if isinstance(k, str):
|
||||
raw_events.add(k)
|
||||
return {_EVENT_MAP[e] for e in raw_events if e in _EVENT_MAP}
|
||||
|
||||
|
||||
def _job_display(jbody: dict, jkey: str) -> str:
|
||||
"""Return job's `name:` if set, else fall back to the job-key.
|
||||
|
||||
Gitea formats status contexts with the job's `name:` when set;
|
||||
when unset it uses the job key. Matches lint-required-no-paths
|
||||
convention.
|
||||
"""
|
||||
n = jbody.get("name") if isinstance(jbody, dict) else None
|
||||
if isinstance(n, str) and n:
|
||||
return n
|
||||
return jkey
|
||||
|
||||
|
||||
def workflow_contexts(doc: Any) -> set[str]:
|
||||
"""Return the set of contexts a workflow emits."""
|
||||
contexts: set[str] = set()
|
||||
if not isinstance(doc, dict):
|
||||
return contexts
|
||||
wf_name = doc.get("name")
|
||||
if not isinstance(wf_name, str) or not wf_name:
|
||||
return contexts # no name => no addressable context
|
||||
events = _on_events(doc)
|
||||
if not events:
|
||||
return contexts
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return contexts
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__": # tolerate line-tracking annotations
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
disp = _job_display(jbody, jkey)
|
||||
for ev in events:
|
||||
contexts.add(f"{wf_name} / {disp} ({ev})")
|
||||
return contexts
|
||||
|
||||
|
||||
def parse_context(ctx: str) -> tuple[str, str, str] | None:
|
||||
m = _CONTEXT_RE.match(ctx)
|
||||
if not m:
|
||||
return None
|
||||
return (m.group("workflow"), m.group("job"), m.group("event"))
|
||||
|
||||
|
||||
def _iter_workflow_files(wf_dir: Path) -> list[Path]:
|
||||
return sorted(list(wf_dir.glob("*.yml")) + list(wf_dir.glob("*.yaml")))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Issue idempotency — search for an open issue with the canonical
|
||||
# title prefix; PATCH if found, POST if not. Mirrors ci-required-drift.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _canonical_title(repo: str, branch: str) -> str:
|
||||
return f"[ci-bp-drift] {repo}/{branch}: BP→emitter mismatch"
|
||||
|
||||
|
||||
def _ensure_labels(repo: str, names: list[str]) -> list[int]:
|
||||
status, labels = api("GET", f"/repos/{repo}/labels", query={"limit": "50"})
|
||||
if status != "ok" or not isinstance(labels, list):
|
||||
return []
|
||||
out: list[int] = []
|
||||
by_name = {l["name"]: l["id"] for l in labels if isinstance(l, dict)}
|
||||
for n in names:
|
||||
if n in by_name:
|
||||
out.append(by_name[n])
|
||||
return out
|
||||
|
||||
|
||||
def file_or_update_issue(
|
||||
repo: str, branch: str, orphans: list[str], emitter_orphans: list[str]
|
||||
) -> None:
|
||||
title = _canonical_title(repo, branch)
|
||||
body_lines = [
|
||||
f"BP→emitter drift detected on `{branch}` at "
|
||||
f"{os.environ.get('GITHUB_RUN_URL', '(run url unavailable)')}.",
|
||||
"",
|
||||
f"## Orphan BP contexts ({len(orphans)})",
|
||||
"",
|
||||
"These contexts are required by branch protection but NO workflow "
|
||||
"emits them. PRs merging into this branch will wait forever for a "
|
||||
"status that never arrives (Gitea treats absent-as-`pending`, NOT "
|
||||
"absent-as-`skipped`). See "
|
||||
"`feedback_phantom_required_check_after_gitea_migration`.",
|
||||
"",
|
||||
]
|
||||
for o in orphans:
|
||||
body_lines.append(f"- `{o}`")
|
||||
if emitter_orphans:
|
||||
body_lines += [
|
||||
"",
|
||||
f"## Workflows emitting contexts NOT in BP ({len(emitter_orphans)})",
|
||||
"",
|
||||
"Informational — Tier 2g handles this direction at PR-time. "
|
||||
"Listed here for completeness.",
|
||||
"",
|
||||
]
|
||||
for o in emitter_orphans:
|
||||
body_lines.append(f"- `{o}`")
|
||||
body_lines += [
|
||||
"",
|
||||
"Fix options:",
|
||||
" 1. PATCH `branch_protections/{branch}.status_check_contexts` "
|
||||
" to remove the orphan.",
|
||||
" 2. Restore the emitting workflow (if it was deleted/renamed).",
|
||||
"",
|
||||
"Linted by `.gitea/workflows/lint-bp-context-emit-match.yml` "
|
||||
"(Tier 2f, internal#350).",
|
||||
]
|
||||
body = "\n".join(body_lines)
|
||||
|
||||
# Idempotency search — find an open issue with the canonical title.
|
||||
status, hits = api(
|
||||
"GET",
|
||||
f"/repos/{repo}/issues",
|
||||
query={
|
||||
"type": "issues",
|
||||
"state": "open",
|
||||
"q": title,
|
||||
},
|
||||
)
|
||||
existing = None
|
||||
if status == "ok" and isinstance(hits, list):
|
||||
for h in hits:
|
||||
if (
|
||||
isinstance(h, dict)
|
||||
and h.get("state") == "open"
|
||||
and isinstance(h.get("title"), str)
|
||||
and h["title"].startswith(title)
|
||||
):
|
||||
existing = h
|
||||
break
|
||||
|
||||
label_ids = _ensure_labels(repo, ["ci-bp-drift", "tier:high"])
|
||||
|
||||
if existing:
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{repo}/issues/{existing['number']}",
|
||||
body={"body": body, "labels": label_ids} if label_ids else {"body": body},
|
||||
)
|
||||
print(
|
||||
f"::notice::Updated existing drift issue "
|
||||
f"#{existing['number']}: {existing.get('html_url', '')}"
|
||||
)
|
||||
else:
|
||||
status, posted = api(
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
body={"title": title, "body": body, "labels": label_ids},
|
||||
)
|
||||
if status == "ok" and isinstance(posted, dict):
|
||||
print(
|
||||
f"::notice::Filed new drift issue "
|
||||
f"#{posted.get('number')}: {posted.get('html_url', '')}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
def run() -> int:
|
||||
_require_env("GITEA_TOKEN")
|
||||
_require_env("GITEA_HOST")
|
||||
repo = _require_env("REPO")
|
||||
branch = _env("BRANCH", "main")
|
||||
wf_dir = Path(_env("WORKFLOWS_DIR", ".gitea/workflows"))
|
||||
|
||||
if not wf_dir.is_dir():
|
||||
sys.stderr.write(f"::error::workflows directory not found: {wf_dir}\n")
|
||||
return 2
|
||||
|
||||
# 1. Pull BP.
|
||||
status, bp = api("GET", f"/repos/{repo}/branch_protections/{branch}")
|
||||
if status == "forbidden":
|
||||
sys.stderr.write(
|
||||
f"::error::GET branch_protections/{branch} returned HTTP 403 — "
|
||||
f"DRIFT_BOT_TOKEN lacks repo-admin scope (Gitea 1.22.6 requires "
|
||||
f"it for this endpoint). Skipping lint with exit 0 to avoid "
|
||||
f"red-X on every run. Fix: grant repo-admin to mc-drift-bot. "
|
||||
f"Per Tier 2a contract.\n"
|
||||
)
|
||||
return 0
|
||||
if status == "not_found":
|
||||
print(
|
||||
f"::notice::branch '{branch}' has no protection configured; "
|
||||
f"nothing to lint."
|
||||
)
|
||||
return 0
|
||||
if status != "ok" or not isinstance(bp, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::branch_protections/{branch} response unexpected; "
|
||||
f"status={status}. Treating as transient; exit 0.\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
bp_contexts: list[str] = list(bp.get("status_check_contexts") or [])
|
||||
if not bp_contexts:
|
||||
print(
|
||||
f"::notice::branch_protections/{branch} has 0 required "
|
||||
f"status_check_contexts; nothing to lint."
|
||||
)
|
||||
return 0
|
||||
|
||||
# 2. Enumerate emitter contexts from all workflows.
|
||||
all_emitter: set[str] = set()
|
||||
for path in _iter_workflow_files(wf_dir):
|
||||
try:
|
||||
doc = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(
|
||||
f"::error file={path}::YAML parse error: {e}; skipping.\n"
|
||||
)
|
||||
continue
|
||||
all_emitter |= workflow_contexts(doc)
|
||||
|
||||
print(
|
||||
f"::notice::Linting {len(bp_contexts)} BP context(s) for {branch} "
|
||||
f"against {len(all_emitter)} workflow-emitted context(s)."
|
||||
)
|
||||
|
||||
bp_set = set(bp_contexts)
|
||||
|
||||
# 3. Find orphans (BP-side: required but no emitter).
|
||||
bp_orphans = sorted(bp_set - all_emitter)
|
||||
|
||||
# Informational: workflow emits but BP doesn't list. Tier 2g
|
||||
# territory at PR-time. We list these as NOTICE only.
|
||||
emitter_orphans = sorted(all_emitter - bp_set)
|
||||
|
||||
if bp_orphans:
|
||||
print(
|
||||
f"::error::Found {len(bp_orphans)} BP context(s) with no "
|
||||
f"emitter — these would block merges forever (Gitea treats "
|
||||
f"absent-as-pending, not skipped):"
|
||||
)
|
||||
for o in bp_orphans:
|
||||
# Closest-match hint: name a workflow whose name-part is a
|
||||
# near-match (lev-1 typo, or same workflow with a different
|
||||
# event).
|
||||
parsed = parse_context(o)
|
||||
hint = ""
|
||||
if parsed:
|
||||
wf, _job, _ev = parsed
|
||||
candidates = sorted(
|
||||
{c for c in all_emitter if c.startswith(wf + " / ")}
|
||||
)
|
||||
if candidates:
|
||||
hint = (
|
||||
f" — closest emitter(s): {', '.join(candidates[:3])}"
|
||||
)
|
||||
print(f"::error:: - {o}{hint}")
|
||||
if emitter_orphans:
|
||||
print(
|
||||
f"::notice::Also: {len(emitter_orphans)} workflow-emitted "
|
||||
f"context(s) not in BP (informational; Tier 2g handles at "
|
||||
f"PR-time):"
|
||||
)
|
||||
for o in emitter_orphans:
|
||||
print(f"::notice:: - {o}")
|
||||
# File / patch tracking issue.
|
||||
try:
|
||||
file_or_update_issue(repo, branch, bp_orphans, emitter_orphans)
|
||||
except Exception as e:
|
||||
sys.stderr.write(
|
||||
f"::error::failed to file drift issue: {e}\n"
|
||||
)
|
||||
return 1
|
||||
|
||||
if emitter_orphans:
|
||||
print(
|
||||
f"::notice::{len(emitter_orphans)} workflow-emitted context(s) "
|
||||
f"not in BP (informational; Tier 2g handles at PR-time):"
|
||||
)
|
||||
for o in emitter_orphans:
|
||||
print(f"::notice:: - {o}")
|
||||
|
||||
print(
|
||||
f"::notice::BP/emitter match clean: all {len(bp_contexts)} required "
|
||||
f"context(s) have an emitter."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -1,438 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_continue_on_error_tracking — Tier 2e per internal#350.
|
||||
|
||||
Rule
|
||||
----
|
||||
Every `continue-on-error: true` directive in `.gitea/workflows/*.yml`
|
||||
must be accompanied by a tracker reference comment within 2 lines
|
||||
(above OR below the directive's line). The reference is one of:
|
||||
|
||||
* `# mc#NNNN` — molecule-core issue
|
||||
* `# internal#NNNN` — molecule-ai/internal issue
|
||||
|
||||
The referenced issue must satisfy ALL of:
|
||||
|
||||
1. Exists (HTTP 200 on `/repos/{owner}/{name}/issues/{num}`)
|
||||
2. `state == "open"`
|
||||
3. `created_at` is ≤ MAX_AGE_DAYS days ago (default 14)
|
||||
|
||||
A passing reference establishes an audit trail and a forced renewal
|
||||
cadence — after 14 days the issue must either be CLOSED (the masked
|
||||
defect was fixed) or the comment must point at a NEW tracker
|
||||
(deliberate decision to keep masking, requires a paper-trail).
|
||||
|
||||
The class this prevents
|
||||
-----------------------
|
||||
Phase-3-masked failures. `continue-on-error: true` on `platform-build`
|
||||
had been hiding mc#664-class regressions for ~3 weeks before #656
|
||||
surfaced them on 2026-05-12. A 14-day cap forces a tracker review
|
||||
cycle and surfaces mask-drift within at most 14 days of the original
|
||||
defect.
|
||||
|
||||
Behaviour-based gate
|
||||
--------------------
|
||||
We parse via PyYAML AST (per `feedback_behavior_based_ast_gates`) to
|
||||
detect `continue-on-error: <truthy>` at job-key level, then map each
|
||||
location back to its source line via PyYAML's line-tracking loader.
|
||||
Comments are scanned from the raw text within a 2-line window of
|
||||
that source line. Reformatting (block-scalar vs flow-style) does not
|
||||
break the rule because the source-line anchor is the directive's
|
||||
own line.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
0 — every `continue-on-error: true` has a passing tracker, OR
|
||||
the issue-API endpoint returned 403/404 (token-scope; graceful
|
||||
degrade per Tier 2a contract — surface via ::error:: on stderr
|
||||
but don't red-X every PR over auth).
|
||||
1 — at least one violation (missing/closed/too-old/non-existent
|
||||
tracker).
|
||||
2 — env contract violation, YAML parse error, or workflows-dir
|
||||
missing.
|
||||
|
||||
Env
|
||||
---
|
||||
GITEA_TOKEN — read scope on the configured repos.
|
||||
Auto-injected `GITHUB_TOKEN` works for same-repo
|
||||
issue reads; for `internal#NNN` we need a token
|
||||
with `molecule-ai/internal` read scope. Use
|
||||
DRIFT_BOT_TOKEN (same persona as other Tier 2
|
||||
lints).
|
||||
GITEA_HOST — e.g. git.moleculesai.app
|
||||
REPO — `owner/name` for `mc#NNNN` lookups
|
||||
INTERNAL_REPO — `owner/name` for `internal#NNNN` lookups
|
||||
(defaults to derived `molecule-ai/internal`)
|
||||
WORKFLOWS_DIR — defaults to `.gitea/workflows`
|
||||
MAX_AGE_DAYS — defaults to 14
|
||||
|
||||
Memory cross-links
|
||||
------------------
|
||||
- internal#350 (the RFC that specs this lint)
|
||||
- mc#664 (the masked-3-weeks empirical case)
|
||||
- feedback_chained_defects_in_never_tested_workflows
|
||||
- feedback_behavior_based_ast_gates
|
||||
- feedback_strict_root_only_after_class_a
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write(
|
||||
"::error::PyYAML is required. Install with: pip install PyYAML\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tracker comment regex.
|
||||
# Matches: `# mc#1234`, `# internal#42`, `# mc#1234 - description`
|
||||
# Also matches trackers embedded mid-sentence: `# see mc#1234 for details`
|
||||
# Does NOT match: `# mc1234` (missing inner #), `mc#1234` (no leading
|
||||
# comment `#`), `# MC#1234` (case-sensitive). The search is line-wide,
|
||||
# not just at the comment-marker prefix — fixes false-negative when
|
||||
# the tracker appears mid-sentence (e.g. `internal#350` after prose).
|
||||
TRACKER_RE = re.compile(
|
||||
r"(?P<slug>mc|internal)#(?P<num>\d+)\b"
|
||||
)
|
||||
|
||||
# Truthy continue-on-error values we treat as "true". PyYAML decodes
|
||||
# `continue-on-error: true` to Python `True`. `continue-on-error: "true"`
|
||||
# decodes to the string "true" — Gitea's evaluator coerces strings,
|
||||
# so we treat string-`"true"` (case-insensitive) as truthy too.
|
||||
def _is_truthy_coe(v: Any) -> bool:
|
||||
if v is True:
|
||||
return True
|
||||
if isinstance(v, str) and v.strip().lower() == "true":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env contract
|
||||
# ---------------------------------------------------------------------------
|
||||
def _env(key: str, default: str | None = None) -> str:
|
||||
v = os.environ.get(key, default)
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def _require_env(key: str) -> str:
|
||||
v = os.environ.get(key)
|
||||
if not v:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PyYAML line-tracking loader. yaml.SafeLoader nodes carry
|
||||
# `start_mark.line` (0-based); using construct_mapping with `deep=True`
|
||||
# preserves that on every node. We need the line of each
|
||||
# `continue-on-error` key so we can scan the source for comments
|
||||
# near it.
|
||||
# ---------------------------------------------------------------------------
|
||||
class _LineLoader(yaml.SafeLoader):
|
||||
"""SafeLoader that annotates every dict with `__line__: {key: line}`."""
|
||||
|
||||
|
||||
def _construct_mapping(loader: yaml.SafeLoader, node: yaml.MappingNode) -> dict:
|
||||
mapping = loader.construct_mapping(node, deep=True)
|
||||
# Annotate per-key source lines so we can locate `continue-on-error`.
|
||||
lines: dict[str, int] = {}
|
||||
for k_node, _v_node in node.value:
|
||||
try:
|
||||
key = loader.construct_object(k_node, deep=True)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(key, (str, int, bool)):
|
||||
lines[str(key)] = k_node.start_mark.line + 1 # 1-based
|
||||
if isinstance(mapping, dict):
|
||||
mapping["__lines__"] = lines
|
||||
return mapping
|
||||
|
||||
|
||||
_LineLoader.add_constructor(
|
||||
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Issue lookup
|
||||
# ---------------------------------------------------------------------------
|
||||
def fetch_issue(slug_kind: str, num: int) -> tuple[str, dict | None]:
|
||||
"""Return `(status, payload_or_none)`.
|
||||
|
||||
status ∈ {"ok", "not_found", "forbidden", "error"}.
|
||||
"""
|
||||
repo = (
|
||||
_env("REPO") if slug_kind == "mc" else _env("INTERNAL_REPO")
|
||||
)
|
||||
if not repo:
|
||||
# Fall through gracefully — caller treats as 403 (token-scope).
|
||||
return ("forbidden", None)
|
||||
host = _env("GITEA_HOST")
|
||||
token = _env("GITEA_TOKEN")
|
||||
url = f"https://{host}/api/v1/repos/{repo}/issues/{num}"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return ("ok", json.loads(resp.read()))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return ("not_found", None)
|
||||
if e.code in (401, 403):
|
||||
return ("forbidden", None)
|
||||
return ("error", None)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return ("error", None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Locate every continue-on-error: <truthy> in a workflow doc, with line.
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_coe_truthies(
|
||||
doc: Any, raw_lines: list[str]
|
||||
) -> list[tuple[str, int]]:
|
||||
"""Return list of (job_key, source_line_1based).
|
||||
|
||||
`doc` is the LineLoader-parsed mapping. We descend `jobs.<key>` and
|
||||
return only those whose value is truthy per `_is_truthy_coe`.
|
||||
Job-step continue-on-error is intentionally NOT considered: it
|
||||
suppresses step-level failure rollup only, not job-level. The
|
||||
masking class this lint targets is the job-level rollup.
|
||||
"""
|
||||
out: list[tuple[str, int]] = []
|
||||
if not isinstance(doc, dict):
|
||||
return out
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return out
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__":
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
if "continue-on-error" not in jbody:
|
||||
continue
|
||||
v = jbody["continue-on-error"]
|
||||
if not _is_truthy_coe(v):
|
||||
continue
|
||||
line = jbody.get("__lines__", {}).get("continue-on-error")
|
||||
if not line:
|
||||
# PyYAML line-tracking shouldn't miss but guard for safety.
|
||||
# Fall back to grepping the raw text.
|
||||
line = _grep_first_coe_line(raw_lines, jkey) or 1
|
||||
out.append((str(jkey), int(line)))
|
||||
return out
|
||||
|
||||
|
||||
def _grep_first_coe_line(raw_lines: list[str], jkey: str) -> int | None:
|
||||
"""Fallback: find the first `continue-on-error:` line after a `jkey:` line."""
|
||||
saw_job = False
|
||||
for i, line in enumerate(raw_lines, start=1):
|
||||
if re.match(rf"^\s*{re.escape(jkey)}\s*:", line):
|
||||
saw_job = True
|
||||
continue
|
||||
if saw_job and "continue-on-error" in line:
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scan window for tracker comment
|
||||
# ---------------------------------------------------------------------------
|
||||
WINDOW = 2 # lines above OR below the directive's line (inclusive)
|
||||
|
||||
|
||||
def find_tracker_in_window(
|
||||
raw_lines: list[str], line_1based: int
|
||||
) -> tuple[str, int] | None:
|
||||
"""Return (slug, num) if a `# mc#NNN`/`# internal#NNN` appears
|
||||
in raw_lines within ±WINDOW lines of `line_1based`. None otherwise.
|
||||
|
||||
We scan the directive's own line (it may carry an inline comment
|
||||
like `continue-on-error: true # mc#3`) plus ±WINDOW.
|
||||
"""
|
||||
lo = max(1, line_1based - WINDOW)
|
||||
hi = min(len(raw_lines), line_1based + WINDOW)
|
||||
for i in range(lo, hi + 1):
|
||||
line = raw_lines[i - 1]
|
||||
# Only the comment portion (after `#`) is considered, so
|
||||
# trailing-inline comments on the directive line are matched.
|
||||
m = TRACKER_RE.search(line)
|
||||
if m:
|
||||
return (m.group("slug"), int(m.group("num")))
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tracker validation
|
||||
# ---------------------------------------------------------------------------
|
||||
def validate_tracker(
|
||||
slug: str, num: int, max_age_days: int
|
||||
) -> tuple[bool, str]:
|
||||
"""Return (ok?, reason). On 403, ok=True is returned with reason
|
||||
explaining graceful-degrade — caller treats 403 as a non-fatal
|
||||
skip (same as Tier 2a contract).
|
||||
"""
|
||||
status, payload = fetch_issue(slug, num)
|
||||
if status == "forbidden":
|
||||
sys.stderr.write(
|
||||
f"::error::issue {slug}#{num} unreadable (HTTP 403 — token "
|
||||
f"scope). Cannot validate; skipping this check to avoid "
|
||||
f"red-X on every PR. Fix the token, not the lint.\n"
|
||||
)
|
||||
return (True, "forbidden — skipped")
|
||||
if status == "not_found":
|
||||
return (False, f"{slug}#{num} does not exist (404)")
|
||||
if status == "error":
|
||||
sys.stderr.write(
|
||||
f"::error::issue {slug}#{num} fetch errored — treating as "
|
||||
f"unverified, skipping this check.\n"
|
||||
)
|
||||
return (True, "fetch-error — skipped")
|
||||
|
||||
assert payload is not None
|
||||
state = payload.get("state", "")
|
||||
if state != "open":
|
||||
return (False, f"{slug}#{num} state={state!r} (must be open)")
|
||||
|
||||
created = payload.get("created_at", "")
|
||||
try:
|
||||
# Gitea returns ISO-8601 with timezone; Python 3.11+
|
||||
# fromisoformat handles `Z` suffix natively from 3.11. Older
|
||||
# runtimes need explicit replace.
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return (False, f"{slug}#{num} created_at unparseable: {created!r}")
|
||||
|
||||
age = datetime.now(timezone.utc) - created_dt
|
||||
# Inclusive boundary at MAX_AGE_DAYS: `age.days` truncates to a
|
||||
# whole-day floor, so an issue created 14d 0h 5m ago has
|
||||
# `age.days == 14` and passes; one created 15d 0h 0m ago has
|
||||
# `age.days == 15` and fails. This is the convention specified
|
||||
# in internal#350 ("≤14 days old").
|
||||
if age.days > max_age_days:
|
||||
return (
|
||||
False,
|
||||
f"{slug}#{num} is {age.days} days old (>{max_age_days}d cap). "
|
||||
f"Close-or-renew the tracker.",
|
||||
)
|
||||
return (True, f"{slug}#{num} open, {age.days}d old, ≤{max_age_days}d")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
def _iter_workflow_files(wf_dir: Path) -> list[Path]:
|
||||
return sorted(list(wf_dir.glob("*.yml")) + list(wf_dir.glob("*.yaml")))
|
||||
|
||||
|
||||
def run() -> int:
|
||||
wf_dir = Path(_env("WORKFLOWS_DIR", ".gitea/workflows"))
|
||||
max_age = int(_env("MAX_AGE_DAYS", "14"))
|
||||
# Defaults for INTERNAL_REPO when unset (best-effort guess based on
|
||||
# the convention `mc#` = same repo, `internal#` = molecule-ai/internal).
|
||||
if not os.environ.get("INTERNAL_REPO"):
|
||||
os.environ["INTERNAL_REPO"] = "molecule-ai/internal"
|
||||
|
||||
if not wf_dir.is_dir():
|
||||
sys.stderr.write(
|
||||
f"::error::workflows directory not found: {wf_dir}\n"
|
||||
)
|
||||
return 2
|
||||
|
||||
yml_files = _iter_workflow_files(wf_dir)
|
||||
if not yml_files:
|
||||
print(f"::notice::no workflow files under {wf_dir}; nothing to lint.")
|
||||
return 0
|
||||
|
||||
violations: list[str] = []
|
||||
notices: list[str] = []
|
||||
total_coe_true = 0
|
||||
|
||||
for path in yml_files:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
raw_lines = raw.splitlines()
|
||||
try:
|
||||
doc = yaml.load(raw, Loader=_LineLoader)
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(
|
||||
f"::error file={path}::YAML parse error: {e}. Skipping "
|
||||
f"this file (lint-workflow-yaml will catch separately).\n"
|
||||
)
|
||||
continue
|
||||
|
||||
coe_locs = find_coe_truthies(doc, raw_lines)
|
||||
for jkey, line in coe_locs:
|
||||
total_coe_true += 1
|
||||
tracker = find_tracker_in_window(raw_lines, line)
|
||||
if tracker is None:
|
||||
violations.append(
|
||||
f"::error file={path},line={line}::lint-continue-on-error-"
|
||||
f"tracking (Tier 2e): job '{jkey}' has "
|
||||
f"`continue-on-error: true` at line {line} with no "
|
||||
f"`# mc#NNNN` or `# internal#NNNN` tracker comment "
|
||||
f"within {WINDOW} lines. Add a tracker reference so "
|
||||
f"this mask has a forced 14-day renewal cycle. "
|
||||
f"Memory: feedback_chained_defects_in_never_tested_workflows."
|
||||
)
|
||||
continue
|
||||
slug, num = tracker
|
||||
ok, reason = validate_tracker(slug, num, max_age)
|
||||
if ok:
|
||||
notices.append(
|
||||
f"::notice::{path.name} job '{jkey}' (line {line}): "
|
||||
f"{reason}"
|
||||
)
|
||||
else:
|
||||
violations.append(
|
||||
f"::error file={path},line={line}::lint-continue-on-error-"
|
||||
f"tracking (Tier 2e): job '{jkey}' "
|
||||
f"`continue-on-error: true` references {slug}#{num}, "
|
||||
f"but {reason}. FIX: close/fix the underlying defect "
|
||||
f"and flip continue-on-error: false, OR file a fresh "
|
||||
f"tracker and update the comment."
|
||||
)
|
||||
|
||||
for n in notices:
|
||||
print(n)
|
||||
|
||||
if violations:
|
||||
print(
|
||||
f"::error::lint-continue-on-error-tracking: "
|
||||
f"{len(violations)} violation(s) across {len(yml_files)} "
|
||||
f"workflow file(s) (of {total_coe_true} `continue-on-error: "
|
||||
f"true` directives in total)."
|
||||
)
|
||||
for v in violations:
|
||||
print(v)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"::notice::lint-continue-on-error-tracking: "
|
||||
f"all {total_coe_true} `continue-on-error: true` directive(s) "
|
||||
f"have valid trackers (open, ≤{max_age}d old)."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -1,361 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_mask_pr_atomicity — Tier 2d structural enforcement per internal#350.
|
||||
|
||||
Rule
|
||||
----
|
||||
A PR whose diff touches `.gitea/workflows/ci.yml` AND modifies EITHER:
|
||||
|
||||
- any `continue-on-error:` value, OR
|
||||
- the `all-required` sentinel job's `needs:` block
|
||||
|
||||
must EITHER:
|
||||
|
||||
- Touch BOTH atomically in the same PR (preferred), OR
|
||||
- Cross-link the paired PR via a literal `Paired: #NNN` reference in
|
||||
the PR body OR in any commit message between BASE_SHA and HEAD_SHA.
|
||||
|
||||
The class this prevents
|
||||
-----------------------
|
||||
PR#665 (interim `continue-on-error: true` on `platform-build`) and
|
||||
PR#668 (sentinel-`needs` demotion of the same job) were designed as a
|
||||
pair but merged solo — #665 landed at 04:47Z 2026-05-12, #668 was still
|
||||
open at 05:07Z when the main-red watchdog (#674) fired. Result: ~20
|
||||
minutes of `main` red and a cascade of false-positives on unrelated PRs.
|
||||
|
||||
The lint operates on the YAML AST (PyYAML), not grep, per
|
||||
`feedback_behavior_based_ast_gates`: a refactor that moves `continue-on-error`
|
||||
between job keys, or renames the `all-required` job, would still be
|
||||
detected because we walk the parsed structure.
|
||||
|
||||
Why this works on Gitea 1.22.6
|
||||
------------------------------
|
||||
We don't use any 1.22.6-missing endpoints (no `/actions/runs/*`, no
|
||||
`branch_protections/*` — Tier 2f/g need those; Tier 2d does not). All
|
||||
required inputs come from the workflow `pull_request` event payload
|
||||
(BASE_SHA, HEAD_SHA, PR_BODY) and from local git via `git show`/`git log`.
|
||||
The auto-injected `GITHUB_TOKEN` is enough; we don't need
|
||||
DRIFT_BOT_TOKEN.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
0 — ci.yml not in diff, OR diff is no-op for the rule predicates,
|
||||
OR atomicity satisfied (both touched), OR a valid `Paired: #NNN`
|
||||
reference is present.
|
||||
1 — exactly ONE of {coe, sentinel-needs} touched AND no valid
|
||||
`Paired: #NNN` reference. The split-pair regression class.
|
||||
2 — env contract violation (BASE_SHA / HEAD_SHA missing) or YAML
|
||||
parse error on either side.
|
||||
|
||||
Env
|
||||
---
|
||||
BASE_SHA — PR base (pull_request.base.sha)
|
||||
HEAD_SHA — PR head (pull_request.head.sha)
|
||||
PR_BODY — pull_request.body (may be empty)
|
||||
CI_WORKFLOW_PATH — defaults to `.gitea/workflows/ci.yml`
|
||||
SENTINEL_JOB_KEY — defaults to `all-required`
|
||||
|
||||
Memory cross-links
|
||||
------------------
|
||||
- internal#350 (the RFC that specs this lint)
|
||||
- PR#665 / PR#668 (the empirical split-pair)
|
||||
- mc#664 (the main-red incident)
|
||||
- feedback_strict_root_only_after_class_a
|
||||
- feedback_behavior_based_ast_gates
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write(
|
||||
"::error::PyYAML is required. Install with: pip install PyYAML\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# YAML quirk: bare `on:` at the top level becomes Python `True` because
|
||||
# `on` is a YAML 1.1 boolean. Not used here but documented for future
|
||||
# editors who copy from this module.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# `Paired: #NNN` reference. `#` is mandatory, NNN must be digits. Any
|
||||
# surrounding markdown/whitespace is fine. The match is case-sensitive
|
||||
# on `Paired:` because lower-case `paired:` collides with conversational
|
||||
# prose ("paired: see comment above") and the convention is the exact
|
||||
# capitalisation.
|
||||
PAIRED_RE = re.compile(r"\bPaired:\s*#(?P<num>\d+)\b")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env contract
|
||||
# ---------------------------------------------------------------------------
|
||||
def _env(key: str, default: str | None = None) -> str:
|
||||
v = os.environ.get(key, default)
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def _require_env(key: str) -> str:
|
||||
v = os.environ.get(key)
|
||||
if not v:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# git-show helper. Returns None when the path doesn't exist on that side
|
||||
# (new file, deleted file, or rename — git returns exit 128 with "fatal:
|
||||
# path not in tree"). We treat None as "no rule predicate triggered on
|
||||
# that side".
|
||||
# ---------------------------------------------------------------------------
|
||||
def git_show(sha: str, path: str) -> str | None:
|
||||
r = subprocess.run(
|
||||
["git", "show", f"{sha}:{path}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return r.stdout
|
||||
|
||||
|
||||
def git_log_messages(base_sha: str, head_sha: str) -> str:
|
||||
r = subprocess.run(
|
||||
["git", "log", "--format=%B", f"{base_sha}..{head_sha}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return ""
|
||||
return r.stdout
|
||||
|
||||
|
||||
def git_diff_paths(base_sha: str, head_sha: str) -> list[str]:
|
||||
r = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{base_sha}..{head_sha}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
return [p for p in r.stdout.splitlines() if p.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Predicate 1 — any `continue-on-error` value changed between base and head
|
||||
# ---------------------------------------------------------------------------
|
||||
def _collect_coe(doc: Any) -> dict[str, Any]:
|
||||
"""Walk every job in `jobs.*` and collect its continue-on-error value.
|
||||
|
||||
Returns a dict {job_key: coe_value}. Missing keys are absent from
|
||||
the dict (NOT `False` — distinguishes "added the key" from
|
||||
"unchanged absent"). Job-step `continue-on-error` is NOT considered
|
||||
— only job-level, because that's the value that masks job status
|
||||
rollup, which is the class this lint targets.
|
||||
"""
|
||||
out: dict[str, Any] = {}
|
||||
if not isinstance(doc, dict):
|
||||
return out
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return out
|
||||
for k, j in jobs.items():
|
||||
if not isinstance(j, dict):
|
||||
continue
|
||||
if "continue-on-error" in j:
|
||||
out[k] = j["continue-on-error"]
|
||||
return out
|
||||
|
||||
|
||||
def coe_changed(base_doc: Any, head_doc: Any) -> tuple[bool, list[str]]:
|
||||
"""Return (changed?, [reasons]) describing per-job coe diffs."""
|
||||
base = _collect_coe(base_doc)
|
||||
head = _collect_coe(head_doc)
|
||||
reasons: list[str] = []
|
||||
all_keys = set(base) | set(head)
|
||||
for k in sorted(all_keys):
|
||||
b = base.get(k, "<absent>")
|
||||
h = head.get(k, "<absent>")
|
||||
if b != h:
|
||||
reasons.append(f"job '{k}' continue-on-error: {b!r} → {h!r}")
|
||||
return (bool(reasons), reasons)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Predicate 2 — sentinel job's `needs:` changed
|
||||
# ---------------------------------------------------------------------------
|
||||
def _collect_needs(doc: Any, sentinel_key: str) -> list[str] | None:
|
||||
"""Return the sentinel job's needs list (sorted) or None if absent."""
|
||||
if not isinstance(doc, dict):
|
||||
return None
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return None
|
||||
j = jobs.get(sentinel_key)
|
||||
if not isinstance(j, dict):
|
||||
return None
|
||||
needs = j.get("needs")
|
||||
if needs is None:
|
||||
return []
|
||||
if isinstance(needs, str):
|
||||
return [needs]
|
||||
if isinstance(needs, list):
|
||||
# Sort because `needs:` is order-insensitive at the engine
|
||||
# level; a reorder is not a semantic change and shouldn't
|
||||
# trip the lint.
|
||||
return sorted(str(x) for x in needs)
|
||||
return None
|
||||
|
||||
|
||||
def sentinel_needs_changed(
|
||||
base_doc: Any, head_doc: Any, sentinel_key: str
|
||||
) -> tuple[bool, str]:
|
||||
"""Return (changed?, reason)."""
|
||||
base = _collect_needs(base_doc, sentinel_key)
|
||||
head = _collect_needs(head_doc, sentinel_key)
|
||||
if base == head:
|
||||
return (False, "")
|
||||
return (
|
||||
True,
|
||||
f"sentinel '{sentinel_key}'.needs: {base!r} → {head!r}",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Predicate 3 — `Paired: #NNN` present in body or any commit message
|
||||
# ---------------------------------------------------------------------------
|
||||
def find_paired_refs(pr_body: str, commit_log: str) -> list[str]:
|
||||
"""Return list of `#NNN` strings found (deduped, sorted)."""
|
||||
found: set[str] = set()
|
||||
for src in (pr_body, commit_log):
|
||||
for m in PAIRED_RE.finditer(src or ""):
|
||||
found.add(m.group("num"))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
def _parse(content: str | None, label: str) -> Any:
|
||||
if content is None:
|
||||
return None
|
||||
try:
|
||||
return yaml.safe_load(content)
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(f"::error::YAML parse error on {label}: {e}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def run() -> int:
|
||||
base_sha = _require_env("BASE_SHA")
|
||||
head_sha = _require_env("HEAD_SHA")
|
||||
pr_body = _env("PR_BODY", "")
|
||||
ci_path = _env("CI_WORKFLOW_PATH", ".gitea/workflows/ci.yml")
|
||||
sentinel_key = _env("SENTINEL_JOB_KEY", "all-required")
|
||||
|
||||
# Step 0 — is ci.yml even in the diff? If not, the lint doesn't apply.
|
||||
changed_paths = git_diff_paths(base_sha, head_sha)
|
||||
if ci_path not in changed_paths:
|
||||
print(
|
||||
f"::notice::{ci_path} not in PR diff; lint-mask-pr-atomicity "
|
||||
f"skipped (no atomicity risk)."
|
||||
)
|
||||
return 0
|
||||
|
||||
base_yml = git_show(base_sha, ci_path)
|
||||
head_yml = git_show(head_sha, ci_path)
|
||||
|
||||
base_doc = _parse(base_yml, f"{ci_path}@{base_sha}")
|
||||
head_doc = _parse(head_yml, f"{ci_path}@{head_sha}")
|
||||
|
||||
# If the file is newly added (no base), no flip is possible — every
|
||||
# value is "newly introduced", not "changed". Tier 2e covers the
|
||||
# tracking-issue check for new continue-on-error: true. Exit 0.
|
||||
if base_doc is None:
|
||||
print(
|
||||
f"::notice::{ci_path} newly added in this PR; no flip to "
|
||||
f"analyse — lint-mask-pr-atomicity skipped."
|
||||
)
|
||||
return 0
|
||||
|
||||
# If the file is deleted on head, ditto — no atomicity question.
|
||||
if head_doc is None:
|
||||
print(
|
||||
f"::notice::{ci_path} deleted in this PR; "
|
||||
f"lint-mask-pr-atomicity skipped."
|
||||
)
|
||||
return 0
|
||||
|
||||
coe_yes, coe_reasons = coe_changed(base_doc, head_doc)
|
||||
needs_yes, needs_reason = sentinel_needs_changed(
|
||||
base_doc, head_doc, sentinel_key
|
||||
)
|
||||
|
||||
if not coe_yes and not needs_yes:
|
||||
print(
|
||||
f"::notice::{ci_path} touched but neither continue-on-error "
|
||||
f"nor sentinel '{sentinel_key}'.needs changed — no atomicity "
|
||||
f"risk. OK."
|
||||
)
|
||||
return 0
|
||||
|
||||
if coe_yes and needs_yes:
|
||||
print(
|
||||
f"::notice::Atomic change detected: both continue-on-error "
|
||||
f"AND sentinel '{sentinel_key}'.needs touched in same PR. OK."
|
||||
)
|
||||
for r in coe_reasons:
|
||||
print(f" - {r}")
|
||||
print(f" - {needs_reason}")
|
||||
return 0
|
||||
|
||||
# Exactly one side touched — require Paired: #NNN reference.
|
||||
commit_log = git_log_messages(base_sha, head_sha)
|
||||
paired = find_paired_refs(pr_body, commit_log)
|
||||
|
||||
one_side = "continue-on-error" if coe_yes else f"sentinel '{sentinel_key}'.needs"
|
||||
other_side = (
|
||||
f"sentinel '{sentinel_key}'.needs" if coe_yes else "continue-on-error"
|
||||
)
|
||||
|
||||
if paired:
|
||||
print(
|
||||
f"::notice::Split-pair detected ({one_side} changed without "
|
||||
f"{other_side}), but Paired reference(s) present: "
|
||||
f"{', '.join('#' + n for n in paired)}. OK."
|
||||
)
|
||||
for r in coe_reasons:
|
||||
print(f" - {r}")
|
||||
if needs_reason:
|
||||
print(f" - {needs_reason}")
|
||||
return 0
|
||||
|
||||
# The failure mode this lint exists to prevent.
|
||||
print(
|
||||
f"::error file={ci_path}::lint-mask-pr-atomicity (Tier 2d): "
|
||||
f"PR touches {one_side} in {ci_path} but NOT {other_side}, "
|
||||
f"and no `Paired: #NNN` reference was found in the PR body or "
|
||||
f"in commit messages between {base_sha[:8]}..{head_sha[:8]}. "
|
||||
f"This is the PR#665+#668 split-pair regression class "
|
||||
f"(see internal#350, mc#664). FIX: either (a) include the "
|
||||
f"matching {other_side} change in the same PR (preferred), or "
|
||||
f"(b) add `Paired: #NNN` (literal, capital P, with `#`) to the "
|
||||
f"PR body or a commit message referencing the paired PR."
|
||||
)
|
||||
for r in coe_reasons:
|
||||
print(f" - {r}")
|
||||
if needs_reason:
|
||||
print(f" - {needs_reason}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -1,681 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint-pre-flip-continue-on-error — block a PR that flips a job from
|
||||
``continue-on-error: true`` to ``continue-on-error: false`` (or removes
|
||||
the key while the base had it ``true``) without proof that the job's
|
||||
recent runs on the target branch are actually green.
|
||||
|
||||
Empirical class — PR #656 / mc#664:
|
||||
PR #656 (RFC internal#219 Phase 4) flipped 5 ``platform-build``-class
|
||||
jobs ``continue-on-error: true → false`` on the basis of a
|
||||
"verified green on main via combined-status check". But that "green"
|
||||
was the LIE produced by the prior ``continue-on-error: true``:
|
||||
Gitea Quirk #10 (internal#342 + dup #287) — when a step inside a
|
||||
job marked ``continue-on-error: true`` fails, the job-level status
|
||||
is still rolled up as ``success``. So the precondition the PR
|
||||
claimed to verify was structurally fooled by the bug being
|
||||
flipped.
|
||||
|
||||
mc#664 then captured the surfaced defects (2 unrelated, mutually-
|
||||
masked regressions):
|
||||
|
||||
Class 1: sqlmock helper drift since 2f36bb9a (24 days old)
|
||||
Class 2: OFFSEC-001 contract collision since 7d1a189f (1 day old)
|
||||
|
||||
Codified 04:35Z as hongming-pc2 charter §SOP-N rule (e)
|
||||
"run-log-grep-before-flip": pull the actual run log + grep for
|
||||
``--- FAIL`` / ``FAIL\\s`` BEFORE flipping; don't trust the masked
|
||||
combined-status.
|
||||
|
||||
This script structurally enforces that rule at PR time.
|
||||
|
||||
How it works (one PR tick):
|
||||
1. Parse the diff: compare ``.gitea/workflows/*.yml`` at PR base
|
||||
vs PR head. For each file present in both, parse the YAML AST
|
||||
and walk ``jobs.<key>.continue-on-error`` on each side. A
|
||||
"flip" is base ∈ {true} AND head ∈ {false, None/absent}. We
|
||||
coerce truthy/falsy per YAML semantics (PyYAML normalizes
|
||||
``true``/``True``/``yes`` to ``True``).
|
||||
2. For each flipped job, derive its commit-status context name as
|
||||
``"{workflow.name} / {job.name or job.key} (push)"`` — that's
|
||||
how Gitea Actions emits the context for runs on
|
||||
``main``/``staging`` (push event, see also expected_context()
|
||||
in ci-required-drift.py).
|
||||
3. Pull the last N commits of the target branch (PR base), fetch
|
||||
combined commit-status per commit, scan ``statuses[]`` for
|
||||
contexts matching ANY of the flipped jobs. For each match,
|
||||
fetch the actual run log via the web-UI route
|
||||
``{server_url}/{repo}/actions/runs/{run_id}/jobs/{job_idx}/logs``
|
||||
(per memory ``reference_gitea_actions_log_fetch`` — Gitea 1.22.6
|
||||
lacks REST ``/actions/runs/*`` endpoints; the web-UI route is the
|
||||
only working path; see ``reference_gitea_1_22_6_lacks_rest_rerun_endpoints``).
|
||||
4. Grep each log for the Go-test failure markers ``--- FAIL`` /
|
||||
``FAIL\\s+<package>`` AND the bash-step error sentinel
|
||||
``::error::``. If ANY recent log shows any of these AND the
|
||||
status itself reads ``success``, the job was masked. ``::error::``
|
||||
the flip with the offending test name + offending run URL +
|
||||
the regression commit (HEAD of the run).
|
||||
5. Exit 1 if any flips have at least one masked run; exit 0
|
||||
otherwise.
|
||||
|
||||
Halt-on-noise contract:
|
||||
- If a recent log fetch 404s (already-pruned-via-act_runner-gc,
|
||||
transient gitea-web outage): emit ``::warning::`` and treat the
|
||||
run as "log unavailable" — does NOT block the flip; logged so
|
||||
a curious reviewer can re-run.
|
||||
- If a flipped job has ZERO recent runs on the target branch (newly
|
||||
added workflow): emit ``::warning::`` "no run history to verify"
|
||||
and allow the flip. This is the only way a NEW workflow can ever
|
||||
ship with ``continue-on-error: false``; otherwise we'd have a
|
||||
chicken-and-egg.
|
||||
|
||||
Behavior-based AST gate per ``feedback_behavior_based_ast_gates``:
|
||||
- YAML parsed via PyYAML safe_load on BOTH sides of the diff
|
||||
- No grep-by-line — formatting changes (comment churn, key order)
|
||||
don't false-positive a flip
|
||||
- Job-key match — so a rename ``platform-build → core-be-build``
|
||||
appears as a DELETE + an ADD, not a flip (the delete side has no
|
||||
new value to compare against; the add side has no base side).
|
||||
|
||||
Run locally (works against this repo, requires PyYAML + Gitea token
|
||||
that can read combined-commit-status):
|
||||
|
||||
GITEA_TOKEN=... GITEA_HOST=git.moleculesai.app \\
|
||||
REPO=molecule-ai/molecule-core BASE_REF=main \\
|
||||
BASE_SHA=$(git rev-parse origin/main) \\
|
||||
HEAD_SHA=$(git rev-parse HEAD) \\
|
||||
python3 .gitea/scripts/lint_pre_flip_continue_on_error.py \\
|
||||
--dry-run
|
||||
|
||||
Cross-links: PR#656, mc#664, PR#665 (the interim re-mask),
|
||||
Quirk #10 (internal#342 + dup #287), hongming-pc2 charter §SOP-N
|
||||
rule (e), feedback_strict_root_only_after_class_a,
|
||||
feedback_no_shared_persona_token_use.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
import yaml # PyYAML 6.0.2 — installed by the workflow before this runs.
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment (read at module-import; runtime contract enforced in main())
|
||||
# --------------------------------------------------------------------------
|
||||
def _env(key: str, *, default: str = "") -> str:
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN")
|
||||
GITEA_HOST = _env("GITEA_HOST")
|
||||
REPO = _env("REPO")
|
||||
BASE_REF = _env("BASE_REF", default="main")
|
||||
BASE_SHA = _env("BASE_SHA")
|
||||
HEAD_SHA = _env("HEAD_SHA")
|
||||
# How many recent commits to scan on the target branch. 5 by default;
|
||||
# enough to catch a job that only fails intermittently, not so many
|
||||
# that the script paginates needlessly. Per spec.
|
||||
RECENT_COMMITS_N = int(_env("RECENT_COMMITS_N", default="5"))
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
WEB = f"https://{GITEA_HOST}" if GITEA_HOST else ""
|
||||
|
||||
# Failure markers we grep for in the run log.
|
||||
# --- FAIL — Go test failure marker
|
||||
# FAIL\s — `FAIL github.com/x/y` package-level rollup
|
||||
# ::error:: — bash-step `::error::` lines (the lint-curl-status-capture
|
||||
# pattern: a `python3 <<PY` block writing `::error::` then
|
||||
# sys.exit(1); also any shell `echo "::error::..."` from
|
||||
# jobs that wrap pytest/eslint/etc. and convert
|
||||
# non-zero exits into masked-by-CoE status)
|
||||
FAIL_PATTERNS = (
|
||||
"--- FAIL",
|
||||
"FAIL\t",
|
||||
"FAIL ",
|
||||
"::error::",
|
||||
)
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "BASE_REF", "BASE_SHA", "HEAD_SHA"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper (no requests dependency)
|
||||
# Mirrors the api()/ApiError contract in ci-required-drift.py +
|
||||
# main-red-watchdog.py per feedback_api_helper_must_raise_not_return_dict.
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API/web call cannot be trusted to have succeeded.
|
||||
|
||||
Soft-failure on non-2xx is the duplicate-write bug factory in
|
||||
find-or-create flows (PR #112 Five-Axis). Here it would mean a
|
||||
transient gitea-web 502 silently allows a flip whose recent runs
|
||||
we couldn't actually verify — exactly the regression class this
|
||||
lint exists to close.
|
||||
"""
|
||||
|
||||
|
||||
def http(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
timeout: int = 30,
|
||||
) -> tuple[int, Any, bytes]:
|
||||
"""Tiny HTTP helper around urllib.
|
||||
|
||||
Returns (status, parsed_or_None, raw_bytes). Raises ApiError on any
|
||||
non-2xx response. ``expect_json=False`` returns raw bytes in the
|
||||
parsed slot (for log-fetch from the web-UI which returns text/plain).
|
||||
"""
|
||||
final_headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json" if expect_json else "text/plain",
|
||||
}
|
||||
if headers:
|
||||
final_headers.update(headers)
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
final_headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=final_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read() or b""
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {url} → HTTP {status}: {snippet}")
|
||||
|
||||
if not expect_json:
|
||||
return status, raw, raw
|
||||
if not raw:
|
||||
return status, None, raw
|
||||
try:
|
||||
return status, json.loads(raw), raw
|
||||
except json.JSONDecodeError as e:
|
||||
raise ApiError(f"{method} {url} → HTTP {status} but body is not JSON: {e}") from e
|
||||
|
||||
|
||||
def api(method: str, path: str, *, body: dict | None = None, query: dict[str, str] | None = None) -> tuple[int, Any]:
|
||||
"""Read-shaped Gitea REST helper. Path is API-relative (``/repos/...``)."""
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
status, parsed, _ = http(method, url, body=body, expect_json=True)
|
||||
return status, parsed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# YAML parsing — coerce truthy/falsy for continue-on-error
|
||||
# --------------------------------------------------------------------------
|
||||
def _coerce_coe(val: Any) -> bool:
|
||||
"""Coerce a continue-on-error YAML value to bool.
|
||||
|
||||
PyYAML safe_load normalizes ``true``/``True``/``yes``/``on`` to
|
||||
Python ``True`` and ``false``/``False``/``no``/``off`` / absence
|
||||
to ``False`` (we treat absence/None as False here too — that's the
|
||||
GitHub Actions default semantics).
|
||||
|
||||
Edge cases:
|
||||
- String ``"true"`` (quoted in YAML) — kept as the string
|
||||
``"true"``, falsy under bool() but a flip we DO care about
|
||||
catching. Normalize string forms case-insensitively to bool
|
||||
so the diff is consistent with the runtime behavior of
|
||||
Gitea Actions, which YAML-parses the same way.
|
||||
"""
|
||||
if isinstance(val, bool):
|
||||
return val
|
||||
if val is None:
|
||||
return False
|
||||
if isinstance(val, str):
|
||||
return val.strip().lower() in ("true", "yes", "on", "1")
|
||||
return bool(val)
|
||||
|
||||
|
||||
def jobs_coe_map(workflow_doc: dict) -> dict[str, bool]:
|
||||
"""Return ``{job_key: continue_on_error_bool}`` for every job in
|
||||
the workflow. Job-level ``continue-on-error`` only — does NOT
|
||||
descend into per-step ``continue-on-error`` (step-level CoE
|
||||
masking is a separate class and is handled by the test suite
|
||||
+ reviewer, not by this gate — see Future Work in the workflow
|
||||
YAML).
|
||||
"""
|
||||
out: dict[str, bool] = {}
|
||||
jobs = workflow_doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return out
|
||||
for key, job in jobs.items():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
out[key] = _coerce_coe(job.get("continue-on-error"))
|
||||
return out
|
||||
|
||||
|
||||
def workflow_name(workflow_doc: dict, *, fallback: str = "") -> str:
|
||||
"""Top-level ``name:`` of the workflow. Falls back to the filename
|
||||
(without extension) per Gitea Actions semantics."""
|
||||
n = workflow_doc.get("name")
|
||||
if isinstance(n, str) and n.strip():
|
||||
return n.strip()
|
||||
return fallback
|
||||
|
||||
|
||||
def job_display_name(workflow_doc: dict, job_key: str) -> str:
|
||||
"""``jobs.<key>.name`` if present, else the key. Mirrors
|
||||
expected_context() in ci-required-drift.py."""
|
||||
job = workflow_doc.get("jobs", {}).get(job_key)
|
||||
if isinstance(job, dict):
|
||||
n = job.get("name")
|
||||
if isinstance(n, str) and n.strip():
|
||||
return n.strip()
|
||||
return job_key
|
||||
|
||||
|
||||
def context_name(workflow_name_str: str, job_name_str: str, event: str = "push") -> str:
|
||||
"""Render the commit-status context the way Gitea Actions emits it.
|
||||
Default ``event="push"`` because recent-runs-on-main are push events;
|
||||
callers can override to ``"pull_request"`` for PR-context lookups."""
|
||||
return f"{workflow_name_str} / {job_name_str} ({event})"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Diff detection — flips, not arbitrary changes
|
||||
# --------------------------------------------------------------------------
|
||||
def detect_flips(
|
||||
base_workflows: dict[str, str],
|
||||
head_workflows: dict[str, str],
|
||||
) -> list[dict]:
|
||||
"""Compare per-file CoE maps; return a list of flip records.
|
||||
|
||||
Inputs are ``{path: yaml_text}`` for both sides. Output records
|
||||
have the shape::
|
||||
|
||||
{
|
||||
"workflow_path": ".gitea/workflows/ci.yml",
|
||||
"workflow_name": "CI",
|
||||
"job_key": "platform-build",
|
||||
"job_name": "Platform (Go)",
|
||||
"context": "CI / Platform (Go) (push)",
|
||||
}
|
||||
|
||||
A flip is base[CoE] ∈ {True} AND head[CoE] ∈ {False}. Files
|
||||
only present on one side are skipped — adding a new workflow
|
||||
with ``CoE: false`` is fine (no history to mask), and removing
|
||||
a workflow can't possibly flip anything.
|
||||
"""
|
||||
flips: list[dict] = []
|
||||
for path, base_text in base_workflows.items():
|
||||
if path not in head_workflows:
|
||||
continue
|
||||
try:
|
||||
base_doc = yaml.safe_load(base_text) or {}
|
||||
head_doc = yaml.safe_load(head_workflows[path]) or {}
|
||||
except yaml.YAMLError as e:
|
||||
# Don't block on a parse error — the YAML lint workflows
|
||||
# catch invalid YAML separately. Just warn so the failing
|
||||
# file is visible.
|
||||
sys.stderr.write(f"::warning file={path}::YAML parse error: {e}\n")
|
||||
continue
|
||||
if not isinstance(base_doc, dict) or not isinstance(head_doc, dict):
|
||||
continue
|
||||
base_map = jobs_coe_map(base_doc)
|
||||
head_map = jobs_coe_map(head_doc)
|
||||
wf_name = workflow_name(head_doc, fallback=os.path.basename(path).rsplit(".", 1)[0])
|
||||
for job_key, base_val in base_map.items():
|
||||
if job_key not in head_map:
|
||||
continue # job removed — not a flip
|
||||
if base_val is True and head_map[job_key] is False:
|
||||
flips.append({
|
||||
"workflow_path": path,
|
||||
"workflow_name": wf_name,
|
||||
"job_key": job_key,
|
||||
"job_name": job_display_name(head_doc, job_key),
|
||||
"context": context_name(wf_name, job_display_name(head_doc, job_key), "push"),
|
||||
})
|
||||
return flips
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Git: snapshot every .gitea/workflows/*.yml at a SHA (no checkout)
|
||||
# --------------------------------------------------------------------------
|
||||
def _git(*args: str, cwd: str | None = None) -> str:
|
||||
"""Run ``git`` and return stdout (text)."""
|
||||
result = subprocess.run(
|
||||
["git", *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
cwd=cwd,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"git {args!r} failed: {result.stderr.strip()}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def workflows_at_sha(sha: str, *, repo_dir: str | None = None) -> dict[str, str]:
|
||||
"""Read every ``.gitea/workflows/*.yml`` blob at ``sha``.
|
||||
|
||||
Uses ``git ls-tree`` + ``git show`` so we never need to check out
|
||||
the SHA (the workflow runs on the PR head; the base SHA is
|
||||
fetched, not checked out).
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
listing = _git("ls-tree", "-r", "--name-only", sha, ".gitea/workflows/", cwd=repo_dir)
|
||||
for line in listing.splitlines():
|
||||
line = line.strip()
|
||||
if not line.endswith((".yml", ".yaml")):
|
||||
continue
|
||||
try:
|
||||
blob = _git("show", f"{sha}:{line}", cwd=repo_dir)
|
||||
except RuntimeError:
|
||||
# Symlink or other non-blob; skip.
|
||||
continue
|
||||
out[line] = blob
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea: recent commits + per-commit combined status + log fetch
|
||||
# --------------------------------------------------------------------------
|
||||
def recent_commits_on_branch(branch: str, n: int) -> list[str]:
|
||||
"""Last `n` commit SHAs on ``branch`` (oldest→newest is fine; we
|
||||
treat them as a set). Uses the REST ``/commits`` endpoint with
|
||||
``sha=branch&limit=n``."""
|
||||
_, body = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/commits",
|
||||
query={"sha": branch, "limit": str(n)},
|
||||
)
|
||||
if not isinstance(body, list):
|
||||
raise ApiError(f"/commits for {branch} returned non-list: {type(body).__name__}")
|
||||
out: list[str] = []
|
||||
for c in body:
|
||||
if isinstance(c, dict):
|
||||
sha = c.get("sha") or (c.get("commit", {}) or {}).get("id")
|
||||
if isinstance(sha, str) and len(sha) >= 7:
|
||||
out.append(sha)
|
||||
return out
|
||||
|
||||
|
||||
def combined_status(sha: str) -> dict:
|
||||
"""Combined commit status for a SHA. Same shape as
|
||||
``main-red-watchdog.get_combined_status``."""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"combined-status for {sha} not a dict")
|
||||
return body
|
||||
|
||||
|
||||
def _entry_state(s: dict) -> str:
|
||||
"""Per-entry state — Gitea 1.22.6 schema asymmetry: top-level
|
||||
uses ``state``, per-entry uses ``status``. Defensive fallback per
|
||||
main-red-watchdog.py line 233."""
|
||||
return s.get("status") or s.get("state") or ""
|
||||
|
||||
|
||||
def fetch_log(target_url: str) -> str | None:
|
||||
"""Fetch a job log given its web-UI ``target_url`` (e.g.
|
||||
``/molecule-ai/molecule-core/actions/runs/13494/jobs/0``).
|
||||
|
||||
Per ``reference_gitea_actions_log_fetch``: append ``/logs`` to the
|
||||
job route. Per ``reference_gitea_1_22_6_lacks_rest_rerun_endpoints``:
|
||||
Gitea 1.22.6 lacks the REST ``/api/v1/.../actions/runs/*`` path; the
|
||||
web-UI route is the only working endpoint until 1.24+.
|
||||
|
||||
Returns the log text on success, ``None`` on 404 / log-pruned /
|
||||
network error (caller treats None as "log unavailable, warn-not-fail").
|
||||
"""
|
||||
if not target_url:
|
||||
return None
|
||||
# Normalize: target_url may be relative ("/owner/repo/...") or
|
||||
# absolute. Both need ``/logs`` appended to the job sub-path.
|
||||
if target_url.startswith("/"):
|
||||
url = f"{WEB}{target_url}"
|
||||
else:
|
||||
url = target_url
|
||||
if not url.endswith("/logs"):
|
||||
url = f"{url}/logs"
|
||||
try:
|
||||
_, body, _ = http("GET", url, expect_json=False, timeout=60)
|
||||
except ApiError as e:
|
||||
sys.stderr.write(f"::warning::log fetch failed for {url}: {e}\n")
|
||||
return None
|
||||
if isinstance(body, bytes):
|
||||
return body.decode("utf-8", errors="replace")
|
||||
return None
|
||||
|
||||
|
||||
def grep_fail_markers(log_text: str) -> list[str]:
|
||||
"""Return up to 5 sample matching lines for any FAIL_PATTERNS hit.
|
||||
Empty list = clean log."""
|
||||
matches: list[str] = []
|
||||
for line in log_text.splitlines():
|
||||
for pat in FAIL_PATTERNS:
|
||||
if pat in line:
|
||||
# Truncate to keep error output bounded.
|
||||
matches.append(line.strip()[:240])
|
||||
break
|
||||
if len(matches) >= 5:
|
||||
break
|
||||
return matches
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Verification: for one flip, scan recent runs on BASE_REF
|
||||
# --------------------------------------------------------------------------
|
||||
def verify_flip(flip: dict, branch: str, n: int) -> dict:
|
||||
"""Scan the last ``n`` commits on ``branch``. For each commit whose
|
||||
combined status contains a context matching ``flip["context"]``,
|
||||
fetch the run log and grep for FAIL markers.
|
||||
|
||||
Returns::
|
||||
|
||||
{
|
||||
"flip": flip,
|
||||
"checked_commits": int, # how many commits had a matching context
|
||||
"masked_runs": [ # runs where log shows FAIL despite status==success
|
||||
{"sha": "...", "status": "success", "target_url": "...", "samples": [...]},
|
||||
...
|
||||
],
|
||||
"fail_runs": [ # runs where status itself is failure/error
|
||||
{"sha": "...", "status": "failure", "target_url": "...", "samples": [...]},
|
||||
...
|
||||
],
|
||||
"warnings": [str], # log-unavailable warnings (not blocking)
|
||||
}
|
||||
|
||||
Blocking condition: ``masked_runs`` OR ``fail_runs`` non-empty.
|
||||
A ``success`` status with a clean log is the only "OK to flip"
|
||||
outcome (per hongming-pc2 §SOP-N rule (e)).
|
||||
"""
|
||||
target_context = flip["context"]
|
||||
result = {
|
||||
"flip": flip,
|
||||
"checked_commits": 0,
|
||||
"masked_runs": [],
|
||||
"fail_runs": [],
|
||||
"warnings": [],
|
||||
}
|
||||
|
||||
shas = recent_commits_on_branch(branch, n)
|
||||
if not shas:
|
||||
result["warnings"].append(
|
||||
f"no recent commits on {branch} (cannot verify flip)"
|
||||
)
|
||||
return result
|
||||
|
||||
for sha in shas:
|
||||
try:
|
||||
status_doc = combined_status(sha)
|
||||
except ApiError as e:
|
||||
result["warnings"].append(f"combined-status for {sha}: {e}")
|
||||
continue
|
||||
statuses = status_doc.get("statuses") or []
|
||||
# First entry matching the context name. Newest SHAs come
|
||||
# first; one entry per context per SHA is the usual shape.
|
||||
for s in statuses:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
if s.get("context") != target_context:
|
||||
continue
|
||||
result["checked_commits"] += 1
|
||||
state = _entry_state(s)
|
||||
target_url = s.get("target_url") or ""
|
||||
log_text = fetch_log(target_url)
|
||||
if log_text is None:
|
||||
result["warnings"].append(
|
||||
f"log unavailable for {sha} {target_context}"
|
||||
)
|
||||
# Still record the status itself if it's red — that's
|
||||
# a hard signal that doesn't need log access.
|
||||
if state in ("failure", "error"):
|
||||
result["fail_runs"].append({
|
||||
"sha": sha,
|
||||
"status": state,
|
||||
"target_url": target_url,
|
||||
"samples": ["[log unavailable; status itself is " + state + "]"],
|
||||
})
|
||||
break
|
||||
samples = grep_fail_markers(log_text)
|
||||
if state in ("failure", "error"):
|
||||
result["fail_runs"].append({
|
||||
"sha": sha,
|
||||
"status": state,
|
||||
"target_url": target_url,
|
||||
"samples": samples or ["[no FAIL markers found but status is " + state + "]"],
|
||||
})
|
||||
elif samples and state == "success":
|
||||
# The bug class: status==success while log shows FAIL.
|
||||
# That's exactly Quirk #10 (continue-on-error masking).
|
||||
result["masked_runs"].append({
|
||||
"sha": sha,
|
||||
"status": state,
|
||||
"target_url": target_url,
|
||||
"samples": samples,
|
||||
})
|
||||
# Either way, we matched one context entry for this SHA;
|
||||
# don't keep looping `statuses[]`.
|
||||
break
|
||||
|
||||
if result["checked_commits"] == 0:
|
||||
result["warnings"].append(
|
||||
f"no runs of {target_context!r} found in the last {n} commits on "
|
||||
f"{branch} — cannot verify; allowing flip with warning"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Report rendering
|
||||
# --------------------------------------------------------------------------
|
||||
def render_flip_report(verdict: dict) -> str:
|
||||
flip = verdict["flip"]
|
||||
lines = [
|
||||
f"job: {flip['job_key']} ({flip['context']})",
|
||||
f" workflow: {flip['workflow_path']}",
|
||||
f" checked_commits: {verdict['checked_commits']}",
|
||||
]
|
||||
for run in verdict["fail_runs"]:
|
||||
url = run["target_url"]
|
||||
# target_url may be relative; render the absolute form for
|
||||
# click-through.
|
||||
if url.startswith("/"):
|
||||
url = f"{WEB}{url}"
|
||||
lines.append(f" fail run {run['sha'][:10]} (status={run['status']}): {url}")
|
||||
for sample in run["samples"]:
|
||||
lines.append(f" | {sample}")
|
||||
for run in verdict["masked_runs"]:
|
||||
url = run["target_url"]
|
||||
if url.startswith("/"):
|
||||
url = f"{WEB}{url}"
|
||||
lines.append(
|
||||
f" MASKED run {run['sha'][:10]} (status=success, log shows FAIL): {url}"
|
||||
)
|
||||
for sample in run["samples"]:
|
||||
lines.append(f" | {sample}")
|
||||
for w in verdict["warnings"]:
|
||||
lines.append(f" warning: {w}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main
|
||||
# --------------------------------------------------------------------------
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="lint-pre-flip-continue-on-error",
|
||||
description="Block a PR that flips continue-on-error true→false "
|
||||
"without proof recent runs are actually green.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect + print findings to stdout; never exit non-zero. "
|
||||
"Useful for local testing.",
|
||||
)
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
_require_runtime_env()
|
||||
|
||||
base_workflows = workflows_at_sha(BASE_SHA)
|
||||
head_workflows = workflows_at_sha(HEAD_SHA)
|
||||
flips = detect_flips(base_workflows, head_workflows)
|
||||
|
||||
if not flips:
|
||||
print("::notice::no continue-on-error true→false flips in this PR")
|
||||
return 0
|
||||
|
||||
print(f"::notice::detected {len(flips)} continue-on-error true→false flip(s); verifying recent runs on {BASE_REF}")
|
||||
bad_flips: list[dict] = []
|
||||
for flip in flips:
|
||||
verdict = verify_flip(flip, BASE_REF, RECENT_COMMITS_N)
|
||||
report = render_flip_report(verdict)
|
||||
if verdict["fail_runs"] or verdict["masked_runs"]:
|
||||
print(f"::error file={flip['workflow_path']}::flip of {flip['job_key']} "
|
||||
f"({flip['context']}) blocked — recent runs on {BASE_REF} show "
|
||||
f"FAIL markers OR are red. Pull each run log below + grep "
|
||||
f"`--- FAIL` / `FAIL ` / `::error::` — DON'T trust the masked "
|
||||
f"combined-status. See hongming-pc2 charter §SOP-N rule (e). "
|
||||
f"PR#656 / mc#664 reference class.")
|
||||
bad_flips.append(verdict)
|
||||
else:
|
||||
print(f"::notice::flip of {flip['job_key']} ({flip['context']}) is safe — "
|
||||
f"{verdict['checked_commits']} recent run(s), no FAIL markers")
|
||||
# Always print the per-flip detail block so the human-readable
|
||||
# report is in the run log for both safe and unsafe flips.
|
||||
print(f"::group::flip detail: {flip['job_key']}")
|
||||
print(report)
|
||||
print("::endgroup::")
|
||||
|
||||
if bad_flips and not args.dry_run:
|
||||
print(f"::error::{len(bad_flips)}/{len(flips)} flip(s) failed pre-flip verification")
|
||||
return 1
|
||||
if bad_flips and args.dry_run:
|
||||
print(f"::warning::[dry-run] {len(bad_flips)}/{len(flips)} flip(s) WOULD fail; exit 0 forced")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,526 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_required_context_exists_in_bp — Tier 2g per internal#350.
|
||||
|
||||
Rule
|
||||
----
|
||||
When a PR adds a NEW commit-status emission (a context that didn't
|
||||
exist on the base side), the workflow file must carry one of three
|
||||
directive comments adjacent to the new job:
|
||||
|
||||
(a) `# bp-required: yes`
|
||||
The new context MUST already be in
|
||||
`branch_protections/<branch>.status_check_contexts`. Verified
|
||||
via Gitea API at PR time.
|
||||
|
||||
(b) `# bp-required: pending #NNN`
|
||||
Acknowledged asymmetry; references an OPEN tracking issue that
|
||||
will follow up with the BP PATCH.
|
||||
|
||||
(c) `# bp-exempt: <free-text reason>`
|
||||
Informational job, not intended to be a required gate.
|
||||
|
||||
No directive on a new emitter → FAIL with a 3-option fix-hint.
|
||||
|
||||
The class this prevents
|
||||
-----------------------
|
||||
PR#656 added `CI / all-required (pull_request)` as a sentinel context
|
||||
that workflows emit, but BP did NOT list it. When `platform-build`
|
||||
failed, `all-required` failed, but BP let the PR merge anyway →
|
||||
cascade to mc#664. With this lint, PR#656 would have been blocked
|
||||
until either the BP PATCH ran alongside OR the author added a
|
||||
`bp-required: pending` directive.
|
||||
|
||||
Why directives MUST live in the workflow YAML
|
||||
---------------------------------------------
|
||||
The directive comment lives with the emitter so a scheduled
|
||||
audit (Tier 2f, daily) can read the same source. PR-body-only
|
||||
directives invisibly evaporate on merge — the asymmetry would
|
||||
return to undetected. PR-body claims are advisory; workflow-file
|
||||
comments are the contract.
|
||||
|
||||
How "new emission" is detected
|
||||
------------------------------
|
||||
Diff base..head over `.gitea/workflows/*.yml`. For each YAML file
|
||||
that's added or modified:
|
||||
- Parse both base-side and head-side via PyYAML AST.
|
||||
- Enumerate emitted contexts on each side using the same rules as
|
||||
Tier 2f (workflow.name + job.name|key + event-mapping).
|
||||
- `new_contexts = head_contexts - base_contexts`.
|
||||
|
||||
If `new_contexts` is empty after de-dup, no rule applies → pass.
|
||||
|
||||
Per `feedback_behavior_based_ast_gates`: comment scanning uses raw
|
||||
text in a small window around the job-key line, NOT regex over the
|
||||
full file. This avoids matching `bp-required:` mentioned in a
|
||||
comment unrelated to the new job.
|
||||
|
||||
Exit codes
|
||||
----------
|
||||
0 — no new emissions, all new emissions have valid directives,
|
||||
or BP read errored (graceful-degrade per Tier 2a contract).
|
||||
1 — at least one new emission lacks a directive, or has
|
||||
`bp-required: yes` but the context is missing from BP.
|
||||
2 — env contract violation or YAML parse error.
|
||||
|
||||
Env
|
||||
---
|
||||
BASE_SHA — PR base SHA
|
||||
HEAD_SHA — PR head SHA
|
||||
GITEA_TOKEN — DRIFT_BOT_TOKEN (repo-admin for BP read)
|
||||
GITEA_HOST — e.g. git.moleculesai.app
|
||||
REPO — owner/name
|
||||
BRANCH — defaults to `main`
|
||||
WORKFLOWS_DIR — defaults to `.gitea/workflows`
|
||||
|
||||
Memory cross-links
|
||||
------------------
|
||||
- internal#350 (the RFC that specs this lint)
|
||||
- PR#656 (the empirical case that prompted Tier 2g)
|
||||
- mc#664 (the surfaced cascade)
|
||||
- feedback_phantom_required_check_after_gitea_migration (Tier 2f cousin)
|
||||
- feedback_behavior_based_ast_gates
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write(
|
||||
"::error::PyYAML is required. Install with: pip install PyYAML\n"
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# Directive comment patterns. We match `# bp-required:` OR `# bp-exempt:`,
|
||||
# both with optional surrounding whitespace and case-sensitive on the
|
||||
# `bp-` prefix (convention).
|
||||
BP_REQUIRED_YES_RE = re.compile(
|
||||
r"#\s*bp-required:\s*yes\b", re.IGNORECASE
|
||||
)
|
||||
BP_REQUIRED_PENDING_RE = re.compile(
|
||||
r"#\s*bp-required:\s*pending\s*#(?P<num>\d+)\b", re.IGNORECASE
|
||||
)
|
||||
BP_EXEMPT_RE = re.compile(
|
||||
r"#\s*bp-exempt:\s*\S", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
# Gitea event-mapping (same as Tier 2f).
|
||||
_EVENT_MAP = {
|
||||
"pull_request": "pull_request",
|
||||
"pull_request_target": "pull_request",
|
||||
"push": "push",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env
|
||||
# ---------------------------------------------------------------------------
|
||||
def _env(key: str, default: str | None = None) -> str:
|
||||
v = os.environ.get(key, default)
|
||||
return v if v is not None else ""
|
||||
|
||||
|
||||
def _require_env(key: str) -> str:
|
||||
v = os.environ.get(key)
|
||||
if not v:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API helper (same contract as Tier 2f).
|
||||
# ---------------------------------------------------------------------------
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
) -> tuple[str, Any]:
|
||||
host = _env("GITEA_HOST")
|
||||
token = _env("GITEA_TOKEN")
|
||||
url = f"https://{host}/api/v1{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
if not raw:
|
||||
return ("ok", None)
|
||||
return ("ok", json.loads(raw))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return ("not_found", None)
|
||||
if e.code in (401, 403):
|
||||
return ("forbidden", None)
|
||||
return ("error", None)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
return ("error", None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# git helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def git_show(sha: str, path: str) -> str | None:
|
||||
r = subprocess.run(
|
||||
["git", "show", f"{sha}:{path}"], capture_output=True, text=True
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return r.stdout
|
||||
|
||||
|
||||
def git_diff_paths(base: str, head: str) -> list[str]:
|
||||
r = subprocess.run(
|
||||
["git", "diff", "--name-only", f"{base}..{head}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
return [p for p in r.stdout.splitlines() if p.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workflow context enumeration (mirror Tier 2f).
|
||||
# ---------------------------------------------------------------------------
|
||||
def _get_on(d: Any) -> Any:
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
if "on" in d:
|
||||
return d["on"]
|
||||
if True in d:
|
||||
return d[True]
|
||||
return None
|
||||
|
||||
|
||||
def _on_events(doc: Any) -> set[str]:
|
||||
on = _get_on(doc)
|
||||
raw: set[str] = set()
|
||||
if on is None:
|
||||
return raw
|
||||
if isinstance(on, str):
|
||||
raw.add(on)
|
||||
elif isinstance(on, list):
|
||||
for e in on:
|
||||
if isinstance(e, str):
|
||||
raw.add(e)
|
||||
elif isinstance(on, dict):
|
||||
for k in on:
|
||||
if isinstance(k, str):
|
||||
raw.add(k)
|
||||
return {_EVENT_MAP[e] for e in raw if e in _EVENT_MAP}
|
||||
|
||||
|
||||
def _job_display(jbody: dict, jkey: str) -> str:
|
||||
n = jbody.get("name") if isinstance(jbody, dict) else None
|
||||
if isinstance(n, str) and n:
|
||||
return n
|
||||
return jkey
|
||||
|
||||
|
||||
def workflow_contexts(doc: Any) -> set[str]:
|
||||
if not isinstance(doc, dict):
|
||||
return set()
|
||||
wf_name = doc.get("name")
|
||||
if not isinstance(wf_name, str) or not wf_name:
|
||||
return set()
|
||||
events = _on_events(doc)
|
||||
if not events:
|
||||
return set()
|
||||
jobs = doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
return set()
|
||||
out: set[str] = set()
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__":
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
disp = _job_display(jbody, jkey)
|
||||
for ev in events:
|
||||
out.add(f"{wf_name} / {disp} ({ev})")
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Find the source line of a job-key in a workflow YAML's raw text.
|
||||
# Used to scan for nearby directive comments.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _find_job_key_line(raw_lines: list[str], jkey: str) -> int | None:
|
||||
"""Return 1-based line of `<jkey>:` under jobs:."""
|
||||
in_jobs = False
|
||||
jobs_indent = -1
|
||||
for i, line in enumerate(raw_lines, start=1):
|
||||
stripped = line.lstrip()
|
||||
if stripped.startswith("jobs:"):
|
||||
in_jobs = True
|
||||
jobs_indent = len(line) - len(stripped)
|
||||
continue
|
||||
if in_jobs:
|
||||
# Job key is the next indent level under `jobs:`.
|
||||
indent = len(line) - len(stripped)
|
||||
if stripped and indent <= jobs_indent:
|
||||
# Left the jobs: block
|
||||
in_jobs = False
|
||||
continue
|
||||
if re.match(rf"^\s*{re.escape(jkey)}\s*:", line):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
_DIRECTIVE_WINDOW = 3 # lines above the job-key line (inclusive)
|
||||
|
||||
|
||||
def find_directive_for_job(
|
||||
raw_text: str, jkey: str
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Return (kind, value) tuple for the first directive in a small
|
||||
window above the job-key line.
|
||||
|
||||
kind ∈ {"required-yes", "required-pending", "exempt"}.
|
||||
value is the pending-issue number for required-pending, else None.
|
||||
Returns None if no directive found.
|
||||
|
||||
We scan ABOVE the line only (the convention is the directive
|
||||
precedes the job — matches how `# mc#NNN` comments are placed
|
||||
above `continue-on-error: true`). We don't scan inside the job
|
||||
body because steps can produce false positives.
|
||||
"""
|
||||
lines = raw_text.splitlines()
|
||||
line_no = _find_job_key_line(lines, jkey)
|
||||
if line_no is None:
|
||||
return None
|
||||
lo = max(1, line_no - _DIRECTIVE_WINDOW)
|
||||
for i in range(lo, line_no):
|
||||
line = lines[i - 1]
|
||||
m = BP_REQUIRED_PENDING_RE.search(line)
|
||||
if m:
|
||||
return ("required-pending", m.group("num"))
|
||||
if BP_REQUIRED_YES_RE.search(line):
|
||||
return ("required-yes", None)
|
||||
if BP_EXEMPT_RE.search(line):
|
||||
return ("exempt", None)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Map a context back to its emitting (workflow_path, job_key) pair so
|
||||
# we know WHERE to look for the directive comment.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _resolve_emitter(
|
||||
ctx: str, head_workflows: dict[str, tuple[str, Any]]
|
||||
) -> tuple[str, str] | None:
|
||||
"""Return (file_path, job_key) emitting ctx, or None."""
|
||||
m = re.match(r"^(?P<wf>.+?) / (?P<job>.+) \((?P<event>[^)]+)\)$", ctx)
|
||||
if not m:
|
||||
return None
|
||||
target_wf = m.group("wf")
|
||||
target_job_disp = m.group("job")
|
||||
for path, (_raw, doc) in head_workflows.items():
|
||||
if not isinstance(doc, dict):
|
||||
continue
|
||||
if doc.get("name") != target_wf:
|
||||
continue
|
||||
jobs = doc.get("jobs") or {}
|
||||
if not isinstance(jobs, dict):
|
||||
continue
|
||||
for jkey, jbody in jobs.items():
|
||||
if jkey == "__lines__":
|
||||
continue
|
||||
if not isinstance(jbody, dict):
|
||||
continue
|
||||
disp = _job_display(jbody, jkey)
|
||||
if disp == target_job_disp:
|
||||
return (path, jkey)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver
|
||||
# ---------------------------------------------------------------------------
|
||||
def run() -> int:
|
||||
base_sha = _require_env("BASE_SHA")
|
||||
head_sha = _require_env("HEAD_SHA")
|
||||
_require_env("GITEA_TOKEN")
|
||||
_require_env("GITEA_HOST")
|
||||
repo = _require_env("REPO")
|
||||
branch = _env("BRANCH", "main")
|
||||
wf_dir = _env("WORKFLOWS_DIR", ".gitea/workflows")
|
||||
|
||||
# Step 1 — find workflow files changed in the PR.
|
||||
changed = git_diff_paths(base_sha, head_sha)
|
||||
changed_workflows = [
|
||||
p
|
||||
for p in changed
|
||||
if p.startswith(wf_dir + "/")
|
||||
and (p.endswith(".yml") or p.endswith(".yaml"))
|
||||
]
|
||||
if not changed_workflows:
|
||||
print(
|
||||
"::notice::no workflow file changes in this PR; "
|
||||
"lint-required-context-exists-in-bp skipped."
|
||||
)
|
||||
return 0
|
||||
|
||||
# Step 2 — load base+head + compute new contexts.
|
||||
head_workflows: dict[str, tuple[str, Any]] = {}
|
||||
new_contexts: set[str] = set()
|
||||
for path in changed_workflows:
|
||||
base_raw = git_show(base_sha, path)
|
||||
head_raw = git_show(head_sha, path)
|
||||
if head_raw is None:
|
||||
# File deleted on head — no new emission contribution.
|
||||
continue
|
||||
try:
|
||||
head_doc = yaml.safe_load(head_raw)
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(
|
||||
f"::error file={path}::YAML parse error on head: {e}\n"
|
||||
)
|
||||
return 2
|
||||
head_workflows[path] = (head_raw, head_doc)
|
||||
head_ctx = workflow_contexts(head_doc)
|
||||
base_ctx: set[str] = set()
|
||||
if base_raw is not None:
|
||||
try:
|
||||
base_doc = yaml.safe_load(base_raw)
|
||||
except yaml.YAMLError:
|
||||
base_doc = None
|
||||
if base_doc is not None:
|
||||
base_ctx = workflow_contexts(base_doc)
|
||||
new_contexts |= (head_ctx - base_ctx)
|
||||
|
||||
if not new_contexts:
|
||||
print(
|
||||
"::notice::no new context emissions detected in this PR; "
|
||||
"lint-required-context-exists-in-bp skipped."
|
||||
)
|
||||
return 0
|
||||
|
||||
# Step 3 — fetch BP context list.
|
||||
status, bp = api("GET", f"/repos/{repo}/branch_protections/{branch}")
|
||||
bp_contexts: set[str] = set()
|
||||
if status == "forbidden":
|
||||
sys.stderr.write(
|
||||
f"::error::GET branch_protections/{branch} returned HTTP 403 — "
|
||||
f"DRIFT_BOT_TOKEN lacks repo-admin scope. Cannot verify "
|
||||
f"bp-required directives; skipping lint with exit 0 per "
|
||||
f"Tier 2a contract. Fix the token, not the lint.\n"
|
||||
)
|
||||
return 0
|
||||
elif status == "not_found":
|
||||
# Branch has no protection — nothing to verify against; the
|
||||
# bp-required: yes directive can't be satisfied. Treat as
|
||||
# graceful-skip rather than red-X.
|
||||
print(
|
||||
f"::notice::branch '{branch}' has no protection; cannot verify "
|
||||
f"bp-required directives. Skipping (exit 0)."
|
||||
)
|
||||
return 0
|
||||
elif status == "ok" and isinstance(bp, dict):
|
||||
bp_contexts = set(bp.get("status_check_contexts") or [])
|
||||
else:
|
||||
sys.stderr.write(
|
||||
f"::error::branch_protections/{branch} response unexpected; "
|
||||
f"status={status}. Treating as transient; exit 0.\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Step 4 — validate each new emission's directive.
|
||||
violations: list[str] = []
|
||||
for ctx in sorted(new_contexts):
|
||||
emitter = _resolve_emitter(ctx, head_workflows)
|
||||
if emitter is None:
|
||||
# Shouldn't happen — we just derived ctx from head_workflows.
|
||||
# Belt-and-suspenders fallback.
|
||||
violations.append(
|
||||
f"::error::new emission '{ctx}' (could not resolve emitter "
|
||||
f"file/job — bug in lint?)"
|
||||
)
|
||||
continue
|
||||
file_path, jkey = emitter
|
||||
raw_text, _ = head_workflows[file_path]
|
||||
directive = find_directive_for_job(raw_text, jkey)
|
||||
if directive is None:
|
||||
violations.append(
|
||||
f"::error file={file_path}::lint-required-context-exists-in-bp "
|
||||
f"(Tier 2g): NEW emission `{ctx}` (job '{jkey}') has no "
|
||||
f"directive comment. Add ONE of these comments on the line "
|
||||
f"directly above `{jkey}:` (within {_DIRECTIVE_WINDOW} lines):\n"
|
||||
f" - `# bp-required: yes` — and ensure the context is "
|
||||
f"already in branch_protections/{branch}.status_check_contexts.\n"
|
||||
f" - `# bp-required: pending #NNN` — acknowledged asymmetry, "
|
||||
f"references the tracking issue for the BP PATCH.\n"
|
||||
f" - `# bp-exempt: <reason>` — informational job, not a gate.\n"
|
||||
f"Memory: internal#350 (PR#656 + mc#664 empirical case)."
|
||||
)
|
||||
continue
|
||||
kind, value = directive
|
||||
if kind == "exempt":
|
||||
print(f"::notice::{ctx}: bp-exempt directive present, OK.")
|
||||
continue
|
||||
if kind == "required-pending":
|
||||
print(
|
||||
f"::notice::{ctx}: bp-required: pending #{value} — "
|
||||
f"acknowledged asymmetry, OK."
|
||||
)
|
||||
continue
|
||||
if kind == "required-yes":
|
||||
if ctx in bp_contexts:
|
||||
print(
|
||||
f"::notice::{ctx}: bp-required: yes, and context is in "
|
||||
f"BP, OK."
|
||||
)
|
||||
else:
|
||||
violations.append(
|
||||
f"::error file={file_path}::lint-required-context-exists-in-bp "
|
||||
f"(Tier 2g): job '{jkey}' has `bp-required: yes` "
|
||||
f"directive but its emitted context `{ctx}` is NOT in "
|
||||
f"`branch_protections/{branch}.status_check_contexts`. "
|
||||
f"FIX: either (a) add `{ctx}` to BP (Owners-tier PATCH), "
|
||||
f"or (b) downgrade the directive to "
|
||||
f"`# bp-required: pending #NNN` referencing the tracker "
|
||||
f"for the pending BP PATCH."
|
||||
)
|
||||
|
||||
if violations:
|
||||
print(
|
||||
f"::error::lint-required-context-exists-in-bp: "
|
||||
f"{len(violations)} violation(s) across "
|
||||
f"{len(changed_workflows)} changed workflow file(s)."
|
||||
)
|
||||
for v in violations:
|
||||
print(v)
|
||||
return 1
|
||||
|
||||
print(
|
||||
f"::notice::lint-required-context-exists-in-bp: "
|
||||
f"{len(new_contexts)} new emission(s) all directive-validated."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(run())
|
||||
@@ -61,7 +61,6 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -90,19 +89,6 @@ API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
# match by exact title without parsing.
|
||||
TITLE_PREFIX = "[main-red]"
|
||||
|
||||
# Settling window (seconds) between initial red detection and the
|
||||
# pre-file recheck. The recheck filters out the two largest false-
|
||||
# positive classes seen in mc#1597..1630 (task #394, 2026-05-21):
|
||||
# 1. HEAD moved on (a new commit landed mid-tick) — the prior red SHA
|
||||
# is no longer authoritative; let the next cron tick re-evaluate.
|
||||
# 2. Combined status recovered on the SAME SHA (transient
|
||||
# cancel-cascade rolled forward to success on retry).
|
||||
# 90s is well below the hourly cron cadence; a real failure that
|
||||
# persists past it is the one we want surfaced.
|
||||
# Override with WATCHDOG_RECHECK_DELAY_SECS for tests / local probes
|
||||
# (the test suite stubs time.sleep to a no-op).
|
||||
RECHECK_DELAY_SECS = int(_env("WATCHDOG_RECHECK_DELAY_SECS", default="90"))
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only.
|
||||
@@ -186,49 +172,6 @@ def api(
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# action_run.status resolver — extensibility hook for task #394.
|
||||
# --------------------------------------------------------------------------
|
||||
def _resolve_action_run_status(target_url: str) -> int | None:
|
||||
"""Resolve the underlying Gitea `action_run.status` integer for the
|
||||
run referenced by `target_url`, returning None if the resolver
|
||||
cannot reach an authoritative source from the runner.
|
||||
|
||||
Canonical Gitea 1.22.6 enum (per `models/actions/status.go` +
|
||||
`reference_gitea_action_status_enum_corrected_2026_05_19`):
|
||||
1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
|
||||
5=Waiting, 6=Running, 7=Blocked
|
||||
Only `status == 2` is a real defect; status=3 is cancel-cascade and
|
||||
status=1 is an emission artifact (Gitea wrote a 'failure' commit_status
|
||||
row for a run that actually succeeded — observed empirically on
|
||||
`publish-canvas-image` jobs at SHAs in mc#1597..1630).
|
||||
|
||||
CURRENT STATE (2026-05-20, verified): Gitea 1.22.6 exposes NO REST
|
||||
endpoint for `action_run.status`. Probed:
|
||||
/api/v1/repos/{o}/{r}/actions/runs/{id} → HTTP 404
|
||||
/api/v1/repos/{o}/{r}/actions/jobs/{id} → HTTP 404
|
||||
/api/v1/repos/{o}/{r}/actions/tasks/{id} → HTTP 404
|
||||
/swagger.v1.json paths containing 'actions' → secrets+variables+runners only
|
||||
The SPA backend (`/{repo}/actions/runs/{id}/jobs/{idx}` POST) requires
|
||||
a session CSRF token, unreachable from a runner. The only authoritative
|
||||
source today is direct DB access (`mol_action_status` on op-host,
|
||||
`docker exec molecule-postgres-1 psql ...`), which the runner cannot
|
||||
reach.
|
||||
|
||||
Therefore: this hook returns None on every call. Callers MUST fall
|
||||
back to the description-string filter (existing) plus the HEAD
|
||||
recheck (this PR). When a future Gitea release (>=1.23 expected) or
|
||||
an op-host proxy exposes the endpoint, replace the body of this
|
||||
function with an `api(...)` call — the caller contract is stable.
|
||||
|
||||
See also:
|
||||
- `reference_chronic_red_sweep_cancelled_vs_failed_filter`
|
||||
- `feedback_gitea_status_enum_use_helper_not_raw_int`
|
||||
"""
|
||||
_ = target_url # noqa: F841 — intentional placeholder
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea reads
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -275,70 +218,15 @@ def is_red(status: dict) -> tuple[bool, list[dict]]:
|
||||
|
||||
`failed_statuses` is the list of per-context entries whose own
|
||||
`state` is in the red set; useful for the issue body.
|
||||
|
||||
Cancel-cascade filter (mc#1564, 2026-05-19):
|
||||
Gitea maps BOTH `action_run.status=2 (Failure)` AND
|
||||
`action_run.status=3 (Cancelled)` to commit-status string
|
||||
`"failure"`. On a busy main with
|
||||
`concurrency: cancel-in-progress: true`, every merge burst
|
||||
cancels prior in-flight runs (status=3) — those bubble to the
|
||||
combined-status `failure` and inflate the watchdog's red%,
|
||||
generating phantom `[main-red]` issues (mc#1562/#1552/#1540/...).
|
||||
Canonical Gitea 1.22.6 enum per `models/actions/status.go` +
|
||||
`reference_gitea_action_status_enum_corrected_2026_05_19`:
|
||||
1=Success, 2=Failure, 3=Cancelled, 4=Skipped,
|
||||
5=Waiting, 6=Running, 7=Blocked
|
||||
We only want status=2 (real defects) to file. At the
|
||||
commit-status layer we don't have the integer enum directly
|
||||
(only the `failure` rollup string), so we use the description
|
||||
string Gitea writes when a run is cancelled — empirically
|
||||
`"Has been cancelled"` (verified 2026-05-19 via #1562 body).
|
||||
Real failures show `"Failing after Ns"` and are unaffected.
|
||||
This is option B from mc#1564 (description-string filter, no
|
||||
extra API call). Description-string stability is a soft contract
|
||||
with Gitea; if a future release renames it, the cancel-cascade
|
||||
entries will simply leak back through (visible-not-silent), and
|
||||
we'll either re-pin the string or upgrade to option A (resolve
|
||||
the underlying action_run.status integer via target_url).
|
||||
"""
|
||||
combined = status.get("state")
|
||||
statuses = status.get("statuses") or []
|
||||
red_states = {"failure", "error"}
|
||||
# Schema asymmetry: top-level combined uses `state`, but per-entry
|
||||
# items in `statuses[]` use `status` in Gitea 1.22.6. Prefer
|
||||
# `status`; fall back to `state` defensively. Verified empirically
|
||||
# 2026-05-12 03:42Z. Pre-rev4 code only read `state` from per-entry
|
||||
# items → failed[] always empty → render_body always showed the
|
||||
# "no per-context entries were in a red state" fallback even when
|
||||
# the combined-state correctly flagged red. See
|
||||
# `feedback_smoke_test_vendor_truth_not_shape_match`.
|
||||
def _entry_state(s: dict) -> str:
|
||||
return s.get("status") or s.get("state") or ""
|
||||
|
||||
def _is_cancel_cascade(s: dict) -> bool:
|
||||
"""status=3 entry per Gitea 1.22.6 description-string contract.
|
||||
Match exactly (after strip) — substring match would catch
|
||||
legitimate test names like "Has been cancelled by the user
|
||||
unexpectedly" in failure logs."""
|
||||
desc = (s.get("description") or "").strip()
|
||||
return desc == "Has been cancelled"
|
||||
|
||||
failed = [
|
||||
s for s in statuses
|
||||
if isinstance(s, dict)
|
||||
and _entry_state(s) in red_states
|
||||
and not _is_cancel_cascade(s)
|
||||
if isinstance(s, dict) and s.get("state") in red_states
|
||||
]
|
||||
# Combined state alone is no longer sufficient — combined=failure
|
||||
# may be 100% cancel-cascade. Drive `red` off the FILTERED list:
|
||||
# if every red-shaped per-entry was cancel-cascade, `failed` is
|
||||
# empty and we report green. Combined-failure with no per-entry
|
||||
# detail (empty `statuses[]`) still trips red — that's the
|
||||
# "CI emitter set combined-status directly" edge case from
|
||||
# render_body's fallback path; we keep filing on it so the
|
||||
# operator sees the breadcrumb.
|
||||
combined_red_no_detail = combined in red_states and not statuses
|
||||
return (bool(failed) or combined_red_no_detail, failed)
|
||||
return (combined in red_states or bool(failed), failed)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -425,9 +313,7 @@ def render_body(sha: str, failed: list[dict], debug: dict) -> str:
|
||||
else:
|
||||
for s in failed:
|
||||
ctx = s.get("context", "(no context)")
|
||||
# Per-entry key is `status` in Gitea 1.22.6, not `state`
|
||||
# (see _entry_state in is_red). Fallback for forward-compat.
|
||||
state = s.get("status") or s.get("state") or "(no state)"
|
||||
state = s.get("state", "(no state)")
|
||||
url = s.get("target_url") or ""
|
||||
desc = (s.get("description") or "").strip()
|
||||
entry = f"- **{ctx}** — `{state}`"
|
||||
@@ -660,67 +546,13 @@ def run_once(*, dry_run: bool = False) -> int:
|
||||
"combined_state": status.get("state"),
|
||||
"failed_contexts": [s.get("context") for s in failed],
|
||||
"all_contexts": [
|
||||
# Per-entry key is `status` in Gitea 1.22.6, not `state`.
|
||||
# Pre-rev4 debug output reported `state: None` for every
|
||||
# context, making run logs useless for triage.
|
||||
{"context": s.get("context"),
|
||||
"state": s.get("status") or s.get("state")}
|
||||
{"context": s.get("context"), "state": s.get("state")}
|
||||
for s in (status.get("statuses") or [])
|
||||
if isinstance(s, dict)
|
||||
],
|
||||
}
|
||||
|
||||
if red:
|
||||
# HEAD recheck (task #394 — guards mc#1597..1630 false-positive
|
||||
# cluster). After the initial detection, wait RECHECK_DELAY_SECS
|
||||
# (default 90s; tests stub time.sleep) and re-evaluate:
|
||||
#
|
||||
# 1. Re-fetch HEAD SHA. If HEAD moved, a new commit landed
|
||||
# mid-tick — the prior red SHA is no longer authoritative
|
||||
# and the next cron run will re-evaluate against the new
|
||||
# HEAD. Skip-file.
|
||||
#
|
||||
# 2. If HEAD unchanged, re-fetch the combined status. If it
|
||||
# recovered (combined state no longer in {failure,error}
|
||||
# after the cancel-cascade filter), a transient retry
|
||||
# rolled the run forward. Skip-file.
|
||||
#
|
||||
# Both paths emit a Loki event distinguishable from the real
|
||||
# `main_red_detected` so obs queries can track filter activity.
|
||||
# The settling window is well below the hourly cron cadence —
|
||||
# genuine failures persist past it and are surfaced normally.
|
||||
time.sleep(RECHECK_DELAY_SECS)
|
||||
|
||||
recheck_sha = get_head_sha(WATCH_BRANCH)
|
||||
if recheck_sha != sha:
|
||||
emit_loki_event("main_red_skipped_head_drift", sha, [])
|
||||
print(
|
||||
f"::notice::skip-file (HEAD moved): initial red at "
|
||||
f"{sha[:10]} but HEAD is now {recheck_sha[:10]} on "
|
||||
f"{WATCH_BRANCH}; next cron tick will re-evaluate."
|
||||
)
|
||||
return 0
|
||||
|
||||
recheck_status = get_combined_status(sha)
|
||||
recheck_red, recheck_failed = is_red(recheck_status)
|
||||
if not recheck_red:
|
||||
emit_loki_event("main_red_skipped_recovered", sha, [])
|
||||
print(
|
||||
f"::notice::skip-file (recovered after settling): "
|
||||
f"combined state at {sha[:10]} flipped to "
|
||||
f"{recheck_status.get('state')!r} on recheck; "
|
||||
f"initial red was a transient cancel-cascade."
|
||||
)
|
||||
return 0
|
||||
|
||||
# Still red after settling — file/update. Use the recheck data
|
||||
# as authoritative so the issue body reflects the latest state.
|
||||
failed = recheck_failed
|
||||
debug["recheck_combined_state"] = recheck_status.get("state")
|
||||
debug["recheck_failed_contexts"] = [
|
||||
s.get("context") for s in failed
|
||||
]
|
||||
|
||||
failed_ctxs = [s.get("context") for s in failed if s.get("context")]
|
||||
emit_loki_event("main_red_detected", sha, failed_ctxs)
|
||||
print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
|
||||
|
||||
@@ -1,257 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Production auto-deploy helpers for Gitea Actions.
|
||||
|
||||
The workflow keeps network side effects in shell/curl, but centralizes the
|
||||
release decision shape here so it has unit coverage: disable flag parsing,
|
||||
target tag selection, CP payload construction, and status-context selection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
TRUE_VALUES = {"1", "true", "yes", "on", "disabled", "disable"}
|
||||
PROD_CP_URL = "https://api.moleculesai.app"
|
||||
DEFAULT_REQUIRED_CONTEXTS = [
|
||||
"CI / Platform (Go) (push)",
|
||||
"CI / Canvas (Next.js) (push)",
|
||||
"CI / Shellcheck (E2E scripts) (push)",
|
||||
"CI / Python Lint & Test (push)",
|
||||
"CI / all-required (push)",
|
||||
"Secret scan / Scan diff for credential-shaped strings (push)",
|
||||
]
|
||||
TERMINAL_FAILURE_STATES = {"failure", "error", "cancelled", "canceled", "skipped"}
|
||||
|
||||
|
||||
def truthy_flag(value: str | None) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
return value.strip().lower() in TRUE_VALUES
|
||||
|
||||
|
||||
def _int_env(env: dict[str, str], name: str, default: int, minimum: int = 1) -> int:
|
||||
raw = env.get(name, "")
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{name} must be an integer, got {raw!r}") from exc
|
||||
if value < minimum:
|
||||
raise ValueError(f"{name} must be >= {minimum}, got {value}")
|
||||
return value
|
||||
|
||||
|
||||
def build_plan(env: dict[str, str]) -> dict:
|
||||
sha = env.get("GITHUB_SHA", "").strip()
|
||||
if not sha:
|
||||
raise ValueError("GITHUB_SHA is required")
|
||||
|
||||
disabled_value = env.get("PROD_AUTO_DEPLOY_DISABLED", "")
|
||||
if truthy_flag(disabled_value):
|
||||
return {
|
||||
"enabled": False,
|
||||
"sha": sha,
|
||||
"disabled_reason": f"PROD_AUTO_DEPLOY_DISABLED={disabled_value}",
|
||||
}
|
||||
|
||||
short_sha = sha[:7]
|
||||
target_tag = env.get("PROD_AUTO_DEPLOY_TARGET_TAG", "").strip() or f"staging-{short_sha}"
|
||||
canary_slug = env.get("PROD_AUTO_DEPLOY_CANARY_SLUG", "hongming").strip()
|
||||
body = {
|
||||
"target_tag": target_tag,
|
||||
"soak_seconds": _int_env(env, "PROD_AUTO_DEPLOY_SOAK_SECONDS", 60, minimum=0),
|
||||
"batch_size": _int_env(env, "PROD_AUTO_DEPLOY_BATCH_SIZE", 3),
|
||||
"dry_run": truthy_flag(env.get("PROD_AUTO_DEPLOY_DRY_RUN", "")),
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. This caller is the
|
||||
# production auto-deploy step that rolls every live tenant (canary
|
||||
# + fan-out), no slug scoping, so confirm:true is correct.
|
||||
"confirm": True,
|
||||
}
|
||||
if canary_slug:
|
||||
body["canary_slug"] = canary_slug
|
||||
|
||||
cp_url = env.get("CP_URL", "").strip() or PROD_CP_URL
|
||||
if cp_url != PROD_CP_URL and not truthy_flag(env.get("PROD_ALLOW_NON_PROD_CP_URL", "")):
|
||||
raise ValueError(
|
||||
f"Refusing production deploy to CP_URL={cp_url!r}; "
|
||||
f"set PROD_ALLOW_NON_PROD_CP_URL=true for an explicit non-prod drill"
|
||||
)
|
||||
|
||||
return {
|
||||
"enabled": True,
|
||||
"sha": sha,
|
||||
"short_sha": short_sha,
|
||||
"target_tag": target_tag,
|
||||
"cp_url": cp_url,
|
||||
"body": body,
|
||||
}
|
||||
|
||||
|
||||
def latest_status_for_context(statuses: list[dict], context: str) -> dict | None:
|
||||
"""Return the first matching status.
|
||||
|
||||
Gitea's combined-status response is newest-first in practice. The merge
|
||||
queue relies on the same contract; keeping the selector explicit makes
|
||||
stale duplicate contexts easy to test.
|
||||
"""
|
||||
|
||||
for status in statuses:
|
||||
if status.get("context") == context:
|
||||
return status
|
||||
return None
|
||||
|
||||
|
||||
def ci_context_state(statuses: list[dict], context: str) -> str:
|
||||
status = latest_status_for_context(statuses, context)
|
||||
if not status:
|
||||
return "missing"
|
||||
return str(status.get("status") or status.get("state") or "missing").lower()
|
||||
|
||||
|
||||
def context_is_satisfied(state: str) -> bool:
|
||||
return state == "success"
|
||||
|
||||
|
||||
def context_is_terminal_failure(state: str) -> bool:
|
||||
return state in TERMINAL_FAILURE_STATES
|
||||
|
||||
|
||||
def required_contexts(env: dict[str, str]) -> list[str]:
|
||||
raw = env.get("PROD_AUTO_DEPLOY_REQUIRED_CONTEXTS", "")
|
||||
if not raw.strip():
|
||||
return DEFAULT_REQUIRED_CONTEXTS
|
||||
return [line.strip() for line in raw.replace(",", "\n").splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _api_json(url: str, token: str) -> dict:
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")[:500]
|
||||
raise RuntimeError(f"GET {url} -> HTTP {exc.code}: {body}") from exc
|
||||
|
||||
|
||||
def _api_json_optional(url: str, token: str) -> tuple[int, dict | None]:
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return resp.status, json.loads(resp.read())
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return exc.code, None
|
||||
body = exc.read().decode("utf-8", errors="replace")[:300]
|
||||
print(f"::warning::GET {url} -> HTTP {exc.code}: {body}", file=sys.stderr)
|
||||
return exc.code, None
|
||||
|
||||
|
||||
def live_disable_flag(env: dict[str, str]) -> str:
|
||||
"""Return a live disable value from Gitea variables when readable.
|
||||
|
||||
Gitea evaluates `${{ vars.* }}` once when the job starts. This API read is
|
||||
the emergency re-check immediately before production side effects.
|
||||
"""
|
||||
|
||||
token = env.get("GITEA_TOKEN", "").strip()
|
||||
if not token:
|
||||
return ""
|
||||
host = env.get("GITEA_HOST", "git.moleculesai.app")
|
||||
repo = env.get("GITHUB_REPOSITORY", "molecule-ai/molecule-core")
|
||||
variable = quote("PROD_AUTO_DEPLOY_DISABLED", safe="")
|
||||
url = f"https://{host}/api/v1/repos/{repo}/actions/variables/{variable}"
|
||||
status, body = _api_json_optional(url, token)
|
||||
if status != 200 or not isinstance(body, dict):
|
||||
return ""
|
||||
return str(body.get("data") or body.get("value") or "")
|
||||
|
||||
|
||||
def assert_not_disabled(env: dict[str, str]) -> None:
|
||||
plan = build_plan(env)
|
||||
if not plan.get("enabled"):
|
||||
raise RuntimeError(plan.get("disabled_reason", "production auto-deploy disabled"))
|
||||
live_value = live_disable_flag(env)
|
||||
if truthy_flag(live_value):
|
||||
raise RuntimeError(f"PROD_AUTO_DEPLOY_DISABLED={live_value} (live Gitea variable)")
|
||||
|
||||
|
||||
def wait_for_ci_context(env: dict[str, str]) -> str:
|
||||
host = env.get("GITEA_HOST", "git.moleculesai.app")
|
||||
repo = env.get("GITHUB_REPOSITORY", "molecule-ai/molecule-core")
|
||||
sha = env.get("GITHUB_SHA", "").strip()
|
||||
token = env.get("GITEA_TOKEN", "").strip()
|
||||
contexts = required_contexts(env)
|
||||
interval = _int_env(env, "CI_STATUS_POLL_INTERVAL_SECONDS", 15)
|
||||
timeout = _int_env(env, "CI_STATUS_TIMEOUT_SECONDS", 1800)
|
||||
|
||||
if not sha:
|
||||
raise ValueError("GITHUB_SHA is required")
|
||||
if not token:
|
||||
raise ValueError("GITEA_TOKEN is required to wait for CI status")
|
||||
|
||||
url = f"https://{host}/api/v1/repos/{repo}/commits/{sha}/status"
|
||||
deadline = time.time() + timeout
|
||||
last_states: dict[str, str] = {}
|
||||
while time.time() <= deadline:
|
||||
body = _api_json(url, token)
|
||||
statuses = body.get("statuses") or []
|
||||
states = {context: ci_context_state(statuses, context) for context in contexts}
|
||||
for context, state in states.items():
|
||||
if state != last_states.get(context):
|
||||
print(f"CI context {context!r}: {state}", file=sys.stderr)
|
||||
last_states = states
|
||||
|
||||
failures = [
|
||||
f"{context}={state}"
|
||||
for context, state in states.items()
|
||||
if context_is_terminal_failure(state)
|
||||
]
|
||||
if failures:
|
||||
raise RuntimeError(
|
||||
"Required CI context failed; refusing production deploy: "
|
||||
+ ", ".join(failures)
|
||||
)
|
||||
if all(context_is_satisfied(state) for state in states.values()):
|
||||
return "success"
|
||||
time.sleep(interval)
|
||||
last = ", ".join(f"{context}={state}" for context, state in last_states.items()) or "none"
|
||||
raise TimeoutError(f"Timed out waiting {timeout}s for required CI contexts; last_states={last}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("plan", help="print production deploy plan as JSON")
|
||||
sub.add_parser("assert-enabled", help="fail if production deploy is currently disabled")
|
||||
sub.add_parser("wait-ci", help="block until required CI context is green")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
if args.command == "plan":
|
||||
print(json.dumps(build_plan(dict(os.environ)), sort_keys=True))
|
||||
return 0
|
||||
if args.command == "assert-enabled":
|
||||
assert_not_disabled(dict(os.environ))
|
||||
return 0
|
||||
if args.command == "wait-ci":
|
||||
wait_for_ci_context(dict(os.environ))
|
||||
return 0
|
||||
except Exception as exc: # noqa: BLE001 - CLI should render operator-friendly errors.
|
||||
print(f"::error::{exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract changed-file list from a Gitea push event's commits JSON array.
|
||||
|
||||
Each commit in a push event has `added`, `removed`, and `modified` file lists.
|
||||
This script aggregates all of them and prints unique filenames one per line.
|
||||
|
||||
Usage:
|
||||
push-commits-diff-files.py < COMMITS_JSON
|
||||
|
||||
Exits 0 always (caller handles empty output as "no files").
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except Exception:
|
||||
sys.exit(0) # Don't fail the step — treat malformed JSON as empty
|
||||
|
||||
if not isinstance(data, list):
|
||||
sys.exit(0)
|
||||
|
||||
files: set[str] = set()
|
||||
for commit in data:
|
||||
if not isinstance(commit, dict):
|
||||
continue
|
||||
for key in ("added", "removed", "modified"):
|
||||
for f in commit.get(key) or []:
|
||||
if isinstance(f, str) and f:
|
||||
files.add(f)
|
||||
|
||||
if files:
|
||||
sys.stdout.write("\n".join(sorted(files)))
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,322 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# review-check — evaluate whether a PR satisfies a single team-review gate.
|
||||
#
|
||||
# RFC#324 Step 1 of 5 — qa-review + security-review check workflows.
|
||||
#
|
||||
# This is the shared evaluator invoked by:
|
||||
# .gitea/workflows/qa-review.yml (TEAM=qa, TEAM_ID=20)
|
||||
# .gitea/workflows/security-review.yml (TEAM=security, TEAM_ID=21)
|
||||
#
|
||||
# Pass condition (per RFC#324 v1.1 addendum):
|
||||
# ≥ 1 review on the PR where:
|
||||
# • state == APPROVED
|
||||
# • review.dismissed == false
|
||||
# • review.user.login != PR.user.login (non-author)
|
||||
# • review.user.login ∈ team-members
|
||||
#
|
||||
# Strict mode (default OFF for v1; see RFC trade-off note):
|
||||
# If REVIEW_CHECK_STRICT=1, additionally require review.commit_id == PR.head.sha.
|
||||
# With dismiss_stale_reviews: true at the protection layer, stale reviews
|
||||
# are already dismissed, so the additional commit_id check is belt-and-
|
||||
# suspenders. Keeping it off in v1 simplifies semantics; flip in a follow-up
|
||||
# PR if reviewer telemetry shows residual stale-APPROVE merges.
|
||||
#
|
||||
# Privilege gate (RFC#324 v1.3 §A1.1 — INFORMATIONAL ONLY):
|
||||
# The /qa-recheck and /security-recheck slash-commands can be triggered
|
||||
# by anyone who can comment on the PR. The workflow's privilege step
|
||||
# logs collaborator-status but does NOT gate execution of this script.
|
||||
# Why this is safe: this evaluator is read-only and idempotent —
|
||||
# reading `pulls/{N}/reviews` and `teams/{id}/members/{u}` can't be
|
||||
# influenced by who triggered the run. If a real team-member APPROVE
|
||||
# exists the gate flips green; otherwise it stays red. A
|
||||
# non-collaborator commenting /qa-recheck cannot manufacture a green
|
||||
# gate. Original (v1.2) design with `if:`-gating of this step was
|
||||
# fail-open (skipped-via-`if:` job still publishes the status as
|
||||
# `success`) — corrected in v1.3 per hongming-pc review 1421.
|
||||
#
|
||||
# Trust boundary (RFC A4):
|
||||
# This script is loaded from the BASE branch (sourced via .gitea/scripts/
|
||||
# on the workflow's checkout-of-base). It does NOT execute any PR-HEAD
|
||||
# code. It only reads PR review state via the Gitea API.
|
||||
#
|
||||
# Token scope (RFC A1-α):
|
||||
# The job's own conclusion (exit 0 / exit 1) is what publishes the
|
||||
# `qa-review / approved` / `security-review / approved` status context.
|
||||
# NO `POST /statuses` call here → NO `write:repository` scope on the
|
||||
# token. `read:organization` (for team-membership probe) and
|
||||
# `read:repository` (for PR + reviews) are enough.
|
||||
#
|
||||
# Required env:
|
||||
# GITEA_TOKEN — least-priv read:repository + read:organization. See note
|
||||
# below about the team-membership API requiring the token
|
||||
# owner to be in the queried team (Gitea 1.22.6 quirk).
|
||||
# GITEA_HOST — e.g. git.moleculesai.app
|
||||
# REPO — owner/name (from github.repository)
|
||||
# PR_NUMBER — int (from github.event.pull_request.number or
|
||||
# github.event.issue.number for issue_comment events)
|
||||
# TEAM — short team name (qa | security) for log lines
|
||||
# TEAM_ID — Gitea team id (20=qa, 21=security at time of writing)
|
||||
#
|
||||
# Optional:
|
||||
# REVIEW_CHECK_DEBUG=1 — per-API-call diagnostic lines
|
||||
# REVIEW_CHECK_STRICT=1 — also require review.commit_id == pr.head.sha
|
||||
# DEFAULT_BRANCH=main — branch this gate protects; non-default-base PRs no-op
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# jq is required for JSON parsing. It is pre-baked into the runner-base
|
||||
# image (per RFC#268 workflow-smoke), so the only reason we'd not find it
|
||||
# is a broken runner. The previous fallback dance (apt-get + curl to
|
||||
# /usr/local/bin/jq) cannot succeed on a uid-1001 rootless runner
|
||||
# (#391/#402 + feedback_ci_runner_install_needs_writable_path), so it's
|
||||
# dropped. Fail loud with a clear diagnostic rather than attempt an
|
||||
# install that physically cannot work.
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "::error::jq missing from runner-base image — bake it into the runner image (see RFC#268 workflow-smoke / feedback_ci_runner_install_needs_writable_path). This evaluator cannot run without jq."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN required}"
|
||||
: "${GITEA_HOST:?GITEA_HOST required}"
|
||||
: "${REPO:?REPO required (owner/name)}"
|
||||
: "${PR_NUMBER:?PR_NUMBER required}"
|
||||
: "${TEAM:?TEAM required (qa|security)}"
|
||||
: "${TEAM_ID:?TEAM_ID required (integer)}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
API="https://${GITEA_HOST}/api/v1"
|
||||
|
||||
# Token-in-argv fix (#541): write the Authorization header to a mode-600
|
||||
# temp file instead of passing it via curl -H "$AUTH" (which puts the
|
||||
# secret token value in the process table for any process to read via
|
||||
# /proc/<pid>/cmdline or ps -ef). The curl config file is read by curl
|
||||
# itself and never appears in the argv of the curl subprocess.
|
||||
CURL_AUTH_FILE=$(mktemp "${TMPDIR:-/tmp}/curl-auth.XXXXXX")
|
||||
chmod 600 "$CURL_AUTH_FILE"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$CURL_AUTH_FILE"
|
||||
|
||||
# Pre-create temp files so cleanup trap can reference them by name
|
||||
# (bash trap 'function' EXIT expands variables at trap-fire time, not def time).
|
||||
PR_JSON=$(mktemp)
|
||||
REVIEWS_JSON=$(mktemp)
|
||||
COMMENTS_JSON=$(mktemp)
|
||||
TEAM_PROBE_TMP=$(mktemp)
|
||||
NA_STATUSES_TMP="" # declared here so cleanup() always has the var
|
||||
|
||||
cleanup() {
|
||||
rm -f "$CURL_AUTH_FILE" "$PR_JSON" "$REVIEWS_JSON" "$COMMENTS_JSON" "$TEAM_PROBE_TMP" "${NA_STATUSES_TMP-}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
debug() {
|
||||
if [ "${REVIEW_CHECK_DEBUG:-}" = "1" ]; then
|
||||
echo " [debug] $*" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
echo "::notice::${TEAM}-review evaluating repo=${OWNER}/${NAME} pr=${PR_NUMBER} team_id=${TEAM_ID}"
|
||||
|
||||
# --- Fetch the PR (for author + head.sha) ---
|
||||
HTTP_CODE=$(curl -sS -o "$PR_JSON" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}")
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::GET /pulls/${PR_NUMBER} returned HTTP ${HTTP_CODE} (token scope?)"
|
||||
cat "$PR_JSON" >&2
|
||||
exit 1
|
||||
fi
|
||||
PR_AUTHOR=$(jq -r '.user.login // ""' "$PR_JSON")
|
||||
PR_HEAD_SHA=$(jq -r '.head.sha // ""' "$PR_JSON")
|
||||
PR_BASE_REF=$(jq -r '.base.ref // ""' "$PR_JSON")
|
||||
PR_STATE=$(jq -r '.state // ""' "$PR_JSON")
|
||||
DEFAULT_BRANCH="${DEFAULT_BRANCH:-main}"
|
||||
debug "pr_author=${PR_AUTHOR} pr_head=${PR_HEAD_SHA:0:7} pr_base=${PR_BASE_REF} pr_state=${PR_STATE}"
|
||||
|
||||
if [ "$PR_STATE" != "open" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} is ${PR_STATE} — exiting 0 (closed PRs do not gate)"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$PR_BASE_REF" != "$DEFAULT_BRANCH" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} targets ${PR_BASE_REF:-<unknown>} not ${DEFAULT_BRANCH} — ${TEAM}-review gate not applicable"
|
||||
exit 0
|
||||
fi
|
||||
if [ -z "$PR_AUTHOR" ] || [ -z "$PR_HEAD_SHA" ]; then
|
||||
echo "::error::PR ${PR_NUMBER} missing user.login or head.sha — webhook payload malformed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- RFC#324 §N/A follow-up: check N/A declarations status ---
|
||||
# sop-checklist.py posts `sop-checklist / na-declarations (pull_request)`
|
||||
# status when a peer posts /sop-n/a <gate>. If our gate is declared N/A,
|
||||
# the requirement for a Gitea APPROVE review is waived.
|
||||
NA_STATUSES_TMP=$(mktemp)
|
||||
HTTP_CODE=$(curl -sS -o "$NA_STATUSES_TMP" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/statuses/${PR_HEAD_SHA}")
|
||||
debug "statuses/${PR_HEAD_SHA} → HTTP ${HTTP_CODE}"
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
# Gitea returns statuses as array; look for the na-declarations context.
|
||||
# jq: find all statuses where context == "sop-checklist / na-declarations (pull_request)"
|
||||
# and state == "success". Extract the description field.
|
||||
NA_DESC=$(jq -r '
|
||||
.[] |
|
||||
select(.context == "sop-checklist / na-declarations (pull_request)") |
|
||||
select(.state == "success") |
|
||||
.description
|
||||
' "$NA_STATUSES_TMP" 2>/dev/null | head -1)
|
||||
|
||||
if [ -n "$NA_DESC" ] && [ "$NA_DESC" != "null" ]; then
|
||||
debug "na-declarations status found: ${NA_DESC}"
|
||||
# Check if our gate appears in the N/A description.
|
||||
# The description format is "N/A: qa-review, security-review" or similar.
|
||||
if echo "$NA_DESC" | grep -iq "\\b${TEAM}-review\\b"; then
|
||||
echo "::notice::${TEAM}-review N/A — gate declared not-applicable via /sop-n/a: ${NA_DESC}"
|
||||
echo "::notice::PR ${PR_NUMBER} passes ${TEAM}-review via N/A declaration"
|
||||
rm -f "$NA_STATUSES_TMP"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
else
|
||||
debug "could not fetch statuses (HTTP ${HTTP_CODE}) — proceeding with normal eval"
|
||||
fi
|
||||
rm -f "$NA_STATUSES_TMP"
|
||||
|
||||
# --- Fetch all reviews on the PR ---
|
||||
HTTP_CODE=$(curl -sS -o "$REVIEWS_JSON" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews")
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::GET /pulls/${PR_NUMBER}/reviews returned HTTP ${HTTP_CODE}"
|
||||
cat "$REVIEWS_JSON" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Filter: state=APPROVED, not-dismissed, non-author. Optionally strict-mode
|
||||
# adds commit_id==head.sha (off by default; see header).
|
||||
JQ_FILTER='.[]
|
||||
| select(.state == "APPROVED")
|
||||
| select(.dismissed != true)
|
||||
| select(.user.login != $author)'
|
||||
if [ "${REVIEW_CHECK_STRICT:-}" = "1" ]; then
|
||||
JQ_FILTER="${JQ_FILTER}
|
||||
| select(.commit_id == \$head)"
|
||||
fi
|
||||
JQ_FILTER="${JQ_FILTER}
|
||||
| .user.login"
|
||||
|
||||
CANDIDATES=$(jq -r --arg author "$PR_AUTHOR" --arg head "$PR_HEAD_SHA" "$JQ_FILTER" "$REVIEWS_JSON" | sort -u)
|
||||
debug "candidate non-author approvers: $(echo "$CANDIDATES" | tr '\n' ' ')"
|
||||
|
||||
if [ -z "$CANDIDATES" ]; then
|
||||
# --- Guardrail (internal#503): explain the most common false
|
||||
# "no candidates" red. Gitea's review event enum is EXACTLY
|
||||
# APPROVED/REQUEST_CHANGES/COMMENT/PENDING. A wrong value ("APPROVE",
|
||||
# lowercase, ...) is silently accepted (HTTP 200) and stored as
|
||||
# state=PENDING. A correctly-started draft review has an EMPTY body;
|
||||
# a NON-empty body + state==PENDING by a non-author == an intended
|
||||
# verdict mis-filed by a wrong event string. Surface it actionably.
|
||||
# This does NOT change the gate result (still fail-closed below) — it
|
||||
# only converts a mystery red into a named, self-fixing error.
|
||||
MISFILED_FILTER='.[]
|
||||
| select(.state == "PENDING")
|
||||
| select(.dismissed != true)
|
||||
| select(.user.login != $author)
|
||||
| select(((.body // "") | gsub("^\\s+|\\s+$";"") | length) > 0)
|
||||
| "\(.id)\t\(.user.login)"'
|
||||
MISFILED=$(jq -r --arg author "$PR_AUTHOR" "$MISFILED_FILTER" "$REVIEWS_JSON" 2>/dev/null || true)
|
||||
if [ -n "$MISFILED" ]; then
|
||||
echo "::error::${TEAM}-review: non-author review(s) were SUBMITTED but stored as PENDING — almost certainly the wrong Gitea review event string (internal#503)."
|
||||
echo "::error::Gitea accepts ONLY the exact enum APPROVED / REQUEST_CHANGES / COMMENT. 'APPROVE' or lowercase is silently (HTTP 200) filed as PENDING and is invisible to this gate."
|
||||
printf '%s\n' "$MISFILED" | while IFS="$(printf '\t')" read -r _rid _rl; do
|
||||
[ -n "${_rid:-}" ] && echo "::error:: review id=${_rid} by '${_rl}': RE-SUBMIT via POST ${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews with {\"event\":\"APPROVED\"} (correct enum) — do NOT edit the DB."
|
||||
done
|
||||
fi
|
||||
|
||||
# --- Fallback (internal#348): check issue comments for agent-approval ---
|
||||
# core-qa-agent and core-security-agent approve via issue comments, NOT
|
||||
# the reviews API. The reviews API returns zero entries for comment-only
|
||||
# approvals. This fallback reads PR issue comments and extracts logins that:
|
||||
# 1. Posted a comment matching the agent-prefix pattern for this gate:
|
||||
# qa → "[core-qa-agent] APPROVED"
|
||||
# security → "[core-security-agent] APPROVED"
|
||||
# OR posted a generic approval keyword (word-anchored, case-insensitive):
|
||||
# APPROVED / LGTM / ACCEPTED
|
||||
# 2. Are not the PR author
|
||||
# 3. The team-membership probe below is the authoritative filter.
|
||||
AGENT_PATTERN=""
|
||||
case "$TEAM" in
|
||||
qa) AGENT_PATTERN="\\[core-qa-agent\\]" ;;
|
||||
security) AGENT_PATTERN="\\[core-security-agent\\]" ;;
|
||||
esac
|
||||
HTTP_CODE=$(curl -sS -o "$COMMENTS_JSON" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/comments")
|
||||
debug "GET /issues/${PR_NUMBER}/comments → HTTP ${HTTP_CODE}"
|
||||
if [ "$HTTP_CODE" = "200" ]; then
|
||||
# JQ expression: select non-author comments that match either the
|
||||
# agent-prefix pattern (case-insensitive) OR a generic approval keyword.
|
||||
JQ_APPROVALS='
|
||||
.[] |
|
||||
select(.user.login != $author) |
|
||||
. as $cmt |
|
||||
if ($agent_pattern | length) > 0 and ($cmt.body // "" | test($agent_pattern; "i")) then
|
||||
$cmt.user.login
|
||||
elif ($cmt.body // "" | test("\\b(APPROVED|LGTM|ACCEPTED)\\b"; "i")) then
|
||||
$cmt.user.login
|
||||
else
|
||||
empty
|
||||
end
|
||||
'
|
||||
CANDIDATES=$(jq -r \
|
||||
--arg author "$PR_AUTHOR" \
|
||||
--arg agent_pattern "$AGENT_PATTERN" \
|
||||
"$JQ_APPROVALS" \
|
||||
"$COMMENTS_JSON" 2>/dev/null | sort -u)
|
||||
debug "comment-based approval candidates: $(echo "$CANDIDATES" | tr '\n' ' ')"
|
||||
|
||||
if [ -n "$CANDIDATES" ]; then
|
||||
echo "::notice::${TEAM}-review: reviews API found no APPROVED reviews; found $(echo "$CANDIDATES" | wc -w | xargs) comment-based approval candidate(s) — verifying team membership..."
|
||||
fi
|
||||
else
|
||||
debug "could not fetch issue comments (HTTP ${HTTP_CODE})"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "${CANDIDATES:-}" ]; then
|
||||
echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (no candidates from reviews API or issue comments)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Probe team membership per candidate ---
|
||||
# Endpoint: GET /api/v1/teams/{id}/members/{username}
|
||||
# 200/204 → is member
|
||||
# 403 → token owner is not in this team (Gitea 1.22.6 'Must be a team
|
||||
# member' constraint — see follow-up issue for token-provisioning)
|
||||
# 404 → not a member
|
||||
for U in $CANDIDATES; do
|
||||
CODE=$(curl -sS -o "$TEAM_PROBE_TMP" -w '%{http_code}' \
|
||||
-K "$CURL_AUTH_FILE" "${API}/teams/${TEAM_ID}/members/${U}")
|
||||
debug "probe ${U} in team ${TEAM} (id=${TEAM_ID}) → HTTP ${CODE}"
|
||||
case "$CODE" in
|
||||
200|204)
|
||||
echo "::notice::${TEAM}-review APPROVED by ${U} (team=${TEAM})"
|
||||
exit 0
|
||||
;;
|
||||
403)
|
||||
# Token owner is not in the team being probed; the API refuses to
|
||||
# confirm membership. This is the RFC#324 follow-up token-scope gap.
|
||||
# Fail closed — never grant approval on a 403; surface clearly.
|
||||
echo "::error::team-probe for ${U} in ${TEAM} returned 403 (token owner not in ${TEAM} team — RFC#324 token-scope follow-up). Cannot confirm membership; failing closed."
|
||||
cat "$TEAM_PROBE_TMP" >&2
|
||||
exit 1
|
||||
;;
|
||||
404)
|
||||
debug "${U} not a member of ${TEAM}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::team-probe for ${U} in ${TEAM} returned unexpected HTTP ${CODE}"
|
||||
cat "$TEAM_PROBE_TMP" >&2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "::error::${TEAM}-review awaiting non-author APPROVE from ${TEAM} team (candidates: $(echo "$CANDIDATES" | tr '\n' ',' | sed 's/,$//') — none are in team)"
|
||||
exit 1
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-run review-check.sh for a slash-command refire and post the protected
|
||||
# pull_request status context to the PR head SHA.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN required}"
|
||||
: "${GITEA_HOST:?GITEA_HOST required}"
|
||||
: "${REPO:?REPO required}"
|
||||
: "${PR_NUMBER:?PR_NUMBER required}"
|
||||
: "${TEAM:?TEAM required}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
API="https://${GITEA_HOST}/api/v1"
|
||||
CONTEXT="${TEAM}-review / approved (pull_request)"
|
||||
TARGET_URL="https://${GITEA_HOST}/${OWNER}/${NAME}/pulls/${PR_NUMBER}"
|
||||
|
||||
authfile=$(mktemp)
|
||||
prfile=$(mktemp)
|
||||
postfile=$(mktemp)
|
||||
# shellcheck disable=SC2329 # invoked by EXIT trap
|
||||
cleanup() {
|
||||
rm -f "$authfile" "$prfile" "$postfile"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
|
||||
code=$(curl -sS -o "$prfile" -w '%{http_code}' -K "$authfile" \
|
||||
"${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::GET /pulls/${PR_NUMBER} returned HTTP ${code}"
|
||||
head -c 200 "$prfile" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
head_sha=$(jq -r '.head.sha // ""' "$prfile")
|
||||
state=$(jq -r '.state // ""' "$prfile")
|
||||
if [ -z "$head_sha" ] || [ "$head_sha" = "null" ]; then
|
||||
echo "::error::Could not resolve PR head SHA for PR ${PR_NUMBER}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$state" != "open" ]; then
|
||||
echo "::notice::PR ${PR_NUMBER} is ${state}; ${TEAM}-review refire is a no-op"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set +e
|
||||
bash .gitea/scripts/review-check.sh
|
||||
rc=$?
|
||||
set -e
|
||||
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
status_state="success"
|
||||
description="Refired via /${TEAM}-recheck by ${COMMENT_AUTHOR:-unknown}"
|
||||
else
|
||||
status_state="failure"
|
||||
description="Refired via /${TEAM}-recheck; ${TEAM}-review failed"
|
||||
fi
|
||||
|
||||
body=$(jq -nc \
|
||||
--arg state "$status_state" \
|
||||
--arg context "$CONTEXT" \
|
||||
--arg description "$description" \
|
||||
--arg target_url "$TARGET_URL" \
|
||||
'{state:$state, context:$context, description:$description, target_url:$target_url}')
|
||||
|
||||
code=$(curl -sS -o "$postfile" -w '%{http_code}' -X POST \
|
||||
-K "$authfile" -H "Content-Type: application/json" \
|
||||
-d "$body" \
|
||||
"${API}/repos/${OWNER}/${NAME}/statuses/${head_sha}")
|
||||
if [ "$code" != "200" ] && [ "$code" != "201" ]; then
|
||||
echo "::error::POST /statuses/${head_sha} returned HTTP ${code}"
|
||||
head -c 200 "$postfile" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::posted ${status_state} for context=\"${CONTEXT}\" on sha=${head_sha}"
|
||||
exit "$rc"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -96,27 +96,16 @@ API="https://${GITEA_HOST}/api/v1"
|
||||
AUTH="Authorization: token ${GITEA_TOKEN}"
|
||||
echo "::notice::tier-check start: repo=$OWNER/$NAME pr=$PR_NUMBER author=$PR_AUTHOR"
|
||||
|
||||
# Sanity: token resolves to a user.
|
||||
# Use || true on the jq pipeline so that set -euo pipefail (line 45) does not
|
||||
# cause the script to exit prematurely when the token is empty/invalid — the
|
||||
# if check below handles that case gracefully. Without || true, a 401 from an
|
||||
# empty/invalid token causes jq to exit 1, triggering set -e and exiting the
|
||||
# entire script before SOP_FAIL_OPEN can be evaluated (the check is in the jq-
|
||||
# install block; if jq is already on PATH, that block is skipped entirely).
|
||||
WHOAMI=$(curl -sS -H "$AUTH" "${API}/user" | jq -r '.login // ""') || true
|
||||
# Sanity: token resolves to a user
|
||||
WHOAMI=$(curl -sS -H "$AUTH" "${API}/user" | jq -r '.login // ""')
|
||||
if [ -z "$WHOAMI" ]; then
|
||||
echo "::error::GITEA_TOKEN cannot resolve a user via /api/v1/user — check the token scope and that the secret is wired correctly."
|
||||
if [ "${SOP_FAIL_OPEN:-}" = "1" ]; then
|
||||
echo "::warning::SOP_FAIL_OPEN=1 — exiting 0 so CI does not block."
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::token resolves to user: $WHOAMI"
|
||||
|
||||
# 1. Read tier label. || true ensures set -euo pipefail does not abort the
|
||||
# script if curl or jq fails (e.g. 401 from empty token).
|
||||
LABELS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/labels" | jq -r '.[].name') || true
|
||||
# 1. Read tier label
|
||||
LABELS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/labels" | jq -r '.[].name')
|
||||
TIER=""
|
||||
for L in $LABELS; do
|
||||
case "$L" in
|
||||
@@ -187,25 +176,17 @@ fi
|
||||
# 4. Resolve all team names → IDs
|
||||
# /orgs/{org}/teams/{slug}/... endpoints don't exist on Gitea 1.22;
|
||||
# we use /teams/{id}.
|
||||
# set +e prevents set -e from aborting the script if curl fails (e.g. empty token).
|
||||
ORG_TEAMS_FILE=$(mktemp)
|
||||
trap 'rm -f "$ORG_TEAMS_FILE"' EXIT
|
||||
set +e
|
||||
HTTP_CODE=$(curl -sS -o "$ORG_TEAMS_FILE" -w '%{http_code}' -H "$AUTH" \
|
||||
"${API}/orgs/${OWNER}/teams")
|
||||
_HTTP_EXIT=$?
|
||||
set -e
|
||||
debug "teams-list HTTP=$HTTP_CODE (curl exit=$_HTTP_EXIT) size=$(wc -c <"$ORG_TEAMS_FILE")"
|
||||
debug "teams-list HTTP=$HTTP_CODE size=$(wc -c <"$ORG_TEAMS_FILE")"
|
||||
if [ "${SOP_DEBUG:-}" = "1" ]; then
|
||||
echo " [debug] teams-list body (first 300 chars):" >&2
|
||||
head -c 300 "$ORG_TEAMS_FILE" >&2; echo >&2
|
||||
fi
|
||||
if [ "$_HTTP_EXIT" -ne 0 ] || [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::GET /orgs/${OWNER}/teams failed (curl exit=$_HTTP_EXIT HTTP=$HTTP_CODE) — token may lack read:org scope or be invalid."
|
||||
if [ "${SOP_FAIL_OPEN:-}" = "1" ]; then
|
||||
echo "::warning::SOP_FAIL_OPEN=1 — exiting 0 so CI does not block."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::GET /orgs/${OWNER}/teams returned HTTP $HTTP_CODE — token likely lacks read:org scope."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -250,22 +231,9 @@ for _t in $_all_teams; do
|
||||
debug "team-id: $_t → $_id"
|
||||
done
|
||||
|
||||
# 5. Read approving reviewers. set +e disables set -e temporarily so that curl
|
||||
# failures (e.g. empty/invalid token → HTTP 401) do not abort the script before
|
||||
# SOP_FAIL_OPEN is evaluated. set -e is restored immediately after.
|
||||
set +e
|
||||
# 5. Read approving reviewers
|
||||
REVIEWS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews")
|
||||
_REVIEWS_EXIT=$?
|
||||
set -e
|
||||
if [ $_REVIEWS_EXIT -ne 0 ] || [ -z "$REVIEWS" ]; then
|
||||
echo "::error::Failed to fetch reviews (curl exit=$_REVIEWS_EXIT) — token may be invalid or unreachable."
|
||||
if [ "${SOP_FAIL_OPEN:-}" = "1" ]; then
|
||||
echo "::warning::SOP_FAIL_OPEN=1 — exiting 0 so CI does not block."
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
APPROVERS=$(echo "$REVIEWS" | jq -r '[.[] | select(.state=="APPROVED") | .user.login] | unique | .[]') || true
|
||||
APPROVERS=$(echo "$REVIEWS" | jq -r '[.[] | select(.state=="APPROVED") | .user.login] | unique | .[]')
|
||||
if [ -z "$APPROVERS" ]; then
|
||||
echo "::error::No approving reviews on this PR. Set SOP_DEBUG=1 and re-run for diagnostics."
|
||||
exit 1
|
||||
|
||||
@@ -104,13 +104,10 @@ if [ "${SOP_REFIRE_DISABLE_RATE_LIMIT:-}" != "1" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects.
|
||||
# The canonical workflow intentionally fail-opens the job conclusion
|
||||
# (`bash .gitea/scripts/sop-tier-check.sh || true`) while Gitea branch
|
||||
# protection enforces reviewer approvals separately. Keep the refire path
|
||||
# aligned with that workflow status behavior; otherwise /refire-tier-check can
|
||||
# post a hard failure that the canonical pull_request_target workflow would
|
||||
# not publish.
|
||||
# 3. Invoke sop-tier-check.sh with the env it expects. Capture exit code.
|
||||
# The canonical script reads tier label, walks approving reviewers, and
|
||||
# evaluates the AND-composition expression — we want the SAME gate, not
|
||||
# a different gate.
|
||||
#
|
||||
# SOP_REFIRE_TIER_CHECK_SCRIPT env var lets tests substitute a mock —
|
||||
# sop-tier-check.sh uses bash 4+ associative arrays which trigger a known
|
||||
@@ -126,6 +123,7 @@ fi
|
||||
|
||||
# Re-invoke. Pipe stdout/stderr through so the runner log shows the
|
||||
# tier-check decision inline.
|
||||
set +e
|
||||
GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
GITEA_HOST="$GITEA_HOST" \
|
||||
REPO="$REPO" \
|
||||
@@ -133,8 +131,9 @@ GITEA_TOKEN="$GITEA_TOKEN" \
|
||||
PR_AUTHOR="$PR_AUTHOR" \
|
||||
SOP_DEBUG="${SOP_DEBUG:-0}" \
|
||||
SOP_LEGACY_CHECK="${SOP_LEGACY_CHECK:-0}" \
|
||||
bash "$SCRIPT" || true
|
||||
TIER_EXIT=0
|
||||
bash "$SCRIPT"
|
||||
TIER_EXIT=$?
|
||||
set -e
|
||||
debug "sop-tier-check.sh exit=$TIER_EXIT"
|
||||
|
||||
# 4. POST the resulting status.
|
||||
|
||||
@@ -1,826 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""status-reaper — Option B compensating-status POST for Gitea 1.22.6's
|
||||
hardcoded `(push)` suffix on default-branch commit statuses.
|
||||
|
||||
Tracking: this PR (workflow + script + tests + audit issue). Sibling
|
||||
bots: internal#327 (publish-runtime-bot), internal#328 (mc-drift-bot).
|
||||
Upstream RFC: internal#80. Persona provisioned by sub-agent aefaac1b
|
||||
(2026-05-11 21:39Z; Gitea uid 94, scope=write:repository).
|
||||
|
||||
What this script does, per `.gitea/workflows/status-reaper.yml` invocation:
|
||||
|
||||
1. Walk `.gitea/workflows/*.yml`. For each file, build the workflow_id
|
||||
using this resolution (per hongming-pc 22:08Z review):
|
||||
- If YAML has top-level `name:` → use that.
|
||||
- Else → use filename stem (basename minus `.yml`).
|
||||
Fail-LOUD on:
|
||||
- Two workflows resolving to the SAME identifier (collision).
|
||||
- Any identifier containing `/` (it would break context parsing
|
||||
downstream — Gitea uses ` / ` as the workflow/job separator).
|
||||
Classify each by whether `on:` contains a `push:` trigger.
|
||||
|
||||
2. List the last N (=30, rev3 — widened from 10) commits on
|
||||
WATCH_BRANCH via GET /repos/{o}/{r}/commits?sha={branch}&limit={N}.
|
||||
rev2 sweeps N commits per tick instead of HEAD only — schedule
|
||||
workflows post `failure` to whatever SHA was HEAD when they
|
||||
COMPLETED, so by the next */5 tick main has often moved forward
|
||||
and the red gets stranded on a stale commit. rev3 widens the
|
||||
window from 10 → 30 because schedule workflows post `failure`
|
||||
RETROACTIVELY (5-15 min after their merge); a 10-commit window
|
||||
is narrower than the merge-cadence during a burst, so reds land
|
||||
OUTSIDE the window before reaper sees them (Phase 1+2 evidence:
|
||||
rev2 run 17057 at 02:46Z saw 185/0 contexts on 10 SHAs; direct
|
||||
probe ~30min later showed ~25 fails on those same 10 SHAs).
|
||||
|
||||
3. For EACH SHA in the list:
|
||||
- GET combined commit status. Per-SHA error isolation
|
||||
(refinement #7): if this call raises ApiError or any 5xx,
|
||||
LOG `::warning::` + continue to the next SHA. Different from
|
||||
the single-HEAD pre-rev2 path where fail-loud was correct;
|
||||
the sweep is best-effort across historical commits, so one
|
||||
transient blip on a stale SHA must not strand reds on the
|
||||
OTHER stale SHAs.
|
||||
- If combined.state == "success": skip — cost optimization
|
||||
(refinement #2), common case (most commits are green).
|
||||
- Otherwise iterate per-context entries. For each entry where:
|
||||
state == "failure" AND context.endswith(" (push)")
|
||||
Parse context as `<workflow_name> / <job_name> (push)`.
|
||||
Look up workflow_name in the trigger map:
|
||||
- missing → log ::notice:: and skip (conservative).
|
||||
- has_push_trigger=True and description == "Has been cancelled"
|
||||
→ compensate cancelled/superseded push noise.
|
||||
- has_push_trigger=True otherwise → preserve (real defect signal).
|
||||
- has_push_trigger=False → POST a compensating
|
||||
`state=success` status to /statuses/{sha} with the same
|
||||
context (Gitea de-dups by context) and a description
|
||||
documenting the workaround + this script's path.
|
||||
|
||||
4. Exit 0. Re-running is idempotent — Gitea's commit-status table
|
||||
stores the LATEST state-per-context, so the success POST sticks
|
||||
even if another tick happens before the runner finishes.
|
||||
|
||||
What it does NOT do:
|
||||
- Touch ` (pull_request)` contexts unless the exact same
|
||||
workflow/job has a successful ` (push)` context on the same
|
||||
default-branch SHA. That case is post-merge status pollution, not
|
||||
an unproven PR gate.
|
||||
- Compensate `error`/`pending` states. Only `failure` — the only one
|
||||
Gitea emits for the hardcoded-suffix bug.
|
||||
- Write to non-default branches. WATCH_BRANCH is sourced from
|
||||
`github.event.repository.default_branch` in the workflow.
|
||||
- Mutate workflows or runs. The Actions UI still shows the
|
||||
underlying schedule-triggered run as failed; this script edits
|
||||
the commit-status surface only.
|
||||
|
||||
Halt conditions (script-level — orchestrator-level halts are in the
|
||||
workflow comments):
|
||||
- PyYAML missing → fail-loud at import (no fallback parse).
|
||||
- Workflow `name:` collision → exit 1 with ::error:: message.
|
||||
- Workflow `name:` containing `/` → exit 1 with ::error:: message.
|
||||
- Ambiguous `on:` shape (e.g. neither str/list/dict) → treat as
|
||||
"has_push_trigger=True" and log ::notice:: (preserve, never
|
||||
compensate the unknown).
|
||||
- api() non-2xx → raise ApiError, fail the workflow run loudly so
|
||||
a subsequent tick retries (per
|
||||
`feedback_api_helper_must_raise_not_return_dict`).
|
||||
|
||||
Local dry-run (no network):
|
||||
GITEA_TOKEN=... GITEA_HOST=git.moleculesai.app REPO=owner/repo \\
|
||||
WATCH_BRANCH=main WORKFLOWS_DIR=.gitea/workflows \\
|
||||
python3 .gitea/scripts/status-reaper.py --dry-run
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml # PyYAML 6.0.2 — installed by the workflow before this runs.
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment
|
||||
# --------------------------------------------------------------------------
|
||||
def _env(key: str, *, default: str = "") -> str:
|
||||
"""Read an env var with a default. Module-import-safe — tests can
|
||||
import this script without setting the full env contract."""
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN")
|
||||
GITEA_HOST = _env("GITEA_HOST")
|
||||
REPO = _env("REPO")
|
||||
WATCH_BRANCH = _env("WATCH_BRANCH", default="main")
|
||||
WORKFLOWS_DIR = _env("WORKFLOWS_DIR", default=".gitea/workflows")
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
API_TIMEOUT_SEC = int(_env("STATUS_REAPER_API_TIMEOUT_SEC", default="30") or "30")
|
||||
API_RETRIES = int(_env("STATUS_REAPER_API_RETRIES", default="3") or "3")
|
||||
API_RETRY_SLEEP_SEC = float(_env("STATUS_REAPER_API_RETRY_SLEEP_SEC", default="2") or "2")
|
||||
|
||||
# Compensating-status description prefix. Used as the marker so a human
|
||||
# auditing commit statuses can tell at a glance that the green was
|
||||
# synthetic, not a real CI pass. Kept stable; downstream tooling
|
||||
# (e.g. main-red-watchdog visual diff) MAY key on it.
|
||||
PUSH_COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (workflow has no push: trigger; "
|
||||
"Gitea 1.22.6 hardcoded-suffix bug — see .gitea/scripts/status-reaper.py)"
|
||||
)
|
||||
# Backward-compatible alias for older tests/tooling that predate the split
|
||||
# between push-suffix compensation and pull-request-shadow compensation.
|
||||
COMPENSATION_DESCRIPTION = PUSH_COMPENSATION_DESCRIPTION
|
||||
PR_SHADOW_COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (default-branch pull_request status "
|
||||
"shadowed by successful push status on same SHA; see "
|
||||
".gitea/scripts/status-reaper.py)"
|
||||
)
|
||||
CANCELLED_PUSH_COMPENSATION_DESCRIPTION = (
|
||||
"Compensated by status-reaper (push run was cancelled/superseded; "
|
||||
"Gitea 1.22.6 reports cancelled runs as failure statuses)"
|
||||
)
|
||||
CANCELLED_DESCRIPTION = "Has been cancelled"
|
||||
|
||||
# Context suffix the reaper acts on. Gitea hardcodes this for ALL
|
||||
# default-branch workflow runs.
|
||||
PUSH_SUFFIX = " (push)"
|
||||
PULL_REQUEST_SUFFIX = " (pull_request)"
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only.
|
||||
|
||||
Tests import individual functions without setting the full env
|
||||
contract. Mirrors `main-red-watchdog.py`/`ci-required-drift.py`.
|
||||
"""
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "WATCH_BRANCH", "WORKFLOWS_DIR"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper — raises on non-2xx + on JSON-decode-of-expected-JSON.
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API call cannot be trusted to have succeeded.
|
||||
|
||||
Per `feedback_api_helper_must_raise_not_return_dict`: soft-failure is
|
||||
opt-in via `expect_json=False`, never the default. A pre-fix
|
||||
implementation that returned `{}` on non-2xx would skip the
|
||||
compensating POST on a transient outage AND silently lose the
|
||||
failed-status enumeration, painting main green via omission.
|
||||
"""
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
"""Tiny HTTP helper around urllib. Same contract as
|
||||
`main-red-watchdog.py` and `ci-required-drift.py` so behaviour
|
||||
is cross-checkable."""
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
attempts = max(API_RETRIES, 1)
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=API_TIMEOUT_SEC) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
break
|
||||
except (TimeoutError, socket.timeout, urllib.error.URLError, OSError) as e:
|
||||
if attempt >= attempts:
|
||||
raise ApiError(
|
||||
f"{method} {path} failed after {attempts} attempts: {e}"
|
||||
) from e
|
||||
print(
|
||||
f"::warning::{method} {path} transient API error "
|
||||
f"(attempt {attempt}/{attempts}): {e}; retrying"
|
||||
)
|
||||
time.sleep(API_RETRY_SLEEP_SEC)
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {path} -> HTTP {status}: {snippet}")
|
||||
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
if expect_json:
|
||||
raise ApiError(
|
||||
f"{method} {path} -> HTTP {status} but body is not JSON: {e}"
|
||||
) from e
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Workflow scan + classification
|
||||
# --------------------------------------------------------------------------
|
||||
def _on_block(doc: dict) -> Any:
|
||||
"""Extract the `on:` block from a parsed YAML doc.
|
||||
|
||||
PyYAML parses bareword `on:` as Python `True` (YAML 1.1 boolean
|
||||
spec — `on/off/yes/no` are booleans). The actual key in the dict
|
||||
is therefore `True`, NOT the string `"on"`. We accept both for
|
||||
forward-compat with YAML 1.2 loaders (which keep it as `"on"`).
|
||||
"""
|
||||
if True in doc:
|
||||
return doc[True]
|
||||
return doc.get("on")
|
||||
|
||||
|
||||
def _has_push_trigger(on_block: Any, workflow_id: str) -> bool:
|
||||
"""Return True if `on:` block declares a `push` trigger.
|
||||
|
||||
Accepts the three common shapes:
|
||||
- str: `on: push` → True only if == "push"
|
||||
- list: `on: [push, pull_request]` → True if "push" in list
|
||||
- dict: `on: { push: {...}, schedule: ... }` → True if "push" key
|
||||
|
||||
Defensive: for anything else (including None/empty), return True
|
||||
so we preserve rather than over-compensate. Logged via ::notice::.
|
||||
"""
|
||||
if isinstance(on_block, str):
|
||||
return on_block == "push"
|
||||
if isinstance(on_block, list):
|
||||
return "push" in on_block
|
||||
if isinstance(on_block, dict):
|
||||
return "push" in on_block
|
||||
# None or unexpected shape — preserve, log.
|
||||
print(
|
||||
f"::notice::ambiguous on: for {workflow_id}; preserving "
|
||||
f"(value={on_block!r}, type={type(on_block).__name__})"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def scan_workflows(workflows_dir: str) -> dict[str, bool]:
|
||||
"""Walk `workflows_dir` and return `{workflow_id: has_push_trigger}`.
|
||||
|
||||
Workflow ID resolution (per hongming-pc 22:08Z review):
|
||||
- Top-level `name:` if present.
|
||||
- Else filename stem (basename minus `.yml`).
|
||||
|
||||
Fail-LOUD on:
|
||||
- Two workflows resolving to the same ID (collision).
|
||||
- Any ID containing `/` (would break ` / `-separated context
|
||||
parsing on the downstream side).
|
||||
|
||||
Returns a dict for O(1) lookup in the per-status loop.
|
||||
"""
|
||||
path = Path(workflows_dir)
|
||||
if not path.is_dir():
|
||||
# Workflow dir missing → no workflows to classify. Empty map is
|
||||
# safe: per-status loop will hit "unknown workflow; skip" for
|
||||
# every entry, which is correct (we cannot tell if a push
|
||||
# trigger exists, so we preserve).
|
||||
print(f"::warning::workflows dir not found: {workflows_dir}")
|
||||
return {}
|
||||
|
||||
out: dict[str, bool] = {}
|
||||
sources: dict[str, str] = {} # workflow_id -> source file (for collision msg)
|
||||
|
||||
for yml in sorted(path.glob("*.yml")):
|
||||
try:
|
||||
with yml.open() as f:
|
||||
doc = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
# A malformed YAML in the workflows dir is a real defect
|
||||
# (the workflow wouldn't load on Gitea either). Surface it
|
||||
# and keep going — the reaper's job is to compensate the
|
||||
# OTHER workflows even if one is broken.
|
||||
print(f"::warning::yaml parse failed for {yml.name}: {e}; skip")
|
||||
continue
|
||||
if not isinstance(doc, dict):
|
||||
print(f"::warning::workflow {yml.name} not a dict; skip")
|
||||
continue
|
||||
|
||||
# Resolve workflow_id.
|
||||
name_field = doc.get("name")
|
||||
if isinstance(name_field, str) and name_field.strip():
|
||||
workflow_id = name_field.strip()
|
||||
else:
|
||||
workflow_id = yml.stem # basename minus .yml
|
||||
|
||||
# Halt-loud: `/` in workflow_id breaks ` / ` context parsing.
|
||||
if "/" in workflow_id:
|
||||
sys.stderr.write(
|
||||
f"::error::workflow name contains '/' which breaks "
|
||||
f"context parsing: {workflow_id} (file={yml.name})\n"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Halt-loud: ID collision.
|
||||
if workflow_id in out:
|
||||
sys.stderr.write(
|
||||
f"::error::workflow name collision detected: {workflow_id} "
|
||||
f"(files: {sources[workflow_id]} + {yml.name})\n"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
on_block = _on_block(doc)
|
||||
out[workflow_id] = _has_push_trigger(on_block, workflow_id)
|
||||
sources[workflow_id] = yml.name
|
||||
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea reads
|
||||
# --------------------------------------------------------------------------
|
||||
def get_head_sha(branch: str) -> str:
|
||||
"""HEAD SHA of `branch`. Raises ApiError on non-2xx."""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/branches/{branch}")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"branch {branch} response not a JSON object")
|
||||
commit = body.get("commit")
|
||||
if not isinstance(commit, dict):
|
||||
raise ApiError(f"branch {branch} response missing `commit` object")
|
||||
sha = commit.get("id") or commit.get("sha")
|
||||
if not isinstance(sha, str) or len(sha) < 7:
|
||||
raise ApiError(f"branch {branch} response has no usable commit SHA")
|
||||
return sha
|
||||
|
||||
|
||||
def get_combined_status(sha: str) -> dict:
|
||||
"""Combined commit status for `sha`. Gitea returns:
|
||||
{
|
||||
"state": "success" | "failure" | "pending" | "error",
|
||||
"statuses": [
|
||||
{"context": "...", "state": "...", "target_url": "...",
|
||||
"description": "..."},
|
||||
...
|
||||
],
|
||||
...
|
||||
}
|
||||
Raises ApiError on non-2xx.
|
||||
"""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"status for {sha} response not a JSON object")
|
||||
return body
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Context parsing
|
||||
# --------------------------------------------------------------------------
|
||||
def parse_suffixed_context(context: str, suffix: str) -> tuple[str, str] | None:
|
||||
"""Parse `<workflow_name> / <job_name> (<event>)` into
|
||||
(workflow_name, job_name).
|
||||
|
||||
Returns None if the context doesn't match the shape (caller skips).
|
||||
Strict: requires the trailing suffix and at least one ` / `
|
||||
separator. Anything else is left alone.
|
||||
"""
|
||||
if not context.endswith(suffix):
|
||||
return None
|
||||
head = context[: -len(suffix)]
|
||||
if " / " not in head:
|
||||
return None
|
||||
workflow_name, job_name = head.split(" / ", 1)
|
||||
return workflow_name, job_name
|
||||
|
||||
|
||||
def parse_push_context(context: str) -> tuple[str, str] | None:
|
||||
"""Parse `<workflow_name> / <job_name> (push)` into
|
||||
(workflow_name, job_name)."""
|
||||
return parse_suffixed_context(context, PUSH_SUFFIX)
|
||||
|
||||
|
||||
def push_equivalent_context(context: str) -> str | None:
|
||||
"""Return the matching `(push)` context for a `(pull_request)` context."""
|
||||
parsed = parse_suffixed_context(context, PULL_REQUEST_SUFFIX)
|
||||
if parsed is None:
|
||||
return None
|
||||
workflow_name, job_name = parsed
|
||||
return f"{workflow_name} / {job_name}{PUSH_SUFFIX}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Compensating POST
|
||||
# --------------------------------------------------------------------------
|
||||
def post_compensating_status(
|
||||
sha: str,
|
||||
context: str,
|
||||
target_url: str | None,
|
||||
*,
|
||||
description: str = PUSH_COMPENSATION_DESCRIPTION,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""POST a `state=success` to /repos/{o}/{r}/statuses/{sha} with the
|
||||
given context. Gitea de-dups by context (latest write wins).
|
||||
|
||||
Description references this script so the compensation is
|
||||
self-documenting on the commit's status view.
|
||||
"""
|
||||
payload: dict[str, Any] = {
|
||||
"context": context,
|
||||
"state": "success",
|
||||
"description": description,
|
||||
}
|
||||
# Echo the original target_url when present so a human auditing
|
||||
# the (now-green) compensated status can still reach the run logs
|
||||
# that produced the original red.
|
||||
if target_url:
|
||||
payload["target_url"] = target_url
|
||||
|
||||
if dry_run:
|
||||
print(
|
||||
f"::notice::[dry-run] would compensate {context!r} on {sha[:10]} "
|
||||
f"with state=success"
|
||||
)
|
||||
return
|
||||
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/statuses/{sha}", body=payload)
|
||||
print(f"::notice::compensated {context!r} on {sha[:10]} (state=success)")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main reap loop
|
||||
# --------------------------------------------------------------------------
|
||||
def reap(
|
||||
workflow_trigger_map: dict[str, bool],
|
||||
combined: dict,
|
||||
sha: str,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Walk `combined.statuses[]` and compensate where appropriate.
|
||||
|
||||
Per-SHA worker. The multi-SHA orchestrator (`reap_branch`) calls
|
||||
this once per stale main commit each tick.
|
||||
|
||||
Returns counters for observability:
|
||||
{compensated, preserved_real_push, preserved_unknown,
|
||||
preserved_non_failure, preserved_non_push_suffix,
|
||||
preserved_unparseable, compensated_pr_shadowed_by_push_success,
|
||||
preserved_pr_without_push_success, compensated_cancelled_push,
|
||||
compensated_contexts: [<context>, ...]}
|
||||
|
||||
`compensated_contexts` is rev2-added so `reap_branch` can build
|
||||
`compensated_per_sha` without re-deriving it from the POST stream.
|
||||
"""
|
||||
counters: dict[str, Any] = {
|
||||
"compensated": 0,
|
||||
"preserved_real_push": 0,
|
||||
"preserved_unknown": 0,
|
||||
"preserved_non_failure": 0,
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_contexts": [],
|
||||
}
|
||||
|
||||
statuses = combined.get("statuses") or []
|
||||
successful_contexts = {
|
||||
(s.get("context") or "")
|
||||
for s in statuses
|
||||
if isinstance(s, dict) and (s.get("status") or s.get("state") or "") == "success"
|
||||
}
|
||||
for s in statuses:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
context = s.get("context") or ""
|
||||
# Schema asymmetry: Gitea 1.22.6 returns the TOP-LEVEL combined
|
||||
# aggregate as `combined.state` but each per-context entry in
|
||||
# `combined.statuses[]` uses the key `status`, NOT `state`.
|
||||
# Prefer `status`; fall back to `state` so a future Gitea
|
||||
# version (or a test fixture written against the wrong key)
|
||||
# still flows through the compensation path. Verified empirically
|
||||
# via direct API probe 2026-05-12 03:42Z:
|
||||
# /repos/.../commits/{sha}/status entries → key is "status".
|
||||
# Pre-rev4 code read "state" only → returned "" → bypassed the
|
||||
# `state != "failure"` guard → compensation path unreachable.
|
||||
# See `feedback_smoke_test_vendor_truth_not_shape_match`.
|
||||
state = s.get("status") or s.get("state") or ""
|
||||
|
||||
# Only `failure` is the bug shape. `error`/`pending`/`success`
|
||||
# left alone — they have other meanings.
|
||||
if state != "failure":
|
||||
counters["preserved_non_failure"] += 1
|
||||
continue
|
||||
|
||||
# Default-branch `pull_request` contexts can be stale shadows of
|
||||
# the exact same workflow/job already proven by the successful
|
||||
# `push` context on the same SHA. Compensate only that narrow
|
||||
# shape; a missing or failed push equivalent remains a real gate
|
||||
# signal and is preserved.
|
||||
push_equivalent = push_equivalent_context(context)
|
||||
if push_equivalent is not None:
|
||||
if push_equivalent in successful_contexts:
|
||||
post_compensating_status(
|
||||
sha,
|
||||
context,
|
||||
s.get("target_url"),
|
||||
description=PR_SHADOW_COMPENSATION_DESCRIPTION,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
counters["compensated"] += 1
|
||||
counters["compensated_pr_shadowed_by_push_success"] += 1
|
||||
counters["compensated_contexts"].append(context)
|
||||
else:
|
||||
counters["preserved_pr_without_push_success"] += 1
|
||||
continue
|
||||
|
||||
# Only `(push)`-suffix contexts hit the hardcoded-suffix bug.
|
||||
# Other failed contexts are preserved unless handled by the
|
||||
# pull-request-shadow rule above.
|
||||
if not context.endswith(PUSH_SUFFIX):
|
||||
counters["preserved_non_push_suffix"] += 1
|
||||
continue
|
||||
|
||||
parsed = parse_push_context(context)
|
||||
if parsed is None:
|
||||
# Has ` (push)` suffix but missing ` / ` separator — not
|
||||
# the bug shape. Preserve.
|
||||
counters["preserved_unparseable"] += 1
|
||||
continue
|
||||
workflow_name, _job_name = parsed
|
||||
|
||||
if workflow_name not in workflow_trigger_map:
|
||||
# Real workflow but renamed/deleted/external — we can't
|
||||
# tell if it has push trigger. Conservative: preserve.
|
||||
print(f"::notice::unknown workflow {workflow_name!r}; skip")
|
||||
counters["preserved_unknown"] += 1
|
||||
continue
|
||||
|
||||
if (s.get("description") or "").strip() == CANCELLED_DESCRIPTION:
|
||||
# Gitea 1.22.6 maps cancelled action runs to failure commit
|
||||
# statuses. During merge bursts, older push runs can be
|
||||
# superseded and cancelled even though a newer run for the
|
||||
# same branch is the real signal. Compensate only the exact
|
||||
# Gitea cancellation description; real push failures remain red.
|
||||
post_compensating_status(
|
||||
sha,
|
||||
context,
|
||||
s.get("target_url"),
|
||||
description=CANCELLED_PUSH_COMPENSATION_DESCRIPTION,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
counters["compensated"] += 1
|
||||
counters["compensated_cancelled_push"] += 1
|
||||
counters["compensated_contexts"].append(context)
|
||||
continue
|
||||
|
||||
if workflow_trigger_map[workflow_name]:
|
||||
# Real push trigger with a non-cancelled failure description
|
||||
# remains a defect signal. Preserve.
|
||||
counters["preserved_real_push"] += 1
|
||||
continue
|
||||
|
||||
# Class-O: schedule/dispatch/etc.-only workflow with a fake
|
||||
# (push) status from Gitea's hardcoded-suffix bug. Compensate.
|
||||
post_compensating_status(
|
||||
sha, context, s.get("target_url"), dry_run=dry_run
|
||||
)
|
||||
counters["compensated"] += 1
|
||||
counters["compensated_contexts"].append(context)
|
||||
|
||||
return counters
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# rev2: multi-SHA sweep over the last N commits on WATCH_BRANCH
|
||||
# --------------------------------------------------------------------------
|
||||
# How many main commits to sweep per tick. Sized to cover a burst-merge
|
||||
# window where multiple PRs land in the 5-min interval between reaper
|
||||
# ticks. Older reds falling off the window is acceptable — they were
|
||||
# already stale enough that the schedule-run that posted them has long
|
||||
# since been overwritten by a real push trigger. See `reference_post_
|
||||
# suspension_pipeline` for the merge-cadence baseline.
|
||||
#
|
||||
# rev3 (2026-05-12, hongming-pc2 GO 03:25Z): widened from 10 → 30.
|
||||
# rev2 (limit=10) shipped 01:48Z and ran 6/6 ticks post-merge with
|
||||
# `compensated:0` despite ~25 stranded reds visible on those same 10
|
||||
# SHAs ~30min later. Root cause: schedule workflows post `failure`
|
||||
# RETROACTIVELY 5-15 min after their merge, so by the time reaper's
|
||||
# next */5 tick lands, the stranded red is on a SHA that has already
|
||||
# fallen out of a 10-commit window during a burst-merge period.
|
||||
# Trades window-width-cheap for cadence-loady (per hongming-pc2):
|
||||
# kept `*/5` cron unchanged; only the window-N is widened.
|
||||
DEFAULT_SWEEP_LIMIT = 30
|
||||
|
||||
|
||||
def list_recent_commit_shas(branch: str, limit: int) -> list[str]:
|
||||
"""List the most recent `limit` commit SHAs on `branch`, newest
|
||||
first.
|
||||
|
||||
Wraps GET /repos/{o}/{r}/commits?sha={branch}&limit={limit}. Gitea
|
||||
1.22.6 returns a JSON list of commit objects each with a `sha` key
|
||||
(verified via vendor-truth probe 2026-05-11 against
|
||||
git.moleculesai.app — `feedback_smoke_test_vendor_truth_not_shape_match`).
|
||||
|
||||
Raises ApiError on non-2xx OR on unexpected response shape. The
|
||||
branch-level caller soft-skips this tick because the next scheduled
|
||||
tick can safely retry the listing. Per-SHA status/write errors remain
|
||||
separate and must not be mislabeled as commit-list outages.
|
||||
"""
|
||||
_, body = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/commits",
|
||||
query={"sha": branch, "limit": str(limit)},
|
||||
)
|
||||
if not isinstance(body, list):
|
||||
raise ApiError(
|
||||
f"commits listing for {branch} not a JSON array "
|
||||
f"(got {type(body).__name__})"
|
||||
)
|
||||
shas: list[str] = []
|
||||
for entry in body:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
sha = entry.get("sha")
|
||||
if isinstance(sha, str) and len(sha) >= 7:
|
||||
shas.append(sha)
|
||||
if not shas:
|
||||
raise ApiError(
|
||||
f"commits listing for {branch} returned no usable SHAs"
|
||||
)
|
||||
return shas
|
||||
|
||||
|
||||
def reap_branch(
|
||||
workflow_trigger_map: dict[str, bool],
|
||||
branch: str,
|
||||
*,
|
||||
limit: int = DEFAULT_SWEEP_LIMIT,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Sweep the last `limit` commits on `branch`, applying `reap()`
|
||||
to each (with per-SHA error isolation).
|
||||
|
||||
Returns aggregated counters PLUS rev2 observability fields:
|
||||
- scanned_shas: how many SHAs we actually iterated
|
||||
- compensated_per_sha: {<sha_full>: [<context>, ...]} — only
|
||||
SHAs that actually got at least one compensation are included
|
||||
"""
|
||||
try:
|
||||
shas = list_recent_commit_shas(branch, limit)
|
||||
except ApiError as e:
|
||||
print(
|
||||
"::warning::status-reaper skipped this tick because the "
|
||||
f"commit list could not be read after retries: {e}"
|
||||
)
|
||||
return {
|
||||
"scanned_shas": 0,
|
||||
"compensated": 0,
|
||||
"preserved_real_push": 0,
|
||||
"preserved_unknown": 0,
|
||||
"preserved_non_failure": 0,
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_per_sha": {},
|
||||
"skipped": True,
|
||||
"skip_reason": "commit-list-api-error",
|
||||
}
|
||||
|
||||
aggregate: dict[str, Any] = {
|
||||
"scanned_shas": 0,
|
||||
"compensated": 0,
|
||||
"preserved_real_push": 0,
|
||||
"preserved_unknown": 0,
|
||||
"preserved_non_failure": 0,
|
||||
"preserved_non_push_suffix": 0,
|
||||
"preserved_unparseable": 0,
|
||||
"compensated_pr_shadowed_by_push_success": 0,
|
||||
"compensated_cancelled_push": 0,
|
||||
"preserved_pr_without_push_success": 0,
|
||||
"compensated_per_sha": {},
|
||||
}
|
||||
|
||||
for sha in shas:
|
||||
aggregate["scanned_shas"] += 1
|
||||
|
||||
# Per-SHA error isolation (refinement #7). One transient blip
|
||||
# on a historical commit must NOT abort the whole tick — the
|
||||
# OTHER stale SHAs may still hold strandable reds.
|
||||
try:
|
||||
combined = get_combined_status(sha)
|
||||
except ApiError as e:
|
||||
print(
|
||||
f"::warning::get_combined_status({sha[:10]}) failed; "
|
||||
f"skipping this SHA: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
# Cost optimization (refinement #2): the common case is a green
|
||||
# commit. Skip the per-context loop entirely when combined is
|
||||
# already success — saves a tight loop over ~20 statuses per SHA
|
||||
# on green commits, the dominant majority.
|
||||
if combined.get("state") == "success":
|
||||
continue
|
||||
|
||||
per_sha = reap(
|
||||
workflow_trigger_map, combined, sha, dry_run=dry_run
|
||||
)
|
||||
|
||||
# Aggregate scalar counters.
|
||||
for key in (
|
||||
"compensated",
|
||||
"preserved_real_push",
|
||||
"preserved_unknown",
|
||||
"preserved_non_failure",
|
||||
"preserved_non_push_suffix",
|
||||
"preserved_unparseable",
|
||||
"compensated_pr_shadowed_by_push_success",
|
||||
"compensated_cancelled_push",
|
||||
"preserved_pr_without_push_success",
|
||||
):
|
||||
aggregate[key] += per_sha[key]
|
||||
|
||||
# Record per-SHA compensated contexts (only when non-empty —
|
||||
# keep the summary readable when most SHAs are no-ops).
|
||||
contexts = per_sha.get("compensated_contexts") or []
|
||||
if contexts:
|
||||
aggregate["compensated_per_sha"][sha] = list(contexts)
|
||||
|
||||
return aggregate
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Skip the compensating POST; print what would be done.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=DEFAULT_SWEEP_LIMIT,
|
||||
help=(
|
||||
"How many recent commits on WATCH_BRANCH to sweep per tick "
|
||||
f"(default: {DEFAULT_SWEEP_LIMIT})."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
_require_runtime_env()
|
||||
|
||||
workflow_trigger_map = scan_workflows(WORKFLOWS_DIR)
|
||||
print(
|
||||
f"::notice::scanned {len(workflow_trigger_map)} workflows; "
|
||||
f"push-triggered={sum(1 for v in workflow_trigger_map.values() if v)}, "
|
||||
f"class-O candidates={sum(1 for v in workflow_trigger_map.values() if not v)}"
|
||||
)
|
||||
|
||||
counters = reap_branch(
|
||||
workflow_trigger_map,
|
||||
WATCH_BRANCH,
|
||||
limit=args.limit,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
# Observability: print one JSON line summarising the tick. Loki
|
||||
# ingestion via the runner's stdout (`source="gitea-actions"`).
|
||||
print(
|
||||
"status-reaper summary: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"branch": WATCH_BRANCH,
|
||||
"dry_run": args.dry_run,
|
||||
"limit": args.limit,
|
||||
**counters,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,176 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stub Gitea API for review-check.sh test scenarios.
|
||||
|
||||
Reads $FIXTURE_STATE_DIR/scenario to decide what to return for each
|
||||
endpoint the review-check.sh script calls.
|
||||
Reads $FIXTURE_STATE_DIR/token_owner_in_teams to decide whether
|
||||
the team membership probe returns 200/204 (member) or 403 (not in team).
|
||||
|
||||
Scenarios:
|
||||
T1_pr_open — open PR, author=alice, sha=deadbeef → continue
|
||||
T2_pr_closed — closed PR → script exits 0 (no-op)
|
||||
T3_reviews_approved_non_author — one APPROVED from non-author → candidates exist
|
||||
T4_reviews_empty — zero APPROVED non-author → exit 1 (no candidates)
|
||||
T5_reviews_only_author — only author reviews → exit 1 (no candidates)
|
||||
T6_reviews_dismissed — dismissed APPROVED → treated as no approval
|
||||
T7_team_member — team membership → 204 (member) → exit 0
|
||||
T8_team_not_member — team membership → 404 (not a member) → exit 1
|
||||
T9_team_403 — team membership → 403 (token not in team) → exit 1
|
||||
T14_non_default_base — open PR targeting staging → script exits 0 (no-op)
|
||||
T15_comments_agent_approval — reviews empty; comments have "[core-qa-agent] APPROVED" → exit 0
|
||||
T16_comments_generic_approval — reviews empty; comments have "APPROVED" by team member → exit 0
|
||||
T17_comments_no_approval — reviews empty; comments have no approval keywords → exit 1
|
||||
|
||||
Usage:
|
||||
FIXTURE_STATE_DIR=/tmp/x python3 _review_check_fixture.py 8080
|
||||
"""
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
|
||||
STATE_DIR = os.environ.get("FIXTURE_STATE_DIR", "/tmp")
|
||||
|
||||
|
||||
def scenario() -> str:
|
||||
p = os.path.join(STATE_DIR, "scenario")
|
||||
if not os.path.isfile(p):
|
||||
return "T1_pr_open"
|
||||
with open(p) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *args, **kwargs):
|
||||
pass # keep stdout for explicit logs only
|
||||
|
||||
def _json(self, code: int, body: dict) -> None:
|
||||
payload = json.dumps(body).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def _empty(self, code: int) -> None:
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def _text(self, code: int, body: str) -> None:
|
||||
payload = body.encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "text/plain")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self):
|
||||
u = urllib.parse.urlparse(self.path)
|
||||
path = u.path
|
||||
sc = scenario()
|
||||
|
||||
if path == "/_ping":
|
||||
return self._json(200, {"ok": True})
|
||||
|
||||
# GET /repos/{owner}/{name}/pulls/{pr_number}
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/pulls/(\d+)$", path)
|
||||
if m:
|
||||
owner, name, pr_num = m.group(1), m.group(2), m.group(3)
|
||||
if sc == "T2_pr_closed":
|
||||
return self._json(200, {
|
||||
"number": int(pr_num),
|
||||
"state": "closed",
|
||||
"head": {"sha": "deadbeef0000111122223333444455556666"},
|
||||
"base": {"ref": "main"},
|
||||
"user": {"login": "alice"},
|
||||
})
|
||||
return self._json(200, {
|
||||
"number": int(pr_num),
|
||||
"state": "open",
|
||||
"head": {"sha": "deadbeef0000111122223333444455556666"},
|
||||
"base": {"ref": "staging" if sc == "T14_non_default_base" else "main"},
|
||||
"user": {"login": "alice"},
|
||||
})
|
||||
|
||||
# GET /repos/{owner}/{name}/pulls/{pr_number}/reviews
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/pulls/(\d+)/reviews$", path)
|
||||
if m:
|
||||
if sc in ("T4_reviews_empty", "T5_reviews_only_author",
|
||||
"T15_comments_agent_approval", "T16_comments_generic_approval",
|
||||
"T17_comments_no_approval"):
|
||||
return self._json(200, [])
|
||||
if sc == "T6_reviews_dismissed":
|
||||
return self._json(200, [{
|
||||
"state": "APPROVED",
|
||||
"dismissed": True,
|
||||
"user": {"login": "core-devops"},
|
||||
"commit_id": "abc1234",
|
||||
}])
|
||||
if sc == "T3_reviews_approved_non_author":
|
||||
return self._json(200, [
|
||||
{"state": "CHANGES_REQUESTED", "dismissed": False, "user": {"login": "bob"}, "commit_id": "abc1234"},
|
||||
{"state": "APPROVED", "dismissed": False, "user": {"login": "core-devops"}, "commit_id": "abc1234"},
|
||||
])
|
||||
# Default: one non-author APPROVED
|
||||
return self._json(200, [
|
||||
{"state": "APPROVED", "dismissed": False, "user": {"login": "core-devops"}, "commit_id": "abc1234"},
|
||||
])
|
||||
|
||||
# GET /repos/{owner}/{name}/issues/{pr_number}/comments
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/issues/(\d+)/comments$", path)
|
||||
if m:
|
||||
if sc == "T15_comments_agent_approval":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "core-qa-agent"}, "body": "[core-qa-agent] APPROVED this PR. Good changes.", "id": 1},
|
||||
{"user": {"login": "alice"}, "body": "I authored this PR", "id": 2},
|
||||
{"user": {"login": "random-user"}, "body": "Looks okay to me", "id": 3},
|
||||
])
|
||||
if sc == "T16_comments_generic_approval":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "core-qa-agent"}, "body": "APPROVED — all acceptance criteria met", "id": 1},
|
||||
{"user": {"login": "alice"}, "body": "-authored", "id": 2},
|
||||
])
|
||||
if sc == "T17_comments_no_approval":
|
||||
return self._json(200, [
|
||||
{"user": {"login": "alice"}, "body": "I authored this PR", "id": 1},
|
||||
{"user": {"login": "random-user"}, "body": "Looks okay to me", "id": 2},
|
||||
])
|
||||
# Default scenarios (T1–T9, T14): no comments
|
||||
return self._json(200, [])
|
||||
|
||||
# GET /teams/{team_id}/members/{username}
|
||||
m = re.match(r"^/api/v1/teams/(\d+)/members/([^/]+)$", path)
|
||||
if m:
|
||||
team_id, login = m.group(1), m.group(2)
|
||||
if sc == "T8_team_not_member":
|
||||
return self._empty(404)
|
||||
if sc == "T9_team_403":
|
||||
return self._empty(403)
|
||||
# T7_team_member: member
|
||||
return self._empty(204)
|
||||
|
||||
# GET /repos/{owner}/{name}/statuses/{sha} — for N/A declaration check
|
||||
m = re.match(r"^/api/v1/repos/([^/]+)/([^/]+)/statuses/([a-f0-9]+)$", path)
|
||||
if m:
|
||||
# All comment-based scenarios have no N/A declarations
|
||||
return self._json(200, [])
|
||||
|
||||
return self._json(404, {"path": path, "msg": "fixture: no route"})
|
||||
|
||||
def do_POST(self):
|
||||
self._json(404, {"path": self.path, "msg": "fixture: no POST routes"})
|
||||
|
||||
|
||||
def main():
|
||||
port = int(sys.argv[1])
|
||||
srv = http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler)
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,130 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "gitea-merge-queue.py"
|
||||
spec = importlib.util.spec_from_file_location("gitea_merge_queue", SCRIPT)
|
||||
mq = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = mq
|
||||
spec.loader.exec_module(mq)
|
||||
|
||||
|
||||
def test_latest_statuses_dedupes_by_context_newest_first():
|
||||
statuses = [
|
||||
{"context": "CI / all-required (pull_request)", "status": "failure"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "state": "success"},
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
]
|
||||
|
||||
latest = mq.latest_statuses_by_context(statuses)
|
||||
|
||||
assert latest["CI / all-required (pull_request)"]["status"] == "failure"
|
||||
assert latest["sop-checklist / all-items-acked (pull_request)"]["state"] == "success"
|
||||
|
||||
|
||||
def test_required_contexts_green_rejects_missing_and_pending():
|
||||
latest = mq.latest_statuses_by_context([
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
{"context": "sop-checklist / all-items-acked (pull_request)", "status": "pending"},
|
||||
])
|
||||
|
||||
ok, missing_or_bad = mq.required_contexts_green(
|
||||
latest,
|
||||
[
|
||||
"CI / all-required (pull_request)",
|
||||
"sop-checklist / all-items-acked (pull_request)",
|
||||
"qa-review / approved (pull_request)",
|
||||
],
|
||||
)
|
||||
|
||||
assert ok is False
|
||||
assert missing_or_bad == [
|
||||
"sop-checklist / all-items-acked (pull_request)=pending",
|
||||
"qa-review / approved (pull_request)=missing",
|
||||
]
|
||||
|
||||
|
||||
def test_choose_next_pr_sorts_by_queue_label_timestamp_then_number():
|
||||
issues = [
|
||||
{
|
||||
"number": 12,
|
||||
"pull_request": {},
|
||||
"labels": [{"name": "merge-queue"}],
|
||||
"created_at": "2026-05-13T05:00:00Z",
|
||||
"updated_at": "2026-05-13T06:00:00Z",
|
||||
},
|
||||
{
|
||||
"number": 9,
|
||||
"pull_request": {},
|
||||
"labels": [{"name": "merge-queue"}],
|
||||
"created_at": "2026-05-13T04:00:00Z",
|
||||
"updated_at": "2026-05-13T07:00:00Z",
|
||||
},
|
||||
{
|
||||
"number": 7,
|
||||
"labels": [{"name": "merge-queue"}],
|
||||
"created_at": "2026-05-13T03:00:00Z",
|
||||
},
|
||||
]
|
||||
|
||||
selected = mq.choose_next_queued_issue(issues, queue_label="merge-queue")
|
||||
|
||||
assert selected["number"] == 9
|
||||
|
||||
|
||||
def test_pr_needs_update_when_base_sha_absent_from_commits():
|
||||
commits = [
|
||||
{"sha": "head"},
|
||||
{"sha": "parent"},
|
||||
]
|
||||
|
||||
assert mq.pr_contains_base_sha(commits, "mainsha") is False
|
||||
assert mq.pr_contains_base_sha(commits, "parent") is True
|
||||
|
||||
|
||||
def test_merge_decision_requires_main_green_pr_green_and_current_base():
|
||||
required = ["CI / all-required (pull_request)"]
|
||||
main_status = {
|
||||
"state": "success",
|
||||
"statuses": [{"context": "CI / all-required (push)", "status": "success"}],
|
||||
}
|
||||
pr_status = {
|
||||
"state": "success",
|
||||
"statuses": [{"context": "CI / all-required (pull_request)", "status": "success"}],
|
||||
}
|
||||
|
||||
decision = mq.evaluate_merge_readiness(
|
||||
main_status=main_status,
|
||||
pr_status=pr_status,
|
||||
required_contexts=required,
|
||||
pr_has_current_base=True,
|
||||
)
|
||||
|
||||
assert decision.ready is True
|
||||
assert decision.action == "merge"
|
||||
|
||||
|
||||
def test_merge_decision_updates_stale_pr_before_merge():
|
||||
decision = mq.evaluate_merge_readiness(
|
||||
main_status={
|
||||
"state": "success",
|
||||
"statuses": [{"context": "CI / all-required (push)", "status": "success"}],
|
||||
},
|
||||
pr_status={"state": "success", "statuses": [{"context": "CI / all-required (pull_request)", "status": "success"}]},
|
||||
required_contexts=["CI / all-required (pull_request)"],
|
||||
pr_has_current_base=False,
|
||||
)
|
||||
|
||||
assert decision.ready is False
|
||||
assert decision.action == "update"
|
||||
|
||||
|
||||
def test_MergePermissionError_inherits_from_ApiError():
|
||||
assert issubclass(mq.MergePermissionError, mq.ApiError)
|
||||
|
||||
|
||||
def test_MergePermissionError_message_preserved():
|
||||
exc = mq.MergePermissionError("POST /merge -> HTTP 405: User not allowed")
|
||||
assert "405" in str(exc)
|
||||
assert "User not allowed" in str(exc)
|
||||
@@ -1,505 +0,0 @@
|
||||
"""Unit tests for .gitea/scripts/lint_pre_flip_continue_on_error.py.
|
||||
|
||||
These tests pin the pure-logic surface (flip detection + per-flip
|
||||
verdict aggregation) without making real HTTP calls. The end-to-end
|
||||
git ls-tree + Gitea API path is exercised by running the workflow
|
||||
against real PRs.
|
||||
|
||||
Run locally::
|
||||
|
||||
python3 -m unittest .gitea/scripts/tests/test_lint_pre_flip_continue_on_error.py -v
|
||||
|
||||
Mirrors the pattern in scripts/ops/test_check_migration_collisions.py
|
||||
+ scripts/test_build_runtime_package.py.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
# Load the script as a module without invoking main(). Tests must NOT
|
||||
# depend on the full runtime env contract (GITEA_TOKEN etc.), so we
|
||||
# import individual functions and stub the network surface explicitly.
|
||||
SCRIPT_PATH = Path(__file__).resolve().parent.parent / "lint_pre_flip_continue_on_error.py"
|
||||
spec = importlib.util.spec_from_file_location("lpfc", SCRIPT_PATH)
|
||||
lpfc = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(lpfc)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Fixtures: minimal valid workflow YAML on each side of a "diff"
|
||||
# --------------------------------------------------------------------------
|
||||
CI_YML_BASE = """\
|
||||
name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
jobs:
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- run: echo platform
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- run: echo canvas
|
||||
all-required:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
needs: [platform-build, canvas-build]
|
||||
steps:
|
||||
- run: echo ok
|
||||
"""
|
||||
|
||||
CI_YML_HEAD_FLIPPED = """\
|
||||
name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
jobs:
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- run: echo platform
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: false
|
||||
steps:
|
||||
- run: echo canvas
|
||||
all-required:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
needs: [platform-build, canvas-build]
|
||||
steps:
|
||||
- run: echo ok
|
||||
"""
|
||||
|
||||
CI_YML_HEAD_NO_DIFF = CI_YML_BASE # identical to base, no flip
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 1. CoE coercion (truthy/falsy/quoted/absent)
|
||||
# --------------------------------------------------------------------------
|
||||
class TestCoerceCoE(unittest.TestCase):
|
||||
def test_python_bool_true(self):
|
||||
self.assertTrue(lpfc._coerce_coe(True))
|
||||
|
||||
def test_python_bool_false(self):
|
||||
self.assertFalse(lpfc._coerce_coe(False))
|
||||
|
||||
def test_none_is_false(self):
|
||||
# GitHub Actions default: absent == false.
|
||||
self.assertFalse(lpfc._coerce_coe(None))
|
||||
|
||||
def test_string_true_lowercase(self):
|
||||
# Quoted "true" in YAML — Gitea Actions normalizes to True.
|
||||
self.assertTrue(lpfc._coerce_coe("true"))
|
||||
|
||||
def test_string_True_titlecase(self):
|
||||
self.assertTrue(lpfc._coerce_coe("True"))
|
||||
|
||||
def test_string_yes(self):
|
||||
# YAML 1.1 truthy form.
|
||||
self.assertTrue(lpfc._coerce_coe("yes"))
|
||||
|
||||
def test_string_false(self):
|
||||
self.assertFalse(lpfc._coerce_coe("false"))
|
||||
|
||||
def test_string_random_falsy(self):
|
||||
# An unrecognized string is treated as falsy — safer than
|
||||
# silently coercing "maybe" to True and false-positiving a
|
||||
# flip.
|
||||
self.assertFalse(lpfc._coerce_coe("maybe"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2. Diff detection — flips, not arbitrary changes
|
||||
# --------------------------------------------------------------------------
|
||||
class TestDetectFlips(unittest.TestCase):
|
||||
def test_no_flip_in_diff_passes(self):
|
||||
# Acceptance test #1: PR doesn't flip continue-on-error → 0 flips.
|
||||
flips = lpfc.detect_flips(
|
||||
{".gitea/workflows/ci.yml": CI_YML_BASE},
|
||||
{".gitea/workflows/ci.yml": CI_YML_HEAD_NO_DIFF},
|
||||
)
|
||||
self.assertEqual(flips, [])
|
||||
|
||||
def test_flip_detected_in_one_file(self):
|
||||
flips = lpfc.detect_flips(
|
||||
{".gitea/workflows/ci.yml": CI_YML_BASE},
|
||||
{".gitea/workflows/ci.yml": CI_YML_HEAD_FLIPPED},
|
||||
)
|
||||
# Two jobs flipped: platform-build, canvas-build. all-required
|
||||
# is still true on both sides.
|
||||
self.assertEqual(len(flips), 2)
|
||||
keys = sorted(f["job_key"] for f in flips)
|
||||
self.assertEqual(keys, ["canvas-build", "platform-build"])
|
||||
|
||||
def test_context_name_render(self):
|
||||
flips = lpfc.detect_flips(
|
||||
{".gitea/workflows/ci.yml": CI_YML_BASE},
|
||||
{".gitea/workflows/ci.yml": CI_YML_HEAD_FLIPPED},
|
||||
)
|
||||
platform = next(f for f in flips if f["job_key"] == "platform-build")
|
||||
self.assertEqual(platform["context"], "CI / Platform (Go) (push)")
|
||||
self.assertEqual(platform["workflow_name"], "CI")
|
||||
|
||||
def test_context_falls_back_to_job_key_when_no_name(self):
|
||||
base = "name: WF\njobs:\n foo:\n continue-on-error: true\n runs-on: x\n steps: []\n"
|
||||
head = "name: WF\njobs:\n foo:\n continue-on-error: false\n runs-on: x\n steps: []\n"
|
||||
flips = lpfc.detect_flips({"a.yml": base}, {"a.yml": head})
|
||||
self.assertEqual(len(flips), 1)
|
||||
self.assertEqual(flips[0]["context"], "WF / foo (push)")
|
||||
|
||||
def test_no_flip_when_only_one_side_has_file(self):
|
||||
# Newly added workflow file — head has CoE:false, base has no
|
||||
# file. Adding a new workflow with CoE:false is fine; there's
|
||||
# nothing to mask.
|
||||
flips = lpfc.detect_flips(
|
||||
{}, # base has no workflow files
|
||||
{".gitea/workflows/new.yml": CI_YML_HEAD_FLIPPED},
|
||||
)
|
||||
self.assertEqual(flips, [])
|
||||
|
||||
def test_no_flip_when_job_removed(self):
|
||||
# Job exists on base, not on head — a removal, not a flip.
|
||||
head = """\
|
||||
name: CI
|
||||
jobs:
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
steps: []
|
||||
"""
|
||||
flips = lpfc.detect_flips(
|
||||
{".gitea/workflows/ci.yml": CI_YML_BASE},
|
||||
{".gitea/workflows/ci.yml": head},
|
||||
)
|
||||
self.assertEqual(flips, [])
|
||||
|
||||
def test_no_flip_when_job_added_with_false(self):
|
||||
# New job on head with CoE:false — no base side; not a flip.
|
||||
head_with_new = CI_YML_BASE.replace(
|
||||
" all-required:",
|
||||
" newjob:\n name: New Job\n continue-on-error: false\n"
|
||||
" runs-on: x\n steps: []\n"
|
||||
" all-required:",
|
||||
)
|
||||
flips = lpfc.detect_flips(
|
||||
{".gitea/workflows/ci.yml": CI_YML_BASE},
|
||||
{".gitea/workflows/ci.yml": head_with_new},
|
||||
)
|
||||
self.assertEqual(flips, [])
|
||||
|
||||
def test_yaml_parse_error_warns_not_raises(self):
|
||||
# Malformed YAML on head — should warn (stderr) and skip,
|
||||
# not raise.
|
||||
bad_head = "name: CI\njobs:\n :::\n"
|
||||
# Capture stderr so the test isn't noisy.
|
||||
with mock.patch.object(sys, "stderr"):
|
||||
flips = lpfc.detect_flips(
|
||||
{".gitea/workflows/ci.yml": CI_YML_BASE},
|
||||
{".gitea/workflows/ci.yml": bad_head},
|
||||
)
|
||||
self.assertEqual(flips, [])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 3. grep_fail_markers — the regex / substring matcher
|
||||
# --------------------------------------------------------------------------
|
||||
class TestGrepFailMarkers(unittest.TestCase):
|
||||
def test_clean_log_returns_empty(self):
|
||||
log = "===== test run starting =====\nPASS\nok example.com/foo 1.234s\n"
|
||||
self.assertEqual(lpfc.grep_fail_markers(log), [])
|
||||
|
||||
def test_go_minus_minus_minus_fail_caught(self):
|
||||
log = "ok example.com/foo 1.234s\n--- FAIL: TestBar (0.01s)\n bar_test.go:42:\n"
|
||||
matches = lpfc.grep_fail_markers(log)
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertIn("FAIL: TestBar", matches[0])
|
||||
|
||||
def test_go_package_fail_caught(self):
|
||||
log = "FAIL\texample.com/baz\t1.234s\n"
|
||||
matches = lpfc.grep_fail_markers(log)
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertIn("FAIL", matches[0])
|
||||
|
||||
def test_bash_error_directive_caught(self):
|
||||
# `lint-curl-status-capture` pattern: a python heredoc inside a
|
||||
# bash step that prints `::error::` then sys.exit(1). With
|
||||
# continue-on-error:true the job rolls up as success despite
|
||||
# this line. THAT's the masking we're trying to catch.
|
||||
log = "Running scan...\n::error::Found 3 curl-status-capture pollution site(s):\n"
|
||||
matches = lpfc.grep_fail_markers(log)
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertIn("::error::", matches[0])
|
||||
|
||||
def test_caps_matches_at_max_5(self):
|
||||
log = "\n".join(["--- FAIL: T%d" % i for i in range(20)])
|
||||
matches = lpfc.grep_fail_markers(log)
|
||||
self.assertEqual(len(matches), 5)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 4. verify_flip — single-flip verdict assembly (network surface stubbed)
|
||||
# --------------------------------------------------------------------------
|
||||
def _stub_status(context: str, state: str, target_url: str = "/owner/repo/actions/runs/1/jobs/0") -> dict:
|
||||
"""Build a single-context combined-status response."""
|
||||
return {
|
||||
"state": state,
|
||||
"statuses": [
|
||||
{"context": context, "status": state, "target_url": target_url, "description": ""}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
FLIP_FIXTURE = {
|
||||
"workflow_path": ".gitea/workflows/ci.yml",
|
||||
"workflow_name": "CI",
|
||||
"job_key": "platform-build",
|
||||
"job_name": "Platform (Go)",
|
||||
"context": "CI / Platform (Go) (push)",
|
||||
}
|
||||
|
||||
|
||||
class TestVerifyFlip(unittest.TestCase):
|
||||
def test_flip_with_clean_history_passes(self):
|
||||
# Acceptance test #2: flip detected, last 5 runs clean → exit 0.
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=["sha1", "sha2", "sha3"]):
|
||||
with mock.patch.object(
|
||||
lpfc, "combined_status",
|
||||
side_effect=[_stub_status(FLIP_FIXTURE["context"], "success") for _ in range(3)],
|
||||
):
|
||||
with mock.patch.object(lpfc, "fetch_log", return_value="ok example.com/foo 1s\nPASS\n"):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(verdict["fail_runs"], [])
|
||||
self.assertEqual(verdict["masked_runs"], [])
|
||||
self.assertEqual(verdict["checked_commits"], 3)
|
||||
self.assertEqual(verdict["warnings"], [])
|
||||
|
||||
def test_flip_with_recent_fail_blocks(self):
|
||||
# Acceptance test #3: flip detected, recent run has --- FAIL → exit 1.
|
||||
# Setup: 3 commits, the most recent run's log shows --- FAIL
|
||||
# but the STATUS is success (Quirk #10 mask). That's the
|
||||
# masked_runs case.
|
||||
log_with_fail = "ok example.com/foo 1s\n--- FAIL: TestSqlmock (0.01s)\n sqlmock_test.go:42:\n"
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=["sha1", "sha2", "sha3"]):
|
||||
with mock.patch.object(
|
||||
lpfc, "combined_status",
|
||||
side_effect=[_stub_status(FLIP_FIXTURE["context"], "success") for _ in range(3)],
|
||||
):
|
||||
with mock.patch.object(lpfc, "fetch_log", side_effect=[log_with_fail, "PASS\n", "PASS\n"]):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(len(verdict["masked_runs"]), 1)
|
||||
self.assertEqual(verdict["masked_runs"][0]["sha"], "sha1")
|
||||
self.assertTrue(any("TestSqlmock" in s for s in verdict["masked_runs"][0]["samples"]))
|
||||
self.assertEqual(verdict["fail_runs"], [])
|
||||
|
||||
def test_red_status_alone_blocks(self):
|
||||
# Status itself is `failure` — block without needing log
|
||||
# markers. (Belt-and-braces: even with a clean log, a `failure`
|
||||
# status means the job's exit code was non-zero.)
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=["sha1"]):
|
||||
with mock.patch.object(
|
||||
lpfc, "combined_status",
|
||||
return_value=_stub_status(FLIP_FIXTURE["context"], "failure"),
|
||||
):
|
||||
with mock.patch.object(lpfc, "fetch_log", return_value="some unrelated text\n"):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(len(verdict["fail_runs"]), 1)
|
||||
self.assertEqual(verdict["fail_runs"][0]["status"], "failure")
|
||||
|
||||
def test_unreadable_log_warns_not_blocks(self):
|
||||
# Acceptance test #5: log fetch 404 (None) → warn, not block.
|
||||
# Status is `success`, log is None — we can't tell, so we warn
|
||||
# and allow.
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=["sha1"]):
|
||||
with mock.patch.object(
|
||||
lpfc, "combined_status",
|
||||
return_value=_stub_status(FLIP_FIXTURE["context"], "success"),
|
||||
):
|
||||
with mock.patch.object(lpfc, "fetch_log", return_value=None):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(verdict["fail_runs"], [])
|
||||
self.assertEqual(verdict["masked_runs"], [])
|
||||
self.assertTrue(any("log unavailable" in w for w in verdict["warnings"]))
|
||||
|
||||
def test_unreadable_log_with_failure_status_still_blocks(self):
|
||||
# Edge case: log fetch fails BUT the status itself is `failure`.
|
||||
# We can still block — the status alone is sufficient signal,
|
||||
# we don't need the log to confirm.
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=["sha1"]):
|
||||
with mock.patch.object(
|
||||
lpfc, "combined_status",
|
||||
return_value=_stub_status(FLIP_FIXTURE["context"], "failure"),
|
||||
):
|
||||
with mock.patch.object(lpfc, "fetch_log", return_value=None):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(len(verdict["fail_runs"]), 1)
|
||||
self.assertIn("log unavailable", verdict["fail_runs"][0]["samples"][0])
|
||||
|
||||
def test_zero_runs_history_warns_allows(self):
|
||||
# No commits with a matching context — newly added workflow.
|
||||
# Allow with warning.
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=["sha1", "sha2"]):
|
||||
with mock.patch.object(
|
||||
lpfc, "combined_status",
|
||||
return_value={"state": "success", "statuses": []}, # no matching context
|
||||
):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(verdict["checked_commits"], 0)
|
||||
self.assertEqual(verdict["fail_runs"], [])
|
||||
self.assertEqual(verdict["masked_runs"], [])
|
||||
self.assertTrue(any("no runs of" in w for w in verdict["warnings"]))
|
||||
|
||||
def test_zero_commits_warns_allows(self):
|
||||
# Empty branch (newly created repo, e.g.). Allow with warning.
|
||||
with mock.patch.object(lpfc, "recent_commits_on_branch", return_value=[]):
|
||||
verdict = lpfc.verify_flip(FLIP_FIXTURE, "main", 5)
|
||||
self.assertEqual(verdict["checked_commits"], 0)
|
||||
self.assertEqual(verdict["fail_runs"], [])
|
||||
self.assertEqual(verdict["masked_runs"], [])
|
||||
self.assertTrue(any("no recent commits" in w for w in verdict["warnings"]))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 5. Multiple-flip aggregation in main()
|
||||
# --------------------------------------------------------------------------
|
||||
class TestMainAggregation(unittest.TestCase):
|
||||
"""Tests that `main()` aggregates multiple flips and exits 1 when
|
||||
ANY one of them has a masked or red recent run. Acceptance test #4.
|
||||
|
||||
We stub at the verify_flip + workflows_at_sha + _require_runtime_env
|
||||
boundary so we don't need real git or HTTP.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
# The actual env values are irrelevant — _require_runtime_env
|
||||
# is stubbed out — but the module reads OWNER/NAME at import
|
||||
# time. Patch the runtime env contract to a no-op for the
|
||||
# duration of each test.
|
||||
self._patches = [
|
||||
mock.patch.object(lpfc, "_require_runtime_env", return_value=None),
|
||||
mock.patch.object(lpfc, "BASE_REF", "main"),
|
||||
mock.patch.object(lpfc, "BASE_SHA", "deadbeefcafe"),
|
||||
mock.patch.object(lpfc, "HEAD_SHA", "feedfaceabad"),
|
||||
mock.patch.object(lpfc, "RECENT_COMMITS_N", 5),
|
||||
]
|
||||
for p in self._patches:
|
||||
p.start()
|
||||
self.addCleanup(lambda: [p.stop() for p in self._patches])
|
||||
|
||||
def test_multiple_flips_aggregated_one_bad_blocks(self):
|
||||
# PR flips 3 jobs; 1 has a recent fail → exit 1, naming that job.
|
||||
flips = [
|
||||
{"workflow_path": ".gitea/workflows/ci.yml", "workflow_name": "CI",
|
||||
"job_key": "platform-build", "job_name": "Platform (Go)",
|
||||
"context": "CI / Platform (Go) (push)"},
|
||||
{"workflow_path": ".gitea/workflows/ci.yml", "workflow_name": "CI",
|
||||
"job_key": "canvas-build", "job_name": "Canvas (Next.js)",
|
||||
"context": "CI / Canvas (Next.js) (push)"},
|
||||
{"workflow_path": ".gitea/workflows/ci.yml", "workflow_name": "CI",
|
||||
"job_key": "python-lint", "job_name": "Python Lint & Test",
|
||||
"context": "CI / Python Lint & Test (push)"},
|
||||
]
|
||||
clean = {"flip": flips[0], "checked_commits": 5, "masked_runs": [],
|
||||
"fail_runs": [], "warnings": []}
|
||||
bad = {"flip": flips[1], "checked_commits": 5,
|
||||
"masked_runs": [{"sha": "abc1234567", "status": "success",
|
||||
"target_url": "/x/y/actions/runs/1/jobs/0",
|
||||
"samples": ["--- FAIL: TestSqlmock"]}],
|
||||
"fail_runs": [], "warnings": []}
|
||||
also_clean = {"flip": flips[2], "checked_commits": 5, "masked_runs": [],
|
||||
"fail_runs": [], "warnings": []}
|
||||
|
||||
with mock.patch.object(lpfc, "workflows_at_sha", return_value={}):
|
||||
with mock.patch.object(lpfc, "detect_flips", return_value=flips):
|
||||
with mock.patch.object(lpfc, "verify_flip",
|
||||
side_effect=[clean, bad, also_clean]):
|
||||
# Capture stdout to assert on naming.
|
||||
captured = []
|
||||
with mock.patch("builtins.print", side_effect=lambda *a, **k: captured.append(" ".join(str(x) for x in a))):
|
||||
rc = lpfc.main([])
|
||||
self.assertEqual(rc, 1)
|
||||
# The blocking error message must name the failing job.
|
||||
joined = "\n".join(captured)
|
||||
self.assertIn("canvas-build", joined)
|
||||
# And it must mention the empirical class so a reviewer can
|
||||
# cross-link the right RFC.
|
||||
self.assertTrue("mc#664" in joined or "PR#656" in joined)
|
||||
|
||||
def test_no_flips_in_diff_exits_zero(self):
|
||||
# Acceptance test #1 at main() level: empty flips → exit 0.
|
||||
with mock.patch.object(lpfc, "workflows_at_sha", return_value={}):
|
||||
with mock.patch.object(lpfc, "detect_flips", return_value=[]):
|
||||
rc = lpfc.main([])
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
def test_all_flips_clean_exits_zero(self):
|
||||
flips = [{"workflow_path": ".gitea/workflows/ci.yml", "workflow_name": "CI",
|
||||
"job_key": "platform-build", "job_name": "Platform (Go)",
|
||||
"context": "CI / Platform (Go) (push)"}]
|
||||
clean = {"flip": flips[0], "checked_commits": 5, "masked_runs": [],
|
||||
"fail_runs": [], "warnings": []}
|
||||
with mock.patch.object(lpfc, "workflows_at_sha", return_value={}):
|
||||
with mock.patch.object(lpfc, "detect_flips", return_value=flips):
|
||||
with mock.patch.object(lpfc, "verify_flip", return_value=clean):
|
||||
rc = lpfc.main([])
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
def test_dry_run_forces_exit_zero_even_with_bad_flip(self):
|
||||
# --dry-run never fails, even when verification finds masked runs.
|
||||
flips = [{"workflow_path": ".gitea/workflows/ci.yml", "workflow_name": "CI",
|
||||
"job_key": "platform-build", "job_name": "Platform (Go)",
|
||||
"context": "CI / Platform (Go) (push)"}]
|
||||
bad = {"flip": flips[0], "checked_commits": 5,
|
||||
"masked_runs": [{"sha": "abc1234567", "status": "success",
|
||||
"target_url": "/x/y/actions/runs/1/jobs/0",
|
||||
"samples": ["--- FAIL: TestSqlmock"]}],
|
||||
"fail_runs": [], "warnings": []}
|
||||
with mock.patch.object(lpfc, "workflows_at_sha", return_value={}):
|
||||
with mock.patch.object(lpfc, "detect_flips", return_value=flips):
|
||||
with mock.patch.object(lpfc, "verify_flip", return_value=bad):
|
||||
rc = lpfc.main(["--dry-run"])
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 6. Context-name rendering (the format Gitea Actions actually emits)
|
||||
# --------------------------------------------------------------------------
|
||||
class TestContextName(unittest.TestCase):
|
||||
def test_push_event(self):
|
||||
self.assertEqual(
|
||||
lpfc.context_name("CI", "Platform (Go)", "push"),
|
||||
"CI / Platform (Go) (push)",
|
||||
)
|
||||
|
||||
def test_pull_request_event(self):
|
||||
self.assertEqual(
|
||||
lpfc.context_name("CI", "Platform (Go)", "pull_request"),
|
||||
"CI / Platform (Go) (pull_request)",
|
||||
)
|
||||
|
||||
def test_workflow_name_falls_back_to_filename(self):
|
||||
# No top-level `name:` → falls back to filename minus extension.
|
||||
doc = {"jobs": {"foo": {"continue-on-error": True}}}
|
||||
self.assertEqual(
|
||||
lpfc.workflow_name(doc, fallback="my-workflow"),
|
||||
"my-workflow",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,148 +0,0 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / "prod-auto-deploy.py"
|
||||
spec = importlib.util.spec_from_file_location("prod_auto_deploy", SCRIPT)
|
||||
prod = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = prod
|
||||
spec.loader.exec_module(prod)
|
||||
|
||||
|
||||
def test_truthy_flag_accepts_operator_disable_values():
|
||||
for value in ("1", "true", "TRUE", "yes", "on", "disabled", "disable"):
|
||||
assert prod.truthy_flag(value) is True
|
||||
|
||||
for value in ("", "0", "false", "no", "off", None):
|
||||
assert prod.truthy_flag(value) is False
|
||||
|
||||
|
||||
def test_build_plan_defaults_to_staging_sha_target_and_prod_cp():
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"PROD_AUTO_DEPLOY_DISABLED": "",
|
||||
}
|
||||
)
|
||||
|
||||
assert plan["enabled"] is True
|
||||
assert plan["sha"] == "abcdef1234567890"
|
||||
assert plan["target_tag"] == "staging-abcdef1"
|
||||
assert plan["cp_url"] == "https://api.moleculesai.app"
|
||||
assert plan["body"] == {
|
||||
"target_tag": "staging-abcdef1",
|
||||
"canary_slug": "hongming",
|
||||
"soak_seconds": 60,
|
||||
"batch_size": 3,
|
||||
"dry_run": False,
|
||||
# cp#228 / task #308: fleet-wide intent must carry confirm:true.
|
||||
"confirm": True,
|
||||
}
|
||||
|
||||
|
||||
def test_build_plan_always_sets_confirm_true_for_fleet_intent():
|
||||
"""Regression guard: every plan body MUST carry confirm:true.
|
||||
|
||||
CP /cp/admin/tenants/redeploy-fleet (cp#228) returns 400 on empty
|
||||
body / {confirm:false} / {only_slugs:[]} to prevent accidental
|
||||
fleet-wide mutation. This caller is fleet-wide intent (canary +
|
||||
fan-out, no slug scoping), so the plan MUST carry confirm:true.
|
||||
Pairs with cp#228's TestRedeployFleet_EmptyBodyReturns400 +
|
||||
TestRedeployFleet_ConfirmTrueProceeds.
|
||||
"""
|
||||
plan = prod.build_plan({"GITHUB_SHA": "abcdef1234567890"})
|
||||
assert plan["body"]["confirm"] is True
|
||||
|
||||
# Operator-overridable knobs do NOT drop the ack.
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"PROD_AUTO_DEPLOY_SOAK_SECONDS": "0",
|
||||
"PROD_AUTO_DEPLOY_BATCH_SIZE": "10",
|
||||
"PROD_AUTO_DEPLOY_DRY_RUN": "true",
|
||||
"PROD_AUTO_DEPLOY_CANARY_SLUG": "",
|
||||
}
|
||||
)
|
||||
assert plan["body"]["confirm"] is True
|
||||
|
||||
|
||||
def test_build_plan_rejects_non_prod_cp_without_explicit_override():
|
||||
try:
|
||||
prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"CP_URL": "https://staging-api.moleculesai.app",
|
||||
}
|
||||
)
|
||||
except ValueError as exc:
|
||||
assert "PROD_ALLOW_NON_PROD_CP_URL=true" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected non-prod CP URL rejection")
|
||||
|
||||
|
||||
def test_build_plan_allows_non_prod_cp_only_with_override():
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"CP_URL": "https://staging-api.moleculesai.app",
|
||||
"PROD_ALLOW_NON_PROD_CP_URL": "true",
|
||||
}
|
||||
)
|
||||
|
||||
assert plan["cp_url"] == "https://staging-api.moleculesai.app"
|
||||
|
||||
|
||||
def test_build_plan_disable_flag_short_circuits_before_credentials():
|
||||
plan = prod.build_plan(
|
||||
{
|
||||
"GITHUB_SHA": "abcdef1234567890",
|
||||
"PROD_AUTO_DEPLOY_DISABLED": "true",
|
||||
}
|
||||
)
|
||||
|
||||
assert plan["enabled"] is False
|
||||
assert plan["disabled_reason"] == "PROD_AUTO_DEPLOY_DISABLED=true"
|
||||
|
||||
|
||||
def test_latest_status_for_context_uses_first_matching_status():
|
||||
statuses = [
|
||||
{"context": "CI / all-required (push)", "status": "pending"},
|
||||
{"context": "CI / all-required (pull_request)", "status": "success"},
|
||||
{"context": "CI / all-required (push)", "status": "success"},
|
||||
]
|
||||
|
||||
latest = prod.latest_status_for_context(statuses, "CI / all-required (push)")
|
||||
|
||||
assert latest == {"context": "CI / all-required (push)", "status": "pending"}
|
||||
|
||||
|
||||
def test_ci_context_state_handles_missing_and_gitea_status_key():
|
||||
assert prod.ci_context_state([], "CI / all-required (push)") == "missing"
|
||||
assert (
|
||||
prod.ci_context_state(
|
||||
[{"context": "CI / all-required (push)", "status": "success"}],
|
||||
"CI / all-required (push)",
|
||||
)
|
||||
== "success"
|
||||
)
|
||||
assert (
|
||||
prod.ci_context_state(
|
||||
[{"context": "CI / all-required (push)", "state": "failure"}],
|
||||
"CI / all-required (push)",
|
||||
)
|
||||
== "failure"
|
||||
)
|
||||
|
||||
|
||||
def test_context_is_satisfied_accepts_only_success():
|
||||
assert prod.context_is_satisfied("success") is True
|
||||
for state in ("failure", "error", "cancelled", "canceled", "skipped", "pending", "missing"):
|
||||
assert prod.context_is_satisfied(state) is False
|
||||
|
||||
|
||||
def test_context_is_terminal_failure_rejects_cancelled_and_skipped():
|
||||
for state in ("failure", "error", "cancelled", "canceled", "skipped"):
|
||||
assert prod.context_is_terminal_failure(state) is True
|
||||
for state in ("pending", "missing", "success"):
|
||||
assert prod.context_is_terminal_failure(state) is False
|
||||
@@ -1,368 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regression tests for .gitea/scripts/review-check.sh (RFC#324 Step 1).
|
||||
#
|
||||
# Covers:
|
||||
# T1 — open PR: script fetches PR + reviews, continues to team probe
|
||||
# T2 — closed PR: script exits 0 (no-op)
|
||||
# T3 — APPROVED non-author review exists → candidates exist
|
||||
# T4 — no non-author APPROVED reviews → exit 1 (no candidates)
|
||||
# T5 — only author reviews (no non-author APPROVE) → exit 1
|
||||
# T6 — dismissed APPROVED review → treated as no approval
|
||||
# T7 — team membership probe → 204 (member) → script exits 0
|
||||
# T8 — team membership probe → 404 (not a member) → script exits 1
|
||||
# T9 — team membership probe → 403 (token not in team) → script exits 1 (fail closed)
|
||||
# T10 — CURL_AUTH_FILE created with mode 600 and correct header content
|
||||
# T11 — bash syntax check (bash -n passes)
|
||||
# T12 — jq filter: non-author APPROVED → in candidate list; dismissed → excluded
|
||||
# T13 — missing required env GITEA_TOKEN → exits 1 with error
|
||||
# T14 — non-default-base PR exits 0 without requiring review
|
||||
#
|
||||
# Hostile-self-review (per feedback_assert_exact_not_substring):
|
||||
# this test MUST FAIL if the script is absent. Verified by running
|
||||
# the test before the file exists (covered in the PR body).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
|
||||
SCRIPT="$SCRIPT_DIR/review-check.sh"
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAILED_TESTS=""
|
||||
|
||||
assert_eq() {
|
||||
local label="$1"
|
||||
local expected="$2"
|
||||
local got="$3"
|
||||
if [ "$expected" = "$got" ]; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label"
|
||||
echo " expected: <$expected>"
|
||||
echo " got: <$got>"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local label="$1"
|
||||
local needle="$2"
|
||||
local haystack="$3"
|
||||
if printf '%s' "$haystack" | grep -qF "$needle"; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label"
|
||||
echo " needle: <$needle>"
|
||||
echo " haystack: <$(printf '%s' "$haystack" | head -c 200)>"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_file_mode() {
|
||||
local label="$1"
|
||||
local path="$2"
|
||||
local expected_mode="$3"
|
||||
if [ ! -f "$path" ]; then
|
||||
echo " FAIL $label (file not found: $path)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
return
|
||||
fi
|
||||
local got_mode
|
||||
got_mode=$(stat -c '%a' "$path" 2>/dev/null || stat -f '%Lp' "$path" 2>/dev/null || echo "000")
|
||||
if [ "$expected_mode" = "$got_mode" ]; then
|
||||
echo " PASS $label (mode=$got_mode)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label (expected mode=$expected_mode, got=$got_mode)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_file_contains() {
|
||||
local label="$1"
|
||||
local path="$2"
|
||||
local needle="$3"
|
||||
if [ ! -f "$path" ]; then
|
||||
echo " FAIL $label (file not found: $path)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
return
|
||||
fi
|
||||
if grep -qF "$needle" "$path"; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label (needle not found: <$needle>)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} ${label}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Existence check (foundation)
|
||||
echo
|
||||
echo "== existence =="
|
||||
if [ -f "$SCRIPT" ]; then
|
||||
echo " PASS script exists: $SCRIPT"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL script not found: $SCRIPT"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} script_exists"
|
||||
echo
|
||||
echo "------"
|
||||
echo "PASS=$PASS FAIL=$FAIL (existence)"
|
||||
echo "Cannot proceed without the script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# T11 — bash syntax check
|
||||
echo
|
||||
echo "== T11 bash syntax =="
|
||||
if bash -n "$SCRIPT" 2>&1; then
|
||||
echo " PASS T11 bash -n passes"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T11 bash -n failed"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T11"
|
||||
fi
|
||||
|
||||
# T13 — missing required env
|
||||
echo
|
||||
echo "== T13 missing GITEA_TOKEN =="
|
||||
set +e
|
||||
T13_OUT=$(PATH="/tmp:$PATH" GITEA_TOKEN= GITEA_HOST=git.example.com REPO=x/y PR_NUMBER=1 TEAM=qa TEAM_ID=1 bash "$SCRIPT" 2>&1 || true)
|
||||
set -e
|
||||
assert_contains "T13 exits non-zero when GITEA_TOKEN missing" "GITEA_TOKEN required" "$T13_OUT"
|
||||
|
||||
# Start fixture HTTP server
|
||||
echo
|
||||
echo "== fixture setup =="
|
||||
FIXTURE_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$FIXTURE_DIR"; [ -n "${FIX_PID:-}" ] && kill "$FIX_PID" 2>/dev/null || true' EXIT
|
||||
FIXTURE_PY="$THIS_DIR/_review_check_fixture.py"
|
||||
if [ ! -f "$FIXTURE_PY" ]; then
|
||||
echo "::error::fixture server $FIXTURE_PY missing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FIX_LOG="$FIXTURE_DIR/fixture.log"
|
||||
FIX_STATE_DIR="$FIXTURE_DIR/state"
|
||||
mkdir -p "$FIX_STATE_DIR"
|
||||
|
||||
# Find an unused port
|
||||
FIX_PORT=$(python3 -c 'import socket;s=socket.socket();s.bind(("127.0.0.1",0));print(s.getsockname()[1]);s.close()')
|
||||
|
||||
FIXTURE_STATE_DIR="$FIX_STATE_DIR" python3 "$FIXTURE_PY" "$FIX_PORT" \
|
||||
>"$FIX_LOG" 2>&1 &
|
||||
FIX_PID=$!
|
||||
|
||||
# Wait for fixture readiness
|
||||
for _ in $(seq 1 50); do
|
||||
if curl -fsS "http://127.0.0.1:${FIX_PORT}/_ping" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
if ! curl -fsS "http://127.0.0.1:${FIX_PORT}/_ping" >/dev/null 2>&1; then
|
||||
echo "::error::fixture server failed to start. Log:"
|
||||
cat "$FIX_LOG"
|
||||
exit 1
|
||||
fi
|
||||
echo " fixture running on port $FIX_PORT"
|
||||
|
||||
# Install a curl shim that rewrites https://fixture.local/* -> http://127.0.0.1:$FIX_PORT/*
|
||||
# Use double-quoted heredoc so FIX_PORT is expanded into the shim at creation time.
|
||||
mkdir -p "$FIXTURE_DIR/bin"
|
||||
cat >"$FIXTURE_DIR/bin/curl" <<"CURL_SHIM"
|
||||
#!/usr/bin/env bash
|
||||
# Shim: rewrite https://fixture.local/* -> http://127.0.0.1:FIXPORT/*
|
||||
# Generated at test-run time; FIXPORT is substituted when this file is written.
|
||||
new_args=()
|
||||
for a in "$@"; do
|
||||
if [[ "$a" == https://fixture.local/* ]]; then
|
||||
rest="${a#https://fixture.local}"
|
||||
a="http://127.0.0.1:FIXPORT${rest}"
|
||||
fi
|
||||
new_args+=("$a")
|
||||
done
|
||||
exec /usr/bin/curl "${new_args[@]}"
|
||||
CURL_SHIM
|
||||
# Now substitute FIXPORT with the actual port number. Use perl rather than
|
||||
# sed -i so the test runs on both GNU sed and BSD/macOS sed.
|
||||
perl -0pi -e "s/FIXPORT/${FIX_PORT}/g" "$FIXTURE_DIR/bin/curl"
|
||||
chmod +x "$FIXTURE_DIR/bin/curl"
|
||||
|
||||
# Helper: run the script with fixture environment
|
||||
run_review_check() {
|
||||
local scenario="$1"
|
||||
echo "$scenario" >"$FIX_STATE_DIR/scenario"
|
||||
local out
|
||||
set +e
|
||||
out=$(
|
||||
PATH="$FIXTURE_DIR/bin:/tmp:$PATH" \
|
||||
GITEA_TOKEN="fixture-token" \
|
||||
GITEA_HOST="fixture.local" \
|
||||
REPO="molecule-ai/molecule-core" \
|
||||
PR_NUMBER="999" \
|
||||
DEFAULT_BRANCH="main" \
|
||||
TEAM="qa" \
|
||||
TEAM_ID="20" \
|
||||
REVIEW_CHECK_DEBUG="0" \
|
||||
REVIEW_CHECK_STRICT="0" \
|
||||
bash "$SCRIPT" 2>&1
|
||||
)
|
||||
local rc=$?
|
||||
set -e
|
||||
echo "$out" >"$FIX_STATE_DIR/last_run.log"
|
||||
echo "$rc" >"$FIX_STATE_DIR/last_rc"
|
||||
echo "$out"
|
||||
}
|
||||
|
||||
# T1 — open PR: script fetches PR and continues
|
||||
echo
|
||||
echo "== T1 open PR =="
|
||||
T1_OUT=$(run_review_check "T1_pr_open")
|
||||
T1_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T1 exit code 0 (approver exists + team member)" "0" "$T1_RC"
|
||||
assert_contains "T1 qa-review APPROVED by core-devops" "APPROVED by core-devops" "$T1_OUT"
|
||||
|
||||
# T2 — closed PR: exits 0 immediately (no-op)
|
||||
echo
|
||||
echo "== T2 closed PR =="
|
||||
T2_OUT=$(run_review_check "T2_pr_closed")
|
||||
T2_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T2 exit code 0 (closed PR no-op)" "0" "$T2_RC"
|
||||
|
||||
# T3 — APPROVED non-author reviews exist
|
||||
echo
|
||||
echo "== T3 approved non-author reviews =="
|
||||
T3_OUT=$(run_review_check "T3_reviews_approved_non_author")
|
||||
T3_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T3 exit code 0 (candidates + team member)" "0" "$T3_RC"
|
||||
|
||||
# T4 — no non-author APPROVED reviews → exit 1
|
||||
echo
|
||||
echo "== T4 no non-author APPROVED reviews =="
|
||||
T4_OUT=$(run_review_check "T4_reviews_empty")
|
||||
T4_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T4 exit code 1 (no candidates)" "1" "$T4_RC"
|
||||
assert_contains "T4 awaiting non-author APPROVE" "awaiting non-author APPROVE" "$T4_OUT"
|
||||
|
||||
# T14 — non-default-base PR should not make the default branch red.
|
||||
echo
|
||||
echo "== T14 non-default base PR =="
|
||||
T14_OUT=$(run_review_check "T14_non_default_base")
|
||||
T14_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T14 exit code 0 (non-default base no-op)" "0" "$T14_RC"
|
||||
assert_contains "T14 not applicable notice" "gate not applicable" "$T14_OUT"
|
||||
|
||||
# T5 — only author reviews → exit 1
|
||||
echo
|
||||
echo "== T5 only author reviews =="
|
||||
T5_OUT=$(run_review_check "T5_reviews_only_author")
|
||||
T5_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T5 exit code 1 (only author reviews, no candidates)" "1" "$T5_RC"
|
||||
|
||||
# T6 — dismissed APPROVED review → treated as no approval
|
||||
echo
|
||||
echo "== T6 dismissed APPROVED review =="
|
||||
T6_OUT=$(run_review_check "T6_reviews_dismissed")
|
||||
T6_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T6 exit code 1 (dismissed = no approval)" "1" "$T6_RC"
|
||||
|
||||
# T7 — team member → exit 0
|
||||
echo
|
||||
echo "== T7 team membership 204 (member) =="
|
||||
T7_OUT=$(run_review_check "T7_team_member")
|
||||
T7_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T7 exit code 0 (member, APPROVED)" "0" "$T7_RC"
|
||||
assert_contains "T7 APPROVED by core-devops (team member)" "APPROVED by core-devops" "$T7_OUT"
|
||||
|
||||
# T8 — not a team member → exit 1 (fail closed)
|
||||
echo
|
||||
echo "== T8 team membership 404 (not a member) =="
|
||||
T8_OUT=$(run_review_check "T8_team_not_member")
|
||||
T8_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T8 exit code 1 (not in team)" "1" "$T8_RC"
|
||||
|
||||
# T9 — 403 token-not-in-team → exit 1 (fail closed)
|
||||
echo
|
||||
echo "== T9 team membership 403 (token not in team) =="
|
||||
T9_OUT=$(run_review_check "T9_team_403")
|
||||
T9_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T9 exit code 1 (403 token-not-in-team, fail closed)" "1" "$T9_RC"
|
||||
assert_contains "T9 403 error in output" "403" "$T9_OUT"
|
||||
|
||||
# T10 — token file creation and permissions
|
||||
echo
|
||||
echo "== T10 CURL_AUTH_FILE =="
|
||||
# Verify the token-file logic directly: create a temp file with the
|
||||
# same mktemp pattern, write the header with printf, chmod 600, then assert.
|
||||
T10_TOKEN="secret-test-token-abc123"
|
||||
T10_AUTHFILE=$(mktemp "${TMPDIR:-/tmp}/curl-auth.test.XXXXXX")
|
||||
chmod 600 "$T10_AUTHFILE"
|
||||
printf 'header = "Authorization: token %s"\n' "$T10_TOKEN" > "$T10_AUTHFILE"
|
||||
assert_file_mode "T10a mktemp authfile mode 600 (CURL_AUTH_FILE pattern)" "$T10_AUTHFILE" "600"
|
||||
assert_file_contains "T10b printf header format (CURL_AUTH_FILE content)" "$T10_AUTHFILE" "Authorization: token secret-test-token-abc123"
|
||||
assert_file_contains "T10c 'header =' curl-config syntax" "$T10_AUTHFILE" 'header = "Authorization: token '
|
||||
rm -f "$T10_AUTHFILE"
|
||||
|
||||
# T12 — jq filter: non-author APPROVED included, dismissed excluded
|
||||
echo
|
||||
echo "== T12 jq filter =="
|
||||
# These are tested indirectly via T3 and T6 above, but let's also test
|
||||
# the jq expression directly.
|
||||
JQ_FILTER='.[]
|
||||
| select(.state == "APPROVED")
|
||||
| select(.dismissed != true)
|
||||
| select(.user.login != "alice")
|
||||
| .user.login'
|
||||
|
||||
T12_INPUT='[{"state":"APPROVED","dismissed":false,"user":{"login":"core-devops"}},{"state":"CHANGES_REQUESTED","dismissed":false,"user":{"login":"bob"}},{"state":"APPROVED","dismissed":false,"user":{"login":"alice"}},{"state":"APPROVED","dismissed":true,"user":{"login":"carol"}}]'
|
||||
|
||||
JQ_CMD=$(command -v jq 2>/dev/null || echo /tmp/jq)
|
||||
T12_CANDIDATES=$(echo "$T12_INPUT" | "$JQ_CMD" -r "$JQ_FILTER" 2>/dev/null | sort -u)
|
||||
assert_contains "T12 jq: core-devops (non-author APPROVED) in candidates" "core-devops" "$T12_CANDIDATES"
|
||||
assert_eq "T12 jq: alice (author) NOT in candidates" "" "$(echo "$T12_CANDIDATES" | grep '^alice$' || true)"
|
||||
assert_eq "T12 jq: carol (dismissed) NOT in candidates" "" "$(echo "$T12_CANDIDATES" | grep '^carol$' || true)"
|
||||
|
||||
# T15 — comment-based approval via agent prefix pattern → exit 0
|
||||
echo
|
||||
echo "== T15 comment agent-prefix approval =="
|
||||
T15_OUT=$(run_review_check "T15_comments_agent_approval")
|
||||
T15_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T15 exit code 0 (agent-comment approval + team member)" "0" "$T15_RC"
|
||||
assert_contains "T15 comment fallback notice" "comment-based approval" "$T15_OUT"
|
||||
assert_contains "T15 core-qa-agent APPROVED" "APPROVED by core-qa-agent" "$T15_OUT"
|
||||
|
||||
# T16 — comment-based approval via generic APPROVED keyword → exit 0
|
||||
echo
|
||||
echo "== T16 comment generic keyword approval =="
|
||||
T16_OUT=$(run_review_check "T16_comments_generic_approval")
|
||||
T16_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T16 exit code 0 (generic-approval comment + team member)" "0" "$T16_RC"
|
||||
assert_contains "T16 comment fallback notice" "comment-based approval" "$T16_OUT"
|
||||
|
||||
# T17 — no approval keywords in comments → exit 1
|
||||
echo
|
||||
echo "== T17 comments with no approval keywords =="
|
||||
T17_OUT=$(run_review_check "T17_comments_no_approval")
|
||||
T17_RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
assert_eq "T17 exit code 1 (no candidates from comments)" "1" "$T17_RC"
|
||||
assert_contains "T17 no candidates error" "no candidates from reviews API or issue comments" "$T17_OUT"
|
||||
|
||||
echo
|
||||
echo "------"
|
||||
echo "PASS=$PASS FAIL=$FAIL"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo "Failed:$FAILED_TESTS"
|
||||
fi
|
||||
[ "$FAIL" -eq 0 ]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,10 +6,9 @@
|
||||
# T1: PR open + APPROVED via tier:low → script invokes sop-tier-check
|
||||
# and POSTs status=success.
|
||||
# T2: PR open + missing tier label → sop-tier-check exits non-zero;
|
||||
# refire still POSTs status=success, matching the canonical
|
||||
# pull_request_target workflow's fail-open job conclusion.
|
||||
# refire POSTs status=failure (description mentions failure).
|
||||
# T3: PR open + tier:low but NO approving reviews → sop-tier-check
|
||||
# exits non-zero; refire still POSTs status=success for the same reason.
|
||||
# exits non-zero; refire POSTs status=failure.
|
||||
# T4: PR CLOSED → refire exits 0 with no status POST (no-op on closed).
|
||||
# T5: Rate-limit — recent status update within 30s → refire skips,
|
||||
# no new POST.
|
||||
@@ -33,7 +32,6 @@ THIS_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "$THIS_DIR/.." && pwd)"
|
||||
WORKFLOW_DIR="$(cd "$THIS_DIR/../../workflows" && pwd)"
|
||||
WORKFLOW="$WORKFLOW_DIR/sop-tier-refire.yml"
|
||||
DISPATCH_WORKFLOW="$WORKFLOW_DIR/sop-checklist.yml"
|
||||
SCRIPT="$SCRIPT_DIR/sop-tier-refire.sh"
|
||||
|
||||
PASS=0
|
||||
@@ -89,7 +87,6 @@ assert_file_exists() {
|
||||
echo
|
||||
echo "== existence =="
|
||||
assert_file_exists "workflow file exists" "$WORKFLOW"
|
||||
assert_file_exists "SSOT dispatcher workflow file exists" "$DISPATCH_WORKFLOW"
|
||||
assert_file_exists "script file exists" "$SCRIPT"
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo
|
||||
@@ -107,44 +104,30 @@ echo "== T6/T7 workflow yaml =="
|
||||
PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T7 workflow parses as YAML" "ok" "$PARSE_OUT"
|
||||
|
||||
# The old per-workflow issue_comment listener caused queue storms because
|
||||
# Gitea queues jobs before evaluating job-level `if:`. The script remains,
|
||||
# but comment-triggered refires route through the single dispatcher.
|
||||
# Three required gates in the `if:` expression
|
||||
WORKFLOW_CONTENT=$(cat "$WORKFLOW")
|
||||
if printf '%s' "$WORKFLOW_CONTENT" | grep -q '^ issue_comment:'; then
|
||||
echo " FAIL T6a manual fallback workflow must not listen on issue_comment"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T6a"
|
||||
else
|
||||
echo " PASS T6a manual fallback workflow does not listen on issue_comment"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
assert_contains "T6b workflow exposes workflow_dispatch" \
|
||||
"workflow_dispatch" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6c workflow documents unsupported manual inputs" \
|
||||
"workflow_dispatch inputs" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6a workflow if: contains author_association gate" \
|
||||
"github.event.comment.author_association" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6b workflow if: gates on MEMBER/OWNER/COLLABORATOR" \
|
||||
'["MEMBER","OWNER","COLLABORATOR"]' "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6c workflow if: contains slash-command trigger" \
|
||||
"/refire-tier-check" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6d workflow if: gates on PR-not-issue" \
|
||||
"github.event.issue.pull_request" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6e workflow listens on issue_comment" \
|
||||
"issue_comment" "$WORKFLOW_CONTENT"
|
||||
assert_contains "T6f workflow requests statuses:write permission" \
|
||||
"statuses: write" "$WORKFLOW_CONTENT"
|
||||
# Does NOT check out PR HEAD (security)
|
||||
if grep -q 'ref: \${{ github.event.pull_request.head' "$WORKFLOW"; then
|
||||
echo " FAIL T6d workflow MUST NOT check out PR head (security)"
|
||||
echo " FAIL T6g workflow MUST NOT check out PR head (security)"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T6d"
|
||||
FAILED_TESTS="${FAILED_TESTS} T6g"
|
||||
else
|
||||
echo " PASS T6d workflow does not check out PR head"
|
||||
echo " PASS T6g workflow does not check out PR head"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
|
||||
DISPATCH_PARSE_OUT=$(python3 -c 'import sys,yaml;yaml.safe_load(open(sys.argv[1]).read());print("ok")' "$DISPATCH_WORKFLOW" 2>&1 || true)
|
||||
assert_eq "T6e SSOT dispatcher workflow parses as YAML" "ok" "$DISPATCH_PARSE_OUT"
|
||||
DISPATCH_CONTENT=$(cat "$DISPATCH_WORKFLOW")
|
||||
assert_contains "T6f SSOT dispatcher listens on issue_comment" \
|
||||
"issue_comment" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6g SSOT dispatcher handles /qa-recheck" \
|
||||
"/qa-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6h SSOT dispatcher handles /security-recheck" \
|
||||
"/security-recheck" "$DISPATCH_CONTENT"
|
||||
assert_contains "T6i SSOT dispatcher handles /refire-tier-check" \
|
||||
"/refire-tier-check" "$DISPATCH_CONTENT"
|
||||
|
||||
# T1-T5 — script behavior against a local Gitea-fixture
|
||||
echo
|
||||
echo "== T1-T5 script behavior (vs local fixture) =="
|
||||
@@ -246,21 +229,34 @@ assert_contains "T1 POST context is sop-tier-check / tier-check" \
|
||||
'"context": "sop-tier-check / tier-check (pull_request)"' "$POSTED"
|
||||
assert_contains "T1 description names commenter" "test-runner" "$POSTED"
|
||||
|
||||
# T2: missing tier label → tier-check fails internally, but refire status
|
||||
# matches the canonical workflow's fail-open job conclusion.
|
||||
# T2: missing tier label → tier-check fails → failure status POSTed
|
||||
run_scenario "T2_no_tier_label" "fail_no_label"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T2 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T2 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
# tier-check.sh exits 1; refire script forwards that exit, so RC != 0
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T2 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T2 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T2_rc"
|
||||
fi
|
||||
assert_contains "T2 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T3: tier:low present but ZERO approving reviews → internal tier check fails,
|
||||
# refire status remains aligned with the canonical workflow.
|
||||
# T3: tier:low present but ZERO approving reviews → failure
|
||||
run_scenario "T3_no_approvals" "fail_no_approvals"
|
||||
RC=$(cat "$FIX_STATE_DIR/last_rc")
|
||||
POSTED=$(cat "$FIX_STATE_DIR/posted_statuses.jsonl" 2>/dev/null || true)
|
||||
assert_eq "T3 exit code 0 (canonical fail-open)" "0" "$RC"
|
||||
assert_contains "T3 POSTed state=success" '"state": "success"' "$POSTED"
|
||||
if [ "$RC" -ne 0 ]; then
|
||||
echo " PASS T3 exit code non-zero (got $RC)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL T3 exit code should be non-zero, got 0"
|
||||
FAIL=$((FAIL + 1))
|
||||
FAILED_TESTS="${FAILED_TESTS} T3_rc"
|
||||
fi
|
||||
assert_contains "T3 POSTed state=failure" '"state": "failure"' "$POSTED"
|
||||
|
||||
# T4: closed PR — refire is a no-op (no POST, exit 0)
|
||||
run_scenario "T4_closed" "pass"
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import urllib.error
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "status-reaper.py"
|
||||
|
||||
|
||||
def load_reaper():
|
||||
spec = importlib.util.spec_from_file_location("status_reaper", SCRIPT)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(mod)
|
||||
mod.API = "https://git.example.test/api/v1"
|
||||
mod.GITEA_TOKEN = "test-token"
|
||||
mod.API_TIMEOUT_SEC = 1
|
||||
mod.API_RETRIES = 3
|
||||
mod.API_RETRY_SLEEP_SEC = 0
|
||||
return mod
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
status = 200
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return json.dumps(self.payload).encode("utf-8")
|
||||
|
||||
|
||||
def test_api_retries_transient_timeout(monkeypatch):
|
||||
mod = load_reaper()
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_urlopen(req, timeout):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise TimeoutError("simulated slow Gitea API")
|
||||
return FakeResponse({"ok": True})
|
||||
|
||||
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
status, body = mod.api("GET", "/repos/o/r/commits")
|
||||
|
||||
assert status == 200
|
||||
assert body == {"ok": True}
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_api_raises_after_retry_budget(monkeypatch):
|
||||
mod = load_reaper()
|
||||
|
||||
def fake_urlopen(req, timeout):
|
||||
raise urllib.error.URLError("connection reset")
|
||||
|
||||
monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
try:
|
||||
mod.api("GET", "/repos/o/r/commits")
|
||||
except mod.ApiError as exc:
|
||||
assert "failed after 3 attempts" in str(exc)
|
||||
else:
|
||||
raise AssertionError("expected ApiError")
|
||||
|
||||
|
||||
def test_reap_compensates_failed_pr_context_when_push_equivalent_passed(monkeypatch):
|
||||
mod = load_reaper()
|
||||
posted = []
|
||||
|
||||
def fake_post(sha, context, target_url, *, description="", dry_run=False):
|
||||
posted.append((sha, context, target_url, description, dry_run))
|
||||
|
||||
monkeypatch.setattr(mod, "post_compensating_status", fake_post)
|
||||
|
||||
counters = mod.reap(
|
||||
{"CI": True, "Handlers Postgres Integration": True},
|
||||
{
|
||||
"statuses": [
|
||||
{
|
||||
"context": "CI / Platform (Go) (pull_request)",
|
||||
"status": "failure",
|
||||
"target_url": "https://git.example.test/ci-pr",
|
||||
},
|
||||
{
|
||||
"context": "CI / Platform (Go) (push)",
|
||||
"status": "success",
|
||||
},
|
||||
{
|
||||
"context": (
|
||||
"Handlers Postgres Integration / "
|
||||
"Handlers Postgres Integration (pull_request)"
|
||||
),
|
||||
"status": "failure",
|
||||
"target_url": "https://git.example.test/handlers-pr",
|
||||
},
|
||||
{
|
||||
"context": (
|
||||
"Handlers Postgres Integration / "
|
||||
"Handlers Postgres Integration (push)"
|
||||
),
|
||||
"status": "success",
|
||||
},
|
||||
],
|
||||
},
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
)
|
||||
|
||||
assert counters["compensated_pr_shadowed_by_push_success"] == 2
|
||||
assert posted == [
|
||||
(
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
"CI / Platform (Go) (pull_request)",
|
||||
"https://git.example.test/ci-pr",
|
||||
mod.PR_SHADOW_COMPENSATION_DESCRIPTION,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
"Handlers Postgres Integration / Handlers Postgres Integration (pull_request)",
|
||||
"https://git.example.test/handlers-pr",
|
||||
mod.PR_SHADOW_COMPENSATION_DESCRIPTION,
|
||||
False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_reap_preserves_failed_pr_context_without_push_success(monkeypatch):
|
||||
mod = load_reaper()
|
||||
posted = []
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"post_compensating_status",
|
||||
lambda sha, context, target_url, *, description="", dry_run=False: posted.append(
|
||||
context
|
||||
),
|
||||
)
|
||||
|
||||
counters = mod.reap(
|
||||
{"CI": True},
|
||||
{
|
||||
"statuses": [
|
||||
{
|
||||
"context": "CI / Platform (Go) (pull_request)",
|
||||
"status": "failure",
|
||||
},
|
||||
{
|
||||
"context": "CI / Platform (Go) (push)",
|
||||
"status": "failure",
|
||||
},
|
||||
{
|
||||
"context": "CI / Shellcheck (pull_request)",
|
||||
"status": "failure",
|
||||
},
|
||||
],
|
||||
},
|
||||
"db3b7a93e31adc0cb072a6d177d92dd73275a191",
|
||||
)
|
||||
|
||||
assert counters["preserved_pr_without_push_success"] == 2
|
||||
assert posted == []
|
||||
@@ -1,181 +0,0 @@
|
||||
# SOP-Checklist gate — per-item required reviewer teams.
|
||||
#
|
||||
# RFC#351 v1 starter set. Each item lists:
|
||||
# slug — canonical kebab-case form used in /sop-ack <slug>
|
||||
# pr_section_marker — substring matched in the PR body to detect that
|
||||
# the author filled in this item (case-insensitive)
|
||||
# required_teams — list of Gitea team names; an ack from ANY one of
|
||||
# these teams (logical OR) satisfies the item.
|
||||
# Membership is probed at gate-time via
|
||||
# GET /api/v1/teams/{id}/members/{login}.
|
||||
# Team-id resolution happens at script start via
|
||||
# GET /api/v1/orgs/{org}/teams (cheap, one call).
|
||||
# numeric_alias — 1..7; lets reviewers type `/sop-ack 3` as a
|
||||
# shortcut for `/sop-ack staging-smoke`.
|
||||
#
|
||||
# WHY THESE TEAM MAPPINGS:
|
||||
# The RFC table referenced persona-role names like `core-qa`,
|
||||
# `core-be`, `core-devops` — these are individual Gitea user logins,
|
||||
# not teams. The Gitea team-membership API is /teams/{id}/members/{u},
|
||||
# so we need actual teams. Orchestrator preflight 2026-05-12 verified
|
||||
# only these teams exist on molecule-ai: ceo(5), engineers(2),
|
||||
# managers(6), qa(20), security(21), Owners(1), and bot teams. We
|
||||
# map the RFC roles to the closest existing team and surface the
|
||||
# mapping explicitly so it's reviewable.
|
||||
#
|
||||
# HOW TO EDIT:
|
||||
# - Tightening: replace `engineers` with a smaller team after creating
|
||||
# it (e.g. a new `senior-engineers` team if needed).
|
||||
# - Loosening: add another team to required_teams (OR semantics).
|
||||
# - Add an item: append to items list and document the slug below.
|
||||
#
|
||||
# AUTHOR SELF-ACK IS FORBIDDEN regardless of which team contains them
|
||||
# — the gate script enforces commenter != PR author before checking
|
||||
# team membership.
|
||||
|
||||
version: 1
|
||||
|
||||
# Tier-aware failure mode (RFC#351 open question 2):
|
||||
# For tier:high — hard-fail (status `failure`, blocks merge via BP).
|
||||
# For tier:medium — hard-fail (same as high; medium is non-trivial).
|
||||
# For tier:low — soft-fail (status `pending` with `acked: N/M` in the
|
||||
# description). BP can choose to require the context
|
||||
# or not for low-tier PRs.
|
||||
# If no tier label is present, default to medium (hard-fail) — every PR
|
||||
# should have a tier label per sop-tier-check, and absence indicates
|
||||
# a missing-tier defect we should surface, not silently lower the bar.
|
||||
tier_failure_mode:
|
||||
"tier:high": hard
|
||||
"tier:medium": hard
|
||||
"tier:low": soft
|
||||
default_mode: hard # used when no tier:* label is present
|
||||
|
||||
# High-risk class (RFC#450 Option C, governance-fix for internal#442).
|
||||
#
|
||||
# A PR is "high-risk" when ANY of the listed labels are applied OR when
|
||||
# the PR has `tier:high` (mechanically the strictest existing tier).
|
||||
# High-risk items use `required_teams_high_risk` (when present on the
|
||||
# item); non-high-risk items use the default `required_teams`.
|
||||
#
|
||||
# This closes the inconsistency that the SOP charter already mandates
|
||||
# `tier:high → ceo only` for the sibling `sop-tier-check` gate; the
|
||||
# sop-checklist's `root-cause` and `no-backwards-compat` items now
|
||||
# follow the same risk-classed two-eyes shape:
|
||||
# - Default class (tier:low/medium, not high-risk): a non-author
|
||||
# engineers/managers/ceo ack satisfies the item — 25+ live
|
||||
# identities, no dependency on a dead/inactive senior persona
|
||||
# token.
|
||||
# - High-risk class (tier:high OR any high_risk_label): still
|
||||
# requires a non-author ceo ack (durable human team).
|
||||
#
|
||||
# Tightening: add labels to high_risk_labels.
|
||||
# Loosening: remove labels.
|
||||
high_risk_labels:
|
||||
- "risk:high"
|
||||
- "area:security"
|
||||
- "area:schema"
|
||||
- "area:fleet-image"
|
||||
- "area:identity"
|
||||
- "area:gate-meta"
|
||||
|
||||
items:
|
||||
- slug: comprehensive-testing
|
||||
numeric_alias: 1
|
||||
pr_section_marker: "Comprehensive testing performed"
|
||||
required_teams: [qa, engineers]
|
||||
description: >-
|
||||
What was tested, how, edge cases covered. Ack from any qa-team
|
||||
member (or engineers fallback while qa is small).
|
||||
|
||||
- slug: local-postgres-e2e
|
||||
numeric_alias: 2
|
||||
pr_section_marker: "Local-postgres E2E run"
|
||||
required_teams: [engineers]
|
||||
description: >-
|
||||
Link to local CI artifact, or "N/A: pure-frontend change". Ack
|
||||
from any engineer who can verify the local DB test actually ran.
|
||||
|
||||
- slug: staging-smoke
|
||||
numeric_alias: 3
|
||||
pr_section_marker: "Staging-smoke verified or pending"
|
||||
required_teams: [engineers]
|
||||
description: >-
|
||||
Link to canary run, or "scheduled post-merge". Ack from any
|
||||
engineer (core-devops/infra-sre are members of engineers team).
|
||||
|
||||
- slug: root-cause
|
||||
numeric_alias: 4
|
||||
pr_section_marker: "Root-cause not symptom"
|
||||
required_teams: [engineers, managers, ceo]
|
||||
required_teams_high_risk: [ceo]
|
||||
description: >-
|
||||
One-sentence root-cause statement. Default class: non-author
|
||||
engineers/managers/ceo ack suffices (engineers can attest
|
||||
root-cause-vs-symptom for routine fixes). High-risk class
|
||||
(see `high_risk_labels`): non-author ceo ack required —
|
||||
senior judgment for irreversible/security/identity/gate
|
||||
changes. Closes internal#442 + tracks RFC#450.
|
||||
|
||||
- slug: five-axis-review
|
||||
numeric_alias: 5
|
||||
pr_section_marker: "Five-Axis review walked"
|
||||
required_teams: [engineers]
|
||||
description: >-
|
||||
Correctness / readability / architecture / security / performance.
|
||||
Ack from any non-author engineer.
|
||||
|
||||
- slug: no-backwards-compat
|
||||
numeric_alias: 6
|
||||
pr_section_marker: "No backwards-compat shim / dead code added"
|
||||
required_teams: [engineers, managers, ceo]
|
||||
required_teams_high_risk: [ceo]
|
||||
description: >-
|
||||
Yes/no + justification if no. Default class: non-author
|
||||
engineers/managers/ceo ack suffices. High-risk class
|
||||
(see `high_risk_labels`): non-author ceo ack required —
|
||||
senior judgment for shim-versus-real-fix on irreversible
|
||||
surfaces. Closes internal#442 + tracks RFC#450.
|
||||
|
||||
- slug: memory-consulted
|
||||
numeric_alias: 7
|
||||
pr_section_marker: "Memory/saved-feedback consulted"
|
||||
required_teams: [engineers]
|
||||
description: >-
|
||||
List of feedback memories applicable to this change. Ack from
|
||||
any engineer who has the same memory access.
|
||||
|
||||
# N/A gate declarations (RFC#324 §N/A follow-up).
|
||||
# PRs where a gate genuinely does not apply (e.g., pure-infra with no
|
||||
# qa surface, or docs-only) can be declared N/A by a non-author peer
|
||||
# who is in one of the gate's required_teams. The sop-checklist
|
||||
# posts a `sop-checklist / na-declarations (pull_request)` status that
|
||||
# review-check.sh reads to skip the Gitea-APPROVE requirement.
|
||||
#
|
||||
# Usage: any PR commenter (peer) posts:
|
||||
# /sop-n/a qa-review <reason>
|
||||
# /sop-n/a security-review <reason>
|
||||
#
|
||||
# Slash commands:
|
||||
# /sop-n/a <gate> [reason] — declare gate N/A (most-recent per-user wins)
|
||||
# /sop-revoke <gate> — revoke prior N/A declaration for that gate
|
||||
#
|
||||
# Gate names must match the context strings used by review-check.sh:
|
||||
# qa-review → qa-review / approved (<event>) [TEAM_ID=20]
|
||||
# security-review → security-review / approved (<event>) [TEAM_ID=21]
|
||||
#
|
||||
# required_teams: OR semantics — any team member can declare N/A.
|
||||
# Authors cannot self-declare N/A (enforced by gate script).
|
||||
n/a_gates:
|
||||
qa-review:
|
||||
required_teams: [qa, security, engineers]
|
||||
description: >-
|
||||
QA review N/A when this change has no qa surface (pure-infra,
|
||||
tooling-only, revert, dependency-only). A qa/eng/security member
|
||||
must post /sop-n/a qa-review to activate.
|
||||
|
||||
security-review:
|
||||
required_teams: [security, managers, ceo]
|
||||
description: >-
|
||||
Security review N/A when this change has no security surface
|
||||
(docs-only, pure-frontend, dependency-only). A security/owners
|
||||
member must post /sop-n/a security-review to activate.
|
||||
@@ -1,58 +1,88 @@
|
||||
# audit-force-merge — emit `incident.force_merge` to runner stdout when
|
||||
# a PR is merged with required-status-checks not green. Vector picks
|
||||
# audit-force-merge — emit `incident.force_merge` to the runner log when
|
||||
# a PR is merged with required-status checks NOT all green. Vector picks
|
||||
# the JSON line off docker_logs and ships to Loki on
|
||||
# molecule-canonical-obs (per `reference_obs_stack_phase1`); query as:
|
||||
#
|
||||
# {host="operator"} |= "event_type" |= "incident.force_merge" | json
|
||||
#
|
||||
# Closes the §SOP-6 audit gap (the doc says force-merges write to
|
||||
# `structure_events`, but that table lives in the platform DB, not
|
||||
# Gitea-side; Loki is the practical equivalent for Gitea Actions
|
||||
# events). When the credential / observability stack converges later,
|
||||
# this can sync into structure_events from Loki via a backfill job —
|
||||
# the structured JSON shape is forward-compatible.
|
||||
# Companion to `audit-force-merge.sh` (script-extract pattern, same as
|
||||
# sop-tier-check). The audit observes BOTH UI-merged and REST-merged PRs
|
||||
# uniformly per `feedback_gh_cli_merge_lies_use_rest`.
|
||||
#
|
||||
# Logic in `.gitea/scripts/audit-force-merge.sh` per the same script-
|
||||
# extract pattern as sop-tier-check.
|
||||
# Closes the §SOP-6 audit gap for the molecule-core repo. RFC:
|
||||
# internal#219 §6. Mirrors the same-named workflow in
|
||||
# molecule-controlplane; design rationale lives in the RFC, not here,
|
||||
# to keep the workflow file scannable.
|
||||
|
||||
name: audit-force-merge
|
||||
|
||||
# pull_request_target loads from the base branch — same security model
|
||||
# as sop-tier-check. Without this, an attacker could rewrite the
|
||||
# workflow on a PR and skip the audit emission for their own
|
||||
# force-merge. See `.gitea/workflows/sop-tier-check.yml` for the full
|
||||
# rationale.
|
||||
# as sop-tier-check. Without this, a PR author could rewrite the
|
||||
# workflow on their own PR and skip the audit emission for their own
|
||||
# force-merge. The base-branch checkout below ALSO uses
|
||||
# `base.sha`, not `base.ref`, so a fast-moving base can't slip a
|
||||
# different audit script in under us.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
# `pull-requests: read` + `contents: read` covers everything the script
|
||||
# needs (fetch PR + commit statuses). `issues:` deliberately omitted —
|
||||
# audit fires-and-forgets to stdout, never opens issues.
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
# Skip when PR is closed without merge — saves a runner.
|
||||
if: github.event.pull_request.merged == true
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# base.sha pinning, NOT base.ref — see header rationale.
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Detect force-merge + emit audit event
|
||||
env:
|
||||
# Same org-level secret the sop-tier-check workflow uses.
|
||||
# Same org-level secret the sop-tier-check workflow uses;
|
||||
# falls back to the auto-injected GITHUB_TOKEN if the
|
||||
# org-level SOP_TIER_CHECK_TOKEN isn't set on a transitional
|
||||
# repo.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
# Required-status-check contexts to evaluate at merge time.
|
||||
# Newline-separated. Mirror this against branch protection
|
||||
# (settings → branches → protected branch → required checks).
|
||||
# Newline-separated. MUST mirror branch protection's
|
||||
# status_check_contexts for protected branches
|
||||
# (currently `main`; `staging` protection forthcoming per
|
||||
# RFC internal#219 Phase 4).
|
||||
#
|
||||
# Initialized 2026-05-11 from the current molecule-core `main`
|
||||
# branch protection:
|
||||
#
|
||||
# GET /api/v1/repos/molecule-ai/molecule-core/
|
||||
# branch_protections/main
|
||||
# → status_check_contexts = [
|
||||
# "Secret scan / Scan diff for credential-shaped strings (pull_request)",
|
||||
# "sop-tier-check / tier-check (pull_request)"
|
||||
# ]
|
||||
#
|
||||
# Declared here rather than fetched from /branch_protections
|
||||
# because that endpoint requires admin write — sop-tier-bot is
|
||||
# read-only by design (least-privilege).
|
||||
# because that endpoint requires admin write — sop-tier-bot
|
||||
# is read-only by design (least-privilege per
|
||||
# `feedback_least_privilege_via_workflow_env` / internal#257).
|
||||
# Drift between this env and the real protection list is
|
||||
# auto-detected by `ci-required-drift.yml` (RFC §4 + §6),
|
||||
# which opens a `[ci-drift]` issue within one hour.
|
||||
#
|
||||
# When the protection set changes (e.g. Phase 4 adds the
|
||||
# `ci / all-required (pull_request)` sentinel), update BOTH
|
||||
# branch protection AND this env in the SAME PR; drift-detect
|
||||
# will otherwise file an issue for you.
|
||||
REQUIRED_CHECKS: |
|
||||
CI / all-required (pull_request)
|
||||
sop-checklist / all-items-acked (pull_request)
|
||||
Secret scan / Scan diff for credential-shaped strings (pull_request)
|
||||
sop-tier-check / tier-check (pull_request)
|
||||
run: bash .gitea/scripts/audit-force-merge.sh
|
||||
|
||||
@@ -37,7 +37,6 @@ jobs:
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths reference .gitea/workflows/publish-runtime.yml (the active
|
||||
# Gitea workflow file) instead of .github/workflows/publish-runtime.yml
|
||||
# (which Category A of this sweep deletes).
|
||||
# - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
|
||||
# .gitea/ workflow (the script's default is still .github/... which
|
||||
# will not exist post-Cat-A).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .gitea/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
# Pass the .gitea/ workflow path explicitly — the script's
|
||||
# default still points at .github/... which Category A of this
|
||||
# sweep removes.
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml
|
||||
@@ -45,7 +45,6 @@ jobs:
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
# ci-arm64-advisory — Mac arm64 self-hosted ADVISORY fast-check lane.
|
||||
#
|
||||
# === WHY ===
|
||||
#
|
||||
# The amd64 Gitea runner pool (molecule-runner-1..20) is queue-contended
|
||||
# (internal#418). This lane offloads the *genuinely container-independent*
|
||||
# fast checks (Go build/vet/lint, shellcheck, Python lint) onto the Mac
|
||||
# arm64 self-hosted runner so developers get a fast arm64 signal WITHOUT
|
||||
# adding load to the starved amd64 pool — capability-honestly, as an
|
||||
# additive pilot. Pilot ② of the Mac-CI strategy (CTO-delegated 2026-05-17).
|
||||
#
|
||||
# === NON-NEGOTIABLE SAFETY CONTRACT (the prime directive) ===
|
||||
#
|
||||
# This lane is **ADVISORY ONLY**. It is provably incapable of hanging a
|
||||
# merge. Concretely:
|
||||
#
|
||||
# 1. It is a SEPARATE workflow file. `ci.yml` is byte-for-byte
|
||||
# untouched by this PR. The `CI / all-required` aggregator sentinel
|
||||
# and the five contexts it polls
|
||||
# (`CI / Detect changes|Platform (Go)|Canvas (Next.js)|
|
||||
# Shellcheck (E2E scripts)|Python Lint & Test (pull_request)`)
|
||||
# are unchanged. The canonical required gate stays 100% on the
|
||||
# existing amd64 pool.
|
||||
#
|
||||
# 2. The context this workflow emits is
|
||||
# `ci-arm64-advisory / fast-checks (pull_request)`. That string is
|
||||
# DELIBERATELY NOT present in, and this PR does NOT add it to:
|
||||
# - branch_protections/{main,staging}.status_check_contexts
|
||||
# (DB-verified pb 86/75 = exactly
|
||||
# ["CI / all-required (pull_request)",
|
||||
# "sop-checklist / all-items-acked (pull_request)"])
|
||||
# - audit-force-merge.yml REQUIRED_CHECKS env
|
||||
# - ci.yml `all-required` sentinel's hardcoded `required[]` list
|
||||
# Branch protection therefore never waits on this context. If the
|
||||
# Mac runner is absent / offline / removed, this workflow's status
|
||||
# simply never appears — and because nothing requires it, every
|
||||
# merge proceeds exactly as it does today. There is no path by
|
||||
# which a missing/red arm64 status blocks a merge.
|
||||
#
|
||||
# 3. `continue-on-error: true` on the job — even a genuine arm64-only
|
||||
# failure (toolchain drift, arch-specific test flake) is surfaced
|
||||
# as information, never as a merge blocker, for the duration of
|
||||
# the pilot.
|
||||
#
|
||||
# 4. The job carries a `github.event_name` `if:` gate. Beyond its
|
||||
# functional purpose this also keeps the job OUT of
|
||||
# `ci-required-drift.py:ci_job_names()` (which excludes
|
||||
# `github.event_name`/`github.ref`-gated jobs), so the hourly
|
||||
# ci-required-drift sentinel's F1 ("job not under sentinel needs")
|
||||
# cannot ever flag this advisory job. F2/F3 are untouched because
|
||||
# this context is absent from BP and from REQUIRED_CHECKS.
|
||||
# `lint-bp-context-emit-match` only fails on BP→emitter gaps; an
|
||||
# emitter without a BP context is explicitly informational there.
|
||||
#
|
||||
# === RUNNER TARGETING ===
|
||||
#
|
||||
# The Mac runner is `hongming-pc-runner-1`. The bare `self-hosted`
|
||||
# label is POLLUTED in this Gitea instance: molecule-runner-1..20
|
||||
# (the contended amd64 pool) also advertise `self-hosted`. Targeting
|
||||
# bare `self-hosted` would route back onto the very pool we are trying
|
||||
# to relieve — and onto amd64 hardware. We therefore require an
|
||||
# AND-set of labels that ONLY the Mac satisfies. `macos-self-hosted`
|
||||
# is Mac-exclusive (the amd64 pool does not carry it). Until the
|
||||
# label-install burst (a10862b2) lands `self-hosted`+`macos-self-hosted`
|
||||
# on the Mac, the runner's current unique label `hongming-pc-laptop`
|
||||
# is also listed; AND-semantics over the labels a runner advertises
|
||||
# means a job requiring [self-hosted, macos-self-hosted] can ONLY be
|
||||
# claimed once the Mac advertises both. If neither label set is yet
|
||||
# present on the Mac, the workflow stays queued harmlessly and is
|
||||
# garbage-collected by the normal stale-run reaper — it blocks nothing
|
||||
# (see safety contract point 2).
|
||||
#
|
||||
# === ROLLBACK ===
|
||||
#
|
||||
# Delete this single file (`git rm .gitea/workflows/ci-arm64-advisory.yml`)
|
||||
# and merge. No branch-protection edit, no ci.yml edit, no
|
||||
# REQUIRED_CHECKS edit is required to roll back, because none were made
|
||||
# to roll forward. Zero blast radius either direction.
|
||||
|
||||
name: ci-arm64-advisory
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
# Per-ref cancel: a newer commit on the same ref supersedes the older
|
||||
# advisory run. Distinct from ci.yml's `ci-${ref}` group so this lane
|
||||
# never cancels (or is cancelled by) the canonical required CI.
|
||||
concurrency:
|
||||
group: ci-arm64-advisory-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
fast-checks:
|
||||
name: fast-checks
|
||||
# AND-set: only the Mac arm64 runner advertises macos-self-hosted.
|
||||
# See "RUNNER TARGETING" header note for why bare self-hosted is unsafe.
|
||||
runs-on: [self-hosted, macos-self-hosted]
|
||||
# ADVISORY: never blocks. See safety contract point 3.
|
||||
continue-on-error: true
|
||||
# event_name gate: functional (only meaningful on push/PR) AND keeps
|
||||
# this job out of ci-required-drift.py:ci_job_names() so F1 can never
|
||||
# flag it. See safety contract point 4.
|
||||
if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Provenance — advisory lane, non-gating
|
||||
run: |
|
||||
echo "This is the arm64 ADVISORY fast-check lane."
|
||||
echo "It does NOT gate merges. Canonical required CI is ci.yml"
|
||||
echo "on the amd64 pool. Arch: $(uname -m) on $(uname -s)."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# ---- Go: build + vet + lint (container-independent: needs only the
|
||||
# Go toolchain; no amd64 ECR image, no docker-in-job). Race-detector
|
||||
# unit-test + coverage gates are deliberately NOT duplicated here —
|
||||
# those stay authoritative on amd64 ci.yml `Platform (Go)`. This lane
|
||||
# is fast-feedback for the compile/vet/lint surface only. ----
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- name: Go build + vet (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go mod download
|
||||
go build ./cmd/server
|
||||
go vet ./...
|
||||
- name: golangci-lint (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
"$(go env GOPATH)/bin/golangci-lint" run --timeout 3m ./...
|
||||
|
||||
# ---- Shellcheck (container-independent: shellcheck binary only).
|
||||
# Mirrors ci.yml `Shellcheck (E2E scripts)` bulk pass scope. ----
|
||||
- name: Install shellcheck (arm64)
|
||||
run: |
|
||||
if ! command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "shellcheck not preinstalled on this self-hosted runner."
|
||||
echo "Attempting Homebrew install (Mac arm64)."
|
||||
brew install shellcheck || {
|
||||
echo "::warning::shellcheck unavailable on runner; advisory shellcheck skipped."
|
||||
exit 0
|
||||
}
|
||||
fi
|
||||
shellcheck --version
|
||||
- name: Shellcheck tests/e2e + infra/scripts
|
||||
run: |
|
||||
command -v shellcheck >/dev/null 2>&1 || { echo "skip"; exit 0; }
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
# ---- Python lint/compile (container-independent: CPython only).
|
||||
# Lint + import-compile surface; the authoritative pytest + coverage
|
||||
# floors stay on amd64 ci.yml `Python Lint & Test`. ----
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Python byte-compile (workspace)
|
||||
working-directory: workspace
|
||||
run: |
|
||||
python -m pip install --quiet ruff || true
|
||||
python -m compileall -q .
|
||||
if command -v ruff >/dev/null 2>&1; then
|
||||
ruff check . || echo "::warning::ruff findings (advisory only)"
|
||||
fi
|
||||
|
||||
- name: Advisory summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## arm64 advisory fast-checks complete"
|
||||
echo ""
|
||||
echo "This lane is **advisory** — it does not gate merges."
|
||||
echo "Authoritative required CI remains \`CI / all-required\`"
|
||||
echo "on the amd64 pool (\`ci.yml\`, unchanged by this PR)."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -23,11 +23,11 @@
|
||||
# `feedback_behavior_based_ast_gates` — NOT grep-by-name. That way
|
||||
# job renames or matrix-expansion-induced churn produce honest signal.
|
||||
#
|
||||
# NOTE on protection endpoint scope: `GET /repos/.../branch_protections/{branch}`
|
||||
# requires repo-admin role in Gitea 1.22.6. If DRIFT_BOT_TOKEN lacks it,
|
||||
# the script skips that branch with a clear ::error:: diagnostic and exits 0
|
||||
# (the issue IS the alarm, not a red workflow). See provisioning trail in
|
||||
# the run step's GITEA_TOKEN env comment.
|
||||
# IMPORTANT — TRANSITIONAL STATE: molecule-core's ci.yml does NOT yet
|
||||
# contain the `all-required` sentinel job (RFC §4 Phase 4 adds it).
|
||||
# Until Phase 4 lands the detector will hard-fail with exit 3 on the
|
||||
# missing sentinel. That's intentional: a red workflow on a 5-min cron
|
||||
# is louder than a silent issue and forces Phase 4 to land soon.
|
||||
|
||||
name: ci-required-drift
|
||||
|
||||
@@ -77,18 +77,13 @@ jobs:
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run drift detector
|
||||
env:
|
||||
# DRIFT_BOT_TOKEN is owned by mc-drift-bot, a least-privilege
|
||||
# Gitea persona whose ONLY job is reading branch_protections
|
||||
# and posting the [ci-drift] tracking issue. The endpoint
|
||||
# `GET /repos/.../branch_protections/{branch}` requires
|
||||
# repo-ADMIN role (Gitea 1.22.6) — SOP_TIER_CHECK_TOKEN and the
|
||||
# auto-injected GITHUB_TOKEN do NOT have it (read-only / write
|
||||
# without admin), so the previous fallback chain 403'd.
|
||||
# Mirrors the controlplane fix landed in CP PR#134.
|
||||
# Provisioning trail: internal#329 (audit) + parent pattern
|
||||
# internal#327 (publish-runtime-bot). Per
|
||||
# `feedback_per_agent_gitea_identity_default`.
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
# GITEA_TOKEN reads protection + writes issues. molecule-core
|
||||
# uses `SOP_TIER_CHECK_TOKEN` as the org-level secret name for
|
||||
# read-only Gitea API access from CI (set by audit-force-merge
|
||||
# and sop-tier-check too). Falls back to the auto-injected
|
||||
# GITHUB_TOKEN if the org-level secret isn't set
|
||||
# (transitional repos).
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# Branches whose protection we compare against. molecule-core
|
||||
|
||||
+145
-266
@@ -70,12 +70,10 @@ jobs:
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): all required jobs >=98% green on main.
|
||||
# Flip confirmed 2026-05-12 via combined-status check of latest main
|
||||
# commit (all CI jobs green). `all-required` sentinel hard-fails
|
||||
# when this job fails; no Phase 3 suppression needed.
|
||||
# revert: add `continue-on-error: true` back if regressions appear.
|
||||
continue-on-error: false
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after the surfaced defects
|
||||
# (if any) are triaged.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
platform: ${{ steps.check.outputs.platform }}
|
||||
canvas: ${{ steps.check.outputs.canvas }}
|
||||
@@ -86,92 +84,75 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
env:
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile ci \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "$PR_BASE_SHA" \
|
||||
--base-ref "$PR_BASE_REF" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-$PUSH_BEFORE}"
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Both .github/workflows/ci.yml AND .gitea/workflows/ci.yml count
|
||||
# as "this workflow changed" — either edit should force-run every
|
||||
# downstream job. The Gitea port follows the same shape as the
|
||||
# GitHub original so behavior matches when triggered on either
|
||||
# platform.
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".gitea/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The job always
|
||||
# emits the required context, but expensive steps are path-scoped on every
|
||||
# event so docs/E2E/Canvas-only main pushes do not block deploy on unrelated
|
||||
# Go bootstrap work.
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
|
||||
# + per-step gating shape preserves the GitHub-side required-check name
|
||||
# contract (so when this Gitea port becomes a required check in Phase 4,
|
||||
# the name match works on PRs that don't touch workspace-server/).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
# mc#774 (closed 2026-05-14): Phase 4 flip of the platform-build job.
|
||||
# Phase 4 (#656) originally flipped this to continue-on-error: false based on
|
||||
# Phase-3-masked "green on main 2026-05-12". Two failure classes then surfaced:
|
||||
# (1) 4x delegation_test.go sqlmock gaps (PR #669 / #634 fix-forward, closed).
|
||||
# (2) TestMCPHandler_CommitMemory_GlobalScope_Blocked (mcp_test.go:433):
|
||||
# OFFSEC-001 hardening collided with test assertion; tracked in mc#762.
|
||||
# Fix-forward for (1) landed in PR #669. The mc#762 gap (2) is a separate
|
||||
# issue — it does NOT block this flip because the test is already wrapped in
|
||||
# the diagnostic step with its own continue-on-error: true (line 203).
|
||||
# Flip confirmed by CI / Platform (Go) status = success on main HEAD 363905d3.
|
||||
continue-on-error: false
|
||||
# Job-level ceiling. The go test step below runs with a per-step 10m timeout;
|
||||
# this cap catches any step that leaks past that. Set well above 10m so
|
||||
# the per-step timeout is the active constraint.
|
||||
timeout-minutes: 15
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.platform != 'true' }}
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace-server/** changes — Platform (Go) gate satisfied without running Go build/test/lint."
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: go vet ./...
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Install golangci-lint
|
||||
run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: $(go env GOPATH)/bin/golangci-lint run --timeout 3m ./...
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
name: Diagnostic — per-package verbose 60s
|
||||
run: |
|
||||
set +e
|
||||
go test -race -v -timeout 60s ./internal/handlers/... 2>&1 | tee /tmp/test-handlers.log
|
||||
handlers_exit=$?
|
||||
go test -race -v -timeout 60s ./internal/pendinguploads/... 2>&1 | tee /tmp/test-pu.log
|
||||
pu_exit=$?
|
||||
echo "::group::handlers exit=$handlers_exit (last 100 lines)"
|
||||
tail -100 /tmp/test-handlers.log
|
||||
echo "::endgroup::"
|
||||
echo "::group::pendinguploads exit=$pu_exit (last 100 lines)"
|
||||
tail -100 /tmp/test-pu.log
|
||||
echo "::endgroup::"
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
# Explicit timeout: cold runner cache causes OOM kills at ~4m39s on the
|
||||
# full ./... suite with race detection + coverage. A 10m per-step timeout
|
||||
# lets the suite complete on cold cache (~5-7m) while failing cleanly
|
||||
# instead of OOM-killing. The job-level timeout (15m) is a backstop.
|
||||
run: go test -race -timeout 10m -coverprofile=coverage.out ./...
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
@@ -185,7 +166,7 @@ jobs:
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: ${{ needs.changes.outputs.platform == 'true' }}
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
@@ -275,27 +256,25 @@ jobs:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.canvas != 'true' }}
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — Canvas (Next.js) gate satisfied without running npm build/test."
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
run: npm ci --include=optional --prefer-offline
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
@@ -304,7 +283,7 @@ jobs:
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: ${{ needs.changes.outputs.canvas == 'true' }}
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
@@ -318,19 +297,18 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# Shellcheck (E2E scripts) — required context, path-scoped heavy steps.
|
||||
# Shellcheck (E2E scripts) — required check, always runs.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 4 (RFC #219 §1): confirmed green on main 2026-05-12.
|
||||
continue-on-error: false
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- if: ${{ needs.changes.outputs.scripts != 'true' }}
|
||||
run: echo "No tests/e2e, scripts, or infra/scripts changes — Shellcheck gate satisfied without running script checks."
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
@@ -341,66 +319,31 @@ jobs:
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Test ECR promote-tenant-image script (mock-driven, no live infra)
|
||||
# Covers scripts/promote-tenant-image.sh — the codified
|
||||
# :staging-latest → :latest ECR promote + tenant fleet redeploy
|
||||
# closing molecule-ai/molecule-core#660. 40 mock-driven cases
|
||||
# exercise every exit path (preflight, snapshot, promote, redeploy
|
||||
# 403→SSM-refresh, verify, rollback). No live AWS/CP/SSM calls.
|
||||
run: |
|
||||
bash scripts/test-promote-tenant-image.sh
|
||||
|
||||
- if: ${{ needs.changes.outputs.scripts == 'true' }}
|
||||
name: Shellcheck promote-tenant-image script
|
||||
# scripts/ is excluded from the bulk shellcheck pass above (legacy
|
||||
# SC3040/SC3043 cleanup pending). Run shellcheck explicitly on
|
||||
# the promote script + its test harness so regressions there are
|
||||
# caught by the required check.
|
||||
run: |
|
||||
shellcheck --severity=warning \
|
||||
scripts/promote-tenant-image.sh \
|
||||
scripts/test-promote-tenant-image.sh
|
||||
|
||||
# mc#959 root-fix (sre)
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: docker-host
|
||||
# mc#774 root-fix: added job-level `if:` so ci-required-drift.py's
|
||||
# ci_job_names() detects this as github.ref-gated and skips it from F1.
|
||||
# The step-level exit 0 handles the "not main push" case; the job-level
|
||||
# `if:` makes the gating explicit so the drift script sees it.
|
||||
# Runs on both main and staging pushes; step exits 0 when not applicable.
|
||||
if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' }}
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
CANVAS_CHANGED: ${{ needs.changes.outputs.canvas }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
REF_NAME: ${{ github.ref }}
|
||||
# github.server_url resolves via the workflow-level env override
|
||||
# to the Gitea instance, so the RUN_URL points at the Gitea run
|
||||
# page (not github.com). See feedback_act_runner_github_server_url.
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "$CANVAS_CHANGED" != "true" ] || [ "$EVENT_NAME" != "push" ] || [ "$REF_NAME" != "refs/heads/main" ]; then
|
||||
echo "Canvas deploy reminder not applicable for event=$EVENT_NAME ref=$REF_NAME canvas_changed=$CANVAS_CHANGED."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed — deploy required
|
||||
@@ -430,145 +373,81 @@ jobs:
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs.
|
||||
# Runtime Python moved to molecule-ai-workspace-runtime. Keep this context as
|
||||
# a guard so branch protection still catches attempts to reintroduce an
|
||||
# editable runtime copy under molecule-core/workspace/.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: false
|
||||
continue-on-error: true
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Runtime SSOT guard
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. See issue #2790 for full rationale.
|
||||
run: |
|
||||
set -eu
|
||||
if [ -d workspace ]; then
|
||||
echo "::error file=workspace::Runtime source must live in molecule-ai-workspace-runtime, not molecule-core/workspace."
|
||||
exit 1
|
||||
fi
|
||||
for f in scripts/build_runtime_package.py scripts/test_build_runtime_package.py; do
|
||||
if [ -e "$f" ]; then
|
||||
echo "::error file=$f::Legacy build-from-workspace packaging script must not be restored."
|
||||
exit 1
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text.
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
echo "Runtime SSOT guard passed; core consumes the standalone runtime package."
|
||||
|
||||
all-required:
|
||||
# Aggregator sentinel — RFC internal#219 §2 (Phase 4 — closes internal#286).
|
||||
#
|
||||
# Emits `CI / all-required (<event>)` where <event> is the workflow trigger
|
||||
# (e.g. `CI / all-required (pull_request)`, `CI / all-required (push)`).
|
||||
# Branch protection MUST be updated to require the event-suffixed name —
|
||||
# requiring `CI / all-required` (bare, no suffix) silently blocks all merges
|
||||
# because Gitea treats absent status contexts as pending (not skipped), and
|
||||
# no workflow emits the bare name. Fixed: BP now requires
|
||||
# `CI / all-required (pull_request)` per issue #1473.
|
||||
#
|
||||
# Closes the failure mode where status_check_contexts on molecule-core/main
|
||||
# only listed `Secret scan` + `sop-tier-check` (the 2 meta-gates), so real
|
||||
# `Platform (Go)` / `Canvas (Next.js)` / `Python Lint & Test` / `Shellcheck`
|
||||
# red silently merged through. See internal#286 for the three concrete
|
||||
# tonight-of-2026-05-11 incidents that prompted the emergency bump.
|
||||
#
|
||||
# This job deliberately has no `needs:`. Gitea 1.22/act_runner can mark a
|
||||
# job-level `if: always()` + `needs:` sentinel as skipped before upstream
|
||||
# jobs settle, leaving branch protection with a permanent pending
|
||||
# `CI / all-required` context. Instead, this independent sentinel polls the
|
||||
# required commit-status contexts for this SHA and fails if any fail, skip,
|
||||
# or never emit.
|
||||
#
|
||||
# canvas-deploy-reminder is intentionally NOT included in all-required.needs.
|
||||
# It is an informational main-push reminder, not a PR quality gate. Keeping
|
||||
# it in this dependency list lets a skipped reminder skip the required
|
||||
# sentinel before the `always()` guard can emit a branch-protection status.
|
||||
#
|
||||
continue-on-error: false
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Wait for required CI contexts
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
API_ROOT: ${{ github.server_url }}/api/v1
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
token = os.environ["GITEA_TOKEN"]
|
||||
api_root = os.environ["API_ROOT"].rstrip("/")
|
||||
repo = os.environ["REPOSITORY"]
|
||||
sha = os.environ["COMMIT_SHA"]
|
||||
event = os.environ["EVENT_NAME"]
|
||||
required = [
|
||||
f"CI / Detect changes ({event})",
|
||||
f"CI / Platform (Go) ({event})",
|
||||
f"CI / Canvas (Next.js) ({event})",
|
||||
f"CI / Shellcheck (E2E scripts) ({event})",
|
||||
f"CI / Python Lint & Test ({event})",
|
||||
]
|
||||
terminal_bad = {"failure", "error"}
|
||||
deadline = time.time() + 40 * 60
|
||||
last_summary = None
|
||||
|
||||
def fetch_statuses():
|
||||
statuses = []
|
||||
for page in range(1, 6):
|
||||
url = f"{api_root}/repos/{repo}/commits/{sha}/statuses?page={page}&limit=100"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
chunk = json.load(resp)
|
||||
if not chunk:
|
||||
break
|
||||
statuses.extend(chunk)
|
||||
latest = {}
|
||||
for item in statuses:
|
||||
ctx = item.get("context")
|
||||
if not ctx:
|
||||
continue
|
||||
prev = latest.get(ctx)
|
||||
if prev is None or (item.get("updated_at") or item.get("created_at") or "") >= (prev.get("updated_at") or prev.get("created_at") or ""):
|
||||
latest[ctx] = item
|
||||
return latest
|
||||
|
||||
while True:
|
||||
try:
|
||||
latest = fetch_statuses()
|
||||
except (TimeoutError, OSError, urllib.error.URLError) as exc:
|
||||
if time.time() >= deadline:
|
||||
print(f"FAIL: status polling did not recover before deadline: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"WARN: status poll failed, retrying: {exc}", flush=True)
|
||||
time.sleep(15)
|
||||
continue
|
||||
states = {ctx: (latest.get(ctx) or {}).get("status") or (latest.get(ctx) or {}).get("state") or "missing" for ctx in required}
|
||||
summary = ", ".join(f"{ctx}={state}" for ctx, state in states.items())
|
||||
if summary != last_summary:
|
||||
print(summary, flush=True)
|
||||
last_summary = summary
|
||||
bad = {ctx: state for ctx, state in states.items() if state in terminal_bad}
|
||||
if bad:
|
||||
print("FAIL: required CI context failed:", file=sys.stderr)
|
||||
for ctx, state in bad.items():
|
||||
desc = (latest.get(ctx) or {}).get("description") or ""
|
||||
print(f" - {ctx}: {state} {desc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if all(state == "success" for state in states.values()):
|
||||
print(f"OK: all {len(required)} required CI contexts succeeded")
|
||||
sys.exit(0)
|
||||
if time.time() >= deadline:
|
||||
print("FAIL: timed out waiting for required CI contexts:", file=sys.stderr)
|
||||
for ctx, state in states.items():
|
||||
print(f" - {ctx}: {state}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
time.sleep(15)
|
||||
PY
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -43,18 +43,6 @@ name: Continuous synthetic E2E (staging)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 minutes, on :02 and :32. This keeps a recurring SaaS
|
||||
# behavior probe while cutting runner occupancy from this workflow by
|
||||
# roughly two thirds; fast liveness belongs in the lighter smoke/heartbeat
|
||||
# probes, not in a full tenant/workspace synth every 10 minutes.
|
||||
#
|
||||
# Previous cadence was every 10 minutes (:02 :12 :22 :32 :42 :52).
|
||||
# The current operator-host runner pool is the bottleneck, so full
|
||||
# synth E2E is deliberately lower-cadence until it moves to a dedicated
|
||||
# runner host or warm-runtime pool.
|
||||
#
|
||||
# Historical notes from the 10-minute shape:
|
||||
#
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
@@ -78,7 +66,7 @@ on:
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,32 * * * *'
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
@@ -102,7 +90,6 @@ jobs:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
@@ -118,7 +105,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2 via the template's third-party-
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
@@ -131,9 +118,9 @@ jobs:
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2' }}
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
@@ -145,11 +132,6 @@ jobs:
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
@@ -190,12 +172,6 @@ jobs:
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var secret missing — EC2 leak verification cannot run"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
|
||||
@@ -69,13 +69,6 @@ name: E2E API Smoke Test
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item #1046 (fixed 2026-05-14): Stale platform-server from cancelled runs
|
||||
# lingers on :8080 after "Stop platform" step is skipped (workflow cancelled
|
||||
# before reaching line 335). Added a pre-start "Kill stale platform-server"
|
||||
# step (line 286) that scans /proc for zombie platform-server processes
|
||||
# and kills them before the port probe or bind. Makes the ephemeral port
|
||||
# probe + start sequence deterministic.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
@@ -108,22 +101,8 @@ env:
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
# mc#1529 follow-on: pin to `docker-host` so the e2e-api lane lands
|
||||
# on Linux operator-host runners (molecule-runner-*) that carry the
|
||||
# `molecule-core-net` bridge network + a working `aws ecr get-login-
|
||||
# password | docker login` path. The bare `ubuntu-latest` label is
|
||||
# also accepted by hongming-pc-runner-* (Windows act_runner v1.0.3),
|
||||
# where the docker.sock-bound steps below fail non-deterministically
|
||||
# (e.g. `docker run -d --name pg-e2e-api-...` with port-bind +
|
||||
# `docker exec ... pg_isready` cannot work against a Windows daemon).
|
||||
# detect-changes itself doesn't bind docker.sock, but pinning here too
|
||||
# keeps both jobs on the same lane so we don't re-roll the dice on
|
||||
# workspace-volume cross-host surprises and the routing rule is
|
||||
# discoverable in one place. Mirror of mc#1543 (handlers-postgres-
|
||||
# integration). See internal#512 for the class defect.
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
@@ -132,13 +111,31 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — same pattern PR#372's
|
||||
# ci.yml port used. Diffs against the PR base or push BEFORE SHA,
|
||||
# then matches against the api-relevant path set.
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile e2e-api \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "${{ github.event.pull_request.base.sha }}" \
|
||||
--base-ref "${{ github.event.pull_request.base.ref }}" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-${{ github.event.before }}}"
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
@@ -155,12 +152,8 @@ jobs:
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
# mc#1529 follow-on: must run on operator-host Linux runners (where
|
||||
# docker.sock + `molecule-core-net` + `aws ecr ...` work). See
|
||||
# detect-changes for the full rationale.
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
@@ -171,6 +164,7 @@ jobs:
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
PORT: "8080"
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
@@ -274,49 +268,6 @@ jobs:
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Pick platform port
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Platform host port: ${PLATFORM_PORT}"
|
||||
- name: Kill stale platform-server before start (issue #1046)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Concurrent runs on the same host-network act_runner can leave a
|
||||
# zombie platform-server from a cancelled/timeout run. Cancelled
|
||||
# runs never reach the "Stop platform" step (line 335), so the
|
||||
# old process lingers. Kill it before the ephemeral port probe
|
||||
# or start so the port is definitively free.
|
||||
#
|
||||
# /proc scan — works on any Linux without pkill/lsof/ss.
|
||||
# comm field is truncated to 15 chars: "platform-serve" matches
|
||||
# "platform-server". Verify with cmdline to avoid false positives.
|
||||
killed=0
|
||||
for pid in $(grep -l "platform-serve" /proc/[0-9]*/comm 2>/dev/null); do
|
||||
kpid="${pid%/comm}"
|
||||
kpid="${kpid##*/}"
|
||||
cmdline=$(cat "/proc/${kpid}/cmdline" 2>/dev/null | tr '\0' ' ')
|
||||
if echo "$cmdline" | grep -q "platform-server"; then
|
||||
echo "Killing stale platform-server pid ${kpid}: ${cmdline}"
|
||||
kill "$kpid" 2>/dev/null || true
|
||||
killed=$((killed + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$killed" -gt 0 ]; then
|
||||
sleep 2
|
||||
echo "Killed $killed stale process(es); port(s) released."
|
||||
else
|
||||
echo "No stale platform-server found."
|
||||
fi
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
@@ -329,7 +280,7 @@ jobs:
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "$BASE/health" > /dev/null; then
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
@@ -348,9 +299,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run today's-PR-coverage E2E (mc#1525/1535/1536/1539/1542 fix-specific assertions)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_today_pr_coverage_e2e.sh
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
@@ -360,12 +308,6 @@ jobs:
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Install standalone runtime parser from Gitea registry
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
python3 -m pip install --no-deps \
|
||||
--index-url https://git.moleculesai.app/api/packages/molecule-ai/pypi/simple/ \
|
||||
molecule-ai-workspace-runtime
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
name: E2E Chat
|
||||
|
||||
# Comprehensive Playwright E2E for the unified chat stack (desktop
|
||||
# ChatTab + mobile MobileChat). Heavy browser execution is intentionally
|
||||
# outside the normal required PR path: PRs run it only after entering the
|
||||
# `merge-queue`, while push/main, nightly, and manual dispatch preserve
|
||||
# coverage without making every PR pay the full runtime/browser cost.
|
||||
#
|
||||
# Architecture:
|
||||
# 1. Ephemeral Postgres + Redis (docker, unique container names)
|
||||
# 2. workspace-server built from source, started with
|
||||
# MOLECULE_ENV=development (fail-open auth)
|
||||
# 3. canvas dev server (npm run dev) on :3000
|
||||
# 4. Playwright tests create workspaces via API, point them at an
|
||||
# in-process echo runtime, and exercise the full send/receive
|
||||
# round-trip through the browser.
|
||||
#
|
||||
# Parallel-safety: same pattern as e2e-api.yml — per-run container names
|
||||
# and ephemeral host ports so concurrent jobs on the host-network runner
|
||||
# don't collide.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
schedule:
|
||||
# Nightly at 09:00 UTC. Keeps coverage for the currently non-required
|
||||
# heavy browser lane without spending runner time on every PR.
|
||||
- cron: '0 9 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: e2e-chat-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: helper job; real gate is E2E Chat / E2E Chat (pull_request)
|
||||
detect-changes:
|
||||
# mc#1529 follow-on: pin to `docker-host` (Linux operator-host
|
||||
# runners). The bare `ubuntu-latest` label is also advertised by
|
||||
# hongming-pc-runner-* (Windows act_runner v1.0.3) where the
|
||||
# docker.sock-bound steps below fail. Mirror of mc#1543
|
||||
# (handlers-postgres-integration). See internal#512 for the class
|
||||
# defect.
|
||||
runs-on: docker-host
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
chat: ${{ steps.decide.outputs.chat }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if ! echo "$CHANGED" | grep -qE '^(canvas/|workspace-server/|\.gitea/workflows/e2e-chat\.yml$)'; then
|
||||
echo "chat=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" != "pull_request" ]; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
labels=$(curl -fsS -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
|
||||
| python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
|
||||
rm -f "$authfile"
|
||||
if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
|
||||
echo "chat=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "PR is not in merge-queue; skipping heavy E2E Chat for normal PR path."
|
||||
echo "chat=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# bp-required: pending #1142 — new E2E check; add to branch protection after 3 green runs.
|
||||
e2e-chat:
|
||||
needs: detect-changes
|
||||
name: E2E Chat
|
||||
# mc#1529 follow-on: docker run/exec for postgres + redis containers.
|
||||
# Must land on operator-host Linux (docker-host).
|
||||
runs-on: docker-host
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
PG_CONTAINER: pg-e2e-chat-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-chat-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.chat != 'true'
|
||||
run: |
|
||||
echo "No canvas / workspace-server / workflow changes — E2E Chat gate satisfied without running tests."
|
||||
echo "::notice::E2E Chat no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- if: needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
|
||||
- if: needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "E2E_DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"
|
||||
exit 1
|
||||
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
exit 1
|
||||
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
|
||||
- name: Pick platform port
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PLATFORM_PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "E2E_PLATFORM_URL=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Platform host port: ${PLATFORM_PORT}"
|
||||
|
||||
- name: Pick canvas port
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
CANVAS_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "CANVAS_PORT=${CANVAS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Canvas host port: ${CANVAS_PORT}"
|
||||
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
export MOLECULE_ENV=development
|
||||
export DATABASE_URL="${DATABASE_URL}"
|
||||
export REDIS_URL="${REDIS_URL}"
|
||||
export PORT="${PLATFORM_PORT}"
|
||||
export CORS_ORIGINS="http://localhost:3000,http://localhost:3001,http://localhost:${CANVAS_PORT},http://127.0.0.1:${CANVAS_PORT}"
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://127.0.0.1:${PLATFORM_PORT}/health" > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
|
||||
- name: Install canvas dependencies
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
PREBAKED_PLAYWRIGHT=/ms-playwright
|
||||
if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
|
||||
echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
|
||||
echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
npx playwright install --with-deps chromium
|
||||
|
||||
- name: Start canvas dev server (background)
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
export NEXT_PUBLIC_PLATFORM_URL="http://127.0.0.1:${PLATFORM_PORT}"
|
||||
export NEXT_PUBLIC_WS_URL="ws://127.0.0.1:${PLATFORM_PORT}/ws"
|
||||
npx next dev --turbopack -p "${CANVAS_PORT}" > canvas.log 2>&1 &
|
||||
echo $! > canvas.pid
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://localhost:${CANVAS_PORT}" > /dev/null 2>&1; then
|
||||
echo "Canvas up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Canvas did not start in 30s"
|
||||
cat canvas.log || true
|
||||
exit 1
|
||||
|
||||
- name: Run Playwright E2E tests
|
||||
if: needs.detect-changes.outputs.chat == 'true'
|
||||
working-directory: canvas
|
||||
run: |
|
||||
export E2E_PLATFORM_URL="http://127.0.0.1:${PLATFORM_PORT}"
|
||||
export E2E_DATABASE_URL="${DATABASE_URL}"
|
||||
export PLAYWRIGHT_BASE_URL="http://localhost:${CANVAS_PORT}"
|
||||
npx playwright test e2e/chat-desktop.spec.ts e2e/chat-mobile.spec.ts
|
||||
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
|
||||
- name: Dump canvas log on failure
|
||||
if: failure() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: cat canvas/canvas.log || true
|
||||
|
||||
- name: Upload Playwright report
|
||||
if: failure() && needs.detect-changes.outputs.chat == 'true'
|
||||
uses: actions/upload-artifact@v3.2.2
|
||||
with:
|
||||
name: playwright-report-chat
|
||||
path: canvas/playwright-report/
|
||||
|
||||
- name: Stop canvas
|
||||
if: always() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
if [ -f canvas/canvas.pid ]; then
|
||||
kill "$(cat canvas/canvas.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Stop service containers
|
||||
if: always() && needs.detect-changes.outputs.chat == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -1,402 +0,0 @@
|
||||
name: E2E Peer Visibility (literal MCP list_peers)
|
||||
|
||||
# WHY A DEDICATED WORKFLOW (not folded into e2e-staging-saas.yml)
|
||||
# --------------------------------------------------------------
|
||||
# This is the systemic fix for a real trust failure. Hermes and OpenClaw
|
||||
# were reported "fleet-verified / cascade-complete" because the *proxy*
|
||||
# signals were green (registry registration + heartbeat for Hermes; model
|
||||
# round-trip 200 for OpenClaw). A freshly-provisioned workspace asked on
|
||||
# canvas "can you see your peers" actually FAILS:
|
||||
# - Hermes: 401 on the molecule MCP `list_peers` call
|
||||
# - OpenClaw: native `sessions_list` fallback, sees no platform peers
|
||||
# Tasks #142/#159 were even marked "completed" under this proxy flaw.
|
||||
#
|
||||
# A dedicated workflow (vs extending e2e-staging-saas.yml) because:
|
||||
# - It must provision MULTIPLE distinct runtimes (hermes, openclaw,
|
||||
# claude-code) in ONE org and assert each sees the others. The
|
||||
# full-saas script is single-runtime-per-run (E2E_RUNTIME) and folding
|
||||
# a multi-runtime matrix into it would conflate concerns and bloat its
|
||||
# already-45-min run.
|
||||
# - It needs its own concurrency group so it doesn't fight full-saas /
|
||||
# canvas for the staging org-creation quota.
|
||||
# - It needs an independent, non-required status-context name so it can
|
||||
# be RED today (the in-flight Hermes-401 / OpenClaw-MCP-wiring fixes
|
||||
# have not landed) WITHOUT wedging unrelated merges — and flipped to
|
||||
# REQUIRED in one branch-protection edit once it goes green
|
||||
# (flip-to-required checklist: molecule-core#1296).
|
||||
#
|
||||
# THE ASSERTION IS NOT A PROXY. The driving script
|
||||
# tests/e2e/test_peer_visibility_mcp_staging.sh issues the byte-for-byte
|
||||
# JSON-RPC `tools/call name=list_peers` envelope to `POST
|
||||
# /workspaces/:id/mcp` using each workspace's OWN bearer token, through
|
||||
# the real WorkspaceAuth + MCPRateLimiter middleware chain — the exact
|
||||
# call mcp_molecule_list_peers makes from a canvas agent. It does NOT
|
||||
# read a registry row, /health, the heartbeat table, or
|
||||
# GET /registry/:id/peers.
|
||||
#
|
||||
# HONEST GATE — NO continue-on-error. Per feedback_fix_root_not_symptom a
|
||||
# fake-green mask would defeat the entire purpose. This workflow goes red
|
||||
# on today's broken behavior and green only when the root-cause fixes
|
||||
# actually land. It is intentionally NOT in branch_protections — see PR
|
||||
# body for the required-vs-not decision + flip tracking issue.
|
||||
#
|
||||
# Gitea 1.22.6 / act_runner notes honored:
|
||||
# - No cross-repo `uses:` (feedback_gitea_cross_repo_uses_blocked). The
|
||||
# actions/checkout SHA is the one e2e-staging-canvas.yml already uses
|
||||
# successfully (a mirrored SHA — see #1277/PR#1292 root-cause).
|
||||
# - 2026-05-21 retrigger: verify fresh platform-tenant image after the
|
||||
# publish Buildx DOCKER_CONFIG fix restored staging-latest image updates.
|
||||
# - Per-SHA concurrency, not global (feedback_concurrency_group_per_sha).
|
||||
# - Workflow-level GITHUB_SERVER_URL pinned
|
||||
# (feedback_act_runner_github_server_url).
|
||||
# - pr-validate posts a status under the same check name so a
|
||||
# workflow-only PR is not silently statusless and the context is
|
||||
# flip-to-required-ready (mirrors e2e-staging-saas.yml's proven shape;
|
||||
# real EC2-provisioning E2E is push/dispatch/cron only — it is 30+ min
|
||||
# and cannot run per-PR-update).
|
||||
#
|
||||
# LOCAL BACKEND (added 2026-05-15 — feedback_local_must_mimic_production,
|
||||
# feedback_mandatory_local_e2e_before_ship, feedback_local_test_before_
|
||||
# staging_e2e)
|
||||
# --------------------------------------------------------------------
|
||||
# The standing rule is that the local prod-mimic stack runs a MANDATORY
|
||||
# local-Postgres E2E BEFORE staging E2E. A staging-only peer-visibility
|
||||
# gate caught regressions late + expensively (cold EC2). The
|
||||
# `peer-visibility-local` job below runs the SAME byte-identical
|
||||
# assertion (tests/e2e/lib/peer_visibility_assert.sh) against the local
|
||||
# docker-compose stack — built + booted exactly like e2e-api.yml's
|
||||
# proven E2E API Smoke Test job (ephemeral pg/redis ports, go build,
|
||||
# background platform-server). It runs on PR + push (local boot is
|
||||
# minutes, not the 30+ min cold-EC2 path), so peer-visibility is part of
|
||||
# the local gate that fires before the staging E2E.
|
||||
#
|
||||
# It is its OWN non-required status context `E2E Peer Visibility (local)`.
|
||||
# The local backend uses external-mode workspaces by default so it tests
|
||||
# the literal platform MCP list_peers path without depending on local
|
||||
# template container boot/heartbeat. Container-mode runtime boot remains
|
||||
# available via PV_LOCAL_PROVISION_MODE=container for targeted debugging.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/mcp.go'
|
||||
- 'workspace-server/internal/handlers/mcp_tools.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_local.sh'
|
||||
- 'tests/e2e/lib/peer_visibility_assert.sh'
|
||||
- '.gitea/workflows/e2e-peer-visibility.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/mcp.go'
|
||||
- 'workspace-server/internal/handlers/mcp_tools.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_token_mint_staging.sh'
|
||||
- 'tests/e2e/test_peer_visibility_mcp_local.sh'
|
||||
- 'tests/e2e/lib/peer_visibility_assert.sh'
|
||||
- '.gitea/workflows/e2e-peer-visibility.yml'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 07:30 UTC daily — catches AMI / template-hermes / template-openclaw
|
||||
# drift even on quiet days. Offset 30m from e2e-staging-saas (07:00)
|
||||
# so the two don't collide on the staging org-creation quota.
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA (feedback_concurrency_group_per_sha). A single global group
|
||||
# would let a queued staging/main push behind a PR run get cancelled,
|
||||
# leaving any gate that reads "completed run at SHA" stuck.
|
||||
group: e2e-peer-visibility-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# PR path: post a real status under the required-ready check name so a
|
||||
# workflow-only PR is never silently statusless. The actual EC2 E2E is
|
||||
# push/dispatch/cron only (30+ min). This is NOT a fake-green mask of
|
||||
# the real assertion — it validates the driving script's bash syntax
|
||||
# and inline-python so a broken test script fails at PR time.
|
||||
pr-validate:
|
||||
name: E2E Peer Visibility
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Validate driving scripts + shared assertion lib
|
||||
run: |
|
||||
bash -n tests/e2e/lib/peer_visibility_assert.sh
|
||||
echo "lib/peer_visibility_assert.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_mcp_staging.sh
|
||||
echo "test_peer_visibility_mcp_staging.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_token_mint_staging.sh
|
||||
echo "test_peer_visibility_token_mint_staging.sh — bash syntax OK"
|
||||
bash -n tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
echo "test_peer_visibility_mcp_local.sh — bash syntax OK"
|
||||
if rg -n '/admin/workspaces/.*/test-token|test-token' tests/e2e/test_*staging*.sh; then
|
||||
echo "::error::staging E2E must not use dev-only /admin/workspaces/:id/test-token; use production-safe admin token minting instead"
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging fresh-provision MCP list_peers E2E runs on push to"
|
||||
echo "main / workflow_dispatch / daily cron (30+ min EC2 boot)."
|
||||
echo "The LOCAL backend runs in the peer-visibility-local job"
|
||||
echo "below on this same PR (local docker-compose stack)."
|
||||
|
||||
# LOCAL gate: same byte-identical assertion against the local prod-mimic
|
||||
# docker-compose stack — the MANDATORY local-E2E that must run BEFORE
|
||||
# the staging E2E (feedback_mandatory_local_e2e_before_ship,
|
||||
# feedback_local_test_before_staging_e2e). Bootstrap mirrors
|
||||
# e2e-api.yml's proven E2E API Smoke Test job (per-run container names +
|
||||
# ephemeral host ports so concurrent host-network act_runner runs don't
|
||||
# collide; go build; background platform-server). Its OWN non-required
|
||||
# status context `E2E Peer Visibility (local)` — non-required-by-design
|
||||
# exactly like the staging job (flip-to-required tracked at
|
||||
# molecule-core#1296). HONEST gate, NO continue-on-error mask
|
||||
# (feedback_fix_root_not_symptom). Runs on PR +
|
||||
# push (local boot is minutes, not the 30+ min cold-EC2 path).
|
||||
# bp-required: pending #1296
|
||||
peer-visibility-local:
|
||||
name: E2E Peer Visibility (local)
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# Per-run names + ephemeral ports — same collision-avoidance as
|
||||
# e2e-api.yml (host-network act_runner; feedback_act_runner_*).
|
||||
PG_CONTAINER: pg-e2e-pv-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-pv-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# LLM keys so hermes/openclaw can actually boot. The local script
|
||||
# SKIPs (not fails) any runtime whose key is absent, so a partially
|
||||
# keyed CI env still exercises whatever it can.
|
||||
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.E2E_CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
PV_RUNTIMES: "hermes openclaw claude-code"
|
||||
PV_LOCAL_PROVISION_MODE: external
|
||||
ADMIN_TOKEN: local-e2e-admin-token
|
||||
MOLECULE_ADMIN_TOKEN: local-e2e-admin-token
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network
|
||||
run: |
|
||||
docker pull alpine:latest >/dev/null
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker, ephemeral port)
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
[ -n "$PG_PORT" ] || PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker logs "$PG_CONTAINER" || true; exit 1
|
||||
fi
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1 && { echo "Postgres ready after ${i}s"; exit 0; }
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"; docker logs "$PG_CONTAINER" || true; exit 1
|
||||
- name: Start Redis (docker, ephemeral port)
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
[ -n "$REDIS_PORT" ] || REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker logs "$REDIS_CONTAINER" || true; exit 1
|
||||
fi
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 15); do
|
||||
docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG && { echo "Redis ready after ${i}s"; exit 0; }
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"; docker logs "$REDIS_CONTAINER" || true; exit 1
|
||||
- name: Build platform
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Pick platform port
|
||||
run: |
|
||||
PLATFORM_PORT=$(python3 - <<'PY'
|
||||
import socket
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
print(s.getsockname()[1])
|
||||
PY
|
||||
)
|
||||
echo "PORT=${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "BASE=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Platform host port: ${PLATFORM_PORT}"
|
||||
- name: Kill stale platform-server before start
|
||||
run: |
|
||||
killed=0
|
||||
for pid in $(grep -l "platform-serve" /proc/[0-9]*/comm 2>/dev/null); do
|
||||
kpid="${pid%/comm}"; kpid="${kpid##*/}"
|
||||
cmdline=$(cat "/proc/${kpid}/cmdline" 2>/dev/null | tr '\0' ' ')
|
||||
if echo "$cmdline" | grep -q "platform-server"; then
|
||||
echo "Killing stale platform-server pid ${kpid}"
|
||||
kill "$kpid" 2>/dev/null || true; killed=$((killed + 1))
|
||||
fi
|
||||
done
|
||||
[ "$killed" -gt 0 ] && sleep 2 || true
|
||||
echo "stale-kill done ($killed killed)"
|
||||
- name: Start platform (background)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
curl -sf "$BASE/health" > /dev/null && { echo "Platform up after ${i}s"; exit 0; }
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true; exit 1
|
||||
- name: Run LOCAL fresh-provision peer-visibility E2E (literal MCP list_peers)
|
||||
# HONEST gate — NO continue-on-error. The local backend uses
|
||||
# external-mode workspaces so this context tests the literal MCP
|
||||
# peer-visibility path without coupling to template container boot.
|
||||
run: bash tests/e2e/test_peer_visibility_mcp_local.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure()
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
|
||||
# Real STAGING gate: provisions a throwaway org + sibling-per-runtime,
|
||||
# drives the LITERAL list_peers MCP call per runtime, asserts 200 +
|
||||
# expected peer set, then scoped teardown. push(main)/dispatch/cron only.
|
||||
peer-visibility:
|
||||
name: E2E Peer Visibility
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name != 'pull_request'
|
||||
timeout-minutes: 60
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# LLM provider key so each runtime can authenticate at boot.
|
||||
# Priority MiniMax → direct-Anthropic → OpenAI matches
|
||||
# test_staging_full_saas.sh's secrets-injection chain.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
PV_RUNTIMES: "hermes openclaw claude-code"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present"
|
||||
|
||||
- name: Verify an LLM key present
|
||||
run: |
|
||||
if [ -z "${E2E_MINIMAX_API_KEY:-}" ] && [ -z "${E2E_ANTHROPIC_API_KEY:-}" ] && [ -z "${E2E_OPENAI_API_KEY:-}" ]; then
|
||||
echo "::error::No LLM provider key set — workspaces fail at boot with 'No provider API key found'. Set MOLECULE_STAGING_MINIMAX_API_KEY (or ANTHROPIC / OPENAI)."
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (HTTP $code) — infra, not a workspace bug. Failing loud per feedback_fix_root_not_symptom."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy"
|
||||
|
||||
- name: Run fresh-provision peer-visibility E2E (literal MCP list_peers)
|
||||
run: bash tests/e2e/test_peer_visibility_mcp_staging.sh
|
||||
|
||||
# Belt-and-braces scoped teardown: the script installs an EXIT/INT/
|
||||
# TERM trap, but if the runner itself is cancelled the trap may not
|
||||
# fire. This always() step deletes ONLY the e2e-pv-<run_id> org this
|
||||
# run created — never a cluster-wide sweep
|
||||
# (feedback_never_run_cluster_cleanup_tests_on_live_platform). The
|
||||
# admin DELETE is idempotent so double-invoking is safe;
|
||||
# sweep-stale-e2e-orgs is the final net (slug starts with 'e2e-').
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
except Exception:
|
||||
print(''); sys.exit(0)
|
||||
# ONLY sweep slugs from THIS run. e2e-pv-<YYYYMMDD>-<run_id>-...
|
||||
# Sweep today AND yesterday's UTC date so a midnight-crossing run
|
||||
# still matches its own slug (same bug class as the saas/canvas
|
||||
# safety nets).
|
||||
today = datetime.date.today()
|
||||
yest = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yest.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-pv-{dt}-{run_id}-' for dt in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-pv-{dt}-' for dt in dates)
|
||||
orgs = d if isinstance(d, list) else d.get('orgs', [])
|
||||
cands = [o['slug'] for o in orgs
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('instance_status') not in ('purged',)]
|
||||
print('\n'.join(cands))
|
||||
" 2>/dev/null)
|
||||
for slug in $orgs; do
|
||||
echo "Safety-net teardown: $slug"
|
||||
set +e
|
||||
curl -sS -o /tmp/pv-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/pv-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/pv-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::pv teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within MAX_AGE_MINUTES. Body: $(head -c 300 /tmp/pv-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
@@ -16,9 +16,9 @@ name: E2E Staging Canvas (Playwright)
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main, PR touching canvas sources + this workflow only
|
||||
# after the PR enters `merge-queue`, manual dispatch, and scheduled cron to
|
||||
# catch browser/runtime drift even when canvas is quiet.
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
@@ -37,10 +37,9 @@ on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Nightly at 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * *'
|
||||
workflow_dispatch:
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
@@ -71,7 +70,6 @@ jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
@@ -80,13 +78,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
# Cron and manual triggers always run real work (no diff context).
|
||||
# Cron triggers always run real work (no diff context).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
if [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
@@ -106,26 +101,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if ! echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${{ github.event_name }}" != "pull_request" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
labels=$(curl -fsS -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels" \
|
||||
| python3 -c 'import json,sys; print("\n".join(label.get("name","") for label in json.load(sys.stdin)))')
|
||||
rm -f "$authfile"
|
||||
if printf '%s\n' "$labels" | grep -qx "$QUEUE_LABEL"; then
|
||||
if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "PR is not in merge-queue; skipping heavy E2E Staging Canvas for normal PR path."
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -140,7 +118,6 @@ jobs:
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 40
|
||||
|
||||
@@ -189,15 +166,7 @@ jobs:
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
PREBAKED_PLAYWRIGHT=/ms-playwright
|
||||
if [ -d "${PREBAKED_PLAYWRIGHT}" ] && find "${PREBAKED_PLAYWRIGHT}" -maxdepth 3 -type f -name 'chrome' | grep -q .; then
|
||||
echo "Using prebaked Playwright Chromium from ${PREBAKED_PLAYWRIGHT}"
|
||||
echo "PLAYWRIGHT_BROWSERS_PATH=${PREBAKED_PLAYWRIGHT}" >> "$GITHUB_ENV"
|
||||
exit 0
|
||||
fi
|
||||
npx playwright install --with-deps chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
|
||||
@@ -84,7 +84,6 @@ jobs:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
|
||||
|
||||
@@ -24,22 +24,17 @@ name: E2E Staging SaaS (full lifecycle)
|
||||
# PRs don't need to read.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to main (regression guard — fires on merges to main, not on PR updates)
|
||||
# - pull_request: pr-validate always posts success; real E2E step runs only
|
||||
# when provisioning-critical files change (detect-changes gates the step).
|
||||
# - Push to main (regression guard)
|
||||
# - workflow_dispatch (manual re-run from UI)
|
||||
# - Nightly cron (catches drift even when no pushes land)
|
||||
#
|
||||
# NOTE: A separate pr-validate job handles the pull_request path so this
|
||||
# workflow posts CI status for workflow-only PRs. Without it, a PR that
|
||||
# only touches the workflow file has no status check (workflow only fires
|
||||
# on push, not PR branches), which blocks merge under branch protection.
|
||||
# The E2E step itself only runs when provisioning-critical files change —
|
||||
# pr-validate always posts success, avoiding the double-fire that motivated
|
||||
# the pull_request-trigger removal in PRs #516/#530.
|
||||
# - Changes to any provisioning-critical file under PR review (opt-in
|
||||
# via the same paths watcher that e2e-api.yml uses)
|
||||
|
||||
on:
|
||||
# Trunk-based (Phase 3 of internal#81): main is the only branch.
|
||||
# Previously this fired on staging push too because staging was a
|
||||
# superset of main and ran the gate ahead of auto-promote; with no
|
||||
# staging branch, main is where E2E gates the deploy.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
@@ -49,8 +44,6 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
@@ -61,10 +54,7 @@ on:
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- 'tests/e2e/lib/aws_leak_check.sh'
|
||||
- 'tests/e2e/test_aws_leak_check.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
|
||||
# Cloudflare API regressions, etc. even on quiet days.
|
||||
@@ -82,41 +72,10 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# PR-validation path: always posts success so branch protection can merge
|
||||
# workflow-only PRs. The actual E2E step only runs when provisioning-
|
||||
# critical files change (git-paths filter + if: guard below).
|
||||
# All steps use continue-on-error: true so runner issues do not block merge.
|
||||
pr-validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
|
||||
- name: YAML validation (best-effort)
|
||||
run: |
|
||||
echo "e2e-staging-saas.yml — PR validation: workflow YAML is valid."
|
||||
echo "E2E step runs only when provisioning-critical files change."
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
|
||||
# Actual E2E: runs on trunk pushes (main + staging). NOT the PR-fire-only
|
||||
# path — pr-validate above posts success for workflow-only PRs.
|
||||
e2e-staging-saas:
|
||||
name: E2E Staging SaaS
|
||||
runs-on: ubuntu-latest
|
||||
# Only runs on trunk pushes. PR paths get pr-validate instead.
|
||||
if: github.event.pull_request.base.ref == ''
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
@@ -131,11 +90,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -161,7 +115,7 @@ jobs:
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2' }}
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
@@ -174,12 +128,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
|
||||
@@ -37,7 +37,6 @@ jobs:
|
||||
name: Intentional-failure teardown sanity
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 20
|
||||
|
||||
@@ -47,11 +46,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
E2E_MODE: smoke
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
@@ -66,12 +60,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken.
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
# gate-check-v3 — automated PR gate detector
|
||||
#
|
||||
# Runs on every open PR (push/synchronize) and hourly via cron.
|
||||
# Posts a structured [gate-check-v3] STATUS: comment on the PR.
|
||||
#
|
||||
# Inputs:
|
||||
# PR_NUMBER — set via ${{ github.event.pull_request.number }} from the trigger
|
||||
# POST_COMMENT — "true" to post/update comment on PR
|
||||
#
|
||||
# Gating logic (MVP signals 1,2,3,6):
|
||||
# 1. Author-aware agent-tag comment scan
|
||||
# 2. REQUEST_CHANGES reviews state machine
|
||||
# 3. Staleness detection (SOP-12: review.commit_id != PR.head_sha + >1 working day)
|
||||
# 6. CI required-checks awareness
|
||||
#
|
||||
# Exit code: 0=CLEAR, 1=BLOCKED, 2=ERROR
|
||||
|
||||
name: gate-check-v3
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, synchronize, reopened]
|
||||
schedule:
|
||||
# Hourly: refresh all open PRs
|
||||
- cron: '8 * * * *'
|
||||
# NOTE: `workflow_dispatch.inputs` block intentionally omitted.
|
||||
# Gitea 1.22.6 parser rejects `workflow_dispatch.inputs.X` with
|
||||
# "unknown on type" — it mis-treats the inputs sub-keys as top-level
|
||||
# `on:` event types. Dropping the inputs block restores parsing.
|
||||
# Manual dispatch from the Gitea UI works without the inputs schema
|
||||
# (github.event.inputs.X returns empty); the script falls back to
|
||||
# iterating all open PRs when PR_NUMBER is empty.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
# read: contents — for checkout (base ref, not PR head for security)
|
||||
# read: pull-requests — for reading PR info via API
|
||||
# write: pull-requests — for posting/updating gate-check comments
|
||||
# Without this the token cannot POST/PATCH /issues/comments → 403.
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: PR advisory bot; merge blocking is enforced by CI status and branch protection.
|
||||
gate-check:
|
||||
runs-on: ubuntu-latest
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true # Never block on our own detector failing
|
||||
steps:
|
||||
- name: Check out BASE ref (never PR-head under pull_request_target)
|
||||
# pull_request_target runs with repo secrets-context, so checking out
|
||||
# the PR HEAD would execute PR-branch gate_check.py with secrets.
|
||||
# Fix: always load gate_check.py from the trusted base/default ref.
|
||||
# Bug-1 (self-loop exclusion) + Bug-3 (403→exit0) from #547 are
|
||||
# kept; only this checkout-ref regresses to pre-#547 behavior.
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha || github.ref_name }}
|
||||
|
||||
- name: Run gate-check-v3 (single PR mode)
|
||||
if: github.event_name == 'pull_request_target' || github.event.inputs.pr_number != ''
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
|
||||
POST_COMMENT: ${{ github.event.inputs.post_comment || 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 tools/gate-check-v3/gate_check.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "$PR_NUMBER" \
|
||||
$([ "$POST_COMMENT" = "true" ] && echo "--post-comment")
|
||||
echo "verdict=$?" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run gate-check-v3 (all open PRs — cron mode)
|
||||
if: github.event_name == 'schedule'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch all open PRs and run gate-check on each. This scheduled
|
||||
# refresher is advisory; a transient Gitea list timeout must not turn
|
||||
# main red. PR-specific gate-check runs still use normal failure
|
||||
# semantics.
|
||||
pr_numbers=$(python3 <<'PY'
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
socket.setdefaulttimeout(30)
|
||||
token = os.environ["GITEA_TOKEN"]
|
||||
repo = os.environ["REPO"]
|
||||
url = f"https://git.moleculesai.app/api/v1/repos/{repo}/pulls?state=open&limit=100"
|
||||
last_error = None
|
||||
for attempt in range(1, 4):
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"Authorization": f"token {token}", "Accept": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
prs = json.loads(r.read())
|
||||
break
|
||||
except (TimeoutError, OSError, urllib.error.URLError, urllib.error.HTTPError) as exc:
|
||||
last_error = exc
|
||||
print(f"warning: PR list fetch attempt {attempt}/3 failed: {exc}", file=sys.stderr)
|
||||
if attempt < 3:
|
||||
time.sleep(2 * attempt)
|
||||
else:
|
||||
print(f"warning: skipped scheduled gate-check refresh; failed to list open PRs after 3 attempts: {last_error}", file=sys.stderr)
|
||||
raise SystemExit(0)
|
||||
for pr in prs:
|
||||
print(pr["number"])
|
||||
PY
|
||||
)
|
||||
for pr in $pr_numbers; do
|
||||
echo "Checking PR #$pr..."
|
||||
python3 tools/gate-check-v3/gate_check.py \
|
||||
--repo "${{ github.repository }}" \
|
||||
--pr "$pr" \
|
||||
--post-comment \
|
||||
|| true
|
||||
done
|
||||
@@ -1,64 +0,0 @@
|
||||
name: gitea-merge-queue
|
||||
|
||||
# External serialized merge queue for Gitea 1.22.6.
|
||||
#
|
||||
# Gitea's `pull_auto_merge` table is not a real merge queue: it does not
|
||||
# serialize green PRs against a freshly-tested latest main. This workflow runs
|
||||
# the user-space queue bot, one PR per tick, using the non-bypass merge actor.
|
||||
#
|
||||
# Queue contract:
|
||||
# - add label `merge-queue` to an open same-repo PR
|
||||
# - bot updates stale PR heads with current main, then waits for CI
|
||||
# - bot merges only when current main is green and required PR contexts pass
|
||||
# - add `merge-queue-hold` to pause a queued PR without removing it
|
||||
|
||||
on:
|
||||
# Schedule moved to operator-config:
|
||||
# /etc/cron.d/molecule-core-merge-queue ->
|
||||
# /usr/local/bin/molecule-core-cron-bot.sh merge-queue
|
||||
#
|
||||
# The queue bot still processes one PR per tick, but no longer occupies
|
||||
# one of the shared Actions runners just to poll.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: gitea-merge-queue-${{ github.repository }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
queue:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out queue script from main
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Process one queued PR
|
||||
env:
|
||||
# AUTO_SYNC_TOKEN is the devops-engineer persona PAT. It is the
|
||||
# non-bypass merge actor allowed by branch protection.
|
||||
GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
WATCH_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
QUEUE_LABEL: merge-queue
|
||||
HOLD_LABEL: merge-queue-hold
|
||||
UPDATE_STYLE: merge
|
||||
REQUIRED_CONTEXTS: >-
|
||||
CI / all-required (pull_request),
|
||||
sop-checklist / all-items-acked (pull_request)
|
||||
# Push-side required contexts. Checking CI / all-required (push)
|
||||
# explicitly instead of the combined state avoids false-pause when
|
||||
# non-blocking jobs (continue-on-error: true) have failed — those
|
||||
# failures pollute combined state but do not gate merges.
|
||||
# NOTE: the event-suffixed context name is intentional — branch protection
|
||||
# MUST require `CI / all-required (pull_request)` (with suffix), NOT the
|
||||
# bare `CI / all-required`. Gitea treats absent contexts as pending, not
|
||||
# skipped; requiring the bare name silently blocks all merges (issue #1473).
|
||||
PUSH_REQUIRED_CONTEXTS: CI / all-required (push)
|
||||
run: python3 .gitea/scripts/gitea-merge-queue.py
|
||||
@@ -77,37 +77,39 @@ env:
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: detect-changes
|
||||
# mc#1529 §1: pin to `docker-host` so the integration job runs on the
|
||||
# operator-host runners (molecule-runner-*), which carry the
|
||||
# `molecule-core-net` bridge network this workflow depends on. PC2
|
||||
# runners (hongming-pc-runner-*) also advertise ubuntu-latest but
|
||||
# don't have that network — the previous `runs-on: ubuntu-latest`
|
||||
# rolled the dice and hard-failed the bridge-inspect step ~30% of
|
||||
# the time. detect-changes itself doesn't need the bridge, but keeping
|
||||
# both jobs on the same label avoids workspace-volume cross-host
|
||||
# surprises and keeps the routing rule discoverable in one place.
|
||||
runs-on: docker-host
|
||||
# mc#774 Phase 3 (RFC §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
handlers: ${{ steps.filter.outputs.handlers }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# A full-history checkout can exceed the runner's quiet/startup
|
||||
# window before the path filter emits logs. Fetch the common push
|
||||
# case cheaply; the script below fetches the exact BASE SHA if it is
|
||||
# not present in the shallow checkout.
|
||||
fetch-depth: 2
|
||||
fetch-depth: 0
|
||||
- id: filter
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
run: |
|
||||
python3 .gitea/scripts/detect-changes.py \
|
||||
--profile handlers-postgres \
|
||||
--event-name "${{ github.event_name }}" \
|
||||
--pr-base-sha "${{ github.event.pull_request.base.sha }}" \
|
||||
--base-ref "${{ github.event.pull_request.base.ref }}" \
|
||||
--push-before "${GITHUB_EVENT_BEFORE:-}"
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "handlers=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
@@ -115,11 +117,8 @@ jobs:
|
||||
integration:
|
||||
name: Handlers Postgres Integration
|
||||
needs: detect-changes
|
||||
# mc#1529 §1: must run on operator-host (where `molecule-core-net`
|
||||
# exists). See detect-changes for the full routing rationale.
|
||||
runs-on: docker-host
|
||||
# mc#774 Phase 3 (RFC §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
# Unique name per run so concurrent jobs don't collide on the
|
||||
|
||||
@@ -34,7 +34,7 @@ name: Harness Replays
|
||||
# One job → one check run → branch-protection-clean (the SKIPPED-in-set
|
||||
# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
|
||||
|
||||
"on":
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
@@ -60,41 +60,44 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: change detector only; downstream Harness Replays is the meaningful gate.
|
||||
detect-changes:
|
||||
# mc#1529 follow-on: pin to `docker-host` so this lane lands on
|
||||
# Linux operator-host runners (the only ones with a working
|
||||
# docker.sock + `molecule-core-net`). The bare `ubuntu-latest`
|
||||
# label is also matched by hongming-pc-runner-* (Windows act_runner
|
||||
# v1.0.3), where the `docker compose ...` exec below fails. Mirror
|
||||
# of mc#1543; see internal#512 for class defect.
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Shallow clone — we use the Gitea Compare API for changed-file
|
||||
# detection, not local git diff. The base SHA is supplied via
|
||||
# GitHub event variables, so no local history is needed.
|
||||
fetch-depth: 1
|
||||
- id: decide
|
||||
env:
|
||||
# Pass via env block — env values bypass shell quoting so single
|
||||
# quotes in merge-commit messages (e.g. "Merge pull request 'fix: ...'
|
||||
# from branch into main") cannot break the bash parser. The prior
|
||||
# `echo '${{ toJSON(...) }}'` form broke on every main-push because
|
||||
# every main commit is a merge commit with single quotes in the
|
||||
# message body — the embedded `'` ended the single-quoted shell string
|
||||
# mid-JSON, and a subsequent `(` (e.g. in `(#523)`) was parsed as a
|
||||
# subshell, causing "syntax error near unexpected token `('".
|
||||
COMMITS_JSON: ${{ toJSON(github.event.commits) }}
|
||||
- name: Fetch base branch tip for diff
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# With the default fetch-depth: 1, actions/checkout only fetches the
|
||||
# PR head commit. The base commit is NOT in the local history, so
|
||||
# `git diff "$BASE" "$GITHUB_SHA"` fails. Fetch the base branch at
|
||||
# depth 1 — the base commit is the immediate parent of the PR head
|
||||
# on the base branch, so depth=1 is sufficient.
|
||||
#
|
||||
# Network: Gitea Actions runner (5.78.80.188) cannot reach the git
|
||||
# remote over HTTPS (confirmed: git fetch times out at ~15s). The runner
|
||||
# is on the same host as Gitea, but the container network namespace
|
||||
# cannot reach the Gitea HTTPS endpoint.
|
||||
#
|
||||
# Fallback: if the base commit does not exist locally, skip the diff
|
||||
# and set run=true (always run harness). This is safe: PRs where the
|
||||
# base is unavailable still run the harness (correct), PRs where the
|
||||
# base IS available get the correct path-based diff.
|
||||
#
|
||||
# Timeout: 20s. If the fetch completes, great. If it times out, the
|
||||
# step exits non-zero and we fall through to run=true.
|
||||
if timeout 20 git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1; then
|
||||
echo "::notice::base branch fetched successfully"
|
||||
else
|
||||
echo "::warning::git fetch origin ${{ github.event.pull_request.base.ref }} --depth=1 timed out"
|
||||
echo "::warning::Skipping diff — detect-changes will run the harness unconditionally."
|
||||
fi
|
||||
- id: decide
|
||||
continue-on-error: true
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# workflow_dispatch: always run (manual trigger)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
@@ -102,31 +105,16 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine changed files.
|
||||
# workflow_dispatch: always run.
|
||||
# pull_request: use Compare API (branch-to-branch works fine).
|
||||
# push: use github.event.commits array (Compare API rejects SHA-to-branch).
|
||||
# new-branch: run everything.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
BASE="${{ github.event.pull_request.base.ref }}"
|
||||
HEAD="${{ github.event.pull_request.head.ref }}"
|
||||
# Determine the base commit to diff against.
|
||||
# For pull_request: use base.sha (the merge-base with main/staging).
|
||||
# For push: use github.event.before (the previous tip of the branch).
|
||||
# Fallback for new branches (all-zeros SHA): run everything.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && \
|
||||
[ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "${{ github.event.before }}" ] && \
|
||||
! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
|
||||
# Push event: extract changed files from github.event.commits array.
|
||||
# Gitea Compare API rejects SHA-to-branch comparisons (BaseNotExist),
|
||||
# so we use the commits array instead. This array contains all commits
|
||||
# in the push, each with their added/removed/modified file lists.
|
||||
printf '%s' "$COMMITS_JSON" \
|
||||
| bash .gitea/scripts/push-commits-diff-files.py \
|
||||
> .push-diff-files.txt 2>/dev/null || true
|
||||
DIFF_FILES=$(cat .push-diff-files.txt 2>/dev/null || true)
|
||||
if [ -n "$DIFF_FILES" ] && echo "$DIFF_FILES" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "debug=push-files=$DIFF_FILES" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
BASE="${{ github.event.before }}"
|
||||
else
|
||||
# New branch or github.event.before unavailable — run everything.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
@@ -134,24 +122,17 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Call Gitea Compare API (pull_request path only — branch-to-branch).
|
||||
# Push uses github.event.commits array above.
|
||||
RESP=$(curl -sS --fail --max-time 30 \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/json" \
|
||||
"$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/compare/$BASE...$HEAD") || {
|
||||
# If Gitea's Compare API is slow/unavailable, choose the conservative
|
||||
# behavior: run the harness instead of failing the detector and polluting
|
||||
# main with a red non-gate context.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=compare-api-unavailable base=$BASE head=$HEAD" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
}
|
||||
DIFF_FILES=$(echo "$RESP" | bash .gitea/scripts/compare-api-diff-files.py 2>/dev/null || true)
|
||||
# GitHub Actions and Gitea Actions both expose github.sha for HEAD.
|
||||
# git diff exits 1 when BASE is not in local history (e.g. shallow
|
||||
# checkout where the base commit was never fetched). Capture and
|
||||
# swallow that exit code — the empty diff means "run everything".
|
||||
# The runner network cannot reach the git remote (confirmed: git fetch
|
||||
# times out at ~15s), so a failed fetch is expected and we always fall
|
||||
# through to the unconditional run=true below.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null) || true
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF_FILES" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if echo "$DIFF_FILES" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
|
||||
if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -164,15 +145,11 @@ jobs:
|
||||
# matches e2e-api.yml — see that workflow's comment for why a
|
||||
# job-level `if: false` would block branch protection via the
|
||||
# SKIPPED-in-set bug.
|
||||
# bp-exempt: path-filtered replay suite; CI / all-required is the branch-protection aggregate.
|
||||
harness-replays:
|
||||
needs: detect-changes
|
||||
name: Harness Replays
|
||||
# mc#1529 follow-on: `docker compose ... ps/logs` against tenant-alpha/
|
||||
# beta containers. Must run on operator-host Linux (docker-host).
|
||||
runs-on: docker-host
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
@@ -239,14 +216,12 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::warning::AUTO_SYNC_TOKEN not set — using anonymous clone (repos are public per manifest.json OSS contract)"
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
# Strip JSON5 comments before jq parsing — Integration Tester appends
|
||||
# `// Triggered by ...` which breaks `jq` in clone-manifest.sh.
|
||||
sed '/^[[:space:]]*\/\//d' manifest.json > .manifest-stripped.json
|
||||
bash scripts/clone-manifest.sh \
|
||||
.manifest-stripped.json \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
name: lint-bp-context-emit-match
|
||||
|
||||
# Tier 2f scheduled lint (per mc#774) — detects drift between
|
||||
# `branch_protections/<branch>.status_check_contexts` and the set of
|
||||
# contexts emitted by `.gitea/workflows/*.yml`.
|
||||
#
|
||||
# Rule
|
||||
# ----
|
||||
# For each protected branch context (Source A — BP), there must exist
|
||||
# at least one emitting workflow + job pair (Source B — workflow YAML
|
||||
# + on:-event mapping) whose runtime status-name maps to it. The
|
||||
# inverse direction (emitter without BP context) is informational
|
||||
# only — Tier 2g handles that at PR-time.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# A BP-required context with no emitter blocks merges forever — Gitea
|
||||
# 1.22.6 treats absent-as-`pending`, NOT absent-as-`skipped`. The
|
||||
# phantom-required-check class previously surfaced as
|
||||
# `feedback_phantom_required_check_after_gitea_migration` (a port
|
||||
# kept the GitHub context name after rename to Gitea, but no
|
||||
# workflow emitted under the new name).
|
||||
#
|
||||
# This lint catches the same class structurally + a forward case:
|
||||
# workflow renamed/deleted while still in BP.
|
||||
#
|
||||
# Scope
|
||||
# -----
|
||||
# Scheduled daily. We DON'T run on `pull_request` because (a) the
|
||||
# emitter side moves with PR diffs (transitional state false-flags)
|
||||
# and (b) Tier 2g handles emitter-side drift at PR-time.
|
||||
#
|
||||
# Cross-repo
|
||||
# ----------
|
||||
# Today this runs only on molecule-core/main. Per internal#349
|
||||
# (cross-repo BP sweep) Class-D repos will get the same lint after
|
||||
# their BP rollouts.
|
||||
#
|
||||
# Auth
|
||||
# ----
|
||||
# `GET /repos/.../branch_protections/{branch}` requires repo-admin
|
||||
# role on Gitea 1.22.6. We use DRIFT_BOT_TOKEN (same persona as
|
||||
# ci-required-drift.yml — `internal#329` provisioning trail).
|
||||
# Graceful-degrade per Tier 2a contract: 403/404 → exit 0 with
|
||||
# ::error::.
|
||||
#
|
||||
# Idempotency
|
||||
# -----------
|
||||
# The drift issue is filed with title prefix
|
||||
# `[ci-bp-drift] {repo}/{branch}: BP→emitter mismatch`. The script
|
||||
# searches OPEN issues for an exact title-prefix match and PATCHes
|
||||
# the existing issue (if any) instead of POSTing a duplicate.
|
||||
# Mirrors `ci-required-drift.py`'s contract.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder)
|
||||
# -------------------------------------------
|
||||
# Lands at `continue-on-error: true` (Phase 3). After 7 days of clean
|
||||
# scheduled runs on `main`, flip to `false` so a scheduled failure
|
||||
# becomes a hard CI signal.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#774 (the RFC that specs this lint)
|
||||
# - internal#349 (cross-repo BP sweep)
|
||||
# - feedback_phantom_required_check_after_gitea_migration
|
||||
# - feedback_tier_label_ids_are_per_repo
|
||||
# - ci-required-drift.yml (F2 detector, narrower-scope sibling)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Daily at 03:31 UTC — off-peak, prime-staggered from other
|
||||
# scheduled jobs (ci-required-drift :00 hourly, lint-coe-tracking
|
||||
# 13:11). At 03:31 the CI fleet is quietest in EMEA hours.
|
||||
- cron: '31 3 * * *'
|
||||
workflow_dispatch:
|
||||
# No `push` / `pull_request` here — Tier 2g owns PR-time drift.
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write # needed to file/edit the drift issue
|
||||
|
||||
concurrency:
|
||||
group: lint-bp-context-emit-match-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: lint-bp-context-emit-match
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# Phase 3 (RFC #219 §1): surface drift without blocking. After 7
|
||||
# clean scheduled runs on main, flip to false so a scheduled
|
||||
# failure is a hard CI signal.
|
||||
continue-on-error: true # mc#774 Phase 3 — flip to false after 7 clean main runs
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run lint-bp-context-emit-match
|
||||
env:
|
||||
# DRIFT_BOT_TOKEN — repo-admin on this repo (internal#329
|
||||
# provisioning trail). Required for branch_protections read.
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: main
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
DRIFT_LABEL: ci-bp-drift
|
||||
GITHUB_RUN_URL: https://git.moleculesai.app/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: python3 .gitea/scripts/lint_bp_context_emit_match.py
|
||||
- name: Run lint-bp-context-emit-match unit tests
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_bp_context_emit_match.py -v
|
||||
@@ -1,122 +0,0 @@
|
||||
name: lint-continue-on-error-tracking
|
||||
|
||||
# Tier 2e hard-gate lint (per mc#774) — every
|
||||
# `continue-on-error: true` in `.gitea/workflows/*.yml` must carry a
|
||||
# `# mc#NNNN` or `# internal#NNNN` tracker comment within 2 lines,
|
||||
# the referenced issue must be OPEN, and ≤14 days old.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# `continue-on-error: true` on `platform-build` had been hiding
|
||||
# mc#774-class regressions for ~3 weeks before #656 surfaced them on
|
||||
# 2026-05-12. A 14-day cap on tracker age forces a review cycle and
|
||||
# surfaces mask-drift within at most 14 days of the original defect.
|
||||
# Each `continue-on-error: true` gets a paper trail — close or renew.
|
||||
#
|
||||
# How the gate works
|
||||
# ------------------
|
||||
# 1. Walk `.gitea/workflows/*.yml` via PyYAML's line-tracking loader
|
||||
# (per `feedback_behavior_based_ast_gates`) and find every job
|
||||
# whose `continue-on-error` evaluates truthy (`true` or string
|
||||
# `"true"` — Gitea's evaluator coerces strings).
|
||||
# 2. For each, scan ±2 lines of the directive's source line for a
|
||||
# `# mc#NNNN` or `# internal#NNNN` comment. Inline-trailing
|
||||
# comments on the directive line count.
|
||||
# 3. For each tracker reference, GET the issue from the Gitea API.
|
||||
# Validate: exists, `state == open`, `created_at` ≤ MAX_AGE_DAYS.
|
||||
# 4. Aggregate ALL violations (not short-circuit) and exit 1 if any.
|
||||
#
|
||||
# Triggers
|
||||
# --------
|
||||
# Runs on PR events (paths-filter on `.gitea/workflows/**`) AND on
|
||||
# a daily schedule. PR runs catch the violation at introduction time.
|
||||
# Schedule runs catch the AGE-EXPIRY class: a tracker that was ≤14d
|
||||
# old when the PR landed but is now 20d old, with the underlying
|
||||
# defect still unfixed. Per `feedback_chained_defects_in_never_tested_workflows`,
|
||||
# scheduled drift detection is the second half of the gate.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder)
|
||||
# -------------------------------------------
|
||||
# Lands at `continue-on-error: true` (Phase 3 — surface broken shapes
|
||||
# without blocking). The pre-existing `continue-on-error: true`
|
||||
# directives on `main` will all violate this lint at first
|
||||
# (intentional — they're the masked defects this lint exists to
|
||||
# surface). Each must be triaged: file a fresh tracker comment,
|
||||
# close-and-flip, or document the deliberate keep-mask in a fresh
|
||||
# 14-day-renewable tracker. After main is clean for 3 days,
|
||||
# follow-up PR flips this workflow's continue-on-error to false.
|
||||
# Tracking: mc#774.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#774 (the RFC that specs this lint)
|
||||
# - mc#774 (the empirical masked-3-weeks case)
|
||||
# - feedback_chained_defects_in_never_tested_workflows
|
||||
# - feedback_behavior_based_ast_gates
|
||||
# - feedback_strict_root_only_after_class_a
|
||||
#
|
||||
# Auth: DRIFT_BOT_TOKEN — same persona used by ci-required-drift.yml
|
||||
# (provisioned under internal#329). Auto-injected GITHUB_TOKEN is
|
||||
# insufficient because `internal#NNN` references cross repositories
|
||||
# (molecule-core → molecule-ai/internal).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint_continue_on_error_tracking.py'
|
||||
- 'tests/test_lint_continue_on_error_tracking.py'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint_continue_on_error_tracking.py'
|
||||
schedule:
|
||||
# Daily at 13:11 UTC — off-peak, prime-staggered from the other
|
||||
# Tier-2 lint schedules (ci-required-drift runs hourly :00).
|
||||
- cron: '11 13 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lint-coe-tracking-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: meta-lint for masked jobs; tracked separately until masks are burned down.
|
||||
lint:
|
||||
name: lint-continue-on-error-tracking
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
# Phase 3 (RFC #219 §1): surface masked defects without blocking
|
||||
# PRs. Pre-existing continue-on-error: true directives on main
|
||||
# all violate this lint at first — intentional. Flip to false
|
||||
# follow-up after main is clean for 3 days. mc#774.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true # mc#774 Phase 3 mask — 14d forced-renewal cadence
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run lint-continue-on-error-tracking
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
INTERNAL_REPO: molecule-ai/internal
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
MAX_AGE_DAYS: '14'
|
||||
run: python3 .gitea/scripts/lint_continue_on_error_tracking.py
|
||||
- name: Run lint-continue-on-error-tracking unit tests
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_continue_on_error_tracking.py -v
|
||||
@@ -30,16 +30,10 @@ name: Lint curl status-code capture
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint-curl-status-capture.py'
|
||||
- 'tests/test_lint_curl_status_capture.py'
|
||||
paths: ['.gitea/workflows/**']
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint-curl-status-capture.py'
|
||||
- 'tests/test_lint_curl_status_capture.py'
|
||||
paths: ['.gitea/workflows/**']
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
@@ -51,10 +45,60 @@ jobs:
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
|
||||
run: |
|
||||
python3 .gitea/scripts/lint-curl-status-capture.py
|
||||
set -uo pipefail
|
||||
# Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
|
||||
# subshell where the entire command-substitution wraps a curl that
|
||||
# ends with `|| echo "000"`. Must distinguish from the SAFE shape
|
||||
# `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
|
||||
# tempfile produces empty stdout, no pollution.
|
||||
python3 <<'PY'
|
||||
import os, re, sys, glob
|
||||
|
||||
BAD_FILES = []
|
||||
|
||||
# Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
|
||||
# The `\\n` is the bash line-continuation that lets curl flags span lines.
|
||||
# We collapse continuation lines first, then look for the single-line bad pattern.
|
||||
PATTERN = re.compile(
|
||||
r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Self-skip: this lint workflow contains the literal anti-pattern in
|
||||
# its own docstring — that's intentional, not a bug.
|
||||
SELF = ".gitea/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
for f in sorted(glob.glob(".gitea/workflows/*.yml")):
|
||||
if f == SELF:
|
||||
continue
|
||||
with open(f) as fh:
|
||||
content = fh.read()
|
||||
# Collapse bash line-continuations (\\\n + leading whitespace)
|
||||
# into a single logical line so the regex can see the full
|
||||
# curl invocation as one chunk.
|
||||
flat = re.sub(r'\\\s*\n\s*', ' ', content)
|
||||
for m in PATTERN.finditer(flat):
|
||||
BAD_FILES.append((f, m.group(0)[:120]))
|
||||
|
||||
if not BAD_FILES:
|
||||
print("OK No curl-status-capture pollution patterns detected")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
|
||||
for f, snippet in BAD_FILES:
|
||||
print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
|
||||
print(f" matched: {snippet}...")
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(' set +e')
|
||||
print(' curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
|
||||
print(' set -e')
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
name: Lint forbidden tenant-env keys
|
||||
|
||||
# RFC#523 Layer 3 (task #146): scan workspace_secrets-writer Go code
|
||||
# under workspace-server/ for new code that hardcodes a forbidden
|
||||
# operator-scope env var NAME (GITEA_TOKEN, CP_ADMIN_API_TOKEN,
|
||||
# RAILWAY_TOKEN, INFISICAL_OPERATOR_TOKEN, MOLECULE_OPERATOR_*, …).
|
||||
#
|
||||
# Catches the class "a new writer accidentally widens the propagation
|
||||
# set" — e.g. a future env-mutator plugin that sets envVars["GITEA_TOKEN"]
|
||||
# directly. Today the L1 runtime guard would abort the provision, but
|
||||
# this lint surfaces the offending code at PR review time instead of
|
||||
# at first provision attempt.
|
||||
#
|
||||
# Companion layers:
|
||||
# - L1: workspace-server/internal/handlers/workspace_provision_forbidden_env.go
|
||||
# (fail-closed abort at provision time)
|
||||
# - L2: workspace/entrypoint.sh top-of-file env-grep + exit 1
|
||||
#
|
||||
# Open-source-template-friendly: the deny pattern is generic. A fork
|
||||
# can copy this workflow and replace OPERATOR_KEY_PATTERN with its
|
||||
# own operator-scope key names.
|
||||
#
|
||||
# Path-filter discipline:
|
||||
# This workflow runs on every PR (no paths: filter — see
|
||||
# feedback_path_filtered_workflow_cant_be_required). The scan itself
|
||||
# targets workspace_secrets-writer paths via grep -r; it's fast
|
||||
# (sub-second) so unconditional run is fine.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan workspace_secrets writers for forbidden env keys
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Scan for forbidden operator-scope env key NAMES in writer paths
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Forbidden EXACT-MATCH env var names. Kept in lockstep with
|
||||
# workspace-server/internal/handlers/workspace_provision_forbidden_env.go
|
||||
# forbiddenTenantEnvKeys. The Go-side test
|
||||
# TestIsForbiddenTenantEnvKey_ExactMatches is the source of
|
||||
# truth — if Go-side adds a key, also add it here (and
|
||||
# vice-versa). Drift between the two is the failure mode this
|
||||
# entire 3-layer guardrail is designed to catch.
|
||||
FORBIDDEN_KEYS=(
|
||||
"GITEA_TOKEN" "GITEA_PAT"
|
||||
"GITHUB_TOKEN" "GITHUB_PAT" "GH_TOKEN"
|
||||
"GITLAB_TOKEN" "GL_TOKEN"
|
||||
"BITBUCKET_TOKEN"
|
||||
"CP_ADMIN_API_TOKEN" "CP_ADMIN_TOKEN"
|
||||
"INFISICAL_OPERATOR_TOKEN" "INFISICAL_BOOTSTRAP_TOKEN"
|
||||
"RAILWAY_TOKEN" "RAILWAY_PERSONAL_API_TOKEN"
|
||||
"HETZNER_TOKEN" "HETZNER_API_TOKEN"
|
||||
)
|
||||
|
||||
# Forbidden PREFIX patterns — operator-scope families.
|
||||
FORBIDDEN_PREFIXES=(
|
||||
"MOLECULE_OPERATOR_"
|
||||
)
|
||||
|
||||
# Writer paths: Go source under workspace-server/ that
|
||||
# writes to the env-vars map or to workspace_secrets DB rows.
|
||||
# Tests, the forbidden-env source itself, and the silent-
|
||||
# strip denylist are exempt (they LIST the keys by design).
|
||||
SCAN_ROOT="workspace-server/internal"
|
||||
# Exempt paths fall in two classes:
|
||||
# 1. The deny-set definitions + the silent-strip denylist:
|
||||
# they LIST the forbidden names by design.
|
||||
# 2. Pre-RFC#523 persona-merge / config-read paths that
|
||||
# already handle these names correctly (the silent-
|
||||
# strip downstream + the new L1 fail-closed cover the
|
||||
# runtime risk; these reads are unchanged).
|
||||
# New code MUST NOT be added to this list without reviewer
|
||||
# signoff and a one-line justification in this diff.
|
||||
EXEMPT_PATHS=(
|
||||
# Class 1 — deny-set definitions
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env.go"
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env_test.go"
|
||||
"workspace-server/internal/provisioner/provisioner.go"
|
||||
"workspace-server/internal/provisioner/provisioner_test.go"
|
||||
# Class 2 — pre-existing persona-fallback / org-helper paths
|
||||
# that set the GITEA_TOKEN fallback lane (stripped downstream
|
||||
# by provisioner.buildContainerEnv per forensic #145). The
|
||||
# new L1 fail-closed runs BEFORE these writers, so any
|
||||
# operator-scope leak via global/workspace_secrets is
|
||||
# already caught. See applyAgentGitHTTPCreds doc-comment.
|
||||
"workspace-server/internal/handlers/agent_git_identity.go"
|
||||
"workspace-server/internal/handlers/org_helpers.go"
|
||||
"workspace-server/internal/handlers/org.go"
|
||||
# Class 2 — CP→platform admin auth (NOT a tenant env write;
|
||||
# this is the control-plane HTTP auth header source).
|
||||
"workspace-server/internal/provisioner/cp_provisioner.go"
|
||||
)
|
||||
|
||||
# Build a single grep -F pattern: every forbidden key wrapped
|
||||
# in quotes (Go string-literal form, which is how env-map
|
||||
# writes appear). e.g. envVars["GITEA_TOKEN"] = ... or
|
||||
# `"GITEA_TOKEN":` in a literal-map declaration.
|
||||
#
|
||||
# We deliberately match the quoted form so a comment that
|
||||
# happens to spell the name without quotes (e.g. "see
|
||||
# GITEA_TOKEN below") doesn't trip the lint.
|
||||
PATTERN=""
|
||||
for k in "${FORBIDDEN_KEYS[@]}"; do
|
||||
PATTERN="${PATTERN}\"${k}\"\n"
|
||||
done
|
||||
for p in "${FORBIDDEN_PREFIXES[@]}"; do
|
||||
# Prefix match needs a regex; switch to grep -E below for
|
||||
# this slice. Kept conceptually here so the deny set lives
|
||||
# in one place; scan is run twice (literal + prefix).
|
||||
true
|
||||
done
|
||||
|
||||
# Build exempt-paths grep filter — `grep -v -f` style.
|
||||
EXEMPT_FILTER=$(mktemp)
|
||||
trap 'rm -f "$EXEMPT_FILTER"' EXIT
|
||||
for p in "${EXEMPT_PATHS[@]}"; do
|
||||
echo "$p" >> "$EXEMPT_FILTER"
|
||||
done
|
||||
|
||||
# --- Exact-match scan ---
|
||||
HITS=""
|
||||
for k in "${FORBIDDEN_KEYS[@]}"; do
|
||||
# Only .go files; skip _test.go for the writer-path scan
|
||||
# since tests legitimately reference the names. The
|
||||
# writer-path lint targets PRODUCTION code only.
|
||||
found=$(grep -rn --include='*.go' --exclude='*_test.go' "\"${k}\"" "$SCAN_ROOT" 2>/dev/null \
|
||||
| grep -v -F -f "$EXEMPT_FILTER" || true)
|
||||
if [ -n "$found" ]; then
|
||||
HITS="${HITS}${found}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
# --- Prefix scan ---
|
||||
for prefix in "${FORBIDDEN_PREFIXES[@]}"; do
|
||||
found=$(grep -rnE --include='*.go' --exclude='*_test.go' "\"${prefix}[A-Z0-9_]+\"" "$SCAN_ROOT" 2>/dev/null \
|
||||
| grep -v -F -f "$EXEMPT_FILTER" || true)
|
||||
if [ -n "$found" ]; then
|
||||
HITS="${HITS}${found}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$HITS" ]; then
|
||||
echo "::error::RFC#523 Layer 3: forbidden operator-scope env var name(s) hardcoded in tenant-workspace writer paths:"
|
||||
printf "$HITS"
|
||||
echo ""
|
||||
echo "These env-var NAMES are on the operator-scope deny list (see"
|
||||
echo "workspace-server/internal/handlers/workspace_provision_forbidden_env.go)."
|
||||
echo "If your code legitimately needs to inject one of these for a"
|
||||
echo "non-tenant code path, add the file to EXEMPT_PATHS in this"
|
||||
echo "workflow with a one-line justification — reviewer signoff required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK No forbidden operator-scope env key names hardcoded in writer paths."
|
||||
@@ -1,134 +0,0 @@
|
||||
name: lint-mask-pr-atomicity
|
||||
|
||||
# Tier 2d hard-gate lint (per mc#774) — blocks PRs that touch
|
||||
# `.gitea/workflows/ci.yml` and modify ONLY ONE of {continue-on-error,
|
||||
# all-required.sentinel.needs} without a `Paired: #NNN` reference in
|
||||
# the PR body or in a commit message.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# PR#665 (interim `continue-on-error: true` on `platform-build`) and
|
||||
# PR#668 (sentinel-`needs` demotion of the same job) were designed as a
|
||||
# pair but merged solo — #665 landed at 04:47Z 2026-05-12, #668 was
|
||||
# still open at 05:07Z when the main-red watchdog (#674) fired. Result:
|
||||
# ~20 minutes of `main` red and a cascade of false-positives on
|
||||
# unrelated PRs. This lint structurally prevents that class.
|
||||
#
|
||||
# How the gate works
|
||||
# ------------------
|
||||
# 1. The workflow runs on every PR whose diff touches ci.yml (paths
|
||||
# filter). It is NOT a required check on `main` because the rule is
|
||||
# diff-based — running it on PRs that don't touch ci.yml would
|
||||
# produce a `pending` status forever (per
|
||||
# `feedback_path_filtered_workflow_cant_be_required`).
|
||||
# 2. The script reads `BASE_SHA:ci.yml` and `HEAD_SHA:ci.yml`, parses
|
||||
# both via PyYAML AST (per `feedback_behavior_based_ast_gates` — no
|
||||
# grep, no regex on the raw text — so a YAML-shape refactor still
|
||||
# detects).
|
||||
# 3. Walks `jobs.*.continue-on-error` on each side; flags any value
|
||||
# diff. Reads `jobs.all-required.needs` on each side; flags any
|
||||
# set diff (order-insensitive — `needs:` is engine-unordered).
|
||||
# 4. If both predicates fired → atomic, OK. If neither → no risk, OK.
|
||||
# If exactly one fired → require `Paired: #NNN` in PR body OR in
|
||||
# any commit message between base..head; else fail.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder)
|
||||
# -------------------------------------------
|
||||
# This workflow lands at `continue-on-error: true` (Phase 3 — surface
|
||||
# regressions without blocking PRs while the rule beds in).
|
||||
# Follow-up PR flips to `false` once we have ≥3 days of clean runs on
|
||||
# `main` and no false-positives. Tracking issue: mc#774.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#774 (the RFC that specs this lint)
|
||||
# - PR#665 / PR#668 (the empirical split-pair)
|
||||
# - mc#774 (the main-red incident the split caused)
|
||||
# - feedback_strict_root_only_after_class_a
|
||||
# - feedback_behavior_based_ast_gates
|
||||
#
|
||||
# Auth: only needs the auto-injected GITHUB_TOKEN (read-only, repo
|
||||
# scope). No DRIFT_BOT_TOKEN needed — Tier 2d does NOT call
|
||||
# branch_protections (Tier 2g/f do).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, edited]
|
||||
# `edited` is included because the rule depends on PR_BODY: a user
|
||||
# may add `Paired: #NNN` after first push to satisfy the lint. The
|
||||
# rerun on `edited` lets the PR turn green without an empty
|
||||
# commit. Gitea 1.22.6 fires `edited` on body changes — verified
|
||||
# via gitea-source/models/issues/pull_list.go::triggerNewPRWebhook.
|
||||
paths:
|
||||
- '.gitea/workflows/ci.yml'
|
||||
- '.gitea/scripts/lint_mask_pr_atomicity.py'
|
||||
- '.gitea/workflows/lint-mask-pr-atomicity.yml'
|
||||
- 'tests/test_lint_mask_pr_atomicity.py'
|
||||
|
||||
env:
|
||||
# Belt-and-suspenders against the runner-default trap
|
||||
# (feedback_act_runner_github_server_url). Runners are configured
|
||||
# with this env via /opt/molecule/runners/config.yaml, but pinning
|
||||
# at the workflow level protects against a runner regenerated
|
||||
# without the config file.
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
# Per-PR concurrency — re-pushes cancel previous runs to keep the
|
||||
# queue short. The lint is cheap (one git show + log + a YAML parse).
|
||||
concurrency:
|
||||
group: lint-mask-pr-atomicity-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: meta-lint advisory during mask burn-down; CI / all-required gates merges.
|
||||
scan:
|
||||
name: lint-mask-pr-atomicity
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# Phase 3 (RFC #219 §1): surface broken shapes without blocking
|
||||
# PRs. Follow-up PR flips this to `false` once recent runs on main
|
||||
# are confirmed clean (eat-our-own-dogfood discipline mirrors
|
||||
# PR#673's same-shape comment). Tracking: mc#774.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Check out PR head with full history (need base SHA blobs)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# `git show <base-sha>:<path>` needs the base SHA's blobs.
|
||||
# Shallow=1 would miss it. Same rationale as PR#673 and
|
||||
# check-migration-collisions.yml.
|
||||
fetch-depth: 0
|
||||
- name: Set up Python (PyYAML for AST parsing)
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
# Same pin as ci-required-drift.yml + the rest of the Tier 2
|
||||
# lint family — keep runner-cache hits uniform.
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Ensure base ref is reachable locally
|
||||
# fetch-depth=0 usually pulls the base too, but explicit-fetch
|
||||
# is cheap insurance against runner-version drift (matches the
|
||||
# comment in check-migration-collisions.yml and PR#673).
|
||||
run: |
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
- name: Run lint-mask-pr-atomicity
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
# PR body — the script greps for `Paired: #NNN`.
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
CI_WORKFLOW_PATH: .gitea/workflows/ci.yml
|
||||
SENTINEL_JOB_KEY: all-required
|
||||
run: python3 .gitea/scripts/lint_mask_pr_atomicity.py
|
||||
- name: Run lint-mask-pr-atomicity unit tests
|
||||
# Run the test suite in-CI so the lint's own behaviour is
|
||||
# verified on every change. Matches lint-workflow-yaml.yml.
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_mask_pr_atomicity.py -v
|
||||
@@ -1,182 +0,0 @@
|
||||
name: Lint no tenant GITEA or GITHUB token write
|
||||
|
||||
# Task #146 — CI guardrail companion to RFC#523's `lint-forbidden-env-keys.yml`.
|
||||
#
|
||||
# `lint-forbidden-env-keys.yml` (Layer 3) catches code that hardcodes a
|
||||
# forbidden env-var key NAME as a quoted literal in workspace_secrets
|
||||
# writer paths under workspace-server/internal/.
|
||||
#
|
||||
# This workflow catches a BROADER class: any code path that reads a
|
||||
# repo-host token (GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN) and then writes
|
||||
# it into a TENANT WORKSPACE's env, secret store, user-data, or
|
||||
# provision payload. This is the actual RFC#523 threat-model statement —
|
||||
# the goal is "no tenant workspace ever receives an operator-scope repo
|
||||
# token," not just "no _quoted_ literal `GITEA_TOKEN`." A future writer
|
||||
# could route the value via a variable, a struct field, or a config key
|
||||
# and slip past the existing literal scan; this lint catches those
|
||||
# routing patterns at PR review time.
|
||||
#
|
||||
# Scope
|
||||
# Scans the WHOLE repo's Go sources (not just workspace-server/) for
|
||||
# co-occurrences of:
|
||||
# - a repo-host token NAME (GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN /
|
||||
# GITEA_PAT / GITHUB_PAT) used as os.Getenv argument or string
|
||||
# literal
|
||||
# - within a file that ALSO references a tenant-writer surface
|
||||
# (`tenant`, `workspace_secrets`, `global_secrets`, `seedAllowList`,
|
||||
# `/settings/secrets`, `userData`, `provisionPayload`,
|
||||
# `envVars[`, `containerEnv`).
|
||||
#
|
||||
# Co-occurrence (not single-line) is the false-positive control: a
|
||||
# file that just LOGS the variable name (e.g. "missing GITEA_TOKEN")
|
||||
# without touching any tenant surface won't fire.
|
||||
#
|
||||
# Drift contract with lint-forbidden-env-keys.yml
|
||||
# Both lints share the same FORBIDDEN_KEYS list (a subset — only the
|
||||
# repo-host tokens, since this lint's threat model is "tenant gets
|
||||
# write access to operator's git host"). If RFC#523's deny set grows,
|
||||
# update BOTH this file AND lint-forbidden-env-keys.yml AND the Go
|
||||
# source-of-truth in
|
||||
# workspace-server/internal/handlers/workspace_provision_forbidden_env.go.
|
||||
#
|
||||
# Open-source-template-friendly
|
||||
# The patterns scanned are generic (no MOLECULE_-prefix literals).
|
||||
# A fork can copy this workflow as-is and adjust FORBIDDEN_KEYS.
|
||||
#
|
||||
# Path-filter discipline
|
||||
# No `paths:` filter — required-status workflows must run on every PR
|
||||
# per `feedback_path_filtered_workflow_cant_be_required`. Scan is
|
||||
# sub-second.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: advisory RFC#523 lint; PR review gate is review-driven, not BP-driven.
|
||||
# (Carried with the workflow-name rename in PR mc#1593 so the renamed
|
||||
# context emission satisfies lint_required_context_exists_in_bp Tier 2g.)
|
||||
scan:
|
||||
name: Scan for repo-host token write into tenant workspace surface
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Find Go files referencing a tenant-writer surface AND a repo-host token
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Repo-host token NAMES — the threat-model subset. Operator-fleet
|
||||
# tokens (CP_ADMIN_API_TOKEN, RAILWAY_TOKEN, INFISICAL_*) are
|
||||
# caught by lint-forbidden-env-keys.yml's broader deny set; this
|
||||
# lint focuses on the git-host class so a single co-occurrence
|
||||
# match has a low false-positive rate.
|
||||
FORBIDDEN_KEYS=(
|
||||
"GITEA_TOKEN"
|
||||
"GITEA_PAT"
|
||||
"GITHUB_TOKEN"
|
||||
"GITHUB_PAT"
|
||||
"GH_TOKEN"
|
||||
)
|
||||
|
||||
# Tenant-writer surface markers. A file matches the surface set
|
||||
# if it references ANY of these strings. This is the "is this
|
||||
# code path writing into a tenant workspace?" heuristic.
|
||||
# Curated to catch the actual code shapes used in this repo
|
||||
# (verified by grep against current main 2026-05-19):
|
||||
# - "workspace_secrets" / "global_secrets" → DB table writes
|
||||
# - "seedAllowList" → CP-side seed table
|
||||
# - "/settings/secrets" → tenant HTTP API write
|
||||
# - "envVars[" → in-memory env map write
|
||||
# - "containerEnv" → docker-run env-set
|
||||
# - "userData" → EC2 user-data script
|
||||
# - "provisionPayload" / "provisionContext" → provision-request shape
|
||||
SURFACE_PATTERN='workspace_secrets|global_secrets|seedAllowList|/settings/secrets|envVars\[|containerEnv|userData|provisionPayload|provisionContext'
|
||||
|
||||
# Files that legitimately reference these names AND a surface
|
||||
# marker, but do so for guard / strip / test / doc-comment
|
||||
# reasons. New entries require reviewer signoff and a one-line
|
||||
# justification in the diff.
|
||||
EXEMPT_FILES=(
|
||||
# RFC#523 L1 deny-set source-of-truth + tests
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env.go"
|
||||
"workspace-server/internal/handlers/workspace_provision_forbidden_env_test.go"
|
||||
# Forensic-#145 silent-strip denylist (defense-in-depth, by design lists the names)
|
||||
"workspace-server/internal/provisioner/provisioner.go"
|
||||
"workspace-server/internal/provisioner/provisioner_test.go"
|
||||
# Pre-RFC#523 persona-fallback / org-helper paths. The L1
|
||||
# fail-closed runs BEFORE these writers; downstream silent-strip
|
||||
# also covers them. See applyAgentGitHTTPCreds doc-comment.
|
||||
"workspace-server/internal/handlers/agent_git_identity.go"
|
||||
"workspace-server/internal/handlers/org_helpers.go"
|
||||
"workspace-server/internal/handlers/org.go"
|
||||
# CP→platform admin auth (NOT a tenant env write).
|
||||
"workspace-server/internal/provisioner/cp_provisioner.go"
|
||||
)
|
||||
|
||||
# Build an extended-regex alternation of forbidden keys.
|
||||
KEY_ALT="$(IFS='|'; echo "${FORBIDDEN_KEYS[*]}")"
|
||||
|
||||
# Find candidate files: Go non-test sources that contain a
|
||||
# tenant-writer surface marker.
|
||||
mapfile -t CANDIDATES < <(
|
||||
grep -rlE --include='*.go' --exclude='*_test.go' \
|
||||
"${SURFACE_PATTERN}" . 2>/dev/null \
|
||||
| sed 's|^\./||' \
|
||||
| sort -u
|
||||
)
|
||||
|
||||
if [ "${#CANDIDATES[@]}" -eq 0 ]; then
|
||||
echo "OK No tenant-writer-surface files found in tree (unexpected, but not a lint failure)."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
HITS=""
|
||||
for f in "${CANDIDATES[@]}"; do
|
||||
# Skip exempt files.
|
||||
skip=0
|
||||
for ex in "${EXEMPT_FILES[@]}"; do
|
||||
if [ "$f" = "$ex" ]; then skip=1; break; fi
|
||||
done
|
||||
[ "$skip" = "1" ] && continue
|
||||
|
||||
# File contains a surface marker; now grep for a forbidden
|
||||
# key NAME. We require a QUOTED-literal match to avoid
|
||||
# firing on a comment like "// also handle GITEA_TOKEN".
|
||||
#
|
||||
# The literal form catches:
|
||||
# - os.Getenv("GITEA_TOKEN")
|
||||
# - envVars["GITEA_TOKEN"] = ...
|
||||
# - {envKey: "GITEA_TOKEN", tenantKey: "GITEA_TOKEN"}
|
||||
# but not:
|
||||
# - // see GITEA_TOKEN below (no quotes)
|
||||
found=$(grep -nE "\"(${KEY_ALT})\"" "$f" 2>/dev/null || true)
|
||||
if [ -n "$found" ]; then
|
||||
HITS="${HITS}--- ${f} ---\n${found}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$HITS" ]; then
|
||||
echo "::error::Task #146 lint: repo-host token name(s) quoted in a tenant-writer-surface file:"
|
||||
printf "$HITS"
|
||||
echo ""
|
||||
echo "These files reference a tenant-writer surface (workspace_secrets,"
|
||||
echo "seedAllowList, /settings/secrets, containerEnv, userData, etc.)"
|
||||
echo "AND quote a repo-host token name (GITEA_TOKEN/GITHUB_TOKEN/…)."
|
||||
echo "Per RFC#523 threat model, tenant workspaces MUST NOT receive"
|
||||
echo "operator-scope repo-host tokens. If your code legitimately needs"
|
||||
echo "to reference one of these names in a tenant-writer file (e.g."
|
||||
echo "a deny-set definition or silent-strip list), add the file to"
|
||||
echo "EXEMPT_FILES with a one-line justification — reviewer signoff"
|
||||
echo "required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK No tenant-writer-surface file co-mentions a repo-host token literal."
|
||||
@@ -1,141 +0,0 @@
|
||||
name: Lint pre-flip continue-on-error
|
||||
|
||||
# Pre-merge gate: blocks PRs that flip `continue-on-error: true → false`
|
||||
# on any job in `.gitea/workflows/*.yml` WITHOUT proof that the affected
|
||||
# job's recent runs on the target branch (PR base) are actually green.
|
||||
#
|
||||
# Empirical class: PR #656 / mc#774. PR #656 (RFC internal#219 Phase 4)
|
||||
# flipped 5 platform-build-class jobs `continue-on-error: true → false`
|
||||
# on the basis of a "verified green on main via combined-status check".
|
||||
# But that "green" was the LIE the prior `continue-on-error: true`
|
||||
# produced: Gitea Quirk #10 (internal#342 + dup #287) — a failed step
|
||||
# inside a `continue-on-error: true` job rolls up to a `success`
|
||||
# job-level status. The precondition the PR claimed to verify was
|
||||
# structurally fooled by the bug being flipped.
|
||||
#
|
||||
# mc#774 captured the surfaced defects (2 mutually-masked regressions):
|
||||
# - Class 1: sqlmock helper drift since 2f36bb9a (24 days old)
|
||||
# - Class 2: OFFSEC-001 contract collision since 7d1a189f (1 day old)
|
||||
#
|
||||
# Codified 04:35Z as hongming-pc2 charter §SOP-N rule (e)
|
||||
# "run-log-grep-before-flip" — now structurally enforced here at PR
|
||||
# time, ahead of merge.
|
||||
#
|
||||
# How the gate works:
|
||||
# 1. Read every `.gitea/workflows/*.yml` at the PR base SHA AND at
|
||||
# the PR head SHA via `git show <sha>:<path>` (no checkout
|
||||
# needed).
|
||||
# 2. Parse both sides via PyYAML AST (NOT grep — per
|
||||
# `feedback_behavior_based_ast_gates`). Walk `jobs.<key>.
|
||||
# continue-on-error` on each side. A flip is base=true,
|
||||
# head=false.
|
||||
# 3. For each flipped job, render the commit-status context as
|
||||
# `"{workflow.name} / {job.name or job.key} (push)"` — that's
|
||||
# how Gitea Actions emits the per-context status on `main`/
|
||||
# `staging` runs.
|
||||
# 4. Pull last 5 commits on the PR base branch, fetch combined
|
||||
# commit-status per commit, scan for the target context. For
|
||||
# each match, fetch the run log via the web-UI route
|
||||
# `{server_url}/{repo}/actions/runs/{run_id}/jobs/{job_idx}/logs`
|
||||
# (per `reference_gitea_actions_log_fetch` —
|
||||
# Gitea 1.22.6 lacks REST `/actions/runs/*`; web-UI is the
|
||||
# only working path, see also
|
||||
# `reference_gitea_1_22_6_lacks_rest_rerun_endpoints`).
|
||||
# 5. Grep each log for `--- FAIL`, `FAIL\s`, `::error::`. If
|
||||
# the status is `success` but the log shows any of these,
|
||||
# the job was masked. Block the PR with `::error::`.
|
||||
#
|
||||
# Graceful-degrade contract (per task halt-conditions):
|
||||
# - Log fetch 404 (act_runner pruned the log, transient outage):
|
||||
# emit `::warning::` "log unavailable" — does NOT block.
|
||||
# - Zero recent runs of the flipped job's context on the base
|
||||
# branch (newly added workflow): emit `::warning::` "no run
|
||||
# history to verify" — allow the flip. Chicken-and-egg
|
||||
# exemption.
|
||||
# - YAML parse error in one of the workflow files: warn-only,
|
||||
# don't block — the YAML lint workflows catch this separately.
|
||||
#
|
||||
# Cross-links: PR#656, mc#774, PR#665 (interim re-mask),
|
||||
# Quirk #10 (internal#342 + dup #287), hongming-pc2 charter
|
||||
# §SOP-N rule (e), feedback_strict_root_only_after_class_a,
|
||||
# feedback_no_shared_persona_token_use.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder):
|
||||
# - This workflow lands at `continue-on-error: true` (Phase 3 —
|
||||
# surface defects without blocking). Follow-up PR flips it to
|
||||
# `false` ONLY after this workflow's own recent runs on `main`
|
||||
# are confirmed clean — exactly the discipline the workflow
|
||||
# itself enforces. Eat your own dogfood.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint_pre_flip_continue_on_error.py'
|
||||
- '.gitea/workflows/lint-pre-flip-continue-on-error.yml'
|
||||
|
||||
env:
|
||||
# Per `feedback_act_runner_github_server_url` — without this,
|
||||
# actions/checkout and friends default to github.com → break.
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# Need read on the API to pull combined commit-status + commit list
|
||||
# for the base branch. The job-log fetch uses the same token via
|
||||
# the web-UI route (Gitea 1.22.6 accepts `Authorization: token ...`
|
||||
# there).
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: lint-pre-flip-coe-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Verify continue-on-error flips have run-log proof
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
# Phase 3 (RFC internal#219 §1): surface broken flips without blocking
|
||||
# the PR yet. Follow-up flips this to `false` once the workflow itself
|
||||
# has clean recent runs on main. mc#774 interim — remove when CoE→false.
|
||||
continue-on-error: true # mc#774
|
||||
steps:
|
||||
- name: Check out PR head (full history for base-SHA access)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# `git show <base-sha>:<path>` needs the base SHA's blobs.
|
||||
# Shallow=1 would miss it. Same rationale as
|
||||
# check-migration-collisions.yml.
|
||||
fetch-depth: 0
|
||||
- name: Set up Python (PyYAML for AST parsing)
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
# Same pin as ci-required-drift.yml — keep dependencies
|
||||
# uniform so a Gitea runner cache hits across both jobs.
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Ensure base ref is reachable locally
|
||||
# `actions/checkout@v6 fetch-depth=0` usually pulls the base
|
||||
# too, but explicit-fetch is cheap insurance against the
|
||||
# form-of-ref differences across Gitea runner versions
|
||||
# (mirrors the comment in check-migration-collisions.yml).
|
||||
run: |
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
- name: Run lint
|
||||
env:
|
||||
# Auto-injected by Gitea Actions; sufficient scope for
|
||||
# combined-status + commit-list + log fetch via web-UI
|
||||
# route. NO repo-admin needed (unlike the
|
||||
# branch_protections endpoint).
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
# Last 5 commits on the base branch is the spec default.
|
||||
RECENT_COMMITS_N: '5'
|
||||
run: python3 .gitea/scripts/lint_pre_flip_continue_on_error.py
|
||||
@@ -1,118 +0,0 @@
|
||||
name: lint-required-context-exists-in-bp
|
||||
|
||||
# Tier 2g hard-gate lint (per mc#774) — diff-based PR-time
|
||||
# check. When a PR adds a NEW commit-status emission (workflow YAML
|
||||
# `name:` + job `name:`-or-key + on:-event), the workflow file must
|
||||
# carry one of three directives adjacent to the new job:
|
||||
#
|
||||
# - `# bp-required: yes` — and BP must list the context
|
||||
# - `# bp-required: pending #NNN` — acknowledged asymmetry + tracker
|
||||
# - `# bp-exempt: <reason>` — informational job, not a gate
|
||||
#
|
||||
# Default (no directive on a new emitter) = FAIL.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# PR#656 added `CI / all-required (pull_request)` as a sentinel
|
||||
# context that workflows emit, but BP did NOT list it. When
|
||||
# platform-build failed, all-required failed, but BP let the PR
|
||||
# merge anyway → cascade to mc#774. With this lint, PR#656 would
|
||||
# have been blocked until either the BP PATCH ran alongside OR
|
||||
# the author added a `bp-required: pending` directive.
|
||||
#
|
||||
# Tier 2g vs Tier 2f
|
||||
# ------------------
|
||||
# Tier 2g runs at PR-time (diff-based) and BLOCKS the merge.
|
||||
# Tier 2f runs daily (scheduled) and FILES a drift issue. They
|
||||
# share the workflow-context enumeration helpers
|
||||
# (`_event_map`, `workflow_contexts`, `_job_display`) but the
|
||||
# semantics are intentionally distinct so they're separate scripts.
|
||||
# Co-design is documented in mc#774.
|
||||
#
|
||||
# Directive comment lives in the workflow file (NOT PR body)
|
||||
# ----------------------------------------------------------
|
||||
# A PR-body claim of "BP exempt" evaporates on merge — the
|
||||
# asymmetry returns to undetected state and Tier 2f's daily
|
||||
# scheduled audit can't see it. The directive must live with the
|
||||
# emitter so both PR-time (Tier 2g) and post-merge (Tier 2f)
|
||||
# readers consume the same source.
|
||||
#
|
||||
# Phase contract (RFC internal#219 §1 ladder)
|
||||
# -------------------------------------------
|
||||
# Lands at `continue-on-error: true` (Phase 3 — surface the
|
||||
# pattern without blocking PRs while the directive convention
|
||||
# beds in). After 7 days of clean runs on `main` with no false
|
||||
# positives, follow-up flips to `false`. Tracking: mc#774.
|
||||
#
|
||||
# Cross-links
|
||||
# -----------
|
||||
# - mc#774 (the RFC that specs this lint)
|
||||
# - PR#656 (the empirical case)
|
||||
# - mc#774 (the surfaced cascade)
|
||||
# - feedback_phantom_required_check_after_gitea_migration (Tier 2f cousin)
|
||||
# - feedback_behavior_based_ast_gates
|
||||
#
|
||||
# Auth: DRIFT_BOT_TOKEN (repo-admin for branch_protections read).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint_required_context_exists_in_bp.py'
|
||||
- '.gitea/workflows/lint-required-context-exists-in-bp.yml'
|
||||
- 'tests/test_lint_required_context_exists_in_bp.py'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lint-required-context-exists-in-bp-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: this lint is a PR-time advisory and is not intended to
|
||||
# be a required gate on main. The directive eat-our-own-dogfood
|
||||
# confirms the convention works on the lint that defines it.
|
||||
lint:
|
||||
name: lint-required-context-exists-in-bp
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
# Phase 3 (RFC #219 §1): surface the pattern without blocking PRs
|
||||
# while the directive convention beds in. Follow-up flip to false
|
||||
# after 7 clean days on main. mc#774.
|
||||
continue-on-error: true # mc#774 Phase 3 — flip to false after 7 clean main runs
|
||||
steps:
|
||||
- name: Check out PR head with full history (need base SHA blobs)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# `git show <base-sha>:<path>` needs the base SHA's blobs.
|
||||
# Same rationale as PR#673 and check-migration-collisions.yml.
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Ensure base ref is reachable locally
|
||||
# Cheap insurance against runner-version drift.
|
||||
run: |
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
- name: Run lint-required-context-exists-in-bp
|
||||
env:
|
||||
# DRIFT_BOT_TOKEN — repo-admin (needed for branch_protections).
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: main
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
run: python3 .gitea/scripts/lint_required_context_exists_in_bp.py
|
||||
- name: Run lint-required-context-exists-in-bp unit tests
|
||||
run: |
|
||||
python -m pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_required_context_exists_in_bp.py -v
|
||||
@@ -1,97 +0,0 @@
|
||||
# lint-required-no-paths — structural enforcement of
|
||||
# `feedback_path_filtered_workflow_cant_be_required`.
|
||||
#
|
||||
# Fails the PR if ANY workflow whose status-check context appears in
|
||||
# `branch_protections/main.status_check_contexts` carries a
|
||||
# `paths:` or `paths-ignore:` filter in its `on:` block.
|
||||
#
|
||||
# Why this exists:
|
||||
# A required-check workflow with a paths filter silently degrades the
|
||||
# merge gate. If a PR's diff doesn't touch the filter, the workflow
|
||||
# never fires; Gitea (1.22.6) reports the required context as
|
||||
# `pending` (NOT `skipped == success`), so the PR cannot merge. For a
|
||||
# docs-only PR against `paths: ['**.go']`, the PR is wedged forever.
|
||||
#
|
||||
# Previously prevented only by reviewer vigilance + the saved memory
|
||||
# `feedback_path_filtered_workflow_cant_be_required`. This workflow
|
||||
# makes it a hard CI gate.
|
||||
#
|
||||
# Forward-compat scope:
|
||||
# Today (2026-05-11) molecule-core/main protects 3 contexts:
|
||||
# - "Secret scan / Scan diff for credential-shaped strings (pull_request)"
|
||||
# - "sop-tier-check / tier-check (pull_request)"
|
||||
# - "CI / all-required (pull_request)"
|
||||
# Per RFC#324 Step 2 the required-list expands to ~5 contexts
|
||||
# (qa-review, security-review added). Each new required context's
|
||||
# workflow must remain unconditional. This lint pins that contract.
|
||||
#
|
||||
# Meta-required-check:
|
||||
# This workflow ITSELF deliberately has NO `paths:` filter on its `on:`
|
||||
# block — otherwise a paths-non-matching PR could bypass the check.
|
||||
# Self-evident from this file: only `pull_request` types + no paths.
|
||||
#
|
||||
# Auth:
|
||||
# `GET /repos/.../branch_protections/{branch}` requires repo-admin
|
||||
# role in Gitea 1.22.6. The workflow-default `GITHUB_TOKEN` is
|
||||
# non-admin (read-only), so we re-use `DRIFT_BOT_TOKEN` (same persona
|
||||
# that powers `ci-required-drift.yml` — verified working there).
|
||||
# If `DRIFT_BOT_TOKEN` becomes unavailable, the script exits 0 with a
|
||||
# loud `::error::` rather than red-X every PR — token-scope issues
|
||||
# should be fixed at the token, not surfaced as a gate failure on
|
||||
# every unrelated PR.
|
||||
#
|
||||
# Behavior-based gate per `feedback_behavior_based_ast_gates`:
|
||||
# YAML AST walk (PyYAML), NOT grep. Workflow renames, formatting
|
||||
# changes (block-scalar vs flow-style), or moving `paths:` between
|
||||
# `pull_request:` and `pull_request_target:` all still detect.
|
||||
#
|
||||
# IMPORTANT — Gitea 1.22.6 parser quirk per
|
||||
# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
|
||||
# `inputs:` block to `workflow_dispatch:` — Gitea 1.22.6 rejects the
|
||||
# entire workflow as "unknown on type" and it registers for ZERO events.
|
||||
|
||||
name: lint-required-no-paths
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
workflow_dispatch:
|
||||
|
||||
# Read protection + read local YAML. No writes.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Only one in-flight run per PR — re-pushes cancel the previous run to
|
||||
# keep the queue short. Required-list reads are cheap (one GET); the
|
||||
# cancellation is just hygiene.
|
||||
concurrency:
|
||||
group: lint-required-no-paths-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: meta-lint advisory; CI / all-required is the required aggregate.
|
||||
lint:
|
||||
name: lint-required-no-paths
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out repo (we read the workflow YAML files locally)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Set up Python (PyYAML for AST parsing)
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run lint-required-no-paths
|
||||
env:
|
||||
# DRIFT_BOT_TOKEN is owned by mc-drift-bot, a least-privilege
|
||||
# Gitea persona with repo-admin role for branch_protections
|
||||
# read. Same secret used by ci-required-drift.yml — see that
|
||||
# workflow's header for provisioning trail (internal#329).
|
||||
GITEA_TOKEN: ${{ secrets.DRIFT_BOT_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: main
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
run: python3 .gitea/scripts/lint-required-no-paths.py
|
||||
@@ -1,164 +0,0 @@
|
||||
name: lint-required-workflows-docker-host-pinned
|
||||
|
||||
# Fail-closed lint that catches workflows touching docker.sock without
|
||||
# pinning `runs-on:` to a Linux-only label.
|
||||
#
|
||||
# Class defect (internal#512 + mc#1529 + today's oc#81/82/83 + autogen#8):
|
||||
# the `ubuntu-latest` label is advertised by BOTH the Linux operator-host
|
||||
# runners (molecule-runner-*) AND the Windows act_runner v1.0.3 on
|
||||
# hongming-pc-runner-*. Job placement is non-deterministic. When a docker-
|
||||
# bound job lands on a Windows runner, `docker run`/`docker login`/
|
||||
# `docker compose` fail with platform-specific errors ("protocol not
|
||||
# available", "cannot exec", etc.) — placement-dependent, not transient.
|
||||
#
|
||||
# This lint enforces the convention: any workflow whose YAML body
|
||||
# contains a docker exec (`docker run|build|buildx|compose|pull|push|
|
||||
# exec|tag|login|cp|inspect|ps` OR `docker/build-push-action|docker/
|
||||
# login-action|docker/setup-buildx`) MUST pin every job's `runs-on:` to
|
||||
# one of:
|
||||
# - docker-host (general docker.sock work — molecule-runner-*)
|
||||
# - publish (image build/push — molecule-runner-publish-*)
|
||||
#
|
||||
# Comments and heredoc/markdown bodies that merely MENTION docker are
|
||||
# excluded by the detection rule (see scan.py below).
|
||||
#
|
||||
# Per `feedback_never_skip_ci`: this is fail-closed (exit 1 on miss).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
lint-docker-host-pin:
|
||||
name: Lint docker-host pin on docker-touching workflows
|
||||
runs-on: docker-host
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Scan workflows for docker-bound jobs missing docker-host/publish pin
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Docker-step detection: real exec, not just word-mention in comments.
|
||||
# We strip comment-only lines, then look for the docker subcommand
|
||||
# tokens at word-boundary, OR uses: docker/* actions.
|
||||
DOCKER_EXEC = re.compile(
|
||||
r'(?<!\w)docker\s+(run|build|buildx|compose|pull|push|exec|tag|login|cp|inspect|ps)\b'
|
||||
)
|
||||
DOCKER_ACTION = re.compile(
|
||||
r'uses:\s*docker/(build-push-action|login-action|setup-buildx-action|setup-qemu-action)'
|
||||
)
|
||||
# Detect a job header line like ` myjob:` (2-space indent) AND its runs-on.
|
||||
JOB_HEADER = re.compile(r'^( {2})([a-zA-Z0-9_-]+):\s*$')
|
||||
RUNS_ON = re.compile(r'^( {4})runs-on:\s*(.+?)\s*$')
|
||||
|
||||
ALLOWED_LABELS = {'docker-host', 'publish'}
|
||||
|
||||
fails = []
|
||||
warnings = []
|
||||
|
||||
# Gitea is SSOT for molecule-core CI per task #347 / memory
|
||||
# reference_molecule_core_actions_gitea_only. The legacy
|
||||
# .github/workflows/ tree was deleted in SSOT-Instance-4 (#331).
|
||||
roots = []
|
||||
for root in ('.gitea/workflows',):
|
||||
if os.path.isdir(root):
|
||||
roots.append(root)
|
||||
|
||||
for root in roots:
|
||||
for fn in sorted(os.listdir(root)):
|
||||
if not (fn.endswith('.yml') or fn.endswith('.yaml')):
|
||||
continue
|
||||
path = os.path.join(root, fn)
|
||||
with open(path) as f:
|
||||
raw_lines = f.readlines()
|
||||
|
||||
# Parse job headers + their runs-on. Simple line scan; relies on
|
||||
# 2-space job indent + 4-space runs-on indent under `jobs:`.
|
||||
jobs = []
|
||||
current = None
|
||||
in_jobs = False
|
||||
for i, line in enumerate(raw_lines, 1):
|
||||
if re.match(r'^jobs:\s*$', line):
|
||||
in_jobs = True
|
||||
continue
|
||||
if not in_jobs:
|
||||
continue
|
||||
mh = JOB_HEADER.match(line)
|
||||
if mh:
|
||||
if current:
|
||||
current['end'] = i - 1
|
||||
jobs.append(current)
|
||||
current = {'name': mh.group(2), 'line': i, 'end': len(raw_lines), 'runs_on': None}
|
||||
continue
|
||||
mr = RUNS_ON.match(line)
|
||||
if mr and current and current['runs_on'] is None:
|
||||
current['runs_on'] = mr.group(2).strip()
|
||||
if current:
|
||||
jobs.append(current)
|
||||
|
||||
for j in jobs:
|
||||
# Strip pure-comment lines for docker-exec detection so
|
||||
# documentation comments don't trigger the lint. Scan the
|
||||
# current job body only: a workflow may contain one
|
||||
# docker-bound job and several harmless metadata jobs.
|
||||
job_lines = raw_lines[j['line'] - 1:j['end']]
|
||||
scan_text = ''.join(
|
||||
l for l in job_lines
|
||||
if not re.match(r'^\s*#', l)
|
||||
)
|
||||
has_docker = bool(DOCKER_EXEC.search(scan_text)) or bool(DOCKER_ACTION.search(scan_text))
|
||||
if not has_docker:
|
||||
continue
|
||||
ro = j['runs_on']
|
||||
if ro is None:
|
||||
# Reusable workflow caller (`uses:` instead of `runs-on:`) —
|
||||
# skip; rule enforced in the called workflow.
|
||||
continue
|
||||
# Strip surrounding [ ] and quotes.
|
||||
ro_norm = ro.strip('[]').strip().strip('"\'')
|
||||
# Multi-label "[a, b]" — split.
|
||||
labels = [t.strip().strip('"\'') for t in ro_norm.split(',') if t.strip()]
|
||||
if any(lbl in ALLOWED_LABELS for lbl in labels):
|
||||
continue
|
||||
# Allow caller-supplied label expressions; spell the
|
||||
# marker indirectly so Gitea's expression parser does
|
||||
# not try to parse this Python heredoc.
|
||||
expression_marker = '$' + '{{'
|
||||
if any(expression_marker in lbl for lbl in labels):
|
||||
continue
|
||||
fails.append(
|
||||
f"{path}:{j['line']}: job `{j['name']}` uses docker but runs-on={ro!r} "
|
||||
f"(must be one of {sorted(ALLOWED_LABELS)})"
|
||||
)
|
||||
|
||||
if fails:
|
||||
print("FAIL: docker-bound jobs missing docker-host/publish pin:")
|
||||
for f in fails:
|
||||
print(f" - {f}")
|
||||
print()
|
||||
print("Why this rule exists (internal#512 + mc#1529):")
|
||||
print(" Bare `ubuntu-latest` is advertised by BOTH Linux operator-host")
|
||||
print(" runners AND Windows hongming-pc-runner-* (act_runner v1.0.3).")
|
||||
print(" Docker-bound jobs that land on Windows fail non-deterministically.")
|
||||
print(" Pin to `docker-host` (general) or `publish` (image build/push).")
|
||||
sys.exit(1)
|
||||
|
||||
print("OK: all docker-bound jobs are pinned to docker-host or publish.")
|
||||
PY
|
||||
@@ -1,88 +0,0 @@
|
||||
name: Lint shellcheck (arm64 pilot)
|
||||
|
||||
# Mac-CI dual-track pilot (#233). ADDITIVE / NOT REQUIRED.
|
||||
#
|
||||
# Validates the arm64 self-hosted lane (no docker.sock, no privileged
|
||||
# ops) before any required gate moves onto it. Until a Mac arm64 runner
|
||||
# is registered with the `arm64` label, this workflow sits PENDING —
|
||||
# that is FINE: `arm64` is NOT in branch_protections required contexts.
|
||||
#
|
||||
# Pairs with internal#543 (RFC: Mac arm64 multi-arch runner-base).
|
||||
# No paths: filter on purpose (feedback_path_filtered_workflow_cant_be_required).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- staging
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
shellcheck-arm64:
|
||||
name: shellcheck-arm64 (pilot)
|
||||
runs-on: [self-hosted, arm64]
|
||||
# NOT a required check; safe to sit pending until Mac runner is up.
|
||||
# If the Mac runner has trouble pulling actions/checkout we fall
|
||||
# back to a plain git clone (see step 'fallback clone').
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
steps:
|
||||
- name: Identify runner
|
||||
run: |
|
||||
set -eu
|
||||
echo "arch=$(uname -m)"
|
||||
echo "kernel=$(uname -sr)"
|
||||
echo "shell=$BASH_VERSION"
|
||||
# Sanity: must actually be arm64. If amd64 sneaks in here,
|
||||
# fail fast — that means the label routing is wrong.
|
||||
case "$(uname -m)" in
|
||||
aarch64|arm64) echo "arm64 confirmed" ;;
|
||||
*) echo "ERROR: expected arm64, got $(uname -m)"; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Install shellcheck (arm64)
|
||||
run: |
|
||||
set -eu
|
||||
if command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "shellcheck already present: $(shellcheck --version | head -1)"
|
||||
else
|
||||
# Prefer apt if the runner base ships it; else download arm64 binary.
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends shellcheck
|
||||
else
|
||||
SC_VER=v0.10.0
|
||||
curl -fsSL "https://github.com/koalaman/shellcheck/releases/download/${SC_VER}/shellcheck-${SC_VER}.linux.aarch64.tar.xz" \
|
||||
| tar -xJf - --strip-components=1
|
||||
sudo mv shellcheck /usr/local/bin/
|
||||
fi
|
||||
fi
|
||||
shellcheck --version | head -2
|
||||
|
||||
- name: Run shellcheck on .gitea/scripts/*.sh
|
||||
run: |
|
||||
set -eu
|
||||
# Only the scripts we control under .gitea/scripts. Pilot
|
||||
# scope is intentionally narrow — broaden in a follow-up
|
||||
# once the lane is proven.
|
||||
mapfile -t TARGETS < <(find .gitea/scripts -maxdepth 2 -type f -name '*.sh' | sort)
|
||||
if [ "${#TARGETS[@]}" -eq 0 ]; then
|
||||
echo "No .sh files found under .gitea/scripts — nothing to check"
|
||||
exit 0
|
||||
fi
|
||||
echo "Checking ${#TARGETS[@]} file(s):"
|
||||
printf ' %s\n' "${TARGETS[@]}"
|
||||
# SC1091 = couldn't follow non-constant source; expected for
|
||||
# CI-time analysis without the full runtime layout.
|
||||
shellcheck --severity=error --exclude=SC1091 "${TARGETS[@]}"
|
||||
@@ -1,76 +0,0 @@
|
||||
name: Lint workflow YAML (Gitea-1.22.6-hostile shapes)
|
||||
|
||||
# Tier-2 hard-gate lint (RFC internal#219 §1, charter §SOP-N rule (m)).
|
||||
# Catches six Gitea-1.22.6-hostile workflow-YAML shapes BEFORE they reach
|
||||
# `main`. Each rule maps to a documented incident in saved memory:
|
||||
#
|
||||
# 1. workflow_dispatch.inputs — feedback_gitea_workflow_dispatch_inputs_unsupported
|
||||
# (2026-05-11 PyPI freeze 24h)
|
||||
# 2. on: workflow_run — task #81 (Gitea 1.22.6 lacks the event)
|
||||
# 3. name: containing "/" — breaks status-context tokenization
|
||||
# 4. cross-file name collision — status-reaper rev1 fail-loud class
|
||||
# 5. cross-repo uses: org/r/p@r — feedback_gitea_cross_repo_uses_blocked
|
||||
# (DEFAULT_ACTIONS_URL=github → 404)
|
||||
# 6. (WARN) api.github.com refs — feedback_act_runner_github_server_url
|
||||
# without workflow-level GITHUB_SERVER_URL
|
||||
#
|
||||
# Empirical history this hardens against:
|
||||
# - status-reaper rev1 caught rule-4 (name-collision) class
|
||||
# - sop-tier-refire DOA'd on rule-2 (workflow_run partial)
|
||||
# - #319 bootstrap-paradox (chained-defect class, related)
|
||||
# - internal#329 dispatcher race (adjacent)
|
||||
# - 2026-05-11 publish-runtime: rule-1, 24h PyPI freeze
|
||||
#
|
||||
# Triggers:
|
||||
# - pull_request: pre-merge gate — block hostile shapes before they land
|
||||
# - push: post-merge regression detection — catch direct-to-main edits
|
||||
#
|
||||
# Per RFC internal#219 §1 contract: continue-on-error: true during the
|
||||
# surface-broken-shapes phase. Follow-up PR flips off after surfaced
|
||||
# defects are triaged. The push-trigger ensures we catch regressions
|
||||
# even if the pull_request gate is bypassed by branch-protection drift.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint-workflow-yaml.py'
|
||||
- 'tests/test_lint_workflow_yaml.py'
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/workflows/**'
|
||||
- '.gitea/scripts/lint-workflow-yaml.py'
|
||||
- 'tests/test_lint_workflow_yaml.py'
|
||||
|
||||
# Belt-and-suspenders against runner default
|
||||
# (feedback_act_runner_github_server_url).
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Lint workflow YAML for Gitea-1.22.6-hostile shapes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken shapes without blocking PRs.
|
||||
# Follow-up PR flips this off after the 4 existing-on-main rule-2
|
||||
# (workflow_run) violations are migrated to a supported trigger.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install PyYAML
|
||||
run: pip install --quiet 'PyYAML>=6.0'
|
||||
|
||||
- name: Lint .gitea/workflows/*.yml
|
||||
run: python3 .gitea/scripts/lint-workflow-yaml.py
|
||||
|
||||
- name: Run lint-workflow-yaml unit tests
|
||||
run: |
|
||||
pip install --quiet pytest
|
||||
python3 -m pytest tests/test_lint_workflow_yaml.py -v
|
||||
@@ -37,11 +37,6 @@ name: main-red-watchdog
|
||||
# "unknown on type" when `workflow_dispatch.inputs.X` is present. Revisit
|
||||
# when Gitea ≥ 1.23 is fleet-wide.
|
||||
on:
|
||||
# SCHEDULE RE-ENABLED 2026-05-12 rev3 — interim disable (mc#645) reverted alongside
|
||||
# status-reaper rev3 (widen-window). Job-level timeout-minutes raised 5 → 15 below
|
||||
# to absorb runner-saturation latency without spurious cancels (the original cascade
|
||||
# cause). If runner-saturation root persists, the dedicated-runner-label split
|
||||
# remains the structural next step (tracked separately).
|
||||
schedule:
|
||||
# Hourly at :05 — task spec calls for "off-zero" (`5 * * * *`),
|
||||
# offset from :17 (ci-required-drift) and :00 (peak cron load).
|
||||
@@ -63,12 +58,7 @@ concurrency:
|
||||
jobs:
|
||||
watchdog:
|
||||
runs-on: ubuntu-latest
|
||||
# rev3 (2026-05-12, mc#645 revert): raised 5 → 15 to absorb runner-saturation
|
||||
# latency. Original 5min cap was producing 124-style cancels under load,
|
||||
# which fed the very `[main-red]` issues this workflow files (self-poisoning).
|
||||
# 15min is still well below Gitea-default 6h job ceiling; if a real hang
|
||||
# occurs the issue-file path is still the alarm surface.
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out repo (script lives at .gitea/scripts/)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
@@ -9,12 +9,18 @@ name: publish-canvas-image
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - Retargeted the image push from GHCR to ECR. GHCR was retired during
|
||||
# the 2026-05-06 Gitea migration, and Gitea's GITHUB_TOKEN cannot
|
||||
# authenticate to ghcr.io.
|
||||
# - **Open question for review**: this workflow pushes the canvas
|
||||
# image to `ghcr.io`. GHCR was retired during the 2026-05-06
|
||||
# Gitea migration in favor of ECR (per staging-verify.yml header
|
||||
# notes). The image may not be consumable post-migration. Two
|
||||
# options for follow-up: (a) retarget to
|
||||
# `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas`,
|
||||
# or (b) retire this workflow entirely and route canvas deploys
|
||||
# via the operator-host build path. tier:low + continue-on-error
|
||||
# means failed pushes do not block PRs.
|
||||
#
|
||||
|
||||
# Builds and pushes the canvas Docker image to ECR whenever a commit lands
|
||||
# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
|
||||
# on main that touches canvas code. Previously canvas changes were visible in
|
||||
# CI (npm run build passed) but the live container was never updated —
|
||||
# operators had to manually run `docker compose build canvas` each time.
|
||||
@@ -39,71 +45,32 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
packages: write # required to push to ghcr.io/${{ github.repository_owner }}/*
|
||||
|
||||
env:
|
||||
# SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
|
||||
# sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
|
||||
# bootstrap fallback. When the org var is set, the fallback becomes dead code and
|
||||
# switching accounts/regions is a one-line change at the org level (instead of
|
||||
# touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
|
||||
# use below in this repo's staging-verify.yml.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/canvas
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/canvas
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: post-merge image publication side effect; CI / all-required gates source changes.
|
||||
build-and-push:
|
||||
name: Build & push canvas image
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399). Ship
|
||||
# path (on: push:main, canvas/**) — reserved capacity so a merged
|
||||
# canvas fix's image build never FIFO-queues behind PR required-CI.
|
||||
# The `publish` label resolves ONLY to the molecule-runner-publish-*
|
||||
# sub-pool (config.publish.yaml). HARD DEPENDENCY: this MUST land
|
||||
# AFTER the publish-lane runners are registered/advertising `publish`
|
||||
# — the earlier #599 `docker` label attempt queued indefinitely with
|
||||
# zero eligible runners precisely because the label was targeted
|
||||
# before any runner advertised it (see #576). The lane is registered
|
||||
# in this rollout (internal#462) so the precondition holds.
|
||||
runs-on: publish
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Log in to ECR
|
||||
env:
|
||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Ensure ECR repository exists
|
||||
env:
|
||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
repo_path="${IMAGE_NAME#*/}"
|
||||
if ! aws ecr describe-repositories --repository-names "${repo_path}" --region us-east-2 >/dev/null 2>&1; then
|
||||
aws ecr create-repository \
|
||||
--repository-name "${repo_path}" \
|
||||
--image-scanning-configuration scanOnPush=true \
|
||||
--region us-east-2 >/dev/null
|
||||
fi
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing to the build step
|
||||
@@ -112,15 +79,11 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
echo "Runner: ${HOSTNAME:-unknown}"
|
||||
docker_info="$(docker info 2>&1)" || {
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Runner: ${HOSTNAME:-unknown}"
|
||||
printf '%s\n' "${docker_info}"
|
||||
echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "${docker_info}" | sed -n '1,5p'
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
@@ -154,7 +117,7 @@ jobs:
|
||||
echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build & push canvas image to ECR
|
||||
- name: Build & push canvas image to GHCR
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ./canvas
|
||||
@@ -167,10 +130,9 @@ jobs:
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
|
||||
# Gitea artifact-cache reachability is best-effort on the operator
|
||||
# runner network. Do not let cache export fail an image that already
|
||||
# built and pushed successfully.
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://git.moleculesai.app/${{ github.repository }}
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
name: publish-runtime-autobump
|
||||
|
||||
# Auto-bump-on-workspace-edit half of the publish pipeline.
|
||||
#
|
||||
# Why this file exists (issue #351):
|
||||
# Gitea Actions does not correctly disambiguate `paths:` from `tags:`
|
||||
# when both are bundled under a single `on.push` key. The result is
|
||||
# that tag pushes get filtered out and `publish-runtime.yml` never
|
||||
# fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
|
||||
# because PYPI_TOKEN was absent (publishes would have failed anyway).
|
||||
#
|
||||
# Split design:
|
||||
# - publish-runtime.yml : on.push.tags only (the publisher)
|
||||
# - publish-runtime-autobump.yml: on.push.branches+paths (this file — the version-bumper)
|
||||
#
|
||||
# This file computes the next version from PyPI's latest, pushes a
|
||||
# `runtime-v$VERSION` tag, and exits. The tag push then triggers
|
||||
# publish-runtime.yml via its tags-only trigger.
|
||||
#
|
||||
# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
|
||||
# so concurrent workspace pushes serialize at the bump step. Without
|
||||
# this, two pushes minutes apart could both read PyPI latest=0.1.129
|
||||
# and try to tag 0.1.130 simultaneously, only one of which would land.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- staging
|
||||
paths:
|
||||
- "workspace/**"
|
||||
|
||||
permissions:
|
||||
contents: write # required to push tags back
|
||||
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
autobump-and-tag:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Shallow clone — depth 1 is enough for the workspace-diff check.
|
||||
# Tags needed for the collision check below are fetched explicitly
|
||||
# in the next step, bypassing the runner-network timeout that
|
||||
# full-history fetch triggers on Gitea Actions runners
|
||||
# (runbooks/gitea-operational-quirks.md §runner-network-isolation).
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Fetch tags for collision check
|
||||
# fetch-depth: 1 gets only the most recent commit's refs, not the
|
||||
# tag that points at it. Do a targeted tag fetch so git tag --list
|
||||
# below can detect collision with prior manual pushes.
|
||||
run: git fetch origin --tags --depth=1
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Compute next version from PyPI latest
|
||||
id: bump
|
||||
run: |
|
||||
set -eu
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "PyPI latest=$LATEST -> next=$VERSION"
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
|
||||
exit 1
|
||||
fi
|
||||
if git tag --list | grep -qx "runtime-v$VERSION"; then
|
||||
echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Push runtime-v$VERSION tag
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
VERSION: ${{ steps.bump.outputs.version }}
|
||||
GITEA_URL: https://git.moleculesai.app
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
|
||||
exit 1
|
||||
fi
|
||||
git config user.name "publish-runtime autobump"
|
||||
git config user.email "publish-runtime@moleculesai.app"
|
||||
git tag -a "runtime-v$VERSION" \
|
||||
-m "Auto-bump on workspace/** edit on $GITHUB_REF" \
|
||||
-m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
|
||||
-m "publish-runtime.yml will pick up this tag and upload to PyPI"
|
||||
# Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
|
||||
# ensures the resulting tag-push event is dispatched to
|
||||
# publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
|
||||
# trigger downstream workflows.
|
||||
git remote set-url origin "${GITEA_URL#https://}"
|
||||
git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
|
||||
git push origin "runtime-v$VERSION"
|
||||
echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"
|
||||
@@ -0,0 +1,339 @@
|
||||
name: publish-runtime
|
||||
|
||||
# Gitea Actions port of .github/workflows/publish-runtime.yml.
|
||||
#
|
||||
# Ported 2026-05-10 (issue #206). Key differences from the GitHub version:
|
||||
# - Gitea Actions reads .gitea/workflows/, not .github/workflows/
|
||||
# - Dropped `environment: pypi-publish` — Gitea Actions does not support
|
||||
# named environments or OIDC trusted publishers
|
||||
# - Replaced `pypa/gh-action-pypi-publish@release/v1` (OIDC) with
|
||||
# `twine upload` using PYPI_TOKEN secret — same mechanism as a local
|
||||
# `python -m twine upload` with a PyPI token
|
||||
# - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
|
||||
# — Gitea Actions exposes github.ref (the full ref) but not ref_name
|
||||
# - Dropped `merge_group` trigger (Gitea has no merge queue)
|
||||
#
|
||||
# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
|
||||
# `workspace/**` path-filter trigger in PR #349.
|
||||
#
|
||||
# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
|
||||
# file. Bundling `paths` with `tags` under a single `on.push` key caused
|
||||
# Gitea Actions to never dispatch the workflow for tag-push events (0
|
||||
# runs in `action_run` for workflow_id='publish-runtime.yml' since the
|
||||
# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
|
||||
# 0.1.129 despite a v1.0.0 Gitea tag existing).
|
||||
#
|
||||
# The auto-bump-on-workspace-edit trigger now lives in
|
||||
# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
|
||||
# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
|
||||
# which THIS file then picks up via the tags-only trigger below.
|
||||
#
|
||||
# This decoupling means Gitea's path-vs-tag evaluator never has to
|
||||
# disambiguate — each file has a single unambiguous trigger shape.
|
||||
#
|
||||
# PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
|
||||
# Set via: repo Settings → Actions → Variables and Secrets → New Secret.
|
||||
# The token should be a PyPI API token scoped to molecule-ai-workspace-runtime.
|
||||
#
|
||||
# The DISPATCH_TOKEN cascade (git push to template repos) is unchanged —
|
||||
# it uses the Gitea API directly and was already Gitea-compatible.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "runtime-v*"
|
||||
workflow_dispatch:
|
||||
# 2026-05-11 (root cause of #351 / 0 runs ever):
|
||||
# Gitea 1.22.6's workflow parser rejects `workflow_dispatch.inputs.version`
|
||||
# with "unknown on type" — it mis-treats the inputs sub-keys as top-level
|
||||
# `on:` event types. Log line:
|
||||
# actions/workflows.go:DetectWorkflows() [W] ignore invalid workflow
|
||||
# "publish-runtime.yml": unknown on type: map["version": {...}]
|
||||
# That `[W] ignore invalid workflow` is silent UX — the workflow never
|
||||
# registers, so it never fires for ANY event (push.tags included).
|
||||
# Removing the inputs block restores parsing. Manual dispatch from the
|
||||
# Gitea UI now triggers the PyPI auto-bump fallback in `Derive version`
|
||||
# below (no `inputs.version` to read).
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Serialize publishes so two concurrent tag pushes don't both compute
|
||||
# "latest+1" and race on PyPI upload. The second one waits.
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
|
||||
- name: Derive version (tag or PyPI auto-bump)
|
||||
id: version
|
||||
run: |
|
||||
if echo "$GITHUB_REF" | grep -q "^refs/tags/runtime-v"; then
|
||||
# Tag is `runtime-vX.Y.Z` — strip the prefix.
|
||||
VERSION="${GITHUB_REF#refs/tags/runtime-v}"
|
||||
else
|
||||
# workflow_dispatch path (no inputs supported on Gitea 1.22.6) or
|
||||
# any other non-tag trigger: derive from PyPI latest + patch bump.
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
|
||||
fi
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
|
||||
echo "::error::version $VERSION does not match PEP 440"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Publishing molecule-ai-workspace-runtime $VERSION"
|
||||
|
||||
- name: Install build tooling
|
||||
run: pip install build twine
|
||||
|
||||
- name: Build package from workspace/
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "${{ steps.version.outputs.version }}" \
|
||||
--out "${{ runner.temp }}/runtime-build"
|
||||
|
||||
- name: Build wheel + sdist
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: python -m build
|
||||
|
||||
- name: Capture wheel SHA256 for cascade content-verification
|
||||
id: wheel_hash
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
set -eu
|
||||
WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
|
||||
if [ -z "$WHEEL" ]; then
|
||||
echo "::error::No .whl in dist/ — \`python -m build\` must have failed silently"
|
||||
exit 1
|
||||
fi
|
||||
HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
|
||||
echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
|
||||
echo "Local wheel SHA256 (pre-upload): ${HASH}"
|
||||
echo "Wheel filename: $(basename "$WHEEL")"
|
||||
|
||||
- name: Verify package contents (sanity)
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
python -m twine check dist/*
|
||||
python -m venv /tmp/smoke
|
||||
/tmp/smoke/bin/pip install --quiet dist/*.whl
|
||||
/tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
|
||||
- name: Publish to PyPI
|
||||
# working-directory matches the preceding Build/Verify steps. Without
|
||||
# this, twine runs from the default workspace checkout dir where
|
||||
# `dist/` doesn't exist and fails with:
|
||||
# ERROR InvalidDistribution: Cannot find file (or expand pattern): 'dist/*'
|
||||
# Caught on the first-ever successful dispatch of this workflow
|
||||
# (run 5097, 2026-05-11 02:08Z) — every other step in the publish
|
||||
# job already had this working-directory; Publish was missing it.
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
env:
|
||||
# PYPI_TOKEN: repository secret scoped to molecule-ai-workspace-runtime.
|
||||
# Set via: Settings → Actions → Variables and Secrets → New Secret.
|
||||
# Format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$PYPI_TOKEN" ]; then
|
||||
echo "::error::PYPI_TOKEN secret is not set — set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
echo "::error::Required format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
|
||||
exit 1
|
||||
fi
|
||||
python -m twine upload \
|
||||
--repository pypi \
|
||||
--username __token__ \
|
||||
--password "$PYPI_TOKEN" \
|
||||
dist/*
|
||||
|
||||
cascade:
|
||||
needs: publish
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Wait for PyPI to propagate the new version
|
||||
env:
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
python -m venv /tmp/propagation-probe
|
||||
PROBE=/tmp/propagation-probe/bin
|
||||
$PROBE/pip install --upgrade --quiet pip
|
||||
for i in $(seq 1 30); do
|
||||
if $PROBE/pip install \
|
||||
--quiet \
|
||||
--no-cache-dir \
|
||||
--force-reinstall \
|
||||
--no-deps \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1; then
|
||||
INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
|
||||
| awk -F': ' '/^Version:/{print $2}')
|
||||
if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
|
||||
echo "✓ PyPI resolved $RUNTIME_VERSION (install check)"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "::error::pip install --no-cache-dir molecule-ai-workspace-runtime==${RUNTIME_VERSION} never resolved within ~5 min."
|
||||
echo "::error::Refusing to fan out cascade against a potentially stale PyPI index."
|
||||
exit 1
|
||||
fi
|
||||
echo " [$i/30] waiting for PyPI to propagate ${RUNTIME_VERSION}..."
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Stage (b): download wheel + SHA256 compare against what we built.
|
||||
# Catches Fastly stale-content serving old bytes under a new version URL.
|
||||
#
|
||||
# Caught run 5196 (first-ever successful publish, 2026-05-11): the
|
||||
# previous one-liner `HASH=$(pip download ... && sha256sum ...)`
|
||||
# captured pip's stdout (`Collecting molecule-ai-workspace-runtime
|
||||
# ==X.Y.Z`) into HASH, then the SHA comparison failed against the
|
||||
# leaked `Collecting...` string. `2>/dev/null` silences stderr but
|
||||
# NOT stdout; pip writes its progress to stdout by default.
|
||||
# Fix: split into two steps, silence pip's stdout explicitly, capture
|
||||
# only sha256sum's output into HASH.
|
||||
python -m pip download \
|
||||
--no-deps \
|
||||
--no-cache-dir \
|
||||
--dest /tmp/wheel-probe \
|
||||
--quiet \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1
|
||||
HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
|
||||
if [ "$HASH" != "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
|
||||
echo "::error::Expected: $EXPECTED_SHA256"
|
||||
echo "::error::Got: $HASH"
|
||||
echo "::error::Fastly may be serving stale content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ PyPI CDN verified (SHA256 match)"
|
||||
|
||||
- name: Fan out via push to .runtime-version
|
||||
env:
|
||||
# Gitea PAT with write:repository scope on the 8 cascade-active
|
||||
# template repos. Used for git push to each template repo's main
|
||||
# branch, which trips their `on: push: branches: [main]` trigger
|
||||
# on publish-image.yml.
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
run: |
|
||||
set +e # don't abort on a single repo failure — collect them all
|
||||
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
|
||||
echo "::warning::set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
|
||||
echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version."
|
||||
exit 1
|
||||
fi
|
||||
VERSION="$RUNTIME_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::publish job did not expose a version output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
|
||||
TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
|
||||
FAILED=""
|
||||
SKIPPED=""
|
||||
|
||||
git config --global user.name "publish-runtime cascade"
|
||||
git config --global user.email "publish-runtime@moleculesai.app"
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
for tpl in $TEMPLATES; do
|
||||
REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
|
||||
CLONE="$WORKDIR/$tpl"
|
||||
|
||||
HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $DISPATCH_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "↷ $tpl has no publish-image.yml — soft-skip"
|
||||
SKIPPED="$SKIPPED $tpl"
|
||||
continue
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
success=false
|
||||
while [ $attempt -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
rm -rf "$CLONE"
|
||||
if ! git clone --depth=1 \
|
||||
"https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
|
||||
"$CLONE" >/tmp/clone.log 2>&1; then
|
||||
echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
|
||||
cd "$CLONE"
|
||||
echo "$VERSION" > .runtime-version
|
||||
|
||||
if git diff --quiet -- .runtime-version; then
|
||||
echo "✓ $tpl already at $VERSION — no commit needed"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
git add .runtime-version
|
||||
git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
|
||||
-m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
|
||||
>/dev/null
|
||||
|
||||
if git push origin HEAD:main >/tmp/push.log 2>&1; then
|
||||
echo "✓ $tpl pushed $VERSION on attempt $attempt"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
echo "::warning::push $tpl attempt $attempt failed, pull-rebasing"
|
||||
git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
|
||||
cd - >/dev/null
|
||||
done
|
||||
|
||||
if [ "$success" != "true" ]; then
|
||||
FAILED="$FAILED $tpl"
|
||||
fi
|
||||
done
|
||||
rm -rf "$WORKDIR"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "::error::Cascade incomplete after 3 retries each. Failed:$FAILED"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$SKIPPED" ]; then
|
||||
echo "Cascade complete: pinned $VERSION. Soft-skipped (no publish-image.yml):$SKIPPED"
|
||||
else
|
||||
echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
|
||||
fi
|
||||
@@ -18,90 +18,63 @@ name: publish-workspace-server-image
|
||||
# :staging-<sha> — per-commit digest, stable for canary verify
|
||||
# :staging-latest — tracks most recent build on this branch
|
||||
#
|
||||
# Production auto-deploy:
|
||||
# After both platform and tenant images are pushed, deploy-production waits
|
||||
# for strict required push contexts on the same SHA to go green, then
|
||||
# calls the production CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>. Set repo variable or secret PROD_AUTO_DEPLOY_DISABLED=true
|
||||
# to stop production rollout while keeping image publishing enabled.
|
||||
#
|
||||
# Primary ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Optional staging tenant mirror target:
|
||||
# 004947743811.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Required secrets: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AUTO_SYNC_TOKEN
|
||||
# Staging ECR grants the primary SSOT-managed publisher principal repository
|
||||
# policy access, so no persistent staging AWS access keys are required.
|
||||
#
|
||||
# mc#711: Docker daemon not accessible on ubuntu-latest runner (molecule-canonical-1
|
||||
# shows client-only in `docker info` — daemon not running). DinD mount is present but
|
||||
# daemon doesn't respond. Fix: add diagnostic step showing socket info so ops can
|
||||
# identify which runners have a live daemon. If no daemon is available, the job
|
||||
# fails fast with actionable output rather than silent deep failure.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'manifest.json'
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/publish-workspace-server-image.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
# No `concurrency:` block here. Gitea 1.22.6 can cancel queued runs despite
|
||||
# `cancel-in-progress: false`; that is not acceptable for a workflow with a
|
||||
# production deploy job. Per-SHA image tags are immutable, and staging-latest is
|
||||
# best-effort last-writer-wins metadata.
|
||||
# Serialize per-branch so two rapid staging pushes don't race the same
|
||||
# :staging-latest tag retag. Allow staging and main to run in parallel
|
||||
# (different GITHUB_REF → different concurrency group) since they
|
||||
# produce different :staging-<sha> tags and last-write-wins on
|
||||
# :staging-latest is acceptable across branches.
|
||||
#
|
||||
# 2026-05-20 retrigger: run #86994 on mc#1589 merge sha 0f0f1ba2 failed at
|
||||
# setup-buildx-action with EACCES on PC2 WSL publish runner — the runner's
|
||||
# DOCKER_CONFIG=/home/hongming/.docker-ecr/ dir didn't have a buildx/certs
|
||||
# subdir writable by the container's UID 1001. Hot-patched the dir perms;
|
||||
# this chore push retriggers the workflow. Proper fix (per-runner
|
||||
# DOCKER_CONFIG owned by 1001, internal#597 --env HOME=/home/runner pattern)
|
||||
# is tracked as a CI-hygiene follow-up — not in scope here.
|
||||
# cancel-in-progress: false → in-flight builds finish; the next push's
|
||||
# build queues. This avoids a partially-pushed image.
|
||||
concurrency:
|
||||
group: publish-workspace-server-image-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
# SSOT-Instance-10 (#333): ECR registry triplet (account.dkr.ecr.region.amazonaws.com)
|
||||
# sourced from org/repo var `ECR_REGISTRY` with the current prod-account literal as
|
||||
# bootstrap fallback. When the org var is set, the fallback becomes dead code and
|
||||
# switching accounts/regions is a one-line change at the org level (instead of
|
||||
# touching every workflow). Pattern mirrors `vars.CP_URL || 'literal'` already in
|
||||
# use below in this repo's staging-verify.yml.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
STAGING_TENANT_IMAGE_NAME: ${{ vars.STAGING_ECR_REGISTRY || '004947743811.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399). This
|
||||
# is a post-merge ship job (on: push:main) — it must NOT FIFO-compete
|
||||
# with PR required-CI on the shared pool (PR#1350's prod image build
|
||||
# was delayed ~25min this way). The `publish` label resolves ONLY to
|
||||
# the reserved molecule-runner-publish-* sub-pool (config.publish.yaml,
|
||||
# OUTSIDE the managed 1..20 range) so a merged fix's image build
|
||||
# starts immediately while PR-CI keeps the general pool.
|
||||
runs-on: publish
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing where `docker build`
|
||||
# fails deep in the process with a cryptic ECR auth error.
|
||||
# is inaccessible (e.g. permission change, daemon restart, or group-membership
|
||||
# drift) rather than silently continuing to step 2 where `docker build`
|
||||
# fails deep in the process with a cryptic ECR auth error that doesn't
|
||||
# surface the root cause. Also reports the daemon version so operator
|
||||
# can correlate with runner host logs.
|
||||
- name: Verify Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
echo "Runner: ${HOSTNAME:-unknown}"
|
||||
docker_info="$(docker info 2>&1)" || {
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Runner: ${HOSTNAME:-unknown}"
|
||||
printf '%s\n' "${docker_info}"
|
||||
echo "::error::Check: (1) daemon is running, (2) runner user is in docker group, (3) sock permissions are 660+"
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "${docker_info}" | sed -n '1,5p'
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
@@ -121,12 +94,13 @@ jobs:
|
||||
MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
# Strip JSON5 comments before jq parsing — Integration Tester appends
|
||||
# `// Triggered by ...` which breaks `jq` in clone-manifest.sh.
|
||||
sed '/^[[:space:]]*\/\//d' manifest.json > .manifest-stripped.json
|
||||
bash scripts/clone-manifest.sh \
|
||||
.manifest-stripped.json \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
@@ -140,26 +114,9 @@ jobs:
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Keep Buildx state inside the job temp dir. The publish runner's
|
||||
# inherited DOCKER_CONFIG can point at a host-owned ECR config path
|
||||
# (/home/hongming/.docker-ecr), which caused setup-buildx-action to
|
||||
# fail before image build with EACCES creating buildx/certs.
|
||||
- name: Prepare writable Docker config
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export DOCKER_CONFIG="$RUNNER_TEMP/docker-config"
|
||||
mkdir -p "$DOCKER_CONFIG/buildx/certs"
|
||||
echo "DOCKER_CONFIG=$DOCKER_CONFIG" >> "$GITHUB_ENV"
|
||||
docker buildx version
|
||||
|
||||
# Build + push platform image (inline ECR auth — mirrors the operator-host
|
||||
# approach; credentials come from GITHUB_SECRET_AWS_ACCESS_KEY_ID /
|
||||
# GITHUB_SECRET_AWS_SECRET_ACCESS_KEY in Gitea Actions).
|
||||
# docker buildx bake / build required for `imagetools inspect` digest
|
||||
# capture in the CP pin-update step (RFC internal#229 §X step 4 PR-1).
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build & push platform image to ECR (staging-<sha> + staging-latest)
|
||||
env:
|
||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
||||
@@ -175,26 +132,22 @@ jobs:
|
||||
ECR_REGISTRY="${IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
docker buildx build \
|
||||
docker build \
|
||||
--file ./workspace-server/Dockerfile \
|
||||
--build-arg GIT_SHA="${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.source=https://git.moleculesai.app/molecule-ai/${REPO}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/${REPO}" \
|
||||
--label "org.opencontainers.image.revision=${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--label "molecule.workflow.run_id=${GITHUB_RUN_ID}" \
|
||||
--label "org.opencontainers.image.description=Molecule AI platform — pending canary verify" \
|
||||
--tag "${IMAGE_NAME}:${TAG_SHA}" \
|
||||
--tag "${IMAGE_NAME}:${TAG_LATEST}" \
|
||||
--push .
|
||||
.
|
||||
docker push "${IMAGE_NAME}:${TAG_SHA}"
|
||||
docker push "${IMAGE_NAME}:${TAG_LATEST}"
|
||||
|
||||
# Build + push tenant image (Go platform + Next.js canvas in one image).
|
||||
# Push the same build to the staging account too so fresh staging/E2E
|
||||
# tenants can pull without cross-account ECR reads. The staging ECR repo
|
||||
# policy trusts the primary SSOT-managed publisher principal; do not add
|
||||
# separate persistent staging AWS access keys here.
|
||||
- name: Build & push tenant image to ECR (staging-<sha> + staging-latest)
|
||||
env:
|
||||
TENANT_IMAGE_NAME: ${{ env.TENANT_IMAGE_NAME }}
|
||||
STAGING_TENANT_IMAGE_NAME: ${{ env.STAGING_TENANT_IMAGE_NAME }}
|
||||
TAG_SHA: staging-${{ steps.tags.outputs.sha }}
|
||||
TAG_LATEST: staging-latest
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
@@ -205,198 +158,17 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${TENANT_IMAGE_NAME%%/*}"
|
||||
STAGING_ECR_REGISTRY="${STAGING_TENANT_IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${STAGING_ECR_REGISTRY}"
|
||||
|
||||
build_tags=(
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
--tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
--tag "${STAGING_TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
)
|
||||
|
||||
docker buildx build \
|
||||
docker build \
|
||||
--file ./workspace-server/Dockerfile.tenant \
|
||||
--build-arg NEXT_PUBLIC_PLATFORM_URL= \
|
||||
--build-arg GIT_SHA="${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.source=https://git.moleculesai.app/molecule-ai/${REPO}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/${REPO}" \
|
||||
--label "org.opencontainers.image.revision=${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--label "molecule.workflow.run_id=${GITHUB_RUN_ID}" \
|
||||
"${build_tags[@]}" \
|
||||
--push .
|
||||
|
||||
# bp-exempt: production deploy side-effect; merge is gated by CI / all-required and this job waits for push CI before acting.
|
||||
deploy-production:
|
||||
name: Production auto-deploy
|
||||
needs: build-and-push
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
# Publish/release lane (internal#462) — production deploy of a merged
|
||||
# fix; reserved capacity, never queued behind PR-CI.
|
||||
runs-on: publish
|
||||
timeout-minutes: 75
|
||||
env:
|
||||
CP_URL: ${{ vars.PROD_CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
GITEA_TOKEN: ${{ secrets.PROD_AUTO_DEPLOY_CONTROL_TOKEN || secrets.AUTO_SYNC_TOKEN }}
|
||||
PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
|
||||
PROD_AUTO_DEPLOY_CANARY_SLUG: ${{ vars.PROD_AUTO_DEPLOY_CANARY_SLUG || 'hongming' }}
|
||||
PROD_AUTO_DEPLOY_SOAK_SECONDS: ${{ vars.PROD_AUTO_DEPLOY_SOAK_SECONDS || '60' }}
|
||||
PROD_AUTO_DEPLOY_BATCH_SIZE: ${{ vars.PROD_AUTO_DEPLOY_BATCH_SIZE || '3' }}
|
||||
PROD_AUTO_DEPLOY_DRY_RUN: ${{ vars.PROD_AUTO_DEPLOY_DRY_RUN || '' }}
|
||||
PROD_ALLOW_NON_PROD_CP_URL: ${{ vars.PROD_ALLOW_NON_PROD_CP_URL || '' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Build deploy plan
|
||||
id: plan
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/prod-auto-deploy.py plan > "$RUNNER_TEMP/prod-auto-deploy-plan.json"
|
||||
jq . "$RUNNER_TEMP/prod-auto-deploy-plan.json"
|
||||
enabled="$(jq -r '.enabled' "$RUNNER_TEMP/prod-auto-deploy-plan.json")"
|
||||
echo "enabled=$enabled" >> "$GITHUB_OUTPUT"
|
||||
if [ "$enabled" != "true" ]; then
|
||||
reason="$(jq -r '.disabled_reason' "$RUNNER_TEMP/prod-auto-deploy-plan.json")"
|
||||
echo "::notice::Production auto-deploy disabled: $reason"
|
||||
{
|
||||
echo "## Production auto-deploy skipped"
|
||||
echo ""
|
||||
echo "Reason: \`$reason\`"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is required for production auto-deploy."
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is required so production deploy can wait for green CI."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Self-test production deploy helper
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2'
|
||||
python3 -m pytest .gitea/scripts/tests/test_prod_auto_deploy.py -q
|
||||
python3 .gitea/scripts/lint-workflow-yaml.py --workflow-dir .gitea/workflows
|
||||
|
||||
- name: Wait for green main CI on this SHA
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/prod-auto-deploy.py wait-ci
|
||||
|
||||
- name: Call production CP redeploy-fleet
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/prod-auto-deploy.py assert-enabled
|
||||
PLAN="$RUNNER_TEMP/prod-auto-deploy-plan.json"
|
||||
TARGET_TAG="$(jq -r '.target_tag' "$PLAN")"
|
||||
BODY="$(jq -c '.body' "$PLAN")"
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag: $TARGET_TAG"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE="$RUNNER_TEMP/prod-redeploy-response.json"
|
||||
HTTP_CODE_FILE="$RUNNER_TEMP/prod-redeploy-http-code.txt"
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" > "$HTTP_CODE_FILE"
|
||||
set -e
|
||||
|
||||
HTTP_CODE="$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")"
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
echo "HTTP $HTTP_CODE"
|
||||
jq '{ok, result_count: (.results // [] | length)}' "$HTTP_RESPONSE" || true
|
||||
|
||||
{
|
||||
echo "## Production auto-deploy"
|
||||
echo ""
|
||||
echo "**Commit:** \`${GITHUB_SHA:0:7}\`"
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo "| Slug | Phase | SSM Status | Exit | Healthz | Error present |"
|
||||
echo "|------|-------|------------|------|---------|---------------|"
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \((.error // "") != "") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
exit 1
|
||||
fi
|
||||
OK="$(jq -r '.ok' "$HTTP_RESPONSE")"
|
||||
if [ "$OK" != "true" ]; then
|
||||
echo "::error::redeploy-fleet reported ok=false; production rollout halted."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify reachable tenants report this SHA
|
||||
if: ${{ steps.plan.outputs.enabled == 'true' }}
|
||||
env:
|
||||
TENANT_DOMAIN: moleculesai.app
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RESP="$RUNNER_TEMP/prod-redeploy-response.json"
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::error::No tenants returned from redeploy-fleet; refusing to mark production deploy verified."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
UNHEALTHY_COUNT=0
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
healthz_ok="$(jq -r --arg slug "$slug" '.results[]? | select(.slug == $slug) | .healthz_ok' "$RESP" | tail -1)"
|
||||
if [ "$healthz_ok" != "true" ]; then
|
||||
echo "::error::$slug did not report healthz_ok=true in redeploy-fleet response."
|
||||
UNHEALTHY_COUNT=$((UNHEALTHY_COUNT + 1))
|
||||
continue
|
||||
fi
|
||||
url="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
body="$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$url" || true)"
|
||||
actual="$(echo "$body" | jq -r '.git_sha // ""' 2>/dev/null || echo "")"
|
||||
if [ -z "$actual" ]; then
|
||||
echo "::error::$slug did not return /buildinfo after deploy."
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
continue
|
||||
fi
|
||||
if [ "$actual" != "$GITHUB_SHA" ]; then
|
||||
echo "::error::$slug is stale: actual=${actual:0:7}, expected=${GITHUB_SHA:0:7}"
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
else
|
||||
echo "$slug: ${actual:0:7}"
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Buildinfo verification"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${GITHUB_SHA:0:7}\`"
|
||||
echo "Verified tenants: ${#SLUGS[@]}"
|
||||
echo "Stale tenants: $STALE_COUNT"
|
||||
echo "Unhealthy tenants: $UNHEALTHY_COUNT"
|
||||
echo "Unreachable tenants: $UNREACHABLE_COUNT"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$STALE_COUNT" -gt 0 ] || [ "$UNHEALTHY_COUNT" -gt 0 ] || [ "$UNREACHABLE_COUNT" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
--label "org.opencontainers.image.description=Molecule AI tenant platform + canvas — pending canary verify" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}" \
|
||||
.
|
||||
docker push "${TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
docker push "${TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
# qa-review — non-author APPROVE from the `qa` Gitea team required to merge.
|
||||
#
|
||||
# RFC#324 Step 1 of 5 (workflow-add). Pairs with `security-review.yml` and the
|
||||
# branch-protection flip in Step 2.
|
||||
#
|
||||
# === DESIGN (RFC#324 v1.1 addendum) ===
|
||||
#
|
||||
# A1-α (refire mechanism):
|
||||
# Triggers on:
|
||||
# - `pull_request_target`: opened, synchronize, reopened
|
||||
# → initial status posts when PR opens / re-pushes
|
||||
# - comment refires are handled by `review-refire-comments.yml`
|
||||
# → a single issue_comment dispatcher prevents every SOP/review
|
||||
# comment from enqueueing separate qa/security/tier jobs on
|
||||
# Gitea 1.22.6 before job-level `if:` can skip them.
|
||||
# Workflow name = `qa-review` ; job name = `approved`.
|
||||
# The job's own pass/fail conclusion publishes the status context
|
||||
# `qa-review / approved (<event>)` — NO `POST /statuses` call → NO
|
||||
# write:repository token scope needed. Sidesteps internal#321 defect #2.
|
||||
#
|
||||
# A1.1 (privilege check on slash-comment — INFORMATIONAL ONLY, NOT a gate):
|
||||
# The `issue_comment` event fires for ANY commenter, including
|
||||
# non-collaborators. The original (v1.2) design gated the eval step
|
||||
# behind a collaborator probe → if a non-collaborator commented
|
||||
# /qa-recheck, the eval was `if:`-skipped → the job exited 0 anyway →
|
||||
# the status context published `success` with ZERO real APPROVE.
|
||||
# That was a fail-open: any visitor could green the gate.
|
||||
#
|
||||
# RFC#324 v1.3 §A1.1 correction (option b per hongming-pc 1421):
|
||||
# drop privilege-gating of the evaluation entirely. The eval is
|
||||
# read-only and idempotent — it reads `pulls/{N}/reviews` and
|
||||
# `teams/{id}/members/{u}` (both API-side state that a commenter can't
|
||||
# change). Re-running it on a non-collaborator's comment is harmless
|
||||
# AND correct: if a real team-member APPROVE exists, the eval flips
|
||||
# green; if not, it stays red.
|
||||
#
|
||||
# We KEEP the privilege step as a `::notice::` log line only — useful
|
||||
# for griefer-spotting (one operator spamming /recheck) without
|
||||
# touching the gate. If rate-limiting is needed later, add it as a
|
||||
# separate concern (time-window throttle, not a privilege gate).
|
||||
#
|
||||
# We MUST NOT use `github.event.comment.author_association` (the
|
||||
# field doesn't exist on Gitea 1.22.6 webhook payload — this was
|
||||
# sop-tier-refire's defect #1).
|
||||
#
|
||||
# A4 (no PR-head checkout under pull_request_target):
|
||||
# We check out the BASE ref explicitly so the review-check.sh script is
|
||||
# loaded from trusted source. We NEVER use `ref: ${{ github.event.pull_request.head.sha }}`.
|
||||
# No PR-head code is executed in the runner. Trust boundary preserved.
|
||||
#
|
||||
# A5 (real Gitea team):
|
||||
# `qa` team (id=20) verified by orchestrator preflight 2026-05-11; queried
|
||||
# at run time via /api/v1/teams/20/members/{login}.
|
||||
#
|
||||
# === TOKEN ===
|
||||
#
|
||||
# The workflow reads PR state, PR reviews, and team membership.
|
||||
# Gitea 1.22.6's /api/v1/teams/{id}/members/{u} returns 403 ('Must be a
|
||||
# team member') for tokens whose owner is not in that team. The default
|
||||
# `secrets.GITHUB_TOKEN` is owned by a workflow-scoped identity that is
|
||||
# also not in qa/security teams → also 403.
|
||||
#
|
||||
# Resolution: a dedicated `RFC_324_TEAM_READ_TOKEN` secret, owned by an
|
||||
# identity that IS in both `qa` and `security` teams (Owners-tier
|
||||
# claude-ceo-assistant, or a new service-bot added to both teams).
|
||||
# Provisioning of this secret is tracked as a follow-up issue (filed by
|
||||
# core-devops at PR open).
|
||||
#
|
||||
# Until that secret is provisioned, the job will exit 1 with a clear
|
||||
# 403-on-team-probe error and the `qa-review / approved` status will
|
||||
# stay `failure`. This is the correct fail-closed behavior — the gate
|
||||
# blocks merge until both (a) a QA team member APPROVEs and (b) the
|
||||
# workflow has a token that can confirm their team membership.
|
||||
#
|
||||
# === SLASH-COMMAND CONTRACT ===
|
||||
#
|
||||
# /qa-recheck — re-evaluate the gate (e.g. after an APPROVE lands)
|
||||
#
|
||||
# Open to any PR commenter. The eval is read-only and idempotent, so
|
||||
# unprivileged refires are harmless (RFC#324 v1.3 §A1.1). Collaborator
|
||||
# status is logged for griefer-spotting but does NOT gate execution.
|
||||
|
||||
name: qa-review
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
secrets: read
|
||||
|
||||
jobs:
|
||||
# bp-exempt: PR review bot signal; required merge state is enforced by CI / all-required.
|
||||
approved:
|
||||
# Gate the job:
|
||||
# - On pull_request_target events: always run.
|
||||
# Comment-triggered refires live in review-refire-comments.yml. Keeping
|
||||
# this workflow PR-only avoids comment-triggered queue storms.
|
||||
if: |
|
||||
github.event_name == 'pull_request_target'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Privilege check (A1.1 — INFORMATIONAL log only, NOT a gate)
|
||||
# RFC#324 v1.3 §A1.1: this step does NOT gate subsequent steps.
|
||||
# It exists solely as a log line for griefer-spotting (one
|
||||
# operator spamming /qa-recheck without merit). Re-running the
|
||||
# read-only eval on a non-collaborator comment is harmless;
|
||||
# gating it would be fail-open (skipped steps still publish
|
||||
# `success` for the job's status context).
|
||||
# Only runs on issue_comment events; pull_request_target has
|
||||
# no comment.user.login so the step is a no-op skip there.
|
||||
if: github.event_name == 'issue_comment'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
login="${{ github.event.comment.user.login }}"
|
||||
# Write token to a mode-600 file so it never appears in curl's argv.
|
||||
# (#541: -H "Authorization: token $TOKEN" puts the secret in /proc/<pid>/cmdline)
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/collaborators/${login}")
|
||||
rm -f "$authfile"
|
||||
if [ "$code" = "204" ]; then
|
||||
echo "::notice::Recheck from ${login} (collaborator=true)"
|
||||
else
|
||||
echo "::notice::Recheck from ${login} (collaborator=false, HTTP ${code}) — proceeding with read-only eval anyway"
|
||||
fi
|
||||
|
||||
- name: Check out BASE ref (A4 — never PR-head)
|
||||
# Loads the review-check.sh script from a trusted ref. For
|
||||
# pull_request_target the default checkout is BASE already; we
|
||||
# set ref explicitly for the issue_comment event too so the
|
||||
# script source is always the default-branch version.
|
||||
# NEVER use ref: ${{ github.event.pull_request.head.sha }} —
|
||||
# that would execute PR-head code with secrets-context.
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Evaluate qa-review
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# PR number lives in different places per event:
|
||||
# pull_request_target → github.event.pull_request.number
|
||||
# issue_comment → github.event.issue.number
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: qa
|
||||
TEAM_ID: '20'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: bash .gitea/scripts/review-check.sh
|
||||
@@ -51,7 +51,6 @@ jobs:
|
||||
name: Audit Railway env vars for drift-prone pins
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 10
|
||||
|
||||
|
||||
@@ -9,17 +9,20 @@ name: redeploy-tenants-on-main
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - Dropped unsupported `workflow_run` (task #81).
|
||||
# - Later changed to manual-only after publish-workspace-server-image.yml
|
||||
# gained an integrated ordered production deploy job.
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on .gitea/workflows/publish-workspace-server-image.yml. Until
|
||||
# then continue-on-error+dead-workflow doesn't break anything.
|
||||
#
|
||||
|
||||
# Manual production tenant redeploy/rollback helper.
|
||||
# Auto-refresh prod tenant EC2s after every main merge.
|
||||
#
|
||||
# Why this workflow is manual-only: publish-workspace-server-image now owns
|
||||
# the ordered build -> push -> production auto-deploy sequence in one workflow.
|
||||
# A separate push-triggered redeploy workflow races before the new ECR image
|
||||
# exists and can paint main red with a false deployment failure.
|
||||
# Why this workflow exists: publish-workspace-server-image builds and
|
||||
# pushes a new platform-tenant :<sha> to ECR on every merge to main,
|
||||
# but running tenants pulled their image once at boot and never re-pull.
|
||||
# Users see stale code indefinitely.
|
||||
#
|
||||
# This workflow closes the gap by calling the control-plane admin
|
||||
# endpoint that performs a canary-first, batched, health-gated rolling
|
||||
@@ -32,61 +35,60 @@ name: redeploy-tenants-on-main
|
||||
# Gitea suspension migration. The staging-verify.yml promote step now
|
||||
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
|
||||
#
|
||||
# Runtime ordering for automatic deploys now lives in
|
||||
# publish-workspace-server-image.yml:
|
||||
# 1. build-and-push creates new :staging-<sha> images in ECR.
|
||||
# 2. deploy-production waits for required push contexts on that SHA.
|
||||
# 3. deploy-production calls redeploy-fleet canary-first.
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
|
||||
# 2. This workflow fires via workflow_run, calls redeploy-fleet with
|
||||
# target_tag=staging-<sha>. No CDN propagation wait needed —
|
||||
# ECR image manifest is consistent immediately after push.
|
||||
# 3. Calls redeploy-fleet with canary_slug (if set) and a soak
|
||||
# period. Canary proves the image boots; batches follow.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: set PROD_MANUAL_REDEPLOY_TARGET_TAG as a repo/org
|
||||
# variable or secret, run workflow_dispatch, then unset it after the
|
||||
# rollback. That calls redeploy-fleet with target_tag=<value>,
|
||||
# re-pulling the pinned image on every tenant.
|
||||
# Rollback path: re-run this workflow with a specific SHA pinned via
|
||||
# the workflow_dispatch input. That calls redeploy-fleet with
|
||||
# target_tag=<sha>, re-pulling the older image on every tenant.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize manual redeploys so two operator-triggered rollbacks do not
|
||||
# overlap and cause confusing per-tenant SSM state.
|
||||
# Serialize redeploys so two rapid main pushes' redeploys don't overlap
|
||||
# and cause confusing per-tenant SSM state. Without this, GitHub's
|
||||
# implicit workflow_run queueing would *probably* serialize them, but
|
||||
# the explicit block makes the invariant defensible. Mirrors the
|
||||
# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
|
||||
#
|
||||
# NOTE: cancel-in-progress: false removed (Rule 7 fix). Gitea 1.22.6
|
||||
# cancels queued runs regardless of this setting, so it provides no
|
||||
# actual protection. Each redeploy-fleet call is idempotent (canary-first
|
||||
# + batched + health-gated) so a cancelled predecessor is recovered
|
||||
# automatically by the next run.
|
||||
# cancel-in-progress: false → aborting a half-rolled-out fleet would
|
||||
# leave tenants stuck on whatever image they happened to be on when
|
||||
# cancelled. Better to finish the in-flight rollout before starting
|
||||
# the next one.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-main
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: production redeploy is a side-effect workflow, not a merge gate.
|
||||
redeploy:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399).
|
||||
# Production tenant redeploy — a deploy action, reserved capacity so
|
||||
# it never queues behind PR-CI. `publish` -> molecule-runner-publish-*.
|
||||
runs-on: publish
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
# Rule 9 fix: keep the same operational kill switch surface as the
|
||||
# integrated auto-deploy workflow.
|
||||
PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
|
||||
steps:
|
||||
- name: Kill-switch guard
|
||||
# Rule 9 fix: exit fast if kill switch is set. No redeploy happens.
|
||||
if: env.PROD_AUTO_DEPLOY_DISABLED == 'true'
|
||||
run: |
|
||||
echo "::notice::Production auto-deploy disabled (PROD_AUTO_DEPLOY_DISABLED=true). Skipping redeploy."
|
||||
echo "To re-enable: unset the repo variable or set it to false."
|
||||
- name: Note on ECR propagation
|
||||
# ECR image manifests are consistent immediately after push — no
|
||||
# CDN cache to wait for. The old GHCR-based workflow had a 30s
|
||||
@@ -100,16 +102,21 @@ jobs:
|
||||
# tag) → used verbatim. Lets ops pin `latest` for emergency
|
||||
# rollback to last canary-verified digest, or pin a specific
|
||||
# `staging-<sha>` to roll back to a known-good build.
|
||||
# 2. Default → `staging-<short_head_sha>` for manual reruns from
|
||||
# the current default-branch SHA.
|
||||
# 2. Default → `staging-<short_head_sha>`. The just-published
|
||||
# digest. Bypasses the `:latest` retag path that's currently
|
||||
# dead (staging-verify soft-skips without canary fleet, so
|
||||
# the only thing retagging `:latest` today is the manual
|
||||
# promote-latest.yml — last run 2026-04-28). Auto-trigger
|
||||
# from workflow_run uses workflow_run.head_sha; manual
|
||||
# dispatch with no input falls through to github.sha.
|
||||
env:
|
||||
PROD_MANUAL_REDEPLOY_TARGET_TAG: ${{ vars.PROD_MANUAL_REDEPLOY_TARGET_TAG || secrets.PROD_MANUAL_REDEPLOY_TARGET_TAG || '' }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
INPUT_TAG: ${{ inputs.target_tag }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${PROD_MANUAL_REDEPLOY_TARGET_TAG:-}" ]; then
|
||||
echo "target_tag=$PROD_MANUAL_REDEPLOY_TARGET_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag from PROD_MANUAL_REDEPLOY_TARGET_TAG."
|
||||
if [ -n "${INPUT_TAG:-}" ]; then
|
||||
echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag: $INPUT_TAG"
|
||||
else
|
||||
SHORT="${HEAD_SHA:0:7}"
|
||||
echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
|
||||
@@ -125,37 +132,19 @@ jobs:
|
||||
CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
CANARY_SLUG: ${{ vars.PROD_REDEPLOY_CANARY_SLUG || secrets.PROD_REDEPLOY_CANARY_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.PROD_REDEPLOY_SOAK_SECONDS || secrets.PROD_REDEPLOY_SOAK_SECONDS || '' }}
|
||||
BATCH_SIZE: ${{ vars.PROD_REDEPLOY_BATCH_SIZE || secrets.PROD_REDEPLOY_BATCH_SIZE || '' }}
|
||||
DRY_RUN: ${{ vars.PROD_REDEPLOY_DRY_RUN || secrets.PROD_REDEPLOY_DRY_RUN || '' }}
|
||||
PROD_AUTO_DEPLOY_DISABLED: ${{ vars.PROD_AUTO_DEPLOY_DISABLED || secrets.PROD_AUTO_DEPLOY_DISABLED || '' }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
case "${PROD_AUTO_DEPLOY_DISABLED,,}" in
|
||||
1|true|yes|on)
|
||||
echo "::notice::PROD_AUTO_DEPLOY_DISABLED is set; skipping production redeploy."
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
CANARY_SLUG="${CANARY_SLUG:-hongming}"
|
||||
SOAK_SECONDS="${SOAK_SECONDS:-60}"
|
||||
BATCH_SIZE="${BATCH_SIZE:-3}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. This caller redeploys
|
||||
# the entire prod fleet (canary + fan-out), no slug scoping, so
|
||||
# confirm:true is correct.
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
@@ -167,12 +156,11 @@ jobs:
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry,
|
||||
confirm: true
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag=$TARGET_TAG canary=$CANARY_SLUG soak_seconds=$SOAK_SECONDS batch_size=$BATCH_SIZE dry_run=$DRY_RUN"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
@@ -200,9 +188,7 @@ jobs:
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
# Rule 8 fix: redact raw CP response from CI logs. Print only
|
||||
# safe fields: ok boolean, result count, error presence (no content).
|
||||
jq '{ok, result_count: (.results | length), has_errors: (.results | any(.error != null))}' "$HTTP_RESPONSE" || echo "(jq parse failed)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
# Pretty-print per-tenant results in the job summary so
|
||||
# ops can see which tenants were redeployed without drilling
|
||||
@@ -218,11 +204,9 @@ jobs:
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Errors |'
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
# Rule 8 fix: .error field redacted from CI logs/summary. Print only
|
||||
# presence boolean so ops know whether to look deeper.
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error != null) |"' "$HTTP_RESPONSE" || true
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
@@ -261,11 +245,13 @@ jobs:
|
||||
# fail the workflow, which is what `ok=true` should have
|
||||
# guaranteed all along.
|
||||
#
|
||||
# When the redeploy is triggered manually with a specific tag
|
||||
# (target_tag != "latest"), the expected SHA may not equal
|
||||
# ${{ github.sha }}.
|
||||
# When the redeploy was triggered by workflow_dispatch with a
|
||||
# specific tag (target_tag != "latest"), the expected SHA may
|
||||
# not equal ${{ github.sha }} — in that case we resolve via
|
||||
# GHCR's manifest. For workflow_run (default :latest) the
|
||||
# workflow_run.head_sha is the SHA that just published.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.sha }}
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
# Tenant subdomain template — slugs from the response are
|
||||
# appended. Production CP issues `<slug>.moleculesai.app`;
|
||||
@@ -279,10 +265,10 @@ jobs:
|
||||
if [ "$TARGET_TAG" != "latest" ] \
|
||||
&& [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
|
||||
&& [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
|
||||
# Manual redeploy with a pinned tag that isn't the head
|
||||
# workflow_dispatch with a pinned tag that isn't the head
|
||||
# SHA — operator is rolling back / pinning. Skip the
|
||||
# verification because we don't have the expected SHA in
|
||||
# this context (would need to inspect the ECR
|
||||
# this context (would need to crane-inspect the GHCR
|
||||
# manifest, which is a follow-up). Failing-open here is
|
||||
# safe: the operator chose the tag deliberately.
|
||||
#
|
||||
|
||||
@@ -9,13 +9,12 @@ name: redeploy-tenants-on-staging
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - ~~**Gitea workflow_run trigger limitation**~~ FIXED: replaced with
|
||||
# push+paths filter per this PR. Gitea 1.22.6 does not support
|
||||
# `workflow_run` (task #81). The push trigger fires on every
|
||||
# commit to publish-workspace-server-image.yml which is the
|
||||
# same signal (only successful runs commit to main). Removed
|
||||
# `workflow_run.conclusion==success` job if since push implies
|
||||
# the workflow completed and committed.
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on .gitea/workflows/publish-workspace-server-image.yml. Until
|
||||
# then continue-on-error+dead-workflow doesn't break anything.
|
||||
#
|
||||
|
||||
# Auto-refresh staging tenant EC2s after every staging-branch merge.
|
||||
@@ -51,11 +50,10 @@ name: redeploy-tenants-on-staging
|
||||
# of a known-good build.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [staging]
|
||||
paths:
|
||||
- '.gitea/workflows/publish-workspace-server-image.yml'
|
||||
workflow_dispatch:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
@@ -73,14 +71,15 @@ env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: post-merge staging redeploy side effect; CI / all-required gates source changes.
|
||||
redeploy:
|
||||
# Dedicated publish/release lane (internal#462 / #394 / #399).
|
||||
# Post-merge staging redeploy — a deploy action, reserved capacity.
|
||||
# `publish` -> molecule-runner-publish-* sub-pool.
|
||||
runs-on: publish
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
@@ -123,11 +122,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. Staging IS the
|
||||
# canary, no slug scoping; this rolls the entire staging fleet,
|
||||
# so confirm:true is correct.
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
@@ -139,8 +133,7 @@ jobs:
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry,
|
||||
confirm: true
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
name: review-check-tests
|
||||
|
||||
# Runs review-check.sh regression tests on every PR + push that touches
|
||||
# the evaluator script or its test fixtures.
|
||||
#
|
||||
# Follows RFC#324 follow-up (issue #540):
|
||||
# .gitea/scripts/review-check.sh is load-bearing for PR merge gates.
|
||||
# It has ZERO production CI coverage. This workflow closes that gap.
|
||||
#
|
||||
# Design choices:
|
||||
# - Bash test harness (not bats). The existing test_review_check.sh
|
||||
# uses a custom assert_eq/assert_contains framework that is already
|
||||
# working and covers all 13 acceptance criteria (issue #540 §Acceptance).
|
||||
# Converting to bats would be refactoring, not closing the gap.
|
||||
# - No bats dependency: the runner-base image needs no extra tooling.
|
||||
# - continue-on-error: false — these tests must pass; a failure means
|
||||
# the review-gate evaluator is broken and must not be merged.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/scripts/review-check.sh'
|
||||
- '.gitea/scripts/tests/test_review_check.sh'
|
||||
- '.gitea/scripts/tests/_review_check_fixture.py'
|
||||
- '.gitea/workflows/review-check-tests.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- '.gitea/scripts/review-check.sh'
|
||||
- '.gitea/scripts/tests/test_review_check.sh'
|
||||
- '.gitea/scripts/tests/_review_check_fixture.py'
|
||||
- '.gitea/workflows/review-check-tests.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# bp-exempt: review tooling regression suite; CI / all-required is the required aggregate.
|
||||
test:
|
||||
name: review-check.sh regression tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install jq
|
||||
# Required for T12 jq-filter test case. Gitea Actions runners (ubuntu-latest
|
||||
# label) do not bundle jq. Install via apt-get first (reliable for Ubuntu
|
||||
# runners with internet access to package mirrors). Falls back to GitHub
|
||||
# binary download. GitHub releases may be blocked on some runner networks
|
||||
# (infra#241 follow-up).
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
if apt-get update -qq && apt-get install -y -qq jq; then
|
||||
echo "::notice::jq installed via apt-get: $(jq --version)"
|
||||
elif timeout 120 curl -sSL \
|
||||
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
|
||||
-o /usr/local/bin/jq && chmod +x /usr/local/bin/jq; then
|
||||
echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
|
||||
else
|
||||
echo "::warning::jq install failed — apt-get and GitHub download both failed."
|
||||
fi
|
||||
jq --version 2>/dev/null || echo "::notice::jq not yet available — continuing"
|
||||
|
||||
- name: Run review-check.sh regression suite
|
||||
run: bash .gitea/scripts/tests/test_review_check.sh
|
||||
@@ -1,39 +0,0 @@
|
||||
# DEPRECATED — superseded by `.gitea/workflows/sop-checklist.yml`.
|
||||
#
|
||||
# The review-refire logic (qa/security/tier slash-command dispatch) has been
|
||||
# merged into sop-checklist.yml as the `review-refire` job. This workflow
|
||||
# is kept as a no-op stub to avoid a gap during the transition window where
|
||||
# this file may be deleted while sop-checklist.yml has not yet been merged.
|
||||
#
|
||||
# After sop-checklist.yml lands, this file will be deleted (issue #1280).
|
||||
#
|
||||
# Historical behavior (superseded):
|
||||
# Gitea 1.22 queues one run per workflow subscribed to `issue_comment` before
|
||||
# evaluating job-level `if:`. Previously this workflow was the single
|
||||
# non-SOP comment subscriber for qa/security/tier refire slash commands.
|
||||
|
||||
name: review-refire-comments
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.event.issue.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# No-op stub — all refire logic moved to sop-checklist.yml review-refire job.
|
||||
# Kept to avoid transition gap; will be deleted after sop-checklist.yml merges.
|
||||
dispatch:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deprecated — refire logic moved to sop-checklist.yml
|
||||
run: |
|
||||
echo "::warning::review-refire-comments.yml is deprecated. Refire logic is now in sop-checklist.yml review-refire job. This workflow is a no-op stub pending deletion (issue #1280)."
|
||||
exit 0
|
||||
@@ -0,0 +1,100 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and
|
||||
# `workflow_dispatch:` (no inputs, but the trigger itself is
|
||||
# parser-rejected when inputs are absent in some Gitea 1.22.x
|
||||
# builds; safest to drop entirely — manual runs go via cron-trigger
|
||||
# bump or push-with-paths-filter).
|
||||
# - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -0,0 +1,139 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
|
||||
# (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
|
||||
# finicky without them).
|
||||
# - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
|
||||
# pattern for ci.yml port).
|
||||
# - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on every job (RFC §1 contract).
|
||||
#
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy -> all crash on first boot with ImportError.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
# event_name + sha keeps PR sync and the subsequent staging push on the
|
||||
# same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
run: |
|
||||
# Inline replacement for dorny/paths-filter — same pattern
|
||||
# PR#372's ci.yml port used. Diffs against the PR base or the
|
||||
# previous push SHA, then matches against the wheel-relevant
|
||||
# path set.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
# New branch or no previous SHA: treat as wheel-relevant.
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`.
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -57,7 +57,6 @@ jobs:
|
||||
name: Detect SECRET_PATTERNS drift
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
|
||||
@@ -30,11 +30,6 @@ jobs:
|
||||
scan:
|
||||
name: Scan diff for credential-shaped strings
|
||||
runs-on: ubuntu-latest
|
||||
# Hard CI gate — must complete or the PR is unmergable. 10-minute ceiling
|
||||
# is generous for a diff-scan against a single SHA. If this times out, the
|
||||
# runner is frozen and holding a slot — the step timeout triggers clean
|
||||
# failure, releasing the runner for the next job.
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
@@ -138,14 +133,6 @@ jobs:
|
||||
[ -z "$f" ] && continue
|
||||
[ "$f" = "$SELF_GITHUB" ] && continue
|
||||
[ "$f" = "$SELF_GITEA" ] && continue
|
||||
# Test-fixture exclude (internal#425): the secrets-detector's OWN
|
||||
# unit-test corpus deliberately embeds credential-SHAPED example
|
||||
# strings to exercise the detector. Verified 2026-05-18 synthetic
|
||||
# (fabricated ghp_* fixtures, not real). Without this the scanner
|
||||
# self-trips on its own fixtures and fail-closes every deploy.
|
||||
# Same rationale as the SELF_* excludes above; gate NOT weakened
|
||||
# (all other paths still fully scanned).
|
||||
[ "$f" = "workspace-server/internal/secrets/patterns_test.go" ] && continue
|
||||
if [ -n "$DIFF_RANGE" ]; then
|
||||
ADDED=$(git diff --no-color --unified=0 "$BASE" "$HEAD" -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
|
||||
else
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
# security-review — non-author APPROVE from the `security` Gitea team
|
||||
# required to merge.
|
||||
#
|
||||
# RFC#324 Step 1 of 5 (workflow-add). Mirror of `qa-review.yml`; differs
|
||||
# only in TEAM=security, TEAM_ID=21, and the slash-command name.
|
||||
#
|
||||
# See `qa-review.yml` header for the full A1-α / A1.1 / A4 / A5 design
|
||||
# rationale; everything below is identical in shape.
|
||||
|
||||
name: security-review
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
secrets: read
|
||||
|
||||
jobs:
|
||||
# bp-exempt: PR security review bot signal; required merge state is enforced by CI / all-required.
|
||||
approved:
|
||||
# Comment-triggered refires live in review-refire-comments.yml. Keeping
|
||||
# this workflow PR-only avoids comment-triggered queue storms.
|
||||
if: |
|
||||
github.event_name == 'pull_request_target'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Privilege check (A1.1 — INFORMATIONAL log only, NOT a gate)
|
||||
# RFC#324 v1.3 §A1.1: does NOT gate subsequent steps. See
|
||||
# qa-review.yml for full rationale. Eval is read-only/idempotent
|
||||
# so re-running on a non-collaborator comment is harmless.
|
||||
if: github.event_name == 'issue_comment'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
login="${{ github.event.comment.user.login }}"
|
||||
# Write token to a mode-600 file so it never appears in curl's argv.
|
||||
# (#541: -H "Authorization: token $TOKEN" puts the secret in /proc/<pid>/cmdline)
|
||||
authfile=$(mktemp)
|
||||
chmod 600 "$authfile"
|
||||
printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN" > "$authfile"
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -K "$authfile" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/collaborators/${login}")
|
||||
rm -f "$authfile"
|
||||
if [ "$code" = "204" ]; then
|
||||
echo "::notice::Recheck from ${login} (collaborator=true)"
|
||||
else
|
||||
echo "::notice::Recheck from ${login} (collaborator=false, HTTP ${code}) — proceeding with read-only eval anyway"
|
||||
fi
|
||||
|
||||
- name: Check out BASE ref (A4 — never PR-head)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Evaluate security-review
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: security
|
||||
TEAM_ID: '21'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: bash .gitea/scripts/review-check.sh
|
||||
@@ -1,225 +0,0 @@
|
||||
# sop-checklist — peer-ack merge gate for SOP-checklist items.
|
||||
#
|
||||
# RFC#351 Step 2 of 6 (implementation MVP).
|
||||
#
|
||||
# === CONSOLIDATION (issue #1280) ===
|
||||
#
|
||||
# This workflow is the SINGLE `issue_comment` subscriber — the logic from
|
||||
# `review-refire-comments.yml` has been merged in. Before this change:
|
||||
# - sop-checklist.yml (pre-2026-05-16) → issue_comment:[created,edited,deleted] → runner slot used, job no-oped
|
||||
# - review-refire-comments.yml → issue_comment:[created] → runner slot used, job no-oped
|
||||
# → every non-refire comment occupied 2 runner slots for ~800 s each
|
||||
# (~650 no-op runs/day, ~1,300 runner-slot-occupancy-hours/day).
|
||||
#
|
||||
# Fix (PR #1345 / issue #1280):
|
||||
# - ONE workflow, ONE issue_comment:[created] subscription (no edited/deleted)
|
||||
# - all-items-acked job: pull_request_target OR sop slash-command comments
|
||||
# - review-refire job: qa/security/tier refire slash commands
|
||||
# → ~50% reduction in comment-triggered runner occupancy vs pre-fix.
|
||||
#
|
||||
# Trust boundary (mirrors RFC#324 §A4 + sop-tier-check security note):
|
||||
# `pull_request_target` (not `pull_request`) — workflow def is loaded
|
||||
# from BASE branch, so a PR cannot rewrite this workflow to exfiltrate
|
||||
# the token. The `actions/checkout` step pins `ref: base.sha` so the
|
||||
# script ALSO comes from BASE. PR-HEAD code is never executed in the
|
||||
# runner.
|
||||
#
|
||||
# Token scope:
|
||||
# - read:repository, read:organization for PR + comments + team probes
|
||||
# - write:repository for POST /statuses/{sha}
|
||||
# - The token owner MUST be a member of every team referenced by the
|
||||
# config's required_teams (else /teams/{id}/members/{login} returns
|
||||
# 403 — see review-check.sh same-gotcha doc). For the MVP we use
|
||||
# the dev-lead token (a member of engineers, managers, qa, security)
|
||||
# via a repo secret `SOP_CHECKLIST_GATE_TOKEN`. Provisioning of that
|
||||
# secret is a follow-up authorization step (separate from this PR).
|
||||
#
|
||||
# Failure mode: tier-aware (RFC#351 open question 2):
|
||||
# - tier:high → state=failure (hard-fail; BP blocks merge)
|
||||
# - tier:medium → state=failure (hard-fail; same)
|
||||
# - tier:low → state=pending (soft-fail; BP can choose to require
|
||||
# this context or skip for low-tier PRs)
|
||||
# - missing/no-tier → state=failure (default-mode: hard — never lower
|
||||
# the bar per feedback_fix_root_not_symptom)
|
||||
#
|
||||
# Slash-command contract (RFC#351 v1 + §A1.1-style notes from RFC#324):
|
||||
#
|
||||
# /sop-ack <slug-or-numeric-alias> [optional note]
|
||||
# — register a peer-ack for one checklist item.
|
||||
# — slug accepts kebab-case, snake_case, or natural-spaces
|
||||
# (all normalized to canonical kebab-case).
|
||||
# — numeric 1..7 maps via config.items[*].numeric_alias.
|
||||
# — most-recent (user, slug) directive wins.
|
||||
#
|
||||
# /sop-revoke <slug-or-numeric-alias> [reason]
|
||||
# — invalidate the commenter's own prior /sop-ack for this slug.
|
||||
# — does NOT affect other peers' acks on the same slug.
|
||||
# — most-recent (user, slug) directive wins, so a later /sop-ack
|
||||
# re-restores the ack.
|
||||
#
|
||||
# /sop-n/a <gate> [reason]
|
||||
# — declare a gate (qa-review, security-review) N/A.
|
||||
# — see sop-checklist-config.yaml n/a_gates section.
|
||||
#
|
||||
# /qa-recheck /security-recheck /refire-tier-check
|
||||
# — refire the corresponding status check on the PR head.
|
||||
#
|
||||
# The eval is read-only + idempotent (read PR + comments + team
|
||||
# membership, compute, post status). Re-running on any event is safe —
|
||||
# the new status overwrites the previous one for the same context.
|
||||
|
||||
name: sop-checklist
|
||||
|
||||
# Cancel any in-progress runs for the same PR to prevent
|
||||
# stale runs from overwriting newer status contexts.
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# bp-required: yes ← emits sop-checklist / all-items-acked (pull_request)
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, synchronize, reopened, labeled, unlabeled]
|
||||
issue_comment:
|
||||
types: [created] # NOT [created, edited, deleted] — Gitea 1.22.6 holds a runner slot
|
||||
# at job-parsing time, before job-level if: guards run. edited/deleted events
|
||||
# occupied ~1,300 runner-slot-hours/day on this workflow alone during the
|
||||
# 2026-05-16 freeze. Per PR #1345 fix.
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
secrets: read
|
||||
|
||||
jobs:
|
||||
# sop-checklist gate: runs on PR lifecycle events OR sop slash commands.
|
||||
# All other comment types (no-op text comments) no longer assign a runner
|
||||
# because this job's if: guard short-circuits before runner assignment.
|
||||
all-items-acked:
|
||||
if: |
|
||||
github.event_name == 'pull_request_target' ||
|
||||
(github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null &&
|
||||
(contains(github.event.comment.body, '/sop-ack') ||
|
||||
contains(github.event.comment.body, '/sop-revoke') ||
|
||||
contains(github.event.comment.body, '/sop-n/a')))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out BASE ref (trust boundary — never PR-head)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# For pull_request_target, the default branch is the trust
|
||||
# anchor. For issue_comment the PR base may differ from the
|
||||
# default branch (PR targeting `staging`), so we use the
|
||||
# default-branch ref explicitly — same approach as
|
||||
# qa-review.yml so the script source is always trusted.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Run sop-checklist
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_CHECKLIST_GATE_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO_NAME: ${{ github.event.repository.name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .gitea/scripts/sop-checklist.py \
|
||||
--owner "$OWNER" \
|
||||
--repo "$REPO_NAME" \
|
||||
--pr "$PR_NUMBER" \
|
||||
--config .gitea/sop-checklist-config.yaml \
|
||||
--gitea-host git.moleculesai.app
|
||||
|
||||
# bp-exempt: informational refire handler, not a merge gate. Emits
|
||||
# qa-review/security-review status updates on /qa-recheck et al slash commands.
|
||||
review-refire:
|
||||
if: |
|
||||
github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Classify comment
|
||||
id: classify
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo "run_qa=false"
|
||||
echo "run_security=false"
|
||||
echo "run_tier=false"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
first_line=$(printf '%s\n' "$COMMENT_BODY" | sed -n '1p')
|
||||
case "$first_line" in
|
||||
/qa-recheck*)
|
||||
echo "run_qa=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
/security-recheck*)
|
||||
echo "run_security=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
/refire-tier-check*)
|
||||
echo "run_tier=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
echo "::notice::no supported review refire slash command; no-op"
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Check out BASE ref for trusted scripts
|
||||
if: |
|
||||
steps.classify.outputs.run_qa == 'true' ||
|
||||
steps.classify.outputs.run_security == 'true' ||
|
||||
steps.classify.outputs.run_tier == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Refire qa-review status
|
||||
if: steps.classify.outputs.run_qa == 'true'
|
||||
env:
|
||||
# RFC_324_TEAM_READ_TOKEN is read-only (team membership read scope only).
|
||||
# review-refire-status.sh POSTs to /statuses — requires write scope.
|
||||
# SOP_TIER_CHECK_TOKEN carries write:repository + write:issue + read:organization.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: qa
|
||||
TEAM_ID: '20'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
.gitea/scripts/review-refire-status.sh
|
||||
|
||||
- name: Refire security-review status
|
||||
if: steps.classify.outputs.run_security == 'true'
|
||||
env:
|
||||
# RFC_324_TEAM_READ_TOKEN is read-only (team membership read scope only).
|
||||
# review-refire-status.sh POSTs to /statuses — requires write scope.
|
||||
# SOP_TIER_CHECK_TOKEN carries write:repository + write:issue + read:organization.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
TEAM: security
|
||||
TEAM_ID: '21'
|
||||
REVIEW_CHECK_DEBUG: '0'
|
||||
REVIEW_CHECK_STRICT: '0'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
.gitea/scripts/review-refire-status.sh
|
||||
|
||||
- name: Refire sop-tier-check status
|
||||
if: steps.classify.outputs.run_tier == 'true'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
SOP_DEBUG: '0'
|
||||
run: bash .gitea/scripts/sop-tier-refire.sh
|
||||
@@ -28,16 +28,15 @@
|
||||
#
|
||||
# Environment variables:
|
||||
# SOP_DEBUG=1 — per-API-call diagnostic lines. Default: off.
|
||||
# SOP_LEGACY_CHECK=1 — revert to OR-gate for this run. Intended for
|
||||
# emergency use only; burn-in window closed
|
||||
# 2026-05-17 (internal#189 Phase 1).
|
||||
# SOP_LEGACY_CHECK=1 — revert to OR-gate for this run. Grace window
|
||||
# for PRs in-flight when AND-composition deployed.
|
||||
# Burn-in: remove after 2026-05-17 (7-day window).
|
||||
#
|
||||
# BURN-IN CLOSED 2026-05-17 (internal#189 Phase 1): The 7-day burn-in
|
||||
# window closed. continue-on-error: true has been removed from the
|
||||
# tier-check job; AND-composition is now fully enforced. If you need
|
||||
# to temporarily re-introduce a mask, file a tracker and follow the
|
||||
# mc#774 protocol (Tier 2e lint requires a current tracker within
|
||||
# 2 lines of any continue-on-error: true).
|
||||
# BURN-IN NOTE (internal#189 Phase 1): continue-on-error: true is set on
|
||||
# the tier-check job below. This prevents AND-composition from blocking
|
||||
# PRs during the 7-day burn-in. After 2026-05-17:
|
||||
# 1. Remove `continue-on-error: true` from this job block.
|
||||
# 2. Update this BURN-IN NOTE comment to mark the window closed.
|
||||
|
||||
name: sop-tier-check
|
||||
|
||||
@@ -61,17 +60,15 @@ on:
|
||||
pull_request_review:
|
||||
types: [submitted, dismissed, edited]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.repository }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
tier-check:
|
||||
runs-on: ubuntu-latest
|
||||
# BURN-IN: continue-on-error prevents AND-composition from blocking
|
||||
# PRs during the 7-day window. Remove after 2026-05-17 (internal#189).
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
secrets: read
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -92,7 +89,6 @@ jobs:
|
||||
# runners). The sop-tier-check script has its own fallback as a
|
||||
# third line of defense. continue-on-error: true ensures this step
|
||||
# failing does not block the job.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# apt-get is the primary method — Ubuntu package mirrors are reliably
|
||||
@@ -113,7 +109,6 @@ jobs:
|
||||
# continue-on-error: true at step level — job-level is ignored by Gitea
|
||||
# Actions (quirk #10, internal runbooks). Belt-and-suspenders with
|
||||
# SOP_FAIL_OPEN=1 + || true below.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# sop-tier-refire — manual fallback for sop-tier-check refire.
|
||||
# sop-tier-refire — issue_comment-triggered refire of sop-tier-check.
|
||||
#
|
||||
# Closes internal#292. Gitea 1.22.6 doesn't refire workflows on the
|
||||
# `pull_request_review` event (go-gitea/gitea#33700); the `sop-tier-check`
|
||||
@@ -8,12 +8,12 @@
|
||||
# to merge is the admin force-merge path (audited via `audit-force-merge`
|
||||
# but the audit trail keeps growing; see `feedback_never_admin_merge_bypass`).
|
||||
#
|
||||
# Comment-triggered refires now live in `review-refire-comments.yml`. Gitea
|
||||
# queues issue_comment workflows before evaluating job-level `if:`, so having
|
||||
# qa-review, security-review, sop-checklist, and sop-tier-refire all subscribe
|
||||
# to every comment caused queue storms on SOP-heavy PRs. This workflow is a
|
||||
# non-automatic breadcrumb only; Gitea 1.22.6 does not support
|
||||
# workflow_dispatch inputs, so real refires must use `/refire-tier-check`.
|
||||
# Workaround pattern from `feedback_pull_request_review_no_refire`:
|
||||
# `issue_comment` events DO fire reliably on 1.22.6. When a repo
|
||||
# MEMBER/OWNER/COLLABORATOR comments `/refire-tier-check` on a PR, this
|
||||
# workflow re-runs the sop-tier-check logic and POSTs the resulting
|
||||
# status to the PR head SHA directly. No empty commit, no git history
|
||||
# bloat, no cascade re-fire of every other workflow on the PR.
|
||||
#
|
||||
# SECURITY MODEL:
|
||||
#
|
||||
@@ -37,16 +37,43 @@
|
||||
# Rate-limit: a 1s pre-sleep + a "skip if status posted in last 30s"
|
||||
# guard prevents comment-spam from thrashing the status. See the script.
|
||||
|
||||
name: sop-tier-check refire (manual)
|
||||
name: sop-tier-check refire (issue_comment)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
refire:
|
||||
# Three gates, all required:
|
||||
# - comment is on a PR (not a plain issue)
|
||||
# - commenter is MEMBER, OWNER, or COLLABORATOR
|
||||
# - comment body contains the slash-command trigger
|
||||
if: |
|
||||
github.event.issue.pull_request != null &&
|
||||
contains(fromJson('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) &&
|
||||
contains(github.event.comment.body, '/refire-tier-check')
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
statuses: write
|
||||
steps:
|
||||
- name: Explain supported refire path
|
||||
run: |
|
||||
echo "::error::Gitea 1.22.6 does not support workflow_dispatch inputs here; comment /refire-tier-check on the PR instead."
|
||||
exit 1
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Load the script from the default branch (main), matching the
|
||||
# sop-tier-check.yml security model.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
- name: Re-evaluate sop-tier-check and POST status
|
||||
env:
|
||||
# Same org-level secret sop-tier-check.yml + audit-force-merge.yml use.
|
||||
# Fallback to GITHUB_TOKEN with a clear error if missing.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.issue.number }}
|
||||
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
|
||||
# Set to '1' for diagnostic per-API-call output. Off by default.
|
||||
SOP_DEBUG: '0'
|
||||
run: bash .gitea/scripts/sop-tier-refire.sh
|
||||
|
||||
@@ -81,11 +81,6 @@ jobs:
|
||||
# (dead in org secret store) to CP_STAGING_ADMIN_API_TOKEN per
|
||||
# internal#322 — see this PR for the cross-workflow sweep.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
E2E_AWS_LEAK_CHECK: required
|
||||
E2E_AWS_TERMINATE_LEAKS: '1'
|
||||
# MiniMax is the smoke's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
@@ -112,9 +107,9 @@ jobs:
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the smoke to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). MiniMax-M2 is the
|
||||
# stable staging MiniMax path used by the full-SaaS smoke.
|
||||
E2E_MODEL_SLUG: MiniMax-M2
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "smoke-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the smoke script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
@@ -134,12 +129,6 @@ jobs:
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
echo "::error::$var not set — EC2 leak verification cannot run"
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
|
||||
@@ -11,14 +11,11 @@ name: Staging verify
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - ~~**Gitea workflow_run trigger limitation**~~ FIXED: replaced with
|
||||
# push+paths filter per this PR. Gitea 1.22.6 does not support
|
||||
# `workflow_run` (task #81). The push trigger fires on every
|
||||
# commit to publish-workspace-server-image.yml. Removed the
|
||||
# `workflow_run.conclusion==success` job if since the push trigger
|
||||
# doesn't carry completion state — the smoke test is the safety net
|
||||
# (it will detect and abort on a bad image regardless). Added
|
||||
# workflow_dispatch for manual runs.
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on the same publish workflow's path (i.e. `.gitea/workflows/publish-workspace-server-image.yml`).
|
||||
#
|
||||
|
||||
# Runs the canary smoke suite against the staging canary tenant fleet
|
||||
@@ -62,11 +59,9 @@ name: Staging verify
|
||||
# are populated.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [staging]
|
||||
paths:
|
||||
- '.gitea/workflows/publish-workspace-server-image.yml'
|
||||
workflow_dispatch:
|
||||
workflow_run:
|
||||
workflows: ["publish-workspace-server-image"]
|
||||
types: [completed]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
@@ -75,22 +70,20 @@ permissions:
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
# SSOT-Instance-10 (#333): triplet sourced from org/repo var `ECR_REGISTRY` with
|
||||
# the current prod-account literal as bootstrap fallback. When the org var is set,
|
||||
# the fallback becomes dead code and switching accounts/regions is a one-line
|
||||
# change at the org level. Pattern mirrors `vars.CP_URL || 'literal'` below.
|
||||
IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ${{ vars.ECR_REGISTRY || '153263036946.dkr.ecr.us-east-2.amazonaws.com' }}/molecule-ai/platform-tenant
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# bp-exempt: post-merge staging verification side effect; CI / all-required gates merges.
|
||||
staging-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
# workflow_dispatch trigger dropped in this Gitea port; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
@@ -195,7 +188,6 @@ jobs:
|
||||
echo "assertions in the staging-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# bp-exempt: post-merge image promotion side effect; staging-smoke controls promotion.
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
@@ -212,7 +204,6 @@ jobs:
|
||||
if: ${{ needs.staging-smoke.result == 'success' && needs.staging-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
env:
|
||||
SHA: ${{ needs.staging-smoke.outputs.sha }}
|
||||
@@ -239,11 +230,6 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
# confirm:true ack required by CP /cp/admin/tenants/redeploy-fleet
|
||||
# contract (cp#228 / task #308) for fleet-wide intent. Empty body
|
||||
# / {confirm:false} / {only_slugs:[]} → 400. This caller promotes
|
||||
# the verified staging image across the entire prod fleet (canary
|
||||
# + fan-out), no slug scoping, so confirm:true is correct.
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
@@ -253,8 +239,7 @@ jobs:
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry,
|
||||
confirm: true
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
# status-reaper — Option B (compensating-status POST) for Gitea 1.22.6's
|
||||
# hardcoded `(push)` suffix on default-branch commit statuses.
|
||||
#
|
||||
# Tracking: molecule-core#? (this PR), internal#327 (sibling publish-runtime-bot),
|
||||
# internal#328 (sibling mc-drift-bot), internal#80 (upstream RFC). Sister
|
||||
# bots already deployed under the same per-persona-identity contract
|
||||
# (`feedback_per_agent_gitea_identity_default`).
|
||||
#
|
||||
# Root cause:
|
||||
# Gitea 1.22.6 emits commit-status context as
|
||||
# `<workflow_name> / <job_name> (push)`
|
||||
# for ANY workflow run on the default branch's HEAD commit, REGARDLESS
|
||||
# of the trigger event. Schedule- and workflow_dispatch-triggered runs
|
||||
# on `main` therefore appear as `(push)` failures on the latest main
|
||||
# commit, painting main red via a fake-push status. Verified on runs
|
||||
# 14525 + 14526 via Phase 1 evidence (3 sub-agents). No upstream fix
|
||||
# in 1.23-1.26.1 (sibling a6f20db1 research).
|
||||
#
|
||||
# Why a cron-driven reaper, not workflow_run:
|
||||
# Gitea 1.22.6 does NOT support `on: workflow_run` (verified via
|
||||
# modules/actions/workflows.go enumeration; sister a6f20db1). The
|
||||
# only event-shaped option that fires is cron. 5min is chosen to
|
||||
# sit BETWEEN ci-required-drift (`:17` hourly) and main-red-watchdog
|
||||
# (`:05` hourly) so the reaper sweeps red before the watchdog files
|
||||
# a `[main-red]` issue (would-be false-positive).
|
||||
#
|
||||
# What the reaper does each tick:
|
||||
# 1. Parse `.gitea/workflows/*.yml`, classify each by whether `on:`
|
||||
# contains a `push:` trigger (see script for workflow_id resolution
|
||||
# including `name:` collision and `/`-in-name fail-loud lints).
|
||||
# 2. GET combined status for main HEAD.
|
||||
# 3. For each `failure` status whose context ends ` (push)`:
|
||||
# - if workflow has push trigger: PRESERVE (real defect signal).
|
||||
# - if workflow has no push trigger: POST a compensating
|
||||
# `state=success` with the same context and a description that
|
||||
# documents the workaround.
|
||||
#
|
||||
# What it does NOT do:
|
||||
# - Mutate non-`(push)`-suffix statuses (e.g. `(pull_request)` from
|
||||
# branch_protections required-checks — verified safe 2026-05-11).
|
||||
# - Auto-revert. Same reasoning as main-red-watchdog.
|
||||
# - Cancel runs. The runs themselves stay visible in Actions UI; the
|
||||
# fix is at the commit-status surface only.
|
||||
#
|
||||
# Removal path: drop this workflow when Gitea ≥ 1.24 ships with a
|
||||
# real fix for the hardcoded-suffix bug. Audit issue (filed post-merge)
|
||||
# tracks the deletion as a follow-up sweep.
|
||||
|
||||
name: status-reaper
|
||||
|
||||
# IMPORTANT — Gitea 1.22.6 parser quirk per
|
||||
# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
|
||||
# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
|
||||
# "unknown on type" when `workflow_dispatch.inputs.X` is present.
|
||||
on:
|
||||
# Schedule moved to operator-config:
|
||||
# /etc/cron.d/molecule-core-status-reaper ->
|
||||
# /usr/local/bin/molecule-core-cron-bot.sh status-reaper
|
||||
#
|
||||
# This keeps the 5-minute compensation cadence but stops a maintenance
|
||||
# bot from consuming Gitea Actions runner slots during PR merge waves.
|
||||
workflow_dispatch:
|
||||
|
||||
# Compensating-status POST needs write on repo statuses; no other
|
||||
# write surface is touched. checkout still needs `contents: read`.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# NOTE: NO `concurrency:` block is intentional.
|
||||
# Gitea 1.22.6 doesn't honor `cancel-in-progress: false`: queued ticks
|
||||
# of the same group get cancelled-with-started=0 instead of waiting
|
||||
# (DB-verified 2026-05-12, runs 16053/16085 of status-reaper.yml).
|
||||
# The reaper's POST /statuses/{sha} is idempotent — Gitea de-dups by
|
||||
# context — so concurrent ticks are safe; accept them rather than
|
||||
# serialise via the broken mechanism.
|
||||
|
||||
jobs:
|
||||
reap:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
steps:
|
||||
- name: Check out repo at default-branch HEAD
|
||||
# BASE checkout per `feedback_pull_request_target_workflow_from_base`.
|
||||
# The script reads .gitea/workflows/*.yml from the working tree to
|
||||
# classify trigger sets; we must read main's CURRENT state, not
|
||||
# the SHA a stale schedule fired against.
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
|
||||
- name: Set up Python (PyYAML for workflow `on:` parse)
|
||||
# Pinned to 3.12 to match sibling watchdog / ci-required-drift.
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install PyYAML
|
||||
# PyYAML is needed because shell-grep on `on:` misses list/string
|
||||
# forms and nested `push: { paths: ... }`. Same install pattern
|
||||
# as ci-required-drift.yml (sub-2s install, no wheel cache).
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
|
||||
- name: Compensate operational push-suffix failures on main
|
||||
env:
|
||||
# claude-status-reaper persona token; provisioned by sibling
|
||||
# aefaac1b 2026-05-11. Owns write:repository scope to POST
|
||||
# /statuses/{sha} but NOTHING ELSE
|
||||
# (`feedback_per_agent_gitea_identity_default`).
|
||||
GITEA_TOKEN: ${{ secrets.STATUS_REAPER_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
WATCH_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
WORKFLOWS_DIR: .gitea/workflows
|
||||
STATUS_REAPER_API_RETRIES: "4"
|
||||
STATUS_REAPER_API_TIMEOUT_SEC: "20"
|
||||
STATUS_REAPER_API_RETRY_SLEEP_SEC: "2"
|
||||
run: python3 .gitea/scripts/status-reaper.py
|
||||
@@ -29,11 +29,13 @@ name: Sweep stale AWS Secrets Manager secrets
|
||||
# reconciler enumerator) is filed as a separate controlplane
|
||||
# issue. This sweeper is the immediate cost-relief stopgap.
|
||||
#
|
||||
# AWS credentials: use the dedicated Secrets Manager janitor principal.
|
||||
# Do not fall back to the molecule-cp application principal: it does
|
||||
# not need account-wide ListSecrets, and a 2026-05-12 CI failure proved
|
||||
# that using it here turns a least-privilege production credential into
|
||||
# a red scheduled janitor.
|
||||
# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
|
||||
# This is a DEDICATED principal — the production `molecule-cp` IAM
|
||||
# user lacks `secretsmanager:ListSecrets` (it only has
|
||||
# Get/Create/Update/Delete on specific resources, scoped to its
|
||||
# operational needs). The janitor needs ListSecrets across the
|
||||
# `molecule/tenant/*` prefix, which warrants a separate principal so
|
||||
# we don't broaden the prod-CP policy.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
|
||||
# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
|
||||
@@ -41,12 +43,10 @@ name: Sweep stale AWS Secrets Manager secrets
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30, offset from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45). This janitor is intentionally schedule-only
|
||||
# for deletes; manual dispatch is forced to dry-run below because Gitea
|
||||
# 1.22.6 rejects workflow_dispatch.inputs.
|
||||
# Hourly at :30 — offsets from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45) so the three janitors don't burst the
|
||||
# CP admin endpoints at the same minute.
|
||||
- cron: '30 * * * *'
|
||||
workflow_dispatch:
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
group: sweep-aws-secrets
|
||||
@@ -62,24 +62,21 @@ jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# This is a cost/leak janitor. A scheduled failure must be red so
|
||||
# operators know tenant bootstrap secrets may be leaking.
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
# Keep this literal. Gitea/act_runner 1.22.6 can mis-render
|
||||
# secret-backed expressions with `||`, which produced an invalid
|
||||
# Secrets Manager endpoint in the scheduled janitor.
|
||||
AWS_REGION: us-east-2
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_SECRETS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: 50
|
||||
GRACE_HOURS: 24
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -102,11 +99,13 @@ jobs:
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
@@ -114,25 +113,17 @@ jobs:
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry:
|
||||
# - schedule: execute (the whole point of an hourly janitor).
|
||||
# - workflow_dispatch: dry-run. Gitea 1.22.6 rejects
|
||||
# workflow_dispatch.inputs, so there is no safe manual
|
||||
# "flip it to execute" toggle in this workflow.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) remains the
|
||||
# second line of defense regardless of trigger.
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
|
||||
- name: Notify on sweep failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "::error::sweep-aws-secrets FAILED — AWS tenant bootstrap secrets may be leaking. Check missing Gitea secrets, staging/prod CP admin tokens, AWS janitor IAM permissions, or the script safety gate."
|
||||
exit 1
|
||||
|
||||
@@ -33,11 +33,6 @@ name: Sweep stale Cloudflare DNS records
|
||||
# gate halts before damage. Decision-function unit tests in
|
||||
# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
|
||||
# classifier.
|
||||
#
|
||||
# Secrets: CF_API_TOKEN, CF_ZONE_ID, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
|
||||
# are confirmed existing per issue #425 §425 audit. CP_ADMIN_API_TOKEN and
|
||||
# CP_STAGING_ADMIN_API_TOKEN are unconfirmed — if missing, the verify step
|
||||
# (schedule → hard-fail, dispatch → soft-skip) surfaces it clearly.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -71,7 +66,6 @@ jobs:
|
||||
name: Sweep CF orphans
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
|
||||
# within one cron interval instead of burning a full tick. Realistic
|
||||
|
||||
@@ -28,11 +28,6 @@ name: Sweep stale Cloudflare Tunnels
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
|
||||
# the DNS sweep's 50% because tenant-shaped tunnels are mostly
|
||||
# orphans by design) refuses to nuke past the threshold.
|
||||
#
|
||||
# Secrets: CF_API_TOKEN, CF_ACCOUNT_ID are confirmed existing per
|
||||
# issue #425 §425 audit. CP_ADMIN_API_TOKEN and CP_STAGING_ADMIN_API_TOKEN
|
||||
# are unconfirmed — if missing, the verify step (schedule → hard-fail,
|
||||
# dispatch → soft-skip) surfaces it clearly.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -55,7 +50,6 @@ jobs:
|
||||
name: Sweep CF tunnels
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
# 30 min cap. Was 5 min on the theory that the only thing that
|
||||
# could take >5min is a CF-API hang — but on 2026-05-02 a backlog
|
||||
|
||||
@@ -11,9 +11,8 @@ name: Ops Scripts Tests
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Runs the unittest suite for scripts/ on every PR + push that touches
|
||||
# anything under scripts/ or .gitea/scripts/. Kept separate from the main CI
|
||||
# so a script-only change doesn't trigger the heavier Go/Canvas/Python
|
||||
# pipelines.
|
||||
# anything under scripts/. Kept separate from the main CI so a script-only
|
||||
# change doesn't trigger the heavier Go/Canvas/Python pipelines.
|
||||
#
|
||||
# Discovery layout: tests sit alongside the code they test (see
|
||||
# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
|
||||
@@ -28,13 +27,11 @@ on:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
|
||||
env:
|
||||
@@ -49,31 +46,20 @@ jobs:
|
||||
name: Ops scripts (unittest)
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Install .gitea script test dependencies
|
||||
run: python -m pip install --quiet 'pytest==9.0.2' 'PyYAML==6.0.2'
|
||||
- name: Run scripts/ unittests, if any
|
||||
# Top-level scripts/ tests live alongside their target file. The
|
||||
# runtime packaging tests moved to molecule-ai-workspace-runtime, so
|
||||
# this pass may legitimately find no tests.
|
||||
- name: Run scripts/ unittests (build_runtime_package, ...)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: |
|
||||
set +e
|
||||
python -m unittest discover -t . -p 'test_*.py' -v
|
||||
rc=$?
|
||||
if [ "$rc" -eq 5 ]; then
|
||||
echo "No top-level scripts/ unittest files found; skipping."
|
||||
exit 0
|
||||
fi
|
||||
exit "$rc"
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
- name: Run .gitea/scripts pytest suite
|
||||
run: python -m pytest .gitea/scripts/tests -q
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
name: Weekly Platform-Go Surface
|
||||
|
||||
# Surface latent vet/test errors on main by running the full Platform-Go
|
||||
# suite on a weekly cron regardless of whether the last push touched
|
||||
# workspace-server/.
|
||||
#
|
||||
# Background: ci.yml's `platform-build` job gates real work on
|
||||
# `if: needs.changes.outputs.platform == 'true'`. When no push touches
|
||||
# workspace-server/, the skip fires and the suite never executes on main.
|
||||
# Latent vet errors and test flakes can sit for weeks undetected.
|
||||
#
|
||||
# This workflow runs the full suite (build, vet, golangci-lint, tests with
|
||||
# coverage) every Monday at 04:17 UTC. Results are posted as commit statuses
|
||||
# but continue-on-error: true means they never block anything — they're
|
||||
# purely a noise-reduction signal for when the next workspace-server push
|
||||
# lands and would otherwise trigger the first real suite run.
|
||||
#
|
||||
# Why 04:17 UTC on Monday: off-peak, before the weekly sprint cycle starts.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '17 4 * * 1' # Mondays at 04:17 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
statuses: write
|
||||
|
||||
jobs:
|
||||
weekly-platform-go:
|
||||
name: Weekly Platform-Go Surface
|
||||
runs-on: ubuntu-latest
|
||||
# continue-on-error: surface only, never block
|
||||
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: stable
|
||||
|
||||
- name: Go mod download
|
||||
run: go mod download
|
||||
|
||||
- name: Build
|
||||
run: go build ./cmd/server
|
||||
|
||||
# `go vet` is NOT `|| true`-guarded: surfacing latent vet errors on main is
|
||||
# the whole point of this workflow (issue #567 — the motivating case was a
|
||||
# `go vet` error in org_external.go that sat undetected on main for weeks).
|
||||
# A vet error here fails the step → fails the job → shows red on the weekly
|
||||
# commit. Per Gitea quirk #10 (job-level continue-on-error is ignored), that
|
||||
# red surfaces on main — which is the intended signal, not a regression.
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
||||
# golangci-lint stays `|| true`-guarded: lint is noisier (more false-
|
||||
# positives than vet) and golangci-lint may not be pre-installed on every
|
||||
# runner image — a `|| true` here keeps a missing-binary or lint-noise case
|
||||
# from masking the vet/test signal above. Tighten to match ci.yml's lint
|
||||
# gate if/when ci.yml's lint step becomes hard-failing.
|
||||
- name: golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
|
||||
- name: Tests with race detection + coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- name: Check coverage thresholds
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !(\$TOTAL < \$TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage \${TOTAL}% is below the \${TOTAL_FLOOR}% floor."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
for path in "\${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !(\$pct < 10)}" || continue
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
continue
|
||||
fi
|
||||
echo "::error::Low coverage \${pct}% on \${rel} (below 10% in critical path \${path})"
|
||||
FAILED=$((FAILED + 1))
|
||||
done < <(go tool cover -func=coverage.out | grep -v '^total:' | awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++} END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' | sort)
|
||||
done
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo "::error::\${FAILED} critical paths below 10% coverage — see above."
|
||||
exit 1
|
||||
fi
|
||||
echo "Coverage thresholds: OK"
|
||||
@@ -0,0 +1,154 @@
|
||||
name: Block internal-flavored paths
|
||||
|
||||
# Hard CI gate. Internal content (positioning, competitive briefs, sales
|
||||
# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
|
||||
# this public monorepo must never re-acquire those paths. CEO directive
|
||||
# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
|
||||
#
|
||||
# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
|
||||
# briefs into the easiest path their cwd resolves to (root /research,
|
||||
# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
|
||||
# or a stale gitignore line. This workflow is the mechanical backstop.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
# Required for GitHub merge queue: the queue's pre-merge CI run on
|
||||
# `gh-readonly-queue/...` refs needs this check to fire so the queue
|
||||
# gets a real result instead of stalling forever AWAITING_CHECKS.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Block forbidden paths
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base is github.event.pull_request.base.sha,
|
||||
# which may be many commits behind HEAD and therefore absent from the
|
||||
# shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
# For merge_group events the queue's pre-merge ref is a commit on
|
||||
# `gh-readonly-queue/...` whose parent is the queue's base_sha.
|
||||
# That parent isn't part of the queue branch's shallow clone, so
|
||||
# we fetch it explicitly. Mirrors the equivalent step in
|
||||
# secret-scan.yml (#2120) — same shallow-clone bug class.
|
||||
- name: Fetch merge_group base SHA (merge_group events only)
|
||||
if: github.event_name == 'merge_group'
|
||||
run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
|
||||
|
||||
- name: Refuse if forbidden paths appear
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# merge_group has its own base_sha/head_sha; pull_request has
|
||||
# pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
|
||||
MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Paths that must NEVER live in the public monorepo. Add to this
|
||||
# list narrowly — broader patterns belong in .gitignore so day-to-day
|
||||
# docs work isn't accidentally blocked.
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"^research/"
|
||||
"^marketing/"
|
||||
"^docs/marketing/"
|
||||
"^comment-[0-9]+\.json$"
|
||||
"^test-pmm.*\.(txt|md)$"
|
||||
"^tick-reflections.*\.(txt|md)$"
|
||||
".*-temp\.(md|txt)$"
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
merge_group)
|
||||
BASE="$MG_BASE_SHA"
|
||||
HEAD="$MG_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
# Same recovery shape as secret-scan.yml (#2120 — incident
|
||||
# 2026-04-27 06:50Z block-internal-paths exit 128 with
|
||||
# "fatal: bad object <sha>" on staging push).
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check
|
||||
# the entire tree as if every file were new. Slower but
|
||||
# correct on first push or post-fetch-failure recovery.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFFENDING=""
|
||||
for path in $CHANGED; do
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
if echo "$path" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Forbidden internal-flavored paths detected:"
|
||||
printf "$OFFENDING"
|
||||
echo ""
|
||||
echo "These paths belong in molecule-ai/internal, not this public repo."
|
||||
echo "See docs/internal-content-policy.md for canonical locations."
|
||||
echo ""
|
||||
echo "If your file is genuinely public-facing (e.g. a blog post"
|
||||
echo "ready to ship), use one of these alternatives instead:"
|
||||
echo " • Public-bound blog posts: docs/blog/<slug>.md"
|
||||
echo " • Public-bound tutorials: docs/tutorials/<slug>.md"
|
||||
echo " • Public devrel content: docs/devrel/<slug>.md"
|
||||
echo ""
|
||||
echo "If you legitimately need to add a new top-level path that"
|
||||
echo "happens to match a forbidden pattern, edit"
|
||||
echo ".github/workflows/block-internal-paths.yml and update the"
|
||||
echo "FORBIDDEN_PATTERNS list with reviewer signoff."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No forbidden paths in this change."
|
||||
@@ -0,0 +1,320 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
|
||||
# Minimum viable health check: provisions one Hermes workspace on a fresh
|
||||
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
|
||||
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
|
||||
# issue on the next green run.
|
||||
#
|
||||
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
|
||||
# but runs only on provisioning-critical pushes + nightly — this one
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
- cron: '*/30 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_on_failure:
|
||||
description: >-
|
||||
Skip teardown when the canary fails (debugging only). The
|
||||
tenant org + EC2 + CF tunnel + DNS stay alive so an operator
|
||||
can SSM into the workspace EC2 and capture docker logs of the
|
||||
failing claude-code container. REMEMBER to manually delete
|
||||
via DELETE /cp/admin/tenants/<slug> when done so the org
|
||||
doesn't accumulate cost. Only honored on workflow_dispatch;
|
||||
cron runs always tear down (we don't want unattended cron
|
||||
to leak resources).
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
# Needed to open / close the alerting issue.
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
runs-on: ubuntu-latest
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
# failure mode #1 — capturing the actual exception requires
|
||||
# docker logs from the live container.
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain. Operators only need to set ONE of these
|
||||
# secrets; we don't force a choice between them.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
# subsequent failures; auto-close on next green. Comment-on-existing
|
||||
# de-duplicates so a single open issue accumulates the streak —
|
||||
# ops sees one issue with N comments rather than N issues.
|
||||
#
|
||||
# Why no consecutive-failures threshold (e.g., wait 3 runs before
|
||||
# filing): the prior threshold check used
|
||||
# `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
|
||||
# not expose (returns 404). On Gitea Actions the threshold call
|
||||
# ALWAYS failed, breaking the entire alerting step and going days
|
||||
# silent on real regressions (38h+ chronic red on 2026-05-07/08
|
||||
# before this fix; tracked in molecule-core#129). Filing on first
|
||||
# failure is also better UX — we want to know about the first red,
|
||||
# not wait 90 min for it to "count." Real flakes get one issue +
|
||||
# a quick close-on-green; persistent reds accumulate comments.
|
||||
- name: Open issue on failure
|
||||
if: failure()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const runURL = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
// Find an existing open canary issue (stable title match).
|
||||
// If one exists, this isn't a "first failure" — comment and exit.
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary still failing. ${runURL}`,
|
||||
});
|
||||
core.info(`Commented on existing issue #${match.number}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// No open issue yet — file one on this first failure. The
|
||||
// comment-on-existing branch above means subsequent failures
|
||||
// accumulate as comments on this same issue, so we don't
|
||||
// spam new issues per run.
|
||||
const body =
|
||||
`Canary run failed at ${new Date().toISOString()}.\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`This issue auto-closes on the next green canary run. ` +
|
||||
`Consecutive failures add a comment here rather than a new issue.`;
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['canary-staging', 'bug'],
|
||||
});
|
||||
core.info('Opened canary failure issue (first red)');
|
||||
|
||||
- name: Auto-close canary issue on success
|
||||
if: success()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const { data: open } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = open.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary recovered at ${new Date().toISOString()}. Closing.`,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
state: 'closed',
|
||||
});
|
||||
core.info(`Closed recovered canary issue #${match.number}`);
|
||||
}
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
# crosses midnight still cleans up its own slug — see the
|
||||
# 2026-04-26→27 canvas-safety-net incident.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug DELETE with HTTP-code verification. The previous
|
||||
# `... >/dev/null || true` swallowed every failure, so a 5xx
|
||||
# or timeout from CP looked identical to "successfully cleaned
|
||||
# up" and the tenant kept eating ~2 vCPU until the hourly
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,255 @@
|
||||
name: canary-verify
|
||||
|
||||
# Runs the canary smoke suite against the staging canary tenant fleet
|
||||
# after a new :staging-<sha> image lands in ECR. On green, calls the
|
||||
# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
|
||||
# the prod tenant fleet's 5-minute auto-updater picks up the verified
|
||||
# digest. On red, :latest stays on the prior known-good digest and
|
||||
# prod is untouched.
|
||||
#
|
||||
# Registry note (2026-05-10): This workflow previously used GHCR
|
||||
# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
|
||||
# during the 2026-05-06 Gitea suspension migration when publish-
|
||||
# workspace-server-image.yml switched to the operator's ECR org
|
||||
# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
|
||||
# platform-tenant). The GHCR → ECR migration was never applied to
|
||||
# this file, so canary-verify was silently smoke-testing the stale
|
||||
# GHCR image while the actual staging/prod tenants ran the ECR image.
|
||||
# Result: smoke tests could not catch a broken ECR build. Fix:
|
||||
# - Wait step: reads SHA from running canary /health (tenant-
|
||||
# agnostic, works regardless of registry).
|
||||
# - Promote step: calls CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
|
||||
# No longer attempts GHCR crane ops.
|
||||
#
|
||||
# Dependencies:
|
||||
# - publish-workspace-server-image.yml publishes :staging-<sha>
|
||||
# to ECR on staging and main merges.
|
||||
# - Canary tenants are configured to pull :staging-<sha> from ECR
|
||||
# (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
|
||||
# - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
|
||||
# CANARY_CP_SHARED_SECRET are populated.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["publish-workspace-server-image"]
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
|
||||
jobs:
|
||||
canary-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
smoke_ran: ${{ steps.smoke.outputs.ran }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Compute sha
|
||||
id: compute
|
||||
run: echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Wait for canary tenants to pick up :staging-<sha>
|
||||
# Poll canary health endpoints every 30s for up to 7 min instead
|
||||
# of a fixed 6-min sleep. Exits as soon as ALL canaries report
|
||||
# the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
|
||||
# proceeding after 7 min even if not all canaries responded —
|
||||
# the smoke suite will catch any that didn't update.
|
||||
#
|
||||
# NOTE: The SHA is read from the running tenant's /health response,
|
||||
# NOT from a registry lookup. This is registry-agnostic and works
|
||||
# regardless of whether the tenant pulls from ECR, GHCR, or any
|
||||
# other registry — the canary is telling us what it's actually
|
||||
# running, which is the ground truth for smoke testing.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
|
||||
run: |
|
||||
if [ -z "$CANARY_TENANT_URLS" ]; then
|
||||
echo "No canary URLs configured — falling back to 60s wait"
|
||||
sleep 60
|
||||
exit 0
|
||||
fi
|
||||
IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
|
||||
MAX_WAIT=420 # 7 minutes
|
||||
INTERVAL=30
|
||||
ELAPSED=0
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
ALL_READY=true
|
||||
for url in "${URLS[@]}"; do
|
||||
HEALTH=$(curl -s --max-time 5 "${url}/health" 2>/dev/null || echo "{}")
|
||||
SHA=$(echo "$HEALTH" | grep -o "\"sha\":\"[^\"]*\"" | head -1 | cut -d'"' -f4)
|
||||
if [ "$SHA" != "$EXPECTED_SHA" ]; then
|
||||
ALL_READY=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
if $ALL_READY; then
|
||||
echo "All canaries running staging-${EXPECTED_SHA} after ${ELAPSED}s"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for canaries... (${ELAPSED}s / ${MAX_WAIT}s)"
|
||||
sleep $INTERVAL
|
||||
ELAPSED=$((ELAPSED + INTERVAL))
|
||||
done
|
||||
echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
|
||||
|
||||
- name: Run canary smoke suite
|
||||
id: smoke
|
||||
# Graceful-skip when no canary fleet is configured (Phase 2 not yet
|
||||
# stood up — see molecule-controlplane/docs/canary-tenants.md).
|
||||
# Sets `ran=false` on skip so promote-to-latest stays off (we don't
|
||||
# want every main merge auto-promoting without gating). Manual
|
||||
# promote-latest.yml is the release gate while canary is absent.
|
||||
# Once the fleet is real: delete the early-exit branch.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
|
||||
CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CANARY_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
|
||||
{
|
||||
echo "## ⚠️ canary-verify skipped"
|
||||
echo
|
||||
echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
|
||||
echo "Phase 2 canary fleet has not been stood up yet —"
|
||||
echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
|
||||
echo
|
||||
echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ran=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::canary-verify: skipped — no canary fleet configured"
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/canary-smoke.sh
|
||||
echo "ran=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Summary on failure
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
{
|
||||
echo "## Canary smoke FAILED"
|
||||
echo
|
||||
echo "Canary tenants rejected image \`staging-${{ steps.compute.outputs.sha }}\`."
|
||||
echo ":latest stays pinned to the prior good digest — prod is untouched."
|
||||
echo
|
||||
echo "Fix forward and merge again, or investigate the specific failed"
|
||||
echo "assertions in the canary-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
# mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
|
||||
#
|
||||
# Pre-fix history: the old GHCR promote step used `crane tag` against
|
||||
# ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
|
||||
# image.yml had already migrated to ECR on 2026-05-07 (commit
|
||||
# 10e510f5). The GHCR tags were never updated, so this step was
|
||||
# silently promoting a stale GHCR image while actual prod tenants
|
||||
# pulled from ECR. Canary smoke tests were GHCR-targeted and could
|
||||
# not catch a broken ECR build.
|
||||
needs: canary-smoke
|
||||
if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
SHA: ${{ needs.canary-smoke.outputs.sha }}
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
# CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
|
||||
# Stored at the repo level so all workflows pick it up automatically.
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
# canary_slug pin: deploy the verified :staging-<sha> to the canary
|
||||
# first (soak 120s), then fan out to the rest of the fleet.
|
||||
CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
|
||||
BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
|
||||
steps:
|
||||
- name: Check CP credentials
|
||||
run: |
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
|
||||
echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Promote verified ECR image to :latest
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
--argjson batch "${BATCH_SIZE:-3}" \
|
||||
--argjson dry false \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
|
||||
fi
|
||||
|
||||
echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag: $TARGET_TAG"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
CURL_EXIT=$?
|
||||
set -e
|
||||
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
if [ "$HTTP_CODE" -ge 400 ]; then
|
||||
echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Canary verified — :latest promoted via CP redeploy-fleet"
|
||||
echo ""
|
||||
echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
|
||||
echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
|
||||
echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "- **Batch size:** ${BATCH_SIZE:-3}"
|
||||
echo ""
|
||||
echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
|
||||
echo "The fleet's 5-minute health-check loop will pick up the update automatically."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,39 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Why a gate, not just discipline: PR #2536 pruned the manifest, but the
|
||||
# cascade list wasn't updated for ~weeks before someone (PR #2556)
|
||||
# noticed during an unrelated audit. During that window, codex never
|
||||
# rebuilt on a runtime publish. A structural gate catches the drift
|
||||
# the same day either file changes.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .github/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh
|
||||
@@ -0,0 +1,58 @@
|
||||
name: Check migration collisions
|
||||
|
||||
# Hard gate (#2341): fails a PR that adds a migration prefix already
|
||||
# claimed by the base branch or another open PR. Caught manually 2026-04-30
|
||||
# during PR #2276 rebase: 044_runtime_image_pins collided with
|
||||
# 044_platform_inbound_secret from RFC #2312. This workflow makes that
|
||||
# check automatic.
|
||||
#
|
||||
# Trigger model: pull_request only — there's no value running this on
|
||||
# pushes to staging or main (those are post-merge; the gate must fire
|
||||
# pre-merge to be useful). Path filter scopes to PRs that actually touch
|
||||
# migrations.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'scripts/ops/check_migration_collisions.py'
|
||||
- '.github/workflows/check-migration-collisions.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# gh pr list/diff need read access to other PRs
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Need history to diff against base ref
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect collisions
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: origin/${{ github.event.pull_request.base.ref }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
# gh CLI uses GH_TOKEN from env
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Ensure the named base ref exists locally. checkout@v4 with
|
||||
# fetch-depth=0 pulls full history, but the explicit fetch is
|
||||
# cheap insurance against form-of-ref differences across runs.
|
||||
#
|
||||
# IMPORTANT: do NOT pass --depth=1 here. The script below uses
|
||||
# `git diff origin/<base>...<head>` (three-dot, merge-base form),
|
||||
# which fails with "fatal: no merge base" if the base ref is
|
||||
# shallow. The auto-promote staging→main PR (#2361) was blocked
|
||||
# by exactly this for ~5h on 2026-04-30 — the depth=1 fetch
|
||||
# overwrote checkout@v4's full-history clone with a shallow tip.
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
python3 scripts/ops/check_migration_collisions.py
|
||||
@@ -0,0 +1,443 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# GitHub merge queue fires `merge_group` for the queue's pre-merge CI run.
|
||||
# Required so the queue gets a real check result instead of a false-green
|
||||
# from the absence of a triggered workflow. Safe to add unconditionally —
|
||||
# the event simply doesn't fire until the queue is enabled on the branch.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
# Cancel in-progress CI runs when a new commit arrives on the same ref.
|
||||
# This prevents stale runs from queuing behind each other. The merge_group
|
||||
# refs (refs/heads/gh-readonly-queue/...) get their own concurrency group
|
||||
# automatically because github.ref differs from the PR ref.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Detect which paths changed so downstream jobs can skip when only
|
||||
# docs/markdown files were modified.
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
platform: ${{ steps.check.outputs.platform }}
|
||||
canvas: ${{ steps.check.outputs.canvas }}
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
scripts: ${{ steps.check.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
run: |
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set by GitHub for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".github/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) is a required check on staging. Always-run + per-step
|
||||
# gating (see Canvas (Next.js) for the rationale and the failure mode
|
||||
# this avoids).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: github.com/molecule-ai/molecule-cli
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
# offenders float to the top. Does NOT fail the build; the hard
|
||||
# gate is the threshold check below. (#1823)
|
||||
run: |
|
||||
echo "=== Per-file coverage (worst first) ==="
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
# 2. Per-file floor — non-test .go files in security-critical
|
||||
# paths with coverage <10% fail the build, UNLESS the file
|
||||
# path is listed in .coverage-allowlist.txt (acknowledged
|
||||
# historical debt with a tracking issue + expiry).
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
# Security-critical paths where a 0%-coverage file is a real risk.
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
|
||||
> /tmp/perfile.txt
|
||||
|
||||
# Build allowlist — paths relative to workspace-server, one per line.
|
||||
# Lines starting with # are comments.
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
WARNED=0
|
||||
for path in "${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !($pct < 10)}" || continue
|
||||
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
WARNED=$((WARNED+1))
|
||||
else
|
||||
echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done < /tmp/perfile.txt
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED security-critical file(s) have <10% test coverage and are"
|
||||
echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
|
||||
echo "workspace provisioning — a 0% file here is the exact gap that let"
|
||||
echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
|
||||
echo " (a) add tests to raise coverage above 10%, or"
|
||||
echo " (b) add the path to .coverage-allowlist.txt with an expiry date"
|
||||
echo " and a tracking issue reference."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canvas (Next.js) — required check, always runs. See platform-build
|
||||
# comment above for the rationale.
|
||||
#
|
||||
# Supersedes the canvas-build-noop pattern attempted in PR #2321: two
|
||||
# jobs sharing `name:` doesn't actually satisfy branch protection
|
||||
# because the SKIPPED check run sibling is treated as not-passed
|
||||
# regardless of how many SUCCESS siblings it has. Verified empirically
|
||||
# on PR #2314 — mergeStateStatus stayed BLOCKED until I collapsed to
|
||||
# a single-job-with-conditional-steps shape.
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
# #1815 — wires coverage into CI so we get a baseline visible on
|
||||
# every PR. No threshold gate yet; thresholds dial in (Step 3, also
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
# Per the inline comment in vitest.config.ts: "first land
|
||||
# observability so we can see the baseline, then dial in
|
||||
# thresholds + a hard gate" — this PR ships the observability half.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
# v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
|
||||
# currently supported on GHES`. Drop this pin when Gitea ships
|
||||
# the v4 protocol (tracked: post-Gitea-1.23 followup).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: canvas-coverage-${{ github.run_id }}
|
||||
path: canvas/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# MCP Server + SDK removed from CI — now in standalone repos:
|
||||
# - github.com/molecule-ai/molecule-mcp-server (npm CI)
|
||||
# - github.com/molecule-ai/molecule-sdk-python (PyPI CI)
|
||||
|
||||
# e2e-api job moved to .github/workflows/e2e-api.yml (issue #458).
|
||||
# It now has workflow-level concurrency (cancel-in-progress: false) so
|
||||
# new pushes queue the E2E run rather than cancelling it at the run level.
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs. See
|
||||
# platform-build for the rationale.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
run: |
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
# Asserts every shell E2E test that calls `mktemp` also installs
|
||||
# an EXIT trap. Catches the /tmp-leak class — a missing trap
|
||||
# silently leaks scratch into CI runners (~10-100KB per run).
|
||||
# See tests/e2e/lint_cleanup_traps.sh for the rule + fix pattern.
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
# Pure-bash unit tests for E2E helper libs (lib/*.sh). These pin
|
||||
# behavior of dispatch logic that — when broken — silently masks as
|
||||
# "Could not resolve authentication method" only after a successful
|
||||
# tenant + workspace provision (PR #2571 incident, 2026-05-03). Add
|
||||
# new self-contained unit tests here as the lib/ directory grows;
|
||||
# tests requiring live CP/tenant credentials belong in the dedicated
|
||||
# e2e-staging-* workflows, not this job.
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed ✅ — deploy required
|
||||
|
||||
The `publish-canvas-image` workflow is now building a fresh Docker image
|
||||
(`ghcr.io/molecule-ai/canvas:latest`) in the background.
|
||||
|
||||
Once it completes (~3–5 min), apply on the host machine with:
|
||||
```bash
|
||||
cd <runner-workspace>
|
||||
git pull origin main
|
||||
docker compose pull canvas && docker compose up -d canvas
|
||||
```
|
||||
|
||||
If you need to rebuild from local source instead (e.g. testing unreleased
|
||||
changes or a new `NEXT_PUBLIC_*` URL), use:
|
||||
```bash
|
||||
docker compose build canvas && docker compose up -d canvas
|
||||
```
|
||||
BODY
|
||||
printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
|
||||
"$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
|
||||
|
||||
# Gitea has no commit-comments API (no equivalent of
|
||||
# POST /repos/{owner}/{repo}/commits/{commit_sha}/comments).
|
||||
# Write to GITHUB_STEP_SUMMARY instead — both GitHub Actions and
|
||||
# Gitea Actions render this as the workflow run's summary page,
|
||||
# which is where operators look for post-deploy action items.
|
||||
# (#75 / PR-D)
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs. See platform-build
|
||||
# for the rationale.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. Rationale (issue #2790, after
|
||||
# the PR #2766 → PR #2771 cycle): the total floor averages ~6000
|
||||
# lines, so a single MCP file could regress to ~50% with no
|
||||
# complaint as long as other modules compensate. These five
|
||||
# files handle multi-tenant routing + auth + inbox dispatch —
|
||||
# a coverage drop here is the same risk shape as a Go-side
|
||||
# workspace-server token/secrets file dropping below 10%.
|
||||
#
|
||||
# Floor 75% sits below current actuals (80-96%) so this gate is
|
||||
# strictly additive — no existing PR fails. Ratchet plan in
|
||||
# COVERAGE_FLOOR.md.
|
||||
run: |
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text. --include uses fnmatch, and
|
||||
# the leading "*" allows the file to live anywhere under the
|
||||
# workspace root (today they sit at workspace/<name>.py).
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
# Match by top-level path key (e.g. "a2a_tools.py", not
|
||||
# "builtin_tools/a2a_tools.py" — different file at 100%).
|
||||
# The keys in coverage.json are paths relative to the run
|
||||
# cwd (workspace/), so the critical-path entry sits at the
|
||||
# bare basename.
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# SDK + plugin validation moved to standalone repo:
|
||||
# github.com/molecule-ai/molecule-sdk-python
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
name: Continuous synthetic E2E (staging)
|
||||
|
||||
# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
|
||||
# regressions visible only at runtime — schema drift, deployment-pipeline
|
||||
# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
|
||||
#
|
||||
# Why this gate exists:
|
||||
# PR-time CI catches code-level regressions but not deployment-time or
|
||||
# integration-time ones. Today's empirical data:
|
||||
# • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
|
||||
# JSON-RPC parse layer between sender and receiver. Visible only
|
||||
# to a sender exercising the full path.
|
||||
# • RFC #2312 chat upload — landed on staging-branch but never
|
||||
# reached staging tenants because publish-workspace-server-image
|
||||
# was main-only. Caught by manual dogfooding hours after deploy.
|
||||
# Both would have surfaced within 15-20 min of regression if a
|
||||
# continuous synth-E2E was running.
|
||||
#
|
||||
# Cadence: every 20 min (3x/hour). The script is conservatively
|
||||
# bounded at 10 min wall-clock; even on degraded staging it should
|
||||
# finish before the next firing. cron-overlap is guarded by the
|
||||
# concurrency group below.
|
||||
#
|
||||
# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
|
||||
# Plus a fresh tenant provisioned + torn down each run (Railway +
|
||||
# AWS pennies). Negligible.
|
||||
#
|
||||
# Failure handling: when the run fails, the workflow exits non-zero
|
||||
# and GitHub's standard email/notification path fires. Operators
|
||||
# can subscribe to this workflow's failure channel for paging-grade
|
||||
# alerting.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
|
||||
# Prior history: cron was '0,20,40' (2026-05-02) — only :00
|
||||
# ever survived. Bumped to '10,30,50' (2026-05-03) on the
|
||||
# theory that further-from-:00 wins. Empirically 2026-05-04
|
||||
# that ALSO dropped to ~60 min effective cadence (only ~1
|
||||
# schedule fire per hour — see molecule-core#2726). Detection
|
||||
# latency was claimed 20 min, actual 60 min.
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
# Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
|
||||
# lanes (1-3 min away from any other cron). Even with empirically-
|
||||
# observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runtime:
|
||||
description: "Runtime to provision (claude-code = default + cheapest via MiniMax; langgraph = OpenAI-only; hermes = SDK-native path, slower)"
|
||||
required: false
|
||||
default: "claude-code"
|
||||
type: string
|
||||
model_slug:
|
||||
description: "Model id to provision the workspace with (default MiniMax-M2.7-highspeed; e.g. 'sonnet' to test direct Anthropic, 'openai/gpt-4o' for hermes)"
|
||||
required: false
|
||||
default: "MiniMax-M2.7-highspeed"
|
||||
type: string
|
||||
keep_org:
|
||||
description: "Skip teardown for post-mortem debugging (only manual dispatch — never set this for cron runs)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
# history. If you want auto-issue-on-fail, add a follow-up step that
|
||||
# uses gh issue create gated on `if: failure()`. Keeping the surface
|
||||
# minimal until that's actually wanted.
|
||||
|
||||
# Serialize so two firings can never overlap. Cron firing every 20 min
|
||||
# but scripts conservatively bounded at 10 min — overlap shouldn't
|
||||
# happen in steady state, but if a run hangs we don't want N more
|
||||
# stacking up.
|
||||
concurrency:
|
||||
group: continuous-synth-e2e
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
synth:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
# ssm-agent) runs from raw Ubuntu on every boot — none of it is
|
||||
# pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
|
||||
# across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
|
||||
# latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
|
||||
# A 12min budget leaves only ~2min for the workspace (which needs
|
||||
# ~3.5min for claude-code cold boot) on slow-apt days, blowing the
|
||||
# budget. 20min absorbs the worst tenant tail so the workspace probe
|
||||
# gets the full ~7min it needs even on a slow apt day. Real fix:
|
||||
# pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
# exhaustion class that took the canary down 2026-05-03 (#265).
|
||||
# Operators can pick langgraph / hermes via workflow_dispatch
|
||||
# when they specifically need to exercise the OpenAI or SDK-
|
||||
# native paths.
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
E2E_PROVISION_TIMEOUT_SECS: '600'
|
||||
# Slug suffix — namespaced "synth-" so these runs are
|
||||
# distinguishable from PR-driven runs in CP admin.
|
||||
E2E_RUN_ID: synth-${{ github.run_id }}
|
||||
# Forced false for cron; respected for manual dispatch
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
# tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
|
||||
# which key is present — MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so operators can dispatch with
|
||||
# E2E_RUNTIME=langgraph or =hermes and still have a working
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
run: |
|
||||
# Hard-fail on missing secret REGARDLESS of trigger. Previously
|
||||
# this step soft-skipped on workflow_dispatch via `exit 0`, but
|
||||
# `exit 0` only ends the STEP — subsequent steps still ran with
|
||||
# the empty secret, the synth script fell through to the wrong
|
||||
# SECRETS_JSON branch, and the canary failed 5 min later with a
|
||||
# confusing "Agent error (Exception)" instead of the clean
|
||||
# "secret missing" message at the top. Caught 2026-05-04 by
|
||||
# dispatched run 25296530706: claude-code + missing MINIMAX
|
||||
# silently used OpenAI keys but kept model=MiniMax-M2.7, then
|
||||
# the workspace 401'd against MiniMax once it tried to call.
|
||||
# Fix: exit 1 in both cron and dispatch paths. Operators who
|
||||
# want to verify a YAML change without setting up the secret
|
||||
# can read the verify-secrets step's stderr — the failure is
|
||||
# itself the verification signal.
|
||||
if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
# langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_KEY).
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install required tools
|
||||
run: |
|
||||
# The script depends on jq + curl (already on ubuntu-latest)
|
||||
# and python3 (likewise). Verify they're all present so we
|
||||
# fail fast on a runner image regression rather than mid-script.
|
||||
for cmd in jq curl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "::error::required tool '$cmd' not on PATH — runner image regression?"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
- name: Run synthetic E2E
|
||||
# The script handles its own teardown via EXIT trap; even on
|
||||
# failure (timeout, assertion), the org is deprovisioned and
|
||||
# leaks are reported. Exit code propagates from the script.
|
||||
run: |
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
- name: Failure summary
|
||||
# Runs only on failure. Adds a job summary so the workflow run
|
||||
# page shows a quick "what happened" instead of forcing readers
|
||||
# to scroll through script output.
|
||||
if: failure()
|
||||
run: |
|
||||
{
|
||||
echo "## Continuous synth E2E failed"
|
||||
echo ""
|
||||
echo "**Run ID:** ${{ github.run_id }}"
|
||||
echo "**Trigger:** ${{ github.event_name }}"
|
||||
echo "**Runtime:** ${E2E_RUNTIME}"
|
||||
echo "**Slug:** synth-${{ github.run_id }}"
|
||||
echo ""
|
||||
echo "### What this means"
|
||||
echo ""
|
||||
echo "Staging just regressed on a path that previously worked. Likely classes:"
|
||||
echo "- Schema mismatch between sender and receiver (#2345 class)"
|
||||
echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
|
||||
echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
|
||||
echo "- Staging-CP env var rotation"
|
||||
echo ""
|
||||
echo "### Next steps"
|
||||
echo ""
|
||||
echo "1. Check the script output above for the assertion that failed"
|
||||
echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
|
||||
echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
|
||||
echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,307 @@
|
||||
name: E2E API Smoke Test
|
||||
# Extracted from ci.yml so workflow-level concurrency can protect this job
|
||||
# from run-level cancellation (issue #458).
|
||||
#
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always FIRES on push/pull_request to staging+main. Real work is gated
|
||||
# per-step on `needs.detect-changes.outputs.api` — when paths under
|
||||
# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `E2E API Smoke Test` check, satisfying branch protection without
|
||||
# spending CI cycles. See the in-job comment on the `e2e-api` job for
|
||||
# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
|
||||
# PR #2264 incident that drove the consolidation.
|
||||
#
|
||||
# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
|
||||
# -------------------------------------------------------------------
|
||||
# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
|
||||
# Gitea act_runner runs with `container.network: host` (operator host
|
||||
# `/opt/molecule/runners/config.yaml`), which means:
|
||||
#
|
||||
# * Two concurrent runs both try to bind their `-p 15432:5432` /
|
||||
# `-p 16379:6379` host ports — the second postgres/redis FATALs
|
||||
# with `Address in use` and `docker run` returns exit 125 with
|
||||
# `Conflict. The container name "/molecule-ci-postgres" is already
|
||||
# in use by container ...`. Verified in run a7/2727 on 2026-05-07.
|
||||
# * The fixed container names `molecule-ci-postgres` / `-redis` (the
|
||||
# pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
|
||||
# `docker rm -f` at the start of the second job KILLS the first
|
||||
# job's still-running postgres/redis.
|
||||
#
|
||||
# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
|
||||
# platform-server is a Go binary on the host, not a containerised
|
||||
# step):
|
||||
#
|
||||
# 1. Unique container names per run:
|
||||
# pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
|
||||
# same run_id.
|
||||
# 2. Ephemeral host port per run (`-p 0:5432`), then read the actual
|
||||
# bound port via `docker port` and export DATABASE_URL/REDIS_URL
|
||||
# pointing at it. No fixed host-port → no port collision.
|
||||
# 3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
|
||||
# the original flake fixed in #92 and the script's still IPv6-
|
||||
# enabled.
|
||||
# 4. `if: always()` cleanup so containers don't leak when test steps
|
||||
# fail.
|
||||
#
|
||||
# Issue #94 items #2 + #3 (also fixed here):
|
||||
# * Pre-pull `alpine:latest` so the platform-server's provisioner
|
||||
# (`internal/handlers/container_files.go`) can stand up its
|
||||
# ephemeral token-write helper without a daemon.io round-trip.
|
||||
# * Create `molecule-core-net` bridge network if missing so the
|
||||
# provisioner's container.HostConfig {NetworkMode: ...} attach
|
||||
# succeeds.
|
||||
# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
|
||||
# template-registry resolution issue (ADR-002 / local-build mode) and
|
||||
# belongs in a separate change that touches workspace-server, not
|
||||
# this workflow file.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
|
||||
# same auto-promote-staging brittleness as e2e-staging-canvas — back-
|
||||
# to-back staging pushes share refs/heads/staging, so the older push's
|
||||
# queued run gets cancelled when a newer push lands. Auto-promote-
|
||||
# staging then sees `completed/cancelled` for the older SHA and stays
|
||||
# put; the newer SHA's gates may eventually save the day, but if the
|
||||
# newer push gets cancelled too, we deadlock.
|
||||
#
|
||||
# See e2e-staging-canvas.yml's identical concurrency block for the full
|
||||
# rationale and the 2026-04-28 incident reference.
|
||||
group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
api:
|
||||
- 'workspace-server/**'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/e2e-api.yml'
|
||||
- id: decide
|
||||
# Always run real work for manual dispatch — no diff context to
|
||||
# filter against and ops dispatching this expects the suite to
|
||||
# actually exercise the platform.
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=${{ steps.filter.outputs.api }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
# on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
|
||||
# check run for every job that matches `name:`, and a job-level
|
||||
# `if: false` produces a SKIPPED check run. Branch protection treats
|
||||
# all check runs with a matching context name on the latest commit as a
|
||||
# SET — any SKIPPED in the set fails the required-check eval, even with
|
||||
# SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
|
||||
# 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
|
||||
# promotion despite all real work succeeding. Collapsing to a single
|
||||
# always-running job with conditional steps emits exactly one SUCCESS
|
||||
# check run regardless of paths filter — branch-protection-clean.
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
# Unique per-run container names so concurrent runs on the host-
|
||||
# network act_runner don't collide on name OR port.
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
|
||||
# same run_id. PORT is set later (after docker port lookup) since
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
PORT: "8080"
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
|
||||
echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Provisioner uses alpine:latest for ephemeral token-write
|
||||
# containers (workspace-server/internal/handlers/container_files.go).
|
||||
# Pre-pull so the first provision in test_api.sh doesn't race
|
||||
# the daemon's pull cache. Idempotent — `docker pull` is a no-op
|
||||
# when the image is already present.
|
||||
docker pull alpine:latest >/dev/null
|
||||
# Provisioner attaches workspace containers to
|
||||
# molecule-core-net (workspace-server/internal/provisioner/
|
||||
# provisioner.go::DefaultNetwork). The bridge already exists on
|
||||
# the operator host's docker daemon — `network create` is
|
||||
# idempotent via `|| true`.
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Defensive cleanup — only matches THIS run's container name,
|
||||
# so it cannot kill a sibling run's postgres. (Pre-fix the
|
||||
# name was static and this rm hit other runs' containers.)
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
# `-p 0:5432` requests an ephemeral host port; we read it back
|
||||
# below and export DATABASE_URL.
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
# Resolve the host-side port assignment. `docker port` prints
|
||||
# `0.0.0.0:NNNN` (and on host-net runners may also print an
|
||||
# IPv6 line — take the first IPv4 line).
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
# Fallback: any first line. Some Docker versions print only
|
||||
# one line.
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker port "$PG_CONTAINER" 5432/tcp || true
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
# 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Postgres host port: ${PG_PORT}"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker port "$REDIS_CONTAINER" 6379/tcp || true
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Redis host port: ${REDIS_PORT}"
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
# DATABASE_URL + REDIS_URL exported by the start-postgres /
|
||||
# start-redis steps point at this run's per-run host ports.
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
- name: Assert migrations applied
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
|
||||
if [ "$tables" != "1" ]; then
|
||||
echo "::error::Migrations did not apply"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
- name: Run notify-with-attachments E2E
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_notify_attachments_e2e.sh
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
- name: Run poll-mode chat upload E2E (RFC #2891)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.api == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
# always() so containers don't leak when test steps fail. The
|
||||
# cleanup is best-effort: if the container is already gone
|
||||
# (e.g. concurrent rerun race), don't fail the job.
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -0,0 +1,215 @@
|
||||
name: E2E Staging Canvas (Playwright)
|
||||
|
||||
# Playwright test suite that provisions a fresh staging org per run and
|
||||
# verifies every workspace-panel tab renders without crashing. Complements
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
|
||||
on:
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always fires on push/pull_request; real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `Canvas tabs E2E` check, satisfying branch protection without
|
||||
# spending CI cycles. See e2e-api.yml for the rationale on why this
|
||||
# is a single job rather than two-jobs-sharing-name.
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
# global group made auto-promote-staging brittle: when a staging push
|
||||
# queued behind an in-flight run and a third entrant (a PR run, a
|
||||
# follow-on push) entered the group, the staging push got cancelled —
|
||||
# leaving auto-promote-staging looking at `completed/cancelled` for a
|
||||
# required gate and refusing to advance main. Observed 2026-04-28
|
||||
# 23:51-23:53 on staging tip 3f99fede.
|
||||
#
|
||||
# The original intent of the global group was to throttle parallel
|
||||
# E2E provisions (each spins a fresh EC2). At our scale that throttle
|
||||
# isn't worth the correctness cost — fresh-org-per-run isolates the
|
||||
# state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
|
||||
# is rounding error vs. the cost of a stuck pipeline.
|
||||
#
|
||||
# Per-SHA still dedupes accidental double-triggers for the SAME SHA.
|
||||
# It does NOT cancel obsolete-PR-version runs on force-push; that
|
||||
# wasted CI is acceptable given the alternative is losing staging-tip
|
||||
# data that auto-promote-staging needs.
|
||||
group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
canvas:
|
||||
- 'canvas/**'
|
||||
- '.github/workflows/e2e-staging-canvas.yml'
|
||||
- id: decide
|
||||
# Always run real tests for manual dispatch and the weekly cron —
|
||||
# both exist precisely to exercise the suite, regardless of diff.
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "canvas=${{ steps.filter.outputs.canvas }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `Canvas tabs E2E`. Real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
|
||||
# rationale — same path-filter check-name parity issue blocked PR #2264
|
||||
# (staging→main) on 2026-04-29 because branch protection treats matching-
|
||||
# name check runs as a SET, and any SKIPPED member fails the eval.
|
||||
playwright:
|
||||
needs: detect-changes
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
|
||||
env:
|
||||
CANVAS_E2E_STAGING: '1'
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: |
|
||||
echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
|
||||
echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Set up Node
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Install canvas deps
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright test --config=playwright.staging.config.ts
|
||||
|
||||
- name: Upload Playwright report on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement (see ci.yml upload step for the canonical error
|
||||
# cite). Drop this pin when Gitea ships the v4 protocol.
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-report-staging
|
||||
path: canvas/playwright-report-staging/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload screenshots on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-screenshots
|
||||
path: canvas/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Safety-net teardown — fires only when Playwright's globalTeardown
|
||||
# didn't (worker crash, runner cancel). Reads the slug from
|
||||
# canvas/.playwright-staging-state.json (written by staging-setup
|
||||
# as its first action, before any CP call) and deletes only that
|
||||
# slug.
|
||||
#
|
||||
# Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
|
||||
# orgs to compensate for setup-crash-before-state-file-write. That
|
||||
# over-aggressive cleanup raced concurrent canvas-E2E runs and
|
||||
# poisoned each other's tenants — observed 2026-04-30 when three
|
||||
# real-test runs killed each other mid-test, surfacing as
|
||||
# `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
|
||||
# DNS record. Pattern-sweep removed; setup now writes the state
|
||||
# file before any CP work, so the slug is always recoverable.
|
||||
- name: Teardown safety net
|
||||
if: always() && needs.detect-changes.outputs.canvas == 'true'
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
STATE_FILE=".playwright-staging-state.json"
|
||||
if [ ! -f "$STATE_FILE" ]; then
|
||||
echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
|
||||
exit 0
|
||||
fi
|
||||
slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
|
||||
if [ -z "$slug" ]; then
|
||||
echo "::warning::State file present but slug missing; nothing to clean up."
|
||||
exit 0
|
||||
fi
|
||||
echo "Deleting orphan tenant: $slug"
|
||||
# Verify HTTP 2xx instead of `>/dev/null || true` swallowing
|
||||
# failures. A 5xx or timeout previously looked identical to
|
||||
# success, leaving the tenant alive for up to ~45 min until
|
||||
# sweep-stale-e2e-orgs caught it. Surface failures as
|
||||
# workflow warnings naming the slug. Don't `exit 1` — a single
|
||||
# cleanup miss shouldn't fail-flag the canvas test when the
|
||||
# actual smoke check passed; the sweeper is the safety net.
|
||||
# See molecule-controlplane#420.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,184 @@
|
||||
name: E2E Staging External Runtime
|
||||
|
||||
# Regression for the four/five workspaces.status=awaiting_agent transitions
|
||||
# that silently failed in production for five days before migration 046
|
||||
# extended the workspace_status enum (see
|
||||
# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
|
||||
#
|
||||
# Why this is its own workflow (not folded into e2e-staging-saas.yml):
|
||||
# - The full-saas harness defaults to runtime=hermes, never exercises
|
||||
# external-runtime. Adding an `external` parameter to that script
|
||||
# would force every push to staging through both lifecycles in
|
||||
# series, doubling the EC2 cold-start budget.
|
||||
# - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
|
||||
# window, 90s default + sweep interval), which we wait through
|
||||
# deliberately. Folding it into hermes would make the long path
|
||||
# even longer.
|
||||
# - It can run in parallel with the hermes E2E since both create
|
||||
# fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
|
||||
# `e2e-...`).
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to staging when any source affecting external runtime,
|
||||
# hibernation, or the migration set changes.
|
||||
# - PR review for the same set.
|
||||
# - Manual workflow_dispatch.
|
||||
# - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
|
||||
# 30 min after e2e-staging-saas.yml's 07:00 UTC cron).
|
||||
#
|
||||
# Concurrency: serialized so two staging pushes don't fight for the
|
||||
# same EC2 quota window. cancel-in-progress=false so a half-rolled
|
||||
# tenant always finishes its teardown.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.github/workflows/e2e-staging-external.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.github/workflows/e2e-staging-external.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_org:
|
||||
description: "Skip teardown for debugging (only via manual dispatch)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
stale_wait_secs:
|
||||
description: "Seconds to wait for the heartbeat-staleness sweep (default 180 = 90s window + 90s buffer)"
|
||||
required: false
|
||||
default: "180"
|
||||
schedule:
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-external
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
e2e-staging-external:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
# Schedule + push triggers must hard-fail when the token is
|
||||
# missing — silent skip would mask infra rot. Manual dispatch
|
||||
# gets the same hard-fail; an operator running this on a fork
|
||||
# without secrets configured needs to know up-front.
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run external-runtime E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_external_runtime.sh
|
||||
|
||||
# Mirror the e2e-staging-saas.yml safety net: if the runner is
|
||||
# cancelled (e.g. concurrent staging push), the test script's
|
||||
# EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
|
||||
# *this* run id.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
|
||||
# so concurrent runs and unrelated dev probes are not touched.
|
||||
# Sweep today AND yesterday so a midnight-crossing run still
|
||||
# cleans up its own slug.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if not run_id:
|
||||
# Without a run id we cannot scope safely; bail rather
|
||||
# than risk deleting unrelated tenants.
|
||||
sys.exit(0)
|
||||
prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
|
||||
for o in d.get('orgs', []):
|
||||
s = o.get('slug', '')
|
||||
if s.startswith(prefixes) and o.get('status') != 'purged':
|
||||
print(s)
|
||||
" 2>/dev/null)
|
||||
if [ -n "$orgs" ]; then
|
||||
echo "Safety-net sweep: deleting leftover orgs:"
|
||||
echo "$orgs"
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# `>/dev/null 2>&1` previously hid every failure; surface
|
||||
# non-2xx as workflow warnings so the run page names what
|
||||
# leaked. Sweeper catches the rest within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
else
|
||||
echo "Safety-net sweep: no leftover orgs to clean."
|
||||
fi
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user